HWSVMR0.cpp@ 19169

Last change on this file since 19169 was 19141, checked in by vboxsync, 16 years ago
Action flags breakup. Fixed PGM saved state loading of 2.2.2 images. Reduced hacks in PATM state loading (fixups).
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 94.8 KB

Line
1	/* $Id: HWSVMR0.cpp 19141 2009-04-23 13:52:18Z vboxsync $ */
2	/** @file
3	* HWACCM SVM - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/*******************************************************************************
24	* Header Files *
25	*******************************************************************************/
26	#define LOG_GROUP LOG_GROUP_HWACCM
27	#include <VBox/hwaccm.h>
28	#include "HWACCMInternal.h"
29	#include <VBox/vm.h>
30	#include <VBox/x86.h>
31	#include <VBox/hwacc_svm.h>
32	#include <VBox/pgm.h>
33	#include <VBox/pdm.h>
34	#include <VBox/err.h>
35	#include <VBox/log.h>
36	#include <VBox/selm.h>
37	#include <VBox/iom.h>
38	#include <VBox/dis.h>
39	#include <VBox/dbgf.h>
40	#include <VBox/disopcode.h>
41	#include <iprt/param.h>
42	#include <iprt/assert.h>
43	#include <iprt/asm.h>
44	#include <iprt/cpuset.h>
45	#include <iprt/mp.h>
46	#include "HWSVMR0.h"
47
48	/*******************************************************************************
49	* Internal Functions *
50	*******************************************************************************/
51	static int SVMR0InterpretInvpg(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame, uint32_t uASID);
52
53	/*******************************************************************************
54	* Global Variables *
55	*******************************************************************************/
56	/* IO operation lookup arrays. */
57	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
58
59	/**
60	* Sets up and activates AMD-V on the current CPU
61	*
62	* @returns VBox status code.
63	* @param pCpu CPU info struct
64	* @param pVM The VM to operate on. (can be NULL after a resume!!)
65	* @param pvPageCpu Pointer to the global cpu page
66	* @param pPageCpuPhys Physical address of the global cpu page
67	*/
68	VMMR0DECL(int) SVMR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
69	{
70	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
71	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
72
73	/* We must turn on AMD-V and setup the host state physical address, as those MSRs are per-cpu/core. */
74
75	#if defined(LOG_ENABLED) && !defined(DEBUG_bird)
76	SUPR0Printf("SVMR0EnableCpu cpu %d page (%x) %x\n", pCpu->idCpu, pvPageCpu, (uint32_t)pPageCpuPhys);
77	#endif
78
79	/* Turn on AMD-V in the EFER MSR. */
80	uint64_t val = ASMRdMsr(MSR_K6_EFER);
81	if (!(val & MSR_K6_EFER_SVME))
82	ASMWrMsr(MSR_K6_EFER, val \| MSR_K6_EFER_SVME);
83
84	/* Write the physical page address where the CPU will store the host state while executing the VM. */
85	ASMWrMsr(MSR_K8_VM_HSAVE_PA, pPageCpuPhys);
86
87	return VINF_SUCCESS;
88	}
89
90	/**
91	* Deactivates AMD-V on the current CPU
92	*
93	* @returns VBox status code.
94	* @param pCpu CPU info struct
95	* @param pvPageCpu Pointer to the global cpu page
96	* @param pPageCpuPhys Physical address of the global cpu page
97	*/
98	VMMR0DECL(int) SVMR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
99	{
100	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
101	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
102
103	#if defined(LOG_ENABLED) && !defined(DEBUG_bird)
104	SUPR0Printf("SVMR0DisableCpu cpu %d\n", pCpu->idCpu);
105	#endif
106
107	/* Turn off AMD-V in the EFER MSR. */
108	uint64_t val = ASMRdMsr(MSR_K6_EFER);
109	ASMWrMsr(MSR_K6_EFER, val & ~MSR_K6_EFER_SVME);
110
111	/* Invalidate host state physical address. */
112	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
113
114	return VINF_SUCCESS;
115	}
116
117	/**
118	* Does Ring-0 per VM AMD-V init.
119	*
120	* @returns VBox status code.
121	* @param pVM The VM to operate on.
122	*/
123	VMMR0DECL(int) SVMR0InitVM(PVM pVM)
124	{
125	int rc;
126
127	pVM->hwaccm.s.svm.pMemObjVMCBHost = NIL_RTR0MEMOBJ;
128	pVM->hwaccm.s.svm.pMemObjIOBitmap = NIL_RTR0MEMOBJ;
129	pVM->hwaccm.s.svm.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
130
131	/* Allocate one page for the host context */
132	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjVMCBHost, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
133	if (RT_FAILURE(rc))
134	return rc;
135
136	pVM->hwaccm.s.svm.pVMCBHost = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjVMCBHost);
137	pVM->hwaccm.s.svm.pVMCBHostPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjVMCBHost, 0);
138	ASMMemZeroPage(pVM->hwaccm.s.svm.pVMCBHost);
139
140	/* Allocate 12 KB for the IO bitmap (doesn't seem to be a way to convince SVM not to use it) */
141	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjIOBitmap, 3 << PAGE_SHIFT, true /* executable R0 mapping */);
142	if (RT_FAILURE(rc))
143	return rc;
144
145	pVM->hwaccm.s.svm.pIOBitmap = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjIOBitmap);
146	pVM->hwaccm.s.svm.pIOBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjIOBitmap, 0);
147	/* Set all bits to intercept all IO accesses. */
148	ASMMemFill32(pVM->hwaccm.s.svm.pIOBitmap, PAGE_SIZE*3, 0xffffffff);
149
150	/* Allocate 8 KB for the MSR bitmap (doesn't seem to be a way to convince SVM not to use it) */
151	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjMSRBitmap, 2 << PAGE_SHIFT, true /* executable R0 mapping */);
152	if (RT_FAILURE(rc))
153	return rc;
154
155	pVM->hwaccm.s.svm.pMSRBitmap = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjMSRBitmap);
156	pVM->hwaccm.s.svm.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjMSRBitmap, 0);
157	/* Set all bits to intercept all MSR accesses. */
158	ASMMemFill32(pVM->hwaccm.s.svm.pMSRBitmap, PAGE_SIZE*2, 0xffffffff);
159
160	/* Erratum 170 which requires a forced TLB flush for each world switch:
161	* See http://www.amd.com/us-en/assets/content_type/white_papers_and_tech_docs/33610.pdf
162	*
163	* All BH-G1/2 and DH-G1/2 models include a fix:
164	* Athlon X2: 0x6b 1/2
165	* 0x68 1/2
166	* Athlon 64: 0x7f 1
167	* 0x6f 2
168	* Sempron: 0x7f 1/2
169	* 0x6f 2
170	* 0x6c 2
171	* 0x7c 2
172	* Turion 64: 0x68 2
173	*
174	*/
175	uint32_t u32Dummy;
176	uint32_t u32Version, u32Family, u32Model, u32Stepping, u32BaseFamily;
177	ASMCpuId(1, &u32Version, &u32Dummy, &u32Dummy, &u32Dummy);
178	u32BaseFamily= (u32Version >> 8) & 0xf;
179	u32Family = u32BaseFamily + (u32BaseFamily == 0xf ? ((u32Version >> 20) & 0x7f) : 0);
180	u32Model = ((u32Version >> 4) & 0xf);
181	u32Model = u32Model \| ((u32BaseFamily == 0xf ? (u32Version >> 16) & 0x0f : 0) << 4);
182	u32Stepping = u32Version & 0xf;
183	if ( u32Family == 0xf
184	&& !((u32Model == 0x68 \|\| u32Model == 0x6b \|\| u32Model == 0x7f) && u32Stepping >= 1)
185	&& !((u32Model == 0x6f \|\| u32Model == 0x6c \|\| u32Model == 0x7c) && u32Stepping >= 2))
186	{
187	Log(("SVMR0InitVM: AMD cpu with erratum 170 family %x model %x stepping %x\n", u32Family, u32Model, u32Stepping));
188	pVM->hwaccm.s.svm.fAlwaysFlushTLB = true;
189	}
190
191	/* Allocate VMCBs for all guest CPUs. */
192	for (unsigned i=0;i<pVM->cCPUs;i++)
193	{
194	pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB = NIL_RTR0MEMOBJ;
195
196	/* Allocate one page for the VM control block (VMCB). */
197	rc = RTR0MemObjAllocCont(&pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
198	if (RT_FAILURE(rc))
199	return rc;
200
201	pVM->aCpus[i].hwaccm.s.svm.pVMCB = RTR0MemObjAddress(pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB);
202	pVM->aCpus[i].hwaccm.s.svm.pVMCBPhys = RTR0MemObjGetPagePhysAddr(pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB, 0);
203	ASMMemZeroPage(pVM->aCpus[i].hwaccm.s.svm.pVMCB);
204	}
205
206	return VINF_SUCCESS;
207	}
208
209	/**
210	* Does Ring-0 per VM AMD-V termination.
211	*
212	* @returns VBox status code.
213	* @param pVM The VM to operate on.
214	*/
215	VMMR0DECL(int) SVMR0TermVM(PVM pVM)
216	{
217	for (unsigned i=0;i<pVM->cCPUs;i++)
218	{
219	if (pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB != NIL_RTR0MEMOBJ)
220	{
221	RTR0MemObjFree(pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB, false);
222	pVM->aCpus[i].hwaccm.s.svm.pVMCB = 0;
223	pVM->aCpus[i].hwaccm.s.svm.pVMCBPhys = 0;
224	pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB = NIL_RTR0MEMOBJ;
225	}
226	}
227	if (pVM->hwaccm.s.svm.pMemObjVMCBHost != NIL_RTR0MEMOBJ)
228	{
229	RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjVMCBHost, false);
230	pVM->hwaccm.s.svm.pVMCBHost = 0;
231	pVM->hwaccm.s.svm.pVMCBHostPhys = 0;
232	pVM->hwaccm.s.svm.pMemObjVMCBHost = NIL_RTR0MEMOBJ;
233	}
234	if (pVM->hwaccm.s.svm.pMemObjIOBitmap != NIL_RTR0MEMOBJ)
235	{
236	RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjIOBitmap, false);
237	pVM->hwaccm.s.svm.pIOBitmap = 0;
238	pVM->hwaccm.s.svm.pIOBitmapPhys = 0;
239	pVM->hwaccm.s.svm.pMemObjIOBitmap = NIL_RTR0MEMOBJ;
240	}
241	if (pVM->hwaccm.s.svm.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
242	{
243	RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjMSRBitmap, false);
244	pVM->hwaccm.s.svm.pMSRBitmap = 0;
245	pVM->hwaccm.s.svm.pMSRBitmapPhys = 0;
246	pVM->hwaccm.s.svm.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
247	}
248	return VINF_SUCCESS;
249	}
250
251	/**
252	* Sets up AMD-V for the specified VM
253	*
254	* @returns VBox status code.
255	* @param pVM The VM to operate on.
256	*/
257	VMMR0DECL(int) SVMR0SetupVM(PVM pVM)
258	{
259	int rc = VINF_SUCCESS;
260	SVM_VMCB *pVMCB;
261
262	AssertReturn(pVM, VERR_INVALID_PARAMETER);
263
264	Assert(pVM->hwaccm.s.svm.fSupported);
265
266	for (unsigned i=0;i<pVM->cCPUs;i++)
267	{
268	pVMCB = (SVM_VMCB *)pVM->aCpus[i].hwaccm.s.svm.pVMCB;
269	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
270
271	/* Program the control fields. Most of them never have to be changed again. */
272	/* CR0/3/4 reads must be intercepted, our shadow values are not necessarily the same as the guest's. */
273	/* Note: CR0 & CR4 can be safely read when guest and shadow copies are identical. */
274	if (!pVM->hwaccm.s.fNestedPaging)
275	pVMCB->ctrl.u16InterceptRdCRx = RT_BIT(0) \| RT_BIT(3) \| RT_BIT(4);
276	else
277	pVMCB->ctrl.u16InterceptRdCRx = RT_BIT(0) \| RT_BIT(4);
278
279	/*
280	* CR0/3/4 writes must be intercepted for obvious reasons.
281	*/
282	if (!pVM->hwaccm.s.fNestedPaging)
283	pVMCB->ctrl.u16InterceptWrCRx = RT_BIT(0) \| RT_BIT(3) \| RT_BIT(4);
284	else
285	pVMCB->ctrl.u16InterceptWrCRx = RT_BIT(0) \| RT_BIT(4) \| RT_BIT(8);
286
287	/* Intercept all DRx reads and writes by default. Changed later on. */
288	pVMCB->ctrl.u16InterceptRdDRx = 0xFFFF;
289	pVMCB->ctrl.u16InterceptWrDRx = 0xFFFF;
290
291	/* Currently we don't care about DRx reads or writes. DRx registers are trashed.
292	* All breakpoints are automatically cleared when the VM exits.
293	*/
294
295	pVMCB->ctrl.u32InterceptException = HWACCM_SVM_TRAP_MASK;
296	#ifndef DEBUG
297	if (pVM->hwaccm.s.fNestedPaging)
298	pVMCB->ctrl.u32InterceptException &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
299	#endif
300
301	pVMCB->ctrl.u32InterceptCtrl1 = SVM_CTRL1_INTERCEPT_INTR
302	\| SVM_CTRL1_INTERCEPT_VINTR
303	\| SVM_CTRL1_INTERCEPT_NMI
304	\| SVM_CTRL1_INTERCEPT_SMI
305	\| SVM_CTRL1_INTERCEPT_INIT
306	\| SVM_CTRL1_INTERCEPT_RDPMC
307	\| SVM_CTRL1_INTERCEPT_CPUID
308	\| SVM_CTRL1_INTERCEPT_RSM
309	\| SVM_CTRL1_INTERCEPT_HLT
310	\| SVM_CTRL1_INTERCEPT_INOUT_BITMAP
311	\| SVM_CTRL1_INTERCEPT_MSR_SHADOW
312	\| SVM_CTRL1_INTERCEPT_INVLPG
313	\| SVM_CTRL1_INTERCEPT_INVLPGA /* AMD only */
314	\| SVM_CTRL1_INTERCEPT_TASK_SWITCH
315	\| SVM_CTRL1_INTERCEPT_SHUTDOWN /* fatal */
316	\| SVM_CTRL1_INTERCEPT_FERR_FREEZE; /* Legacy FPU FERR handling. */
317	;
318	/* With nested paging we don't care about invlpg anymore. */
319	if (pVM->hwaccm.s.fNestedPaging)
320	pVMCB->ctrl.u32InterceptCtrl1 &= ~SVM_CTRL1_INTERCEPT_INVLPG;
321
322	pVMCB->ctrl.u32InterceptCtrl2 = SVM_CTRL2_INTERCEPT_VMRUN /* required */
323	\| SVM_CTRL2_INTERCEPT_VMMCALL
324	\| SVM_CTRL2_INTERCEPT_VMLOAD
325	\| SVM_CTRL2_INTERCEPT_VMSAVE
326	\| SVM_CTRL2_INTERCEPT_STGI
327	\| SVM_CTRL2_INTERCEPT_CLGI
328	\| SVM_CTRL2_INTERCEPT_SKINIT
329	\| SVM_CTRL2_INTERCEPT_WBINVD
330	\| SVM_CTRL2_INTERCEPT_MWAIT_UNCOND; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
331	;
332	Log(("pVMCB->ctrl.u32InterceptException = %x\n", pVMCB->ctrl.u32InterceptException));
333	Log(("pVMCB->ctrl.u32InterceptCtrl1 = %x\n", pVMCB->ctrl.u32InterceptCtrl1));
334	Log(("pVMCB->ctrl.u32InterceptCtrl2 = %x\n", pVMCB->ctrl.u32InterceptCtrl2));
335
336	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
337	pVMCB->ctrl.IntCtrl.n.u1VIrqMasking = 1;
338	/* Ignore the priority in the TPR; just deliver it when we tell it to. */
339	pVMCB->ctrl.IntCtrl.n.u1IgnoreTPR = 1;
340
341	/* Set IO and MSR bitmap addresses. */
342	pVMCB->ctrl.u64IOPMPhysAddr = pVM->hwaccm.s.svm.pIOBitmapPhys;
343	pVMCB->ctrl.u64MSRPMPhysAddr = pVM->hwaccm.s.svm.pMSRBitmapPhys;
344
345	/* No LBR virtualization. */
346	pVMCB->ctrl.u64LBRVirt = 0;
347
348	/** The ASID must start at 1; the host uses 0. */
349	pVMCB->ctrl.TLBCtrl.n.u32ASID = 1;
350
351	/** Setup the PAT msr (nested paging only) */
352	pVMCB->guest.u64GPAT = 0x0007040600070406ULL;
353	}
354	return rc;
355	}
356
357
358	/**
359	* Injects an event (trap or external interrupt)
360	*
361	* @param pVM The VM to operate on.
362	* @param pVMCB SVM control block
363	* @param pCtx CPU Context
364	* @param pIntInfo SVM interrupt info
365	*/
366	inline void SVMR0InjectEvent(PVM pVM, SVM_VMCB pVMCB, CPUMCTX pCtx, SVM_EVENT* pEvent)
367	{
368	#ifdef VBOX_STRICT
369	if (pEvent->n.u8Vector == 0xE)
370	Log(("SVM: Inject int %d at %RGv error code=%02x CR2=%RGv intInfo=%08x\n", pEvent->n.u8Vector, (RTGCPTR)pCtx->rip, pEvent->n.u32ErrorCode, (RTGCPTR)pCtx->cr2, pEvent->au64[0]));
371	else
372	if (pEvent->n.u8Vector < 0x20)
373	Log(("SVM: Inject int %d at %RGv error code=%08x\n", pEvent->n.u8Vector, (RTGCPTR)pCtx->rip, pEvent->n.u32ErrorCode));
374	else
375	{
376	Log(("INJ-EI: %x at %RGv\n", pEvent->n.u8Vector, (RTGCPTR)pCtx->rip));
377	Assert(!VMCPU_FF_ISSET(VMMGetCpu(pVM), VMCPU_FF_INHIBIT_INTERRUPTS));
378	Assert(pCtx->eflags.u32 & X86_EFL_IF);
379	}
380	#endif
381
382	/* Set event injection state. */
383	pVMCB->ctrl.EventInject.au64[0] = pEvent->au64[0];
384	}
385
386
387	/**
388	* Checks for pending guest interrupts and injects them
389	*
390	* @returns VBox status code.
391	* @param pVM The VM to operate on.
392	* @param pVCpu The VM CPU to operate on.
393	* @param pVMCB SVM control block
394	* @param pCtx CPU Context
395	*/
396	static int SVMR0CheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, SVM_VMCB pVMCB, CPUMCTX pCtx)
397	{
398	int rc;
399
400	/* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
401	if (pVCpu->hwaccm.s.Event.fPending)
402	{
403	SVM_EVENT Event;
404
405	Log(("Reinjecting event %08x %08x at %RGv\n", pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip));
406	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
407	Event.au64[0] = pVCpu->hwaccm.s.Event.intInfo;
408	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
409
410	pVCpu->hwaccm.s.Event.fPending = false;
411	return VINF_SUCCESS;
412	}
413
414	if (pVM->hwaccm.s.fInjectNMI)
415	{
416	SVM_EVENT Event;
417
418	Event.n.u8Vector = X86_XCPT_NMI;
419	Event.n.u1Valid = 1;
420	Event.n.u32ErrorCode = 0;
421	Event.n.u3Type = SVM_EVENT_NMI;
422
423	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
424	pVM->hwaccm.s.fInjectNMI = false;
425	return VINF_SUCCESS;
426	}
427
428	/* When external interrupts are pending, we should exit the VM when IF is set. */
429	if ( !TRPMHasTrap(pVCpu)
430	&& VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)))
431	{
432	if ( !(pCtx->eflags.u32 & X86_EFL_IF)
433	\|\| VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
434	{
435	if (!pVMCB->ctrl.IntCtrl.n.u1VIrqValid)
436	{
437	if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
438	LogFlow(("Enable irq window exit!\n"));
439	else
440	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS -> irq window exit\n", (RTGCPTR)pCtx->rip));
441
442	/** @todo use virtual interrupt method to inject a pending irq; dispatched as soon as guest.IF is set. */
443	pVMCB->ctrl.u32InterceptCtrl1 \|= SVM_CTRL1_INTERCEPT_VINTR;
444	pVMCB->ctrl.IntCtrl.n.u1VIrqValid = 1;
445	pVMCB->ctrl.IntCtrl.n.u8VIrqVector = 0; /* don't care */
446	}
447	}
448	else
449	{
450	uint8_t u8Interrupt;
451
452	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
453	Log(("Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc\n", u8Interrupt, u8Interrupt, rc));
454	if (RT_SUCCESS(rc))
455	{
456	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
457	AssertRC(rc);
458	}
459	else
460	{
461	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
462	Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)));
463	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
464	/* Just continue */
465	}
466	}
467	}
468
469	#ifdef VBOX_STRICT
470	if (TRPMHasTrap(pVCpu))
471	{
472	uint8_t u8Vector;
473	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
474	AssertRC(rc);
475	}
476	#endif
477
478	if ( pCtx->eflags.u32 & X86_EFL_IF
479	&& (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
480	&& TRPMHasTrap(pVCpu)
481	)
482	{
483	uint8_t u8Vector;
484	int rc;
485	TRPMEVENT enmType;
486	SVM_EVENT Event;
487	RTGCUINT u32ErrorCode;
488
489	Event.au64[0] = 0;
490
491	/* If a new event is pending, then dispatch it now. */
492	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &u32ErrorCode, 0);
493	AssertRC(rc);
494	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
495	Assert(enmType != TRPM_SOFTWARE_INT);
496
497	/* Clear the pending trap. */
498	rc = TRPMResetTrap(pVCpu);
499	AssertRC(rc);
500
501	Event.n.u8Vector = u8Vector;
502	Event.n.u1Valid = 1;
503	Event.n.u32ErrorCode = u32ErrorCode;
504
505	if (enmType == TRPM_TRAP)
506	{
507	switch (u8Vector) {
508	case 8:
509	case 10:
510	case 11:
511	case 12:
512	case 13:
513	case 14:
514	case 17:
515	/* Valid error codes. */
516	Event.n.u1ErrorCodeValid = 1;
517	break;
518	default:
519	break;
520	}
521	if (u8Vector == X86_XCPT_NMI)
522	Event.n.u3Type = SVM_EVENT_NMI;
523	else
524	Event.n.u3Type = SVM_EVENT_EXCEPTION;
525	}
526	else
527	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
528
529	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
530	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
531	} /* if (interrupts can be dispatched) */
532
533	return VINF_SUCCESS;
534	}
535
536	/**
537	* Save the host state
538	*
539	* @returns VBox status code.
540	* @param pVM The VM to operate on.
541	* @param pVCpu The VM CPU to operate on.
542	*/
543	VMMR0DECL(int) SVMR0SaveHostState(PVM pVM, PVMCPU pVCpu)
544	{
545	NOREF(pVM);
546	NOREF(pVCpu);
547	/* Nothing to do here. */
548	return VINF_SUCCESS;
549	}
550
551	/**
552	* Loads the guest state
553	*
554	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
555	*
556	* @returns VBox status code.
557	* @param pVM The VM to operate on.
558	* @param pVCpu The VM CPU to operate on.
559	* @param pCtx Guest context
560	*/
561	VMMR0DECL(int) SVMR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
562	{
563	RTGCUINTPTR val;
564	SVM_VMCB *pVMCB;
565
566	if (pVM == NULL)
567	return VERR_INVALID_PARAMETER;
568
569	/* Setup AMD SVM. */
570	Assert(pVM->hwaccm.s.svm.fSupported);
571
572	pVMCB = (SVM_VMCB *)pVCpu->hwaccm.s.svm.pVMCB;
573	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
574
575	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
576	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
577	{
578	SVM_WRITE_SELREG(CS, cs);
579	SVM_WRITE_SELREG(SS, ss);
580	SVM_WRITE_SELREG(DS, ds);
581	SVM_WRITE_SELREG(ES, es);
582	SVM_WRITE_SELREG(FS, fs);
583	SVM_WRITE_SELREG(GS, gs);
584	}
585
586	/* Guest CPU context: LDTR. */
587	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
588	{
589	SVM_WRITE_SELREG(LDTR, ldtr);
590	}
591
592	/* Guest CPU context: TR. */
593	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
594	{
595	SVM_WRITE_SELREG(TR, tr);
596	}
597
598	/* Guest CPU context: GDTR. */
599	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
600	{
601	pVMCB->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
602	pVMCB->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
603	}
604
605	/* Guest CPU context: IDTR. */
606	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
607	{
608	pVMCB->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
609	pVMCB->guest.IDTR.u64Base = pCtx->idtr.pIdt;
610	}
611
612	/*
613	* Sysenter MSRs (unconditional)
614	*/
615	pVMCB->guest.u64SysEnterCS = pCtx->SysEnter.cs;
616	pVMCB->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
617	pVMCB->guest.u64SysEnterESP = pCtx->SysEnter.esp;
618
619	/* Control registers */
620	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
621	{
622	val = pCtx->cr0;
623	if (!CPUMIsGuestFPUStateActive(pVCpu))
624	{
625	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
626	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
627	}
628	else
629	{
630	/** @todo check if we support the old style mess correctly. */
631	if (!(val & X86_CR0_NE))
632	{
633	Log(("Forcing X86_CR0_NE!!!\n"));
634
635	/* Also catch floating point exceptions as we need to report them to the guest in a different way. */
636	if (!pVCpu->hwaccm.s.fFPUOldStyleOverride)
637	{
638	pVMCB->ctrl.u32InterceptException \|= RT_BIT(X86_XCPT_MF);
639	pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
640	}
641	}
642	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
643	}
644	/* Always enable caching. */
645	val &= ~(X86_CR0_CD\|X86_CR0_NW);
646
647	/* Note: WP is not relevant in nested paging mode as we catch accesses on the (guest) physical level. */
648	/* Note: In nested paging mode the guest is allowed to run with paging disabled; the guest physical to host physical translation will remain active. */
649	if (!pVM->hwaccm.s.fNestedPaging)
650	{
651	val \|= X86_CR0_PG; /* Paging is always enabled; even when the guest is running in real mode or PE without paging. */
652	val \|= X86_CR0_WP; /* Must set this as we rely on protect various pages and supervisor writes must be caught. */
653	}
654	pVMCB->guest.u64CR0 = val;
655	}
656	/* CR2 as well */
657	pVMCB->guest.u64CR2 = pCtx->cr2;
658
659	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
660	{
661	/* Save our shadow CR3 register. */
662	if (pVM->hwaccm.s.fNestedPaging)
663	{
664	PGMMODE enmShwPagingMode;
665
666	#if HC_ARCH_BITS == 32
667	if (CPUMIsGuestInLongModeEx(pCtx))
668	enmShwPagingMode = PGMMODE_AMD64_NX;
669	else
670	#endif
671	enmShwPagingMode = PGMGetHostMode(pVM);
672
673	pVMCB->ctrl.u64NestedPagingCR3 = PGMGetNestedCR3(pVCpu, enmShwPagingMode);
674	Assert(pVMCB->ctrl.u64NestedPagingCR3);
675	pVMCB->guest.u64CR3 = pCtx->cr3;
676	}
677	else
678	{
679	pVMCB->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
680	Assert(pVMCB->guest.u64CR3 \|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
681	}
682	}
683
684	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
685	{
686	val = pCtx->cr4;
687	if (!pVM->hwaccm.s.fNestedPaging)
688	{
689	switch(pVCpu->hwaccm.s.enmShadowMode)
690	{
691	case PGMMODE_REAL:
692	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
693	AssertFailed();
694	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
695
696	case PGMMODE_32_BIT: /* 32-bit paging. */
697	val &= ~X86_CR4_PAE;
698	break;
699
700	case PGMMODE_PAE: /* PAE paging. */
701	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
702	/** @todo use normal 32 bits paging */
703	val \|= X86_CR4_PAE;
704	break;
705
706	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
707	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
708	#ifdef VBOX_ENABLE_64_BITS_GUESTS
709	break;
710	#else
711	AssertFailed();
712	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
713	#endif
714
715	default: /* shut up gcc */
716	AssertFailed();
717	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
718	}
719	}
720	pVMCB->guest.u64CR4 = val;
721	}
722
723	/* Debug registers. */
724	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
725	{
726	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
727	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
728
729	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
730	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
731	pCtx->dr[7] \|= 0x400; /* must be one */
732
733	pVMCB->guest.u64DR7 = pCtx->dr[7];
734	pVMCB->guest.u64DR6 = pCtx->dr[6];
735
736	/* Sync the debug state now if any breakpoint is armed. */
737	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
738	&& !CPUMIsGuestDebugStateActive(pVCpu)
739	&& !DBGFIsStepping(pVM))
740	{
741	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
742
743	/* Disable drx move intercepts. */
744	pVMCB->ctrl.u16InterceptRdDRx = 0;
745	pVMCB->ctrl.u16InterceptWrDRx = 0;
746
747	/* Save the host and load the guest debug state. */
748	int rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
749	AssertRC(rc);
750	}
751	}
752
753	/* EIP, ESP and EFLAGS */
754	pVMCB->guest.u64RIP = pCtx->rip;
755	pVMCB->guest.u64RSP = pCtx->rsp;
756	pVMCB->guest.u64RFlags = pCtx->eflags.u32;
757
758	/* Set CPL */
759	pVMCB->guest.u8CPL = pCtx->csHid.Attr.n.u2Dpl;
760
761	/* RAX/EAX too, as VMRUN uses RAX as an implicit parameter. */
762	pVMCB->guest.u64RAX = pCtx->rax;
763
764	/* vmrun will fail without MSR_K6_EFER_SVME. */
765	pVMCB->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
766
767	/* 64 bits guest mode? */
768	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
769	{
770	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
771	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
772	#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
773	pVCpu->hwaccm.s.svm.pfnVMRun = SVMR0VMSwitcherRun64;
774	#else
775	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
776	if (!pVM->hwaccm.s.fAllow64BitGuests)
777	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
778	# endif
779	pVCpu->hwaccm.s.svm.pfnVMRun = SVMR0VMRun64;
780	#endif
781	/* Unconditionally update these as wrmsr might have changed them. (HWACCM_CHANGED_GUEST_SEGMENT_REGS will not be set) */
782	pVMCB->guest.FS.u64Base = pCtx->fsHid.u64Base;
783	pVMCB->guest.GS.u64Base = pCtx->gsHid.u64Base;
784	}
785	else
786	{
787	/* Filter out the MSR_K6_LME bit or else AMD-V expects amd64 shadow paging. */
788	pVMCB->guest.u64EFER &= ~MSR_K6_EFER_LME;
789
790	pVCpu->hwaccm.s.svm.pfnVMRun = SVMR0VMRun;
791	}
792
793	/* TSC offset. */
794	if (TMCpuTickCanUseRealTSC(pVCpu, &pVMCB->ctrl.u64TSCOffset))
795	{
796	pVMCB->ctrl.u32InterceptCtrl1 &= ~SVM_CTRL1_INTERCEPT_RDTSC;
797	pVMCB->ctrl.u32InterceptCtrl2 &= ~SVM_CTRL2_INTERCEPT_RDTSCP;
798	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
799	}
800	else
801	{
802	pVMCB->ctrl.u32InterceptCtrl1 \|= SVM_CTRL1_INTERCEPT_RDTSC;
803	pVMCB->ctrl.u32InterceptCtrl2 \|= SVM_CTRL2_INTERCEPT_RDTSCP;
804	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
805	}
806
807	/* Sync the various msrs for 64 bits mode. */
808	pVMCB->guest.u64STAR = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
809	pVMCB->guest.u64LSTAR = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
810	pVMCB->guest.u64CSTAR = pCtx->msrCSTAR; /* compatibility mode syscall rip */
811	pVMCB->guest.u64SFMASK = pCtx->msrSFMASK; /* syscall flag mask */
812	pVMCB->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
813
814	#ifdef DEBUG
815	/* Intercept X86_XCPT_DB if stepping is enabled */
816	if (DBGFIsStepping(pVM))
817	pVMCB->ctrl.u32InterceptException \|= RT_BIT(X86_XCPT_DB);
818	else
819	pVMCB->ctrl.u32InterceptException &= ~RT_BIT(X86_XCPT_DB);
820	#endif
821
822	/* Done. */
823	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
824
825	return VINF_SUCCESS;
826	}
827
828
829	/**
830	* Runs guest code in an AMD-V VM.
831	*
832	* @returns VBox status code.
833	* @param pVM The VM to operate on.
834	* @param pVCpu The VM CPU to operate on.
835	* @param pCtx Guest context
836	*/
837	VMMR0DECL(int) SVMR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
838	{
839	int rc = VINF_SUCCESS;
840	uint64_t exitCode = (uint64_t)SVM_EXIT_INVALID;
841	SVM_VMCB *pVMCB;
842	bool fSyncTPR = false;
843	unsigned cResume = 0;
844	uint8_t u8LastVTPR;
845	PHWACCM_CPUINFO pCpu = 0;
846	#ifdef VBOX_STRICT
847	RTCPUID idCpuCheck;
848	#endif
849
850	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
851
852	pVMCB = (SVM_VMCB *)pVCpu->hwaccm.s.svm.pVMCB;
853	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
854
855	/* We can jump to this point to resume execution after determining that a VM-exit is innocent.
856	*/
857	ResumeExecution:
858	Assert(!HWACCMR0SuspendPending());
859
860	/* Safety precaution; looping for too long here can have a very bad effect on the host */
861	if (++cResume > HWACCM_MAX_RESUME_LOOPS)
862	{
863	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
864	rc = VINF_EM_RAW_INTERRUPT;
865	goto end;
866	}
867
868	/* Check for irq inhibition due to instruction fusing (sti, mov ss). */
869	if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
870	{
871	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
872	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
873	{
874	/* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
875	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
876	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
877	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
878	*/
879	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
880	/* Irq inhibition is no longer active; clear the corresponding SVM state. */
881	pVMCB->ctrl.u64IntShadow = 0;
882	}
883	}
884	else
885	{
886	/* Irq inhibition is no longer active; clear the corresponding SVM state. */
887	pVMCB->ctrl.u64IntShadow = 0;
888	}
889
890	/* Check for pending actions that force us to go back to ring 3. */
891	#ifdef DEBUG
892	/* Intercept X86_XCPT_DB if stepping is enabled */
893	if (!DBGFIsStepping(pVM))
894	#endif
895	{
896	if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
897	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
898	{
899	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
900	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
901	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
902	rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
903	goto end;
904	}
905	}
906
907	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
908	if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
909	{
910	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
911	rc = VINF_EM_PENDING_REQUEST;
912	goto end;
913	}
914
915	/* When external interrupts are pending, we should exit the VM when IF is set. */
916	/* Note! After VM_FF_INHIBIT_INTERRUPTS check!!! */
917	rc = SVMR0CheckPendingInterrupt(pVM, pVCpu, pVMCB, pCtx);
918	if (RT_FAILURE(rc))
919	{
920	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
921	goto end;
922	}
923
924	/* TPR caching using CR8 is only available in 64 bits mode */
925	/* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
926	/* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!!!!! */
927	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
928	{
929	bool fPending;
930
931	/* TPR caching in CR8 */
932	int rc = PDMApicGetTPR(pVM, &u8LastVTPR, &fPending);
933	AssertRC(rc);
934	pVMCB->ctrl.IntCtrl.n.u8VTPR = u8LastVTPR;
935
936	if (fPending)
937	{
938	/* A TPR change could activate a pending interrupt, so catch cr8 writes. */
939	pVMCB->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
940	}
941	else
942	/* No interrupts are pending, so we don't need to be explicitely notified.
943	* There are enough world switches for detecting pending interrupts.
944	*/
945	pVMCB->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
946
947	fSyncTPR = !fPending;
948	}
949
950	/* All done! Let's start VM execution. */
951	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatInGC, x);
952
953	/* Enable nested paging if necessary (disabled each time after #VMEXIT). */
954	pVMCB->ctrl.NestedPaging.n.u1NestedPaging = pVM->hwaccm.s.fNestedPaging;
955
956	#ifdef LOG_ENABLED
957	pCpu = HWACCMR0GetCurrentCpu();
958	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
959	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
960	{
961	if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
962	Log(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
963	else
964	Log(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
965	}
966	if (pCpu->fFlushTLB)
967	Log(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
968	#endif
969
970	/*
971	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
972	* (until the actual world switch)
973	*/
974
975	#ifdef VBOX_STRICT
976	idCpuCheck = RTMpCpuId();
977	#endif
978
979	/* Load the guest state; must be here as it sets up the shadow cr0 for lazy fpu syncing! */
980	rc = SVMR0LoadGuestState(pVM, pVCpu, pCtx);
981	if (rc != VINF_SUCCESS)
982	{
983	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
984	goto end;
985	}
986
987	pCpu = HWACCMR0GetCurrentCpu();
988	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
989	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
990	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
991	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
992	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
993	{
994	/* Force a TLB flush on VM entry. */
995	pVCpu->hwaccm.s.fForceTLBFlush = true;
996	}
997	else
998	Assert(!pCpu->fFlushTLB \|\| pVM->hwaccm.s.svm.fAlwaysFlushTLB);
999
1000	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
1001
1002	/* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
1003	if ( pVCpu->hwaccm.s.fForceTLBFlush
1004	&& !pVM->hwaccm.s.svm.fAlwaysFlushTLB)
1005	{
1006	if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
1007	\|\| pCpu->fFlushTLB)
1008	{
1009	pCpu->fFlushTLB = false;
1010	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
1011	pVMCB->ctrl.TLBCtrl.n.u1TLBFlush = 1; /* wrap around; flush TLB */
1012	pCpu->cTLBFlushes++;
1013	}
1014	else
1015	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
1016
1017	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
1018	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
1019	}
1020	else
1021	{
1022	Assert(!pCpu->fFlushTLB \|\| pVM->hwaccm.s.svm.fAlwaysFlushTLB);
1023
1024	/* We never increase uCurrentASID in the fAlwaysFlushTLB (erratum 170) case. */
1025	if (!pCpu->uCurrentASID \|\| !pVCpu->hwaccm.s.uCurrentASID)
1026	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID = 1;
1027
1028	Assert(!pVM->hwaccm.s.svm.fAlwaysFlushTLB \|\| pVCpu->hwaccm.s.fForceTLBFlush);
1029	pVMCB->ctrl.TLBCtrl.n.u1TLBFlush = pVCpu->hwaccm.s.fForceTLBFlush;
1030	}
1031	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
1032	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
1033	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
1034	pVMCB->ctrl.TLBCtrl.n.u32ASID = pVCpu->hwaccm.s.uCurrentASID;
1035
1036	#ifdef VBOX_WITH_STATISTICS
1037	if (pVMCB->ctrl.TLBCtrl.n.u1TLBFlush)
1038	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
1039	else
1040	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
1041	#endif
1042
1043	/* In case we execute a goto ResumeExecution later on. */
1044	pVCpu->hwaccm.s.fResumeVM = true;
1045	pVCpu->hwaccm.s.fForceTLBFlush = pVM->hwaccm.s.svm.fAlwaysFlushTLB;
1046
1047	Assert(sizeof(pVCpu->hwaccm.s.svm.pVMCBPhys) == 8);
1048	Assert(pVMCB->ctrl.IntCtrl.n.u1VIrqMasking);
1049	Assert(pVMCB->ctrl.u64IOPMPhysAddr == pVM->hwaccm.s.svm.pIOBitmapPhys);
1050	Assert(pVMCB->ctrl.u64MSRPMPhysAddr == pVM->hwaccm.s.svm.pMSRBitmapPhys);
1051	Assert(pVMCB->ctrl.u64LBRVirt == 0);
1052
1053	#ifdef VBOX_STRICT
1054	Assert(idCpuCheck == RTMpCpuId());
1055	#endif
1056	TMNotifyStartOfExecution(pVCpu);
1057	pVCpu->hwaccm.s.svm.pfnVMRun(pVM->hwaccm.s.svm.pVMCBHostPhys, pVCpu->hwaccm.s.svm.pVMCBPhys, pCtx, pVM, pVCpu);
1058	TMNotifyEndOfExecution(pVCpu);
1059	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatInGC, x);
1060
1061	/*
1062	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1063	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
1064	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1065	*/
1066
1067	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit1, x);
1068
1069	/* Reason for the VM exit */
1070	exitCode = pVMCB->ctrl.u64ExitCode;
1071
1072	if (exitCode == (uint64_t)SVM_EXIT_INVALID) /* Invalid guest state. */
1073	{
1074	HWACCMDumpRegs(pVM, pVCpu, pCtx);
1075	#ifdef DEBUG
1076	Log(("ctrl.u16InterceptRdCRx %x\n", pVMCB->ctrl.u16InterceptRdCRx));
1077	Log(("ctrl.u16InterceptWrCRx %x\n", pVMCB->ctrl.u16InterceptWrCRx));
1078	Log(("ctrl.u16InterceptRdDRx %x\n", pVMCB->ctrl.u16InterceptRdDRx));
1079	Log(("ctrl.u16InterceptWrDRx %x\n", pVMCB->ctrl.u16InterceptWrDRx));
1080	Log(("ctrl.u32InterceptException %x\n", pVMCB->ctrl.u32InterceptException));
1081	Log(("ctrl.u32InterceptCtrl1 %x\n", pVMCB->ctrl.u32InterceptCtrl1));
1082	Log(("ctrl.u32InterceptCtrl2 %x\n", pVMCB->ctrl.u32InterceptCtrl2));
1083	Log(("ctrl.u64IOPMPhysAddr %RX64\n", pVMCB->ctrl.u64IOPMPhysAddr));
1084	Log(("ctrl.u64MSRPMPhysAddr %RX64\n", pVMCB->ctrl.u64MSRPMPhysAddr));
1085	Log(("ctrl.u64TSCOffset %RX64\n", pVMCB->ctrl.u64TSCOffset));
1086
1087	Log(("ctrl.TLBCtrl.u32ASID %x\n", pVMCB->ctrl.TLBCtrl.n.u32ASID));
1088	Log(("ctrl.TLBCtrl.u1TLBFlush %x\n", pVMCB->ctrl.TLBCtrl.n.u1TLBFlush));
1089	Log(("ctrl.TLBCtrl.u7Reserved %x\n", pVMCB->ctrl.TLBCtrl.n.u7Reserved));
1090	Log(("ctrl.TLBCtrl.u24Reserved %x\n", pVMCB->ctrl.TLBCtrl.n.u24Reserved));
1091
1092	Log(("ctrl.IntCtrl.u8VTPR %x\n", pVMCB->ctrl.IntCtrl.n.u8VTPR));
1093	Log(("ctrl.IntCtrl.u1VIrqValid %x\n", pVMCB->ctrl.IntCtrl.n.u1VIrqValid));
1094	Log(("ctrl.IntCtrl.u7Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u7Reserved));
1095	Log(("ctrl.IntCtrl.u4VIrqPriority %x\n", pVMCB->ctrl.IntCtrl.n.u4VIrqPriority));
1096	Log(("ctrl.IntCtrl.u1IgnoreTPR %x\n", pVMCB->ctrl.IntCtrl.n.u1IgnoreTPR));
1097	Log(("ctrl.IntCtrl.u3Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u3Reserved));
1098	Log(("ctrl.IntCtrl.u1VIrqMasking %x\n", pVMCB->ctrl.IntCtrl.n.u1VIrqMasking));
1099	Log(("ctrl.IntCtrl.u7Reserved2 %x\n", pVMCB->ctrl.IntCtrl.n.u7Reserved2));
1100	Log(("ctrl.IntCtrl.u8VIrqVector %x\n", pVMCB->ctrl.IntCtrl.n.u8VIrqVector));
1101	Log(("ctrl.IntCtrl.u24Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u24Reserved));
1102
1103	Log(("ctrl.u64IntShadow %RX64\n", pVMCB->ctrl.u64IntShadow));
1104	Log(("ctrl.u64ExitCode %RX64\n", pVMCB->ctrl.u64ExitCode));
1105	Log(("ctrl.u64ExitInfo1 %RX64\n", pVMCB->ctrl.u64ExitInfo1));
1106	Log(("ctrl.u64ExitInfo2 %RX64\n", pVMCB->ctrl.u64ExitInfo2));
1107	Log(("ctrl.ExitIntInfo.u8Vector %x\n", pVMCB->ctrl.ExitIntInfo.n.u8Vector));
1108	Log(("ctrl.ExitIntInfo.u3Type %x\n", pVMCB->ctrl.ExitIntInfo.n.u3Type));
1109	Log(("ctrl.ExitIntInfo.u1ErrorCodeValid %x\n", pVMCB->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
1110	Log(("ctrl.ExitIntInfo.u19Reserved %x\n", pVMCB->ctrl.ExitIntInfo.n.u19Reserved));
1111	Log(("ctrl.ExitIntInfo.u1Valid %x\n", pVMCB->ctrl.ExitIntInfo.n.u1Valid));
1112	Log(("ctrl.ExitIntInfo.u32ErrorCode %x\n", pVMCB->ctrl.ExitIntInfo.n.u32ErrorCode));
1113	Log(("ctrl.NestedPaging %RX64\n", pVMCB->ctrl.NestedPaging.au64));
1114	Log(("ctrl.EventInject.u8Vector %x\n", pVMCB->ctrl.EventInject.n.u8Vector));
1115	Log(("ctrl.EventInject.u3Type %x\n", pVMCB->ctrl.EventInject.n.u3Type));
1116	Log(("ctrl.EventInject.u1ErrorCodeValid %x\n", pVMCB->ctrl.EventInject.n.u1ErrorCodeValid));
1117	Log(("ctrl.EventInject.u19Reserved %x\n", pVMCB->ctrl.EventInject.n.u19Reserved));
1118	Log(("ctrl.EventInject.u1Valid %x\n", pVMCB->ctrl.EventInject.n.u1Valid));
1119	Log(("ctrl.EventInject.u32ErrorCode %x\n", pVMCB->ctrl.EventInject.n.u32ErrorCode));
1120
1121	Log(("ctrl.u64NestedPagingCR3 %RX64\n", pVMCB->ctrl.u64NestedPagingCR3));
1122	Log(("ctrl.u64LBRVirt %RX64\n", pVMCB->ctrl.u64LBRVirt));
1123
1124	Log(("guest.CS.u16Sel %04X\n", pVMCB->guest.CS.u16Sel));
1125	Log(("guest.CS.u16Attr %04X\n", pVMCB->guest.CS.u16Attr));
1126	Log(("guest.CS.u32Limit %X\n", pVMCB->guest.CS.u32Limit));
1127	Log(("guest.CS.u64Base %RX64\n", pVMCB->guest.CS.u64Base));
1128	Log(("guest.DS.u16Sel %04X\n", pVMCB->guest.DS.u16Sel));
1129	Log(("guest.DS.u16Attr %04X\n", pVMCB->guest.DS.u16Attr));
1130	Log(("guest.DS.u32Limit %X\n", pVMCB->guest.DS.u32Limit));
1131	Log(("guest.DS.u64Base %RX64\n", pVMCB->guest.DS.u64Base));
1132	Log(("guest.ES.u16Sel %04X\n", pVMCB->guest.ES.u16Sel));
1133	Log(("guest.ES.u16Attr %04X\n", pVMCB->guest.ES.u16Attr));
1134	Log(("guest.ES.u32Limit %X\n", pVMCB->guest.ES.u32Limit));
1135	Log(("guest.ES.u64Base %RX64\n", pVMCB->guest.ES.u64Base));
1136	Log(("guest.FS.u16Sel %04X\n", pVMCB->guest.FS.u16Sel));
1137	Log(("guest.FS.u16Attr %04X\n", pVMCB->guest.FS.u16Attr));
1138	Log(("guest.FS.u32Limit %X\n", pVMCB->guest.FS.u32Limit));
1139	Log(("guest.FS.u64Base %RX64\n", pVMCB->guest.FS.u64Base));
1140	Log(("guest.GS.u16Sel %04X\n", pVMCB->guest.GS.u16Sel));
1141	Log(("guest.GS.u16Attr %04X\n", pVMCB->guest.GS.u16Attr));
1142	Log(("guest.GS.u32Limit %X\n", pVMCB->guest.GS.u32Limit));
1143	Log(("guest.GS.u64Base %RX64\n", pVMCB->guest.GS.u64Base));
1144
1145	Log(("guest.GDTR.u32Limit %X\n", pVMCB->guest.GDTR.u32Limit));
1146	Log(("guest.GDTR.u64Base %RX64\n", pVMCB->guest.GDTR.u64Base));
1147
1148	Log(("guest.LDTR.u16Sel %04X\n", pVMCB->guest.LDTR.u16Sel));
1149	Log(("guest.LDTR.u16Attr %04X\n", pVMCB->guest.LDTR.u16Attr));
1150	Log(("guest.LDTR.u32Limit %X\n", pVMCB->guest.LDTR.u32Limit));
1151	Log(("guest.LDTR.u64Base %RX64\n", pVMCB->guest.LDTR.u64Base));
1152
1153	Log(("guest.IDTR.u32Limit %X\n", pVMCB->guest.IDTR.u32Limit));
1154	Log(("guest.IDTR.u64Base %RX64\n", pVMCB->guest.IDTR.u64Base));
1155
1156	Log(("guest.TR.u16Sel %04X\n", pVMCB->guest.TR.u16Sel));
1157	Log(("guest.TR.u16Attr %04X\n", pVMCB->guest.TR.u16Attr));
1158	Log(("guest.TR.u32Limit %X\n", pVMCB->guest.TR.u32Limit));
1159	Log(("guest.TR.u64Base %RX64\n", pVMCB->guest.TR.u64Base));
1160
1161	Log(("guest.u8CPL %X\n", pVMCB->guest.u8CPL));
1162	Log(("guest.u64CR0 %RX64\n", pVMCB->guest.u64CR0));
1163	Log(("guest.u64CR2 %RX64\n", pVMCB->guest.u64CR2));
1164	Log(("guest.u64CR3 %RX64\n", pVMCB->guest.u64CR3));
1165	Log(("guest.u64CR4 %RX64\n", pVMCB->guest.u64CR4));
1166	Log(("guest.u64DR6 %RX64\n", pVMCB->guest.u64DR6));
1167	Log(("guest.u64DR7 %RX64\n", pVMCB->guest.u64DR7));
1168
1169	Log(("guest.u64RIP %RX64\n", pVMCB->guest.u64RIP));
1170	Log(("guest.u64RSP %RX64\n", pVMCB->guest.u64RSP));
1171	Log(("guest.u64RAX %RX64\n", pVMCB->guest.u64RAX));
1172	Log(("guest.u64RFlags %RX64\n", pVMCB->guest.u64RFlags));
1173
1174	Log(("guest.u64SysEnterCS %RX64\n", pVMCB->guest.u64SysEnterCS));
1175	Log(("guest.u64SysEnterEIP %RX64\n", pVMCB->guest.u64SysEnterEIP));
1176	Log(("guest.u64SysEnterESP %RX64\n", pVMCB->guest.u64SysEnterESP));
1177
1178	Log(("guest.u64EFER %RX64\n", pVMCB->guest.u64EFER));
1179	Log(("guest.u64STAR %RX64\n", pVMCB->guest.u64STAR));
1180	Log(("guest.u64LSTAR %RX64\n", pVMCB->guest.u64LSTAR));
1181	Log(("guest.u64CSTAR %RX64\n", pVMCB->guest.u64CSTAR));
1182	Log(("guest.u64SFMASK %RX64\n", pVMCB->guest.u64SFMASK));
1183	Log(("guest.u64KernelGSBase %RX64\n", pVMCB->guest.u64KernelGSBase));
1184	Log(("guest.u64GPAT %RX64\n", pVMCB->guest.u64GPAT));
1185	Log(("guest.u64DBGCTL %RX64\n", pVMCB->guest.u64DBGCTL));
1186	Log(("guest.u64BR_FROM %RX64\n", pVMCB->guest.u64BR_FROM));
1187	Log(("guest.u64BR_TO %RX64\n", pVMCB->guest.u64BR_TO));
1188	Log(("guest.u64LASTEXCPFROM %RX64\n", pVMCB->guest.u64LASTEXCPFROM));
1189	Log(("guest.u64LASTEXCPTO %RX64\n", pVMCB->guest.u64LASTEXCPTO));
1190
1191	#endif
1192	rc = VERR_SVM_UNABLE_TO_START_VM;
1193	goto end;
1194	}
1195
1196	/* Let's first sync back eip, esp, and eflags. */
1197	pCtx->rip = pVMCB->guest.u64RIP;
1198	pCtx->rsp = pVMCB->guest.u64RSP;
1199	pCtx->eflags.u32 = pVMCB->guest.u64RFlags;
1200	/* eax is saved/restore across the vmrun instruction */
1201	pCtx->rax = pVMCB->guest.u64RAX;
1202
1203	pCtx->msrKERNELGSBASE = pVMCB->guest.u64KernelGSBase; /* swapgs exchange value */
1204
1205	/* Can be updated behind our back in the nested paging case. */
1206	pCtx->cr2 = pVMCB->guest.u64CR2;
1207
1208	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1209	SVM_READ_SELREG(SS, ss);
1210	SVM_READ_SELREG(CS, cs);
1211	SVM_READ_SELREG(DS, ds);
1212	SVM_READ_SELREG(ES, es);
1213	SVM_READ_SELREG(FS, fs);
1214	SVM_READ_SELREG(GS, gs);
1215
1216	/*
1217	* System MSRs
1218	*/
1219	pCtx->SysEnter.cs = pVMCB->guest.u64SysEnterCS;
1220	pCtx->SysEnter.eip = pVMCB->guest.u64SysEnterEIP;
1221	pCtx->SysEnter.esp = pVMCB->guest.u64SysEnterESP;
1222
1223	/* Remaining guest CPU context: TR, IDTR, GDTR, LDTR; must sync everything otherwise we can get out of sync when jumping to ring 3. */
1224	SVM_READ_SELREG(LDTR, ldtr);
1225	SVM_READ_SELREG(TR, tr);
1226
1227	pCtx->gdtr.cbGdt = pVMCB->guest.GDTR.u32Limit;
1228	pCtx->gdtr.pGdt = pVMCB->guest.GDTR.u64Base;
1229
1230	pCtx->idtr.cbIdt = pVMCB->guest.IDTR.u32Limit;
1231	pCtx->idtr.pIdt = pVMCB->guest.IDTR.u64Base;
1232
1233	/* Note: no reason to sync back the CRx and DRx registers. They can't be changed by the guest. */
1234	/* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1235	if ( pVM->hwaccm.s.fNestedPaging
1236	&& pCtx->cr3 != pVMCB->guest.u64CR3)
1237	{
1238	CPUMSetGuestCR3(pVCpu, pVMCB->guest.u64CR3);
1239	PGMUpdateCR3(pVCpu, pVMCB->guest.u64CR3);
1240	}
1241
1242	/* Note! NOW IT'S SAFE FOR LOGGING! */
1243
1244	/* Take care of instruction fusing (sti, mov ss) (see 15.20.5 Interrupt Shadows) */
1245	if (pVMCB->ctrl.u64IntShadow & SVM_INTERRUPT_SHADOW_ACTIVE)
1246	{
1247	Log(("uInterruptState %x rip=%RGv\n", pVMCB->ctrl.u64IntShadow, (RTGCPTR)pCtx->rip));
1248	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
1249	}
1250	else
1251	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1252
1253	Log2(("exitCode = %x\n", exitCode));
1254
1255	/* Sync back DR6 as it could have been changed by hitting breakpoints. */
1256	pCtx->dr[6] = pVMCB->guest.u64DR6;
1257	/* DR7.GD can be cleared by debug exceptions, so sync it back as well. */
1258	pCtx->dr[7] = pVMCB->guest.u64DR7;
1259
1260	/* Check if an injected event was interrupted prematurely. */
1261	pVCpu->hwaccm.s.Event.intInfo = pVMCB->ctrl.ExitIntInfo.au64[0];
1262	if ( pVMCB->ctrl.ExitIntInfo.n.u1Valid
1263	&& pVMCB->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT /* we don't care about 'int xx' as the instruction will be restarted. */)
1264	{
1265	Log(("Pending inject %RX64 at %RGv exit=%08x\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitCode));
1266
1267	#ifdef LOG_ENABLED
1268	SVM_EVENT Event;
1269	Event.au64[0] = pVCpu->hwaccm.s.Event.intInfo;
1270
1271	if ( exitCode == SVM_EXIT_EXCEPTION_E
1272	&& Event.n.u8Vector == 0xE)
1273	{
1274	Log(("Double fault!\n"));
1275	}
1276	#endif
1277
1278	pVCpu->hwaccm.s.Event.fPending = true;
1279	/* Error code present? (redundant) */
1280	if (pVMCB->ctrl.ExitIntInfo.n.u1ErrorCodeValid)
1281	{
1282	pVCpu->hwaccm.s.Event.errCode = pVMCB->ctrl.ExitIntInfo.n.u32ErrorCode;
1283	}
1284	else
1285	pVCpu->hwaccm.s.Event.errCode = 0;
1286	}
1287	#ifdef VBOX_WITH_STATISTICS
1288	if (exitCode == SVM_EXIT_NPF)
1289	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
1290	else
1291	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitCode & MASK_EXITREASON_STAT]);
1292	#endif
1293
1294	if (fSyncTPR)
1295	{
1296	rc = PDMApicSetTPR(pVM, pVMCB->ctrl.IntCtrl.n.u8VTPR);
1297	AssertRC(rc);
1298	}
1299
1300	/* Deal with the reason of the VM-exit. */
1301	switch (exitCode)
1302	{
1303	case SVM_EXIT_EXCEPTION_0: case SVM_EXIT_EXCEPTION_1: case SVM_EXIT_EXCEPTION_2: case SVM_EXIT_EXCEPTION_3:
1304	case SVM_EXIT_EXCEPTION_4: case SVM_EXIT_EXCEPTION_5: case SVM_EXIT_EXCEPTION_6: case SVM_EXIT_EXCEPTION_7:
1305	case SVM_EXIT_EXCEPTION_8: case SVM_EXIT_EXCEPTION_9: case SVM_EXIT_EXCEPTION_A: case SVM_EXIT_EXCEPTION_B:
1306	case SVM_EXIT_EXCEPTION_C: case SVM_EXIT_EXCEPTION_D: case SVM_EXIT_EXCEPTION_E: case SVM_EXIT_EXCEPTION_F:
1307	case SVM_EXIT_EXCEPTION_10: case SVM_EXIT_EXCEPTION_11: case SVM_EXIT_EXCEPTION_12: case SVM_EXIT_EXCEPTION_13:
1308	case SVM_EXIT_EXCEPTION_14: case SVM_EXIT_EXCEPTION_15: case SVM_EXIT_EXCEPTION_16: case SVM_EXIT_EXCEPTION_17:
1309	case SVM_EXIT_EXCEPTION_18: case SVM_EXIT_EXCEPTION_19: case SVM_EXIT_EXCEPTION_1A: case SVM_EXIT_EXCEPTION_1B:
1310	case SVM_EXIT_EXCEPTION_1C: case SVM_EXIT_EXCEPTION_1D: case SVM_EXIT_EXCEPTION_1E: case SVM_EXIT_EXCEPTION_1F:
1311	{
1312	/* Pending trap. */
1313	SVM_EVENT Event;
1314	uint32_t vector = exitCode - SVM_EXIT_EXCEPTION_0;
1315
1316	Log2(("Hardware/software interrupt %d\n", vector));
1317	switch (vector)
1318	{
1319	case X86_XCPT_DB:
1320	{
1321	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
1322
1323	/* Note that we don't support guest and host-initiated debugging at the same time. */
1324	Assert(DBGFIsStepping(pVM));
1325
1326	rc = DBGFR0Trap01Handler(pVM, CPUMCTX2CORE(pCtx), pCtx->dr[6]);
1327	if (rc == VINF_EM_RAW_GUEST_TRAP)
1328	{
1329	Log(("Trap %x (debug) at %016RX64\n", vector, pCtx->rip));
1330
1331	/* Reinject the exception. */
1332	Event.au64[0] = 0;
1333	Event.n.u3Type = SVM_EVENT_EXCEPTION; /* trap or fault */
1334	Event.n.u1Valid = 1;
1335	Event.n.u8Vector = X86_XCPT_DB;
1336
1337	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1338
1339	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1340	goto ResumeExecution;
1341	}
1342	/* Return to ring 3 to deal with the debug exit code. */
1343	break;
1344	}
1345
1346	case X86_XCPT_NM:
1347	{
1348	Log(("#NM fault at %RGv\n", (RTGCPTR)pCtx->rip));
1349
1350	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
1351	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
1352	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
1353	if (rc == VINF_SUCCESS)
1354	{
1355	Assert(CPUMIsGuestFPUStateActive(pVCpu));
1356	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
1357
1358	/* Continue execution. */
1359	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1360	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
1361
1362	goto ResumeExecution;
1363	}
1364
1365	Log(("Forward #NM fault to the guest\n"));
1366	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
1367
1368	Event.au64[0] = 0;
1369	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1370	Event.n.u1Valid = 1;
1371	Event.n.u8Vector = X86_XCPT_NM;
1372
1373	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1374	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1375	goto ResumeExecution;
1376	}
1377
1378	case X86_XCPT_PF: /* Page fault */
1379	{
1380	uint32_t errCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1381	RTGCUINTPTR uFaultAddress = pVMCB->ctrl.u64ExitInfo2; /* EXITINFO2 = fault address */
1382
1383	#ifdef DEBUG
1384	if (pVM->hwaccm.s.fNestedPaging)
1385	{ /* A genuine pagefault.
1386	* Forward the trap to the guest by injecting the exception and resuming execution.
1387	*/
1388	Log(("Guest page fault at %RGv cr2=%RGv error code %x rsp=%RGv\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode, (RTGCPTR)pCtx->rsp));
1389	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
1390
1391	/* Now we must update CR2. */
1392	pCtx->cr2 = uFaultAddress;
1393
1394	Event.au64[0] = 0;
1395	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1396	Event.n.u1Valid = 1;
1397	Event.n.u8Vector = X86_XCPT_PF;
1398	Event.n.u1ErrorCodeValid = 1;
1399	Event.n.u32ErrorCode = errCode;
1400
1401	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1402
1403	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1404	goto ResumeExecution;
1405	}
1406	#endif
1407	Assert(!pVM->hwaccm.s.fNestedPaging);
1408
1409	Log2(("Page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1410	/* Exit qualification contains the linear address of the page fault. */
1411	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
1412	TRPMSetErrorCode(pVCpu, errCode);
1413	TRPMSetFaultAddress(pVCpu, uFaultAddress);
1414
1415	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
1416	rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
1417	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
1418	if (rc == VINF_SUCCESS)
1419	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
1420	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1421	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
1422
1423	TRPMResetTrap(pVCpu);
1424
1425	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1426	goto ResumeExecution;
1427	}
1428	else
1429	if (rc == VINF_EM_RAW_GUEST_TRAP)
1430	{ /* A genuine pagefault.
1431	* Forward the trap to the guest by injecting the exception and resuming execution.
1432	*/
1433	Log2(("Forward page fault to the guest\n"));
1434	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
1435	/* The error code might have been changed. */
1436	errCode = TRPMGetErrorCode(pVCpu);
1437
1438	TRPMResetTrap(pVCpu);
1439
1440	/* Now we must update CR2. */
1441	pCtx->cr2 = uFaultAddress;
1442
1443	Event.au64[0] = 0;
1444	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1445	Event.n.u1Valid = 1;
1446	Event.n.u8Vector = X86_XCPT_PF;
1447	Event.n.u1ErrorCodeValid = 1;
1448	Event.n.u32ErrorCode = errCode;
1449
1450	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1451
1452	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1453	goto ResumeExecution;
1454	}
1455	#ifdef VBOX_STRICT
1456	if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
1457	LogFlow(("PGMTrap0eHandler failed with %d\n", rc));
1458	#endif
1459	/* Need to go back to the recompiler to emulate the instruction. */
1460	TRPMResetTrap(pVCpu);
1461	break;
1462	}
1463
1464	case X86_XCPT_MF: /* Floating point exception. */
1465	{
1466	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
1467	if (!(pCtx->cr0 & X86_CR0_NE))
1468	{
1469	/* old style FPU error reporting needs some extra work. */
1470	/** @todo don't fall back to the recompiler, but do it manually. */
1471	rc = VINF_EM_RAW_EMULATE_INSTR;
1472	break;
1473	}
1474	Log(("Trap %x at %RGv\n", vector, (RTGCPTR)pCtx->rip));
1475
1476	Event.au64[0] = 0;
1477	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1478	Event.n.u1Valid = 1;
1479	Event.n.u8Vector = X86_XCPT_MF;
1480
1481	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1482
1483	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1484	goto ResumeExecution;
1485	}
1486
1487	#ifdef VBOX_STRICT
1488	case X86_XCPT_GP: /* General protection failure exception.*/
1489	case X86_XCPT_UD: /* Unknown opcode exception. */
1490	case X86_XCPT_DE: /* Divide error. */
1491	case X86_XCPT_SS: /* Stack segment exception. */
1492	case X86_XCPT_NP: /* Segment not present exception. */
1493	{
1494	Event.au64[0] = 0;
1495	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1496	Event.n.u1Valid = 1;
1497	Event.n.u8Vector = vector;
1498
1499	switch(vector)
1500	{
1501	case X86_XCPT_GP:
1502	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
1503	Event.n.u1ErrorCodeValid = 1;
1504	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1505	break;
1506	case X86_XCPT_DE:
1507	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
1508	break;
1509	case X86_XCPT_UD:
1510	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
1511	break;
1512	case X86_XCPT_SS:
1513	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
1514	Event.n.u1ErrorCodeValid = 1;
1515	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1516	break;
1517	case X86_XCPT_NP:
1518	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
1519	Event.n.u1ErrorCodeValid = 1;
1520	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1521	break;
1522	}
1523	Log(("Trap %x at %RGv esi=%x\n", vector, (RTGCPTR)pCtx->rip, pCtx->esi));
1524	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1525
1526	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1527	goto ResumeExecution;
1528	}
1529	#endif
1530	default:
1531	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
1532	rc = VERR_EM_INTERNAL_ERROR;
1533	break;
1534
1535	} /* switch (vector) */
1536	break;
1537	}
1538
1539	case SVM_EXIT_NPF:
1540	{
1541	/* EXITINFO1 contains fault errorcode; EXITINFO2 contains the guest physical address causing the fault. */
1542	uint32_t errCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1543	RTGCPHYS uFaultAddress = pVMCB->ctrl.u64ExitInfo2; /* EXITINFO2 = fault address */
1544	PGMMODE enmShwPagingMode;
1545
1546	Assert(pVM->hwaccm.s.fNestedPaging);
1547	Log(("Nested page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1548	/* Exit qualification contains the linear address of the page fault. */
1549	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
1550	TRPMSetErrorCode(pVCpu, errCode);
1551	TRPMSetFaultAddress(pVCpu, uFaultAddress);
1552
1553	/* Handle the pagefault trap for the nested shadow table. */
1554	#if HC_ARCH_BITS == 32
1555	if (CPUMIsGuestInLongModeEx(pCtx))
1556	enmShwPagingMode = PGMMODE_AMD64_NX;
1557	else
1558	#endif
1559	enmShwPagingMode = PGMGetHostMode(pVM);
1560
1561	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmShwPagingMode, errCode, CPUMCTX2CORE(pCtx), uFaultAddress);
1562	Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
1563	if (rc == VINF_SUCCESS)
1564	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
1565	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1566	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
1567
1568	TRPMResetTrap(pVCpu);
1569
1570	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1571	goto ResumeExecution;
1572	}
1573
1574	#ifdef VBOX_STRICT
1575	if (rc != VINF_EM_RAW_EMULATE_INSTR)
1576	LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", rc));
1577	#endif
1578	/* Need to go back to the recompiler to emulate the instruction. */
1579	TRPMResetTrap(pVCpu);
1580	break;
1581	}
1582
1583	case SVM_EXIT_VINTR:
1584	/* A virtual interrupt is about to be delivered, which means IF=1. */
1585	Log(("SVM_EXIT_VINTR IF=%d\n", pCtx->eflags.Bits.u1IF));
1586	pVMCB->ctrl.IntCtrl.n.u1VIrqValid = 0;
1587	pVMCB->ctrl.IntCtrl.n.u8VIrqVector = 0;
1588	goto ResumeExecution;
1589
1590	case SVM_EXIT_FERR_FREEZE:
1591	case SVM_EXIT_INTR:
1592	case SVM_EXIT_NMI:
1593	case SVM_EXIT_SMI:
1594	case SVM_EXIT_INIT:
1595	/* External interrupt; leave to allow it to be dispatched again. */
1596	rc = VINF_EM_RAW_INTERRUPT;
1597	break;
1598
1599	case SVM_EXIT_WBINVD:
1600	case SVM_EXIT_INVD: /* Guest software attempted to execute INVD. */
1601	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
1602	/* Skip instruction and continue directly. */
1603	pCtx->rip += 2; /* Note! hardcoded opcode size! */
1604	/* Continue execution.*/
1605	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1606	goto ResumeExecution;
1607
1608	case SVM_EXIT_CPUID: /* Guest software attempted to execute CPUID. */
1609	{
1610	Log2(("SVM: Cpuid at %RGv for %x\n", (RTGCPTR)pCtx->rip, pCtx->eax));
1611	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
1612	rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
1613	if (rc == VINF_SUCCESS)
1614	{
1615	/* Update EIP and continue execution. */
1616	pCtx->rip += 2; /* Note! hardcoded opcode size! */
1617	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1618	goto ResumeExecution;
1619	}
1620	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", rc));
1621	rc = VINF_EM_RAW_EMULATE_INSTR;
1622	break;
1623	}
1624
1625	case SVM_EXIT_RDTSC: /* Guest software attempted to execute RDTSC. */
1626	{
1627	Log2(("SVM: Rdtsc\n"));
1628	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
1629	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
1630	if (rc == VINF_SUCCESS)
1631	{
1632	/* Update EIP and continue execution. */
1633	pCtx->rip += 2; /* Note! hardcoded opcode size! */
1634	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1635	goto ResumeExecution;
1636	}
1637	rc = VINF_EM_RAW_EMULATE_INSTR;
1638	break;
1639	}
1640
1641	case SVM_EXIT_RDPMC: /* Guest software attempted to execute RDPMC. */
1642	{
1643	Log2(("SVM: Rdpmc %x\n", pCtx->ecx));
1644	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
1645	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
1646	if (rc == VINF_SUCCESS)
1647	{
1648	/* Update EIP and continue execution. */
1649	pCtx->rip += 2; /* Note! hardcoded opcode size! */
1650	goto ResumeExecution;
1651	}
1652	rc = VINF_EM_RAW_EMULATE_INSTR;
1653	break;
1654	}
1655
1656	case SVM_EXIT_RDTSCP: /* Guest software attempted to execute RDTSCP. */
1657	{
1658	Log2(("SVM: Rdtscp\n"));
1659	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
1660	rc = EMInterpretRdtscp(pVM, pVCpu, pCtx);
1661	if (rc == VINF_SUCCESS)
1662	{
1663	/* Update EIP and continue execution. */
1664	pCtx->rip += 3; /* Note! hardcoded opcode size! */
1665	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1666	goto ResumeExecution;
1667	}
1668	AssertMsgFailed(("EMU: rdtscp failed with %Rrc\n", rc));
1669	rc = VINF_EM_RAW_EMULATE_INSTR;
1670	break;
1671	}
1672
1673	case SVM_EXIT_INVLPG: /* Guest software attempted to execute INVPG. */
1674	{
1675	Log2(("SVM: invlpg\n"));
1676	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
1677
1678	Assert(!pVM->hwaccm.s.fNestedPaging);
1679
1680	/* Truly a pita. Why can't SVM give the same information as VT-x? */
1681	rc = SVMR0InterpretInvpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), pVMCB->ctrl.TLBCtrl.n.u32ASID);
1682	if (rc == VINF_SUCCESS)
1683	{
1684	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushPageInvlpg);
1685	goto ResumeExecution; /* eip already updated */
1686	}
1687	break;
1688	}
1689
1690	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
1691	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
1692	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
1693	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
1694	{
1695	uint32_t cbSize;
1696
1697	Log2(("SVM: %RGv mov cr%d, \n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_WRITE_CR0));
1698	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[exitCode - SVM_EXIT_WRITE_CR0]);
1699	rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
1700
1701	switch (exitCode - SVM_EXIT_WRITE_CR0)
1702	{
1703	case 0:
1704	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
1705	break;
1706	case 2:
1707	break;
1708	case 3:
1709	Assert(!pVM->hwaccm.s.fNestedPaging);
1710	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
1711	break;
1712	case 4:
1713	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
1714	break;
1715	case 8:
1716	break;
1717	default:
1718	AssertFailed();
1719	}
1720	/* Check if a sync operation is pending. */
1721	if ( rc == VINF_SUCCESS /* don't bother if we are going to ring 3 anyway */
1722	&& VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
1723	{
1724	rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
1725	AssertRC(rc);
1726
1727	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBCRxChange);
1728
1729	/* Must be set by PGMSyncCR3 */
1730	Assert(rc != VINF_SUCCESS \|\| PGMGetGuestMode(pVCpu) <= PGMMODE_PROTECTED \|\| pVCpu->hwaccm.s.fForceTLBFlush);
1731	}
1732	if (rc == VINF_SUCCESS)
1733	{
1734	/* EIP has been updated already. */
1735
1736	/* Only resume if successful. */
1737	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1738	goto ResumeExecution;
1739	}
1740	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1741	break;
1742	}
1743
1744	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
1745	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
1746	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
1747	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
1748	{
1749	uint32_t cbSize;
1750
1751	Log2(("SVM: %RGv mov x, cr%d\n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_READ_CR0));
1752	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[exitCode - SVM_EXIT_READ_CR0]);
1753	rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
1754	if (rc == VINF_SUCCESS)
1755	{
1756	/* EIP has been updated already. */
1757
1758	/* Only resume if successful. */
1759	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1760	goto ResumeExecution;
1761	}
1762	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1763	break;
1764	}
1765
1766	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
1767	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
1768	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
1769	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
1770	{
1771	uint32_t cbSize;
1772
1773	Log2(("SVM: %RGv mov dr%d, x\n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_WRITE_DR0));
1774	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
1775
1776	if (!DBGFIsStepping(pVM))
1777	{
1778	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
1779
1780	/* Disable drx move intercepts. */
1781	pVMCB->ctrl.u16InterceptRdDRx = 0;
1782	pVMCB->ctrl.u16InterceptWrDRx = 0;
1783
1784	/* Save the host and load the guest debug state. */
1785	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
1786	AssertRC(rc);
1787
1788	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1789	goto ResumeExecution;
1790	}
1791
1792	rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
1793	if (rc == VINF_SUCCESS)
1794	{
1795	/* EIP has been updated already. */
1796	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
1797
1798	/* Only resume if successful. */
1799	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1800	goto ResumeExecution;
1801	}
1802	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1803	break;
1804	}
1805
1806	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
1807	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
1808	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
1809	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
1810	{
1811	uint32_t cbSize;
1812
1813	Log2(("SVM: %RGv mov x, dr%d\n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_READ_DR0));
1814	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
1815
1816	if (!DBGFIsStepping(pVM))
1817	{
1818	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
1819
1820	/* Disable drx move intercepts. */
1821	pVMCB->ctrl.u16InterceptRdDRx = 0;
1822	pVMCB->ctrl.u16InterceptWrDRx = 0;
1823
1824	/* Save the host and load the guest debug state. */
1825	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
1826	AssertRC(rc);
1827
1828	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1829	goto ResumeExecution;
1830	}
1831
1832	rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
1833	if (rc == VINF_SUCCESS)
1834	{
1835	/* EIP has been updated already. */
1836
1837	/* Only resume if successful. */
1838	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1839	goto ResumeExecution;
1840	}
1841	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1842	break;
1843	}
1844
1845	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
1846	case SVM_EXIT_IOIO: /* I/O instruction. */
1847	{
1848	SVM_IOIO_EXIT IoExitInfo;
1849	uint32_t uIOSize, uAndVal;
1850
1851	IoExitInfo.au32[0] = pVMCB->ctrl.u64ExitInfo1;
1852
1853	/** @todo could use a lookup table here */
1854	if (IoExitInfo.n.u1OP8)
1855	{
1856	uIOSize = 1;
1857	uAndVal = 0xff;
1858	}
1859	else
1860	if (IoExitInfo.n.u1OP16)
1861	{
1862	uIOSize = 2;
1863	uAndVal = 0xffff;
1864	}
1865	else
1866	if (IoExitInfo.n.u1OP32)
1867	{
1868	uIOSize = 4;
1869	uAndVal = 0xffffffff;
1870	}
1871	else
1872	{
1873	AssertFailed(); /* should be fatal. */
1874	rc = VINF_EM_RAW_EMULATE_INSTR;
1875	break;
1876	}
1877
1878	if (IoExitInfo.n.u1STR)
1879	{
1880	/* ins/outs */
1881	DISCPUSTATE Cpu;
1882
1883	/* Disassemble manually to deal with segment prefixes. */
1884	rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), &Cpu, NULL);
1885	if (rc == VINF_SUCCESS)
1886	{
1887	if (IoExitInfo.n.u1Type == 0)
1888	{
1889	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
1890	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
1891	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, Cpu.prefix, uIOSize);
1892	}
1893	else
1894	{
1895	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
1896	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
1897	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, Cpu.prefix, uIOSize);
1898	}
1899	}
1900	else
1901	rc = VINF_EM_RAW_EMULATE_INSTR;
1902	}
1903	else
1904	{
1905	/* normal in/out */
1906	Assert(!IoExitInfo.n.u1REP);
1907
1908	if (IoExitInfo.n.u1Type == 0)
1909	{
1910	Log2(("IOMIOPortWrite %RGv %x %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, uIOSize));
1911	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
1912	rc = IOMIOPortWrite(pVM, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, uIOSize);
1913	}
1914	else
1915	{
1916	uint32_t u32Val = 0;
1917
1918	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
1919	rc = IOMIOPortRead(pVM, IoExitInfo.n.u16Port, &u32Val, uIOSize);
1920	if (IOM_SUCCESS(rc))
1921	{
1922	/* Write back to the EAX register. */
1923	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
1924	Log2(("IOMIOPortRead %RGv %x %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, u32Val & uAndVal, uIOSize));
1925	}
1926	}
1927	}
1928	/*
1929	* Handled the I/O return codes.
1930	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
1931	*/
1932	if (IOM_SUCCESS(rc))
1933	{
1934	/* Update EIP and continue execution. */
1935	pCtx->rip = pVMCB->ctrl.u64ExitInfo2; /* RIP/EIP of the next instruction is saved in EXITINFO2. */
1936	if (RT_LIKELY(rc == VINF_SUCCESS))
1937	{
1938	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
1939	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
1940	{
1941	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
1942	for (unsigned i=0;i<4;i++)
1943	{
1944	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
1945
1946	if ( (IoExitInfo.n.u16Port >= pCtx->dr[i] && IoExitInfo.n.u16Port < pCtx->dr[i] + uBPLen)
1947	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
1948	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
1949	{
1950	SVM_EVENT Event;
1951
1952	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1953
1954	/* Clear all breakpoint status flags and set the one we just hit. */
1955	pCtx->dr[6] &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
1956	pCtx->dr[6] \|= (uint64_t)RT_BIT(i);
1957
1958	/* Note: AMD64 Architecture Programmer's Manual 13.1:
1959	* Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
1960	* the contents have been read.
1961	*/
1962	pVMCB->guest.u64DR6 = pCtx->dr[6];
1963
1964	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
1965	pCtx->dr[7] &= ~X86_DR7_GD;
1966
1967	/* Paranoia. */
1968	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1969	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
1970	pCtx->dr[7] \|= 0x400; /* must be one */
1971
1972	pVMCB->guest.u64DR7 = pCtx->dr[7];
1973
1974	/* Inject the exception. */
1975	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
1976
1977	Event.au64[0] = 0;
1978	Event.n.u3Type = SVM_EVENT_EXCEPTION; /* trap or fault */
1979	Event.n.u1Valid = 1;
1980	Event.n.u8Vector = X86_XCPT_DB;
1981
1982	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1983
1984	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1985	goto ResumeExecution;
1986	}
1987	}
1988	}
1989
1990	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1991	goto ResumeExecution;
1992	}
1993	Log2(("EM status from IO at %RGv %x size %d: %Rrc\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize, rc));
1994	break;
1995	}
1996
1997	#ifdef VBOX_STRICT
1998	if (rc == VINF_IOM_HC_IOPORT_READ)
1999	Assert(IoExitInfo.n.u1Type != 0);
2000	else if (rc == VINF_IOM_HC_IOPORT_WRITE)
2001	Assert(IoExitInfo.n.u1Type == 0);
2002	else
2003	AssertMsg(RT_FAILURE(rc) \|\| rc == VINF_EM_RAW_EMULATE_INSTR \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
2004	#endif
2005	Log2(("Failed IO at %RGv %x size %d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
2006	break;
2007	}
2008
2009	case SVM_EXIT_HLT:
2010	/** Check if external interrupts are pending; if so, don't switch back. */
2011	pCtx->rip++; /* skip hlt */
2012	if ( pCtx->eflags.Bits.u1IF
2013	&& VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)))
2014	goto ResumeExecution;
2015
2016	rc = VINF_EM_HALT;
2017	break;
2018
2019	case SVM_EXIT_RSM:
2020	case SVM_EXIT_INVLPGA:
2021	case SVM_EXIT_VMRUN:
2022	case SVM_EXIT_VMMCALL:
2023	case SVM_EXIT_VMLOAD:
2024	case SVM_EXIT_VMSAVE:
2025	case SVM_EXIT_STGI:
2026	case SVM_EXIT_CLGI:
2027	case SVM_EXIT_SKINIT:
2028	{
2029	/* Unsupported instructions. */
2030	SVM_EVENT Event;
2031
2032	Event.au64[0] = 0;
2033	Event.n.u3Type = SVM_EVENT_EXCEPTION;
2034	Event.n.u1Valid = 1;
2035	Event.n.u8Vector = X86_XCPT_UD;
2036
2037	Log(("Forced #UD trap at %RGv\n", (RTGCPTR)pCtx->rip));
2038	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
2039
2040	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
2041	goto ResumeExecution;
2042	}
2043
2044	/* Emulate in ring 3. */
2045	case SVM_EXIT_MSR:
2046	{
2047	uint32_t cbSize;
2048
2049	/* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
2050	Log(("SVM: %s\n", (pVMCB->ctrl.u64ExitInfo1 == 0) ? "rdmsr" : "wrmsr"));
2051	rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
2052	if (rc == VINF_SUCCESS)
2053	{
2054	/* EIP has been updated already. */
2055
2056	/* Only resume if successful. */
2057	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
2058	goto ResumeExecution;
2059	}
2060	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (pVMCB->ctrl.u64ExitInfo1 == 0) ? "rdmsr" : "wrmsr", rc));
2061	break;
2062	}
2063
2064	case SVM_EXIT_MONITOR:
2065	case SVM_EXIT_PAUSE:
2066	case SVM_EXIT_MWAIT_UNCOND:
2067	case SVM_EXIT_MWAIT_ARMED:
2068	case SVM_EXIT_TASK_SWITCH: /* can change CR3; emulate */
2069	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
2070	break;
2071
2072	case SVM_EXIT_SHUTDOWN:
2073	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
2074	break;
2075
2076	case SVM_EXIT_IDTR_READ:
2077	case SVM_EXIT_GDTR_READ:
2078	case SVM_EXIT_LDTR_READ:
2079	case SVM_EXIT_TR_READ:
2080	case SVM_EXIT_IDTR_WRITE:
2081	case SVM_EXIT_GDTR_WRITE:
2082	case SVM_EXIT_LDTR_WRITE:
2083	case SVM_EXIT_TR_WRITE:
2084	case SVM_EXIT_CR0_SEL_WRITE:
2085	default:
2086	/* Unexpected exit codes. */
2087	rc = VERR_EM_INTERNAL_ERROR;
2088	AssertMsgFailed(("Unexpected exit code %x\n", exitCode)); /* Can't happen. */
2089	break;
2090	}
2091
2092	end:
2093
2094	/* Signal changes for the recompiler. */
2095	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR \| CPUM_CHANGED_LDTR \| CPUM_CHANGED_GDTR \| CPUM_CHANGED_IDTR \| CPUM_CHANGED_TR \| CPUM_CHANGED_HIDDEN_SEL_REGS);
2096
2097	/* If we executed vmrun and an external irq was pending, then we don't have to do a full sync the next time. */
2098	if (exitCode == SVM_EXIT_INTR)
2099	{
2100	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
2101	/* On the next entry we'll only sync the host context. */
2102	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
2103	}
2104	else
2105	{
2106	/* On the next entry we'll sync everything. */
2107	/** @todo we can do better than this */
2108	/* Not in the VINF_PGM_CHANGE_MODE though! */
2109	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
2110	}
2111
2112	/* translate into a less severe return code */
2113	if (rc == VERR_EM_INTERPRETER)
2114	rc = VINF_EM_RAW_EMULATE_INSTR;
2115
2116	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
2117	return rc;
2118	}
2119
2120	/**
2121	* Enters the AMD-V session
2122	*
2123	* @returns VBox status code.
2124	* @param pVM The VM to operate on.
2125	* @param pVCpu The VM CPU to operate on.
2126	* @param pCpu CPU info struct
2127	*/
2128	VMMR0DECL(int) SVMR0Enter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
2129	{
2130	Assert(pVM->hwaccm.s.svm.fSupported);
2131
2132	LogFlow(("SVMR0Enter cpu%d last=%d asid=%d\n", pCpu->idCpu, pVCpu->hwaccm.s.idLastCpu, pVCpu->hwaccm.s.uCurrentASID));
2133	pVCpu->hwaccm.s.fResumeVM = false;
2134
2135	/* Force to reload LDTR, so we'll execute VMLoad to load additional guest state. */
2136	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_LDTR;
2137
2138	return VINF_SUCCESS;
2139	}
2140
2141
2142	/**
2143	* Leaves the AMD-V session
2144	*
2145	* @returns VBox status code.
2146	* @param pVM The VM to operate on.
2147	* @param pVCpu The VM CPU to operate on.
2148	* @param pCtx CPU context
2149	*/
2150	VMMR0DECL(int) SVMR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2151	{
2152	SVM_VMCB pVMCB = (SVM_VMCB )pVCpu->hwaccm.s.svm.pVMCB;
2153
2154	Assert(pVM->hwaccm.s.svm.fSupported);
2155
2156	/* Save the guest debug state if necessary. */
2157	if (CPUMIsGuestDebugStateActive(pVCpu))
2158	{
2159	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, false /* skip DR6 */);
2160
2161	/* Intercept all DRx reads and writes again. Changed later on. */
2162	pVMCB->ctrl.u16InterceptRdDRx = 0xFFFF;
2163	pVMCB->ctrl.u16InterceptWrDRx = 0xFFFF;
2164
2165	/* Resync the debug registers the next time. */
2166	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
2167	}
2168	else
2169	Assert(pVMCB->ctrl.u16InterceptRdDRx == 0xFFFF && pVMCB->ctrl.u16InterceptWrDRx == 0xFFFF);
2170
2171	return VINF_SUCCESS;
2172	}
2173
2174
2175	static int svmR0InterpretInvlPg(PVMCPU pVCpu, PDISCPUSTATE pCpu, PCPUMCTXCORE pRegFrame, uint32_t uASID)
2176	{
2177	OP_PARAMVAL param1;
2178	RTGCPTR addr;
2179
2180	int rc = DISQueryParamVal(pRegFrame, pCpu, &pCpu->param1, &param1, PARAM_SOURCE);
2181	if(RT_FAILURE(rc))
2182	return VERR_EM_INTERPRETER;
2183
2184	switch(param1.type)
2185	{
2186	case PARMTYPE_IMMEDIATE:
2187	case PARMTYPE_ADDRESS:
2188	if(!(param1.flags & (PARAM_VAL32\|PARAM_VAL64)))
2189	return VERR_EM_INTERPRETER;
2190	addr = param1.val.val64;
2191	break;
2192
2193	default:
2194	return VERR_EM_INTERPRETER;
2195	}
2196
2197	/** @todo is addr always a flat linear address or ds based
2198	* (in absence of segment override prefixes)????
2199	*/
2200	rc = PGMInvalidatePage(pVCpu, addr);
2201	if (RT_SUCCESS(rc))
2202	{
2203	/* Manually invalidate the page for the VM's TLB. */
2204	Log(("SVMR0InvlpgA %RGv ASID=%d\n", addr, uASID));
2205	SVMR0InvlpgA(addr, uASID);
2206	return VINF_SUCCESS;
2207	}
2208	Assert(rc == VERR_REM_FLUSHED_PAGES_OVERFLOW);
2209	return rc;
2210	}
2211
2212	/**
2213	* Interprets INVLPG
2214	*
2215	* @returns VBox status code.
2216	* @retval VINF_* Scheduling instructions.
2217	* @retval VERR_EM_INTERPRETER Something we can't cope with.
2218	* @retval VERR_* Fatal errors.
2219	*
2220	* @param pVM The VM handle.
2221	* @param pRegFrame The register frame.
2222	* @param ASID Tagged TLB id for the guest
2223	*
2224	* Updates the EIP if an instruction was executed successfully.
2225	*/
2226	static int SVMR0InterpretInvpg(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame, uint32_t uASID)
2227	{
2228	/*
2229	* Only allow 32 & 64 bits code.
2230	*/
2231	DISCPUMODE enmMode = SELMGetCpuModeFromSelector(pVM, pRegFrame->eflags, pRegFrame->cs, &pRegFrame->csHid);
2232	if (enmMode != CPUMODE_16BIT)
2233	{
2234	RTGCPTR pbCode;
2235	int rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->rip, &pbCode);
2236	if (RT_SUCCESS(rc))
2237	{
2238	uint32_t cbOp;
2239	DISCPUSTATE Cpu;
2240
2241	Cpu.mode = enmMode;
2242	rc = EMInterpretDisasOneEx(pVM, pVCpu, pbCode, pRegFrame, &Cpu, &cbOp);
2243	Assert(RT_FAILURE(rc) \|\| Cpu.pCurInstr->opcode == OP_INVLPG);
2244	if (RT_SUCCESS(rc) && Cpu.pCurInstr->opcode == OP_INVLPG)
2245	{
2246	Assert(cbOp == Cpu.opsize);
2247	rc = svmR0InterpretInvlPg(pVCpu, &Cpu, pRegFrame, uASID);
2248	if (RT_SUCCESS(rc))
2249	{
2250	pRegFrame->rip += cbOp; /* Move on to the next instruction. */
2251	}
2252	return rc;
2253	}
2254	}
2255	}
2256	return VERR_EM_INTERPRETER;
2257	}
2258
2259
2260	/**
2261	* Invalidates a guest page
2262	*
2263	* @returns VBox status code.
2264	* @param pVM The VM to operate on.
2265	* @param pVCpu The VM CPU to operate on.
2266	* @param GCVirt Page to invalidate
2267	*/
2268	VMMR0DECL(int) SVMR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
2269	{
2270	bool fFlushPending = pVM->hwaccm.s.svm.fAlwaysFlushTLB \| pVCpu->hwaccm.s.fForceTLBFlush;
2271
2272	/* Skip it if a TLB flush is already pending. */
2273	if (!fFlushPending)
2274	{
2275	SVM_VMCB *pVMCB;
2276
2277	Log2(("SVMR0InvalidatePage %RGv\n", GCVirt));
2278	AssertReturn(pVM, VERR_INVALID_PARAMETER);
2279	Assert(pVM->hwaccm.s.svm.fSupported);
2280
2281	/* @todo SMP */
2282	pVMCB = (SVM_VMCB *)pVM->aCpus[0].hwaccm.s.svm.pVMCB;
2283	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
2284
2285	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushPageManual);
2286	#if HC_ARCH_BITS == 32
2287	/* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invlpga takes only 32 bits addresses. */
2288	if (CPUMIsGuestInLongMode(pVCpu))
2289	pVCpu->hwaccm.s.fForceTLBFlush = true;
2290	else
2291	#endif
2292	SVMR0InvlpgA(GCVirt, pVMCB->ctrl.TLBCtrl.n.u32ASID);
2293	}
2294	return VINF_SUCCESS;
2295	}
2296
2297
2298	/**
2299	* Invalidates a guest page by physical address
2300	*
2301	* @returns VBox status code.
2302	* @param pVM The VM to operate on.
2303	* @param pVCpu The VM CPU to operate on.
2304	* @param GCPhys Page to invalidate
2305	*/
2306	VMMR0DECL(int) SVMR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
2307	{
2308	Assert(pVM->hwaccm.s.fNestedPaging);
2309	/* invlpga only invalidates TLB entries for guest virtual addresses; we have no choice but to force a TLB flush here. */
2310	pVCpu->hwaccm.s.fForceTLBFlush = true;
2311	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBInvlpga);
2312	return VINF_SUCCESS;
2313	}
2314
2315	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2316	/**
2317	* Prepares for and executes VMRUN (64 bits guests from a 32 bits hosts).
2318	*
2319	* @returns VBox status code.
2320	* @param pVMCBHostPhys Physical address of host VMCB.
2321	* @param pVMCBPhys Physical address of the VMCB.
2322	* @param pCtx Guest context.
2323	* @param pVM The VM to operate on.
2324	* @param pVCpu The VMCPU to operate on.
2325	*/
2326	DECLASM(int) SVMR0VMSwitcherRun64(RTHCPHYS pVMCBHostPhys, RTHCPHYS pVMCBPhys, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu)
2327	{
2328	uint32_t aParam[4];
2329
2330	aParam[0] = (uint32_t)(pVMCBHostPhys); /* Param 1: pVMCBHostPhys - Lo. */
2331	aParam[1] = (uint32_t)(pVMCBHostPhys >> 32); /* Param 1: pVMCBHostPhys - Hi. */
2332	aParam[2] = (uint32_t)(pVMCBPhys); /* Param 2: pVMCBPhys - Lo. */
2333	aParam[3] = (uint32_t)(pVMCBPhys >> 32); /* Param 2: pVMCBPhys - Hi. */
2334
2335	return SVMR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnSVMGCVMRun64, 4, &aParam[0]);
2336	}
2337
2338	/**
2339	* Executes the specified handler in 64 mode
2340	*
2341	* @returns VBox status code.
2342	* @param pVM The VM to operate on.
2343	* @param pVCpu The VMCPU to operate on.
2344	* @param pCtx Guest context
2345	* @param pfnHandler RC handler
2346	* @param cbParam Number of parameters
2347	* @param paParam Array of 32 bits parameters
2348	*/
2349	VMMR0DECL(int) SVMR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
2350	{
2351	int rc;
2352	RTHCUINTREG uFlags;
2353
2354	/* @todo This code is not guest SMP safe (hyper stack) */
2355	AssertReturn(pVM->cCPUs == 1, VERR_ACCESS_DENIED);
2356	Assert(pfnHandler);
2357
2358	uFlags = ASMIntDisableFlags();
2359
2360	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVM));
2361	CPUMSetHyperEIP(pVCpu, pfnHandler);
2362	for (int i=(int)cbParam-1;i>=0;i--)
2363	CPUMPushHyper(pVCpu, paParam[i]);
2364
2365	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
2366	/* Call switcher. */
2367	rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM);
2368	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
2369
2370	ASMSetFlags(uFlags);
2371	return rc;
2372	}
2373
2374	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWSVMR0.cpp@ 19169

Download in other formats: