HWSVMR0.cpp@ 11964

Last change on this file since 11964 was 11575, checked in by vboxsync, 16 years ago
Sync back CR2 as it can be changed behind our back in the nested paging case
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 84.3 KB

Line
1	/* $Id: HWSVMR0.cpp 11575 2008-08-22 13:08:12Z vboxsync $ */
2	/** @file
3	* HWACCM SVM - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/*******************************************************************************
24	* Header Files *
25	*******************************************************************************/
26	#define LOG_GROUP LOG_GROUP_HWACCM
27	#include <VBox/hwaccm.h>
28	#include "HWACCMInternal.h"
29	#include <VBox/vm.h>
30	#include <VBox/x86.h>
31	#include <VBox/hwacc_svm.h>
32	#include <VBox/pgm.h>
33	#include <VBox/pdm.h>
34	#include <VBox/err.h>
35	#include <VBox/log.h>
36	#include <VBox/selm.h>
37	#include <VBox/iom.h>
38	#include <VBox/dis.h>
39	#include <VBox/dbgf.h>
40	#include <VBox/disopcode.h>
41	#include <iprt/param.h>
42	#include <iprt/assert.h>
43	#include <iprt/asm.h>
44	#include <iprt/cpuset.h>
45	#include <iprt/mp.h>
46	#include "HWSVMR0.h"
47
48	static int SVMR0InterpretInvpg(PVM pVM, PCPUMCTXCORE pRegFrame, uint32_t uASID);
49
50	/**
51	* Sets up and activates AMD-V on the current CPU
52	*
53	* @returns VBox status code.
54	* @param pCpu CPU info struct
55	* @param pVM The VM to operate on.
56	* @param pvPageCpu Pointer to the global cpu page
57	* @param pPageCpuPhys Physical address of the global cpu page
58	*/
59	HWACCMR0DECL(int) SVMR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
60	{
61	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
62	AssertReturn(pVM, VERR_INVALID_PARAMETER);
63	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
64
65	/* We must turn on AMD-V and setup the host state physical address, as those MSRs are per-cpu/core. */
66
67	#ifdef LOG_ENABLED
68	SUPR0Printf("SVMR0EnableCpu cpu %d page (%x) %x\n", pCpu->idCpu, pvPageCpu, (uint32_t)pPageCpuPhys);
69	#endif
70
71	/* Turn on AMD-V in the EFER MSR. */
72	uint64_t val = ASMRdMsr(MSR_K6_EFER);
73	if (!(val & MSR_K6_EFER_SVME))
74	ASMWrMsr(MSR_K6_EFER, val \| MSR_K6_EFER_SVME);
75
76	/* Write the physical page address where the CPU will store the host state while executing the VM. */
77	ASMWrMsr(MSR_K8_VM_HSAVE_PA, pPageCpuPhys);
78
79	pCpu->uCurrentASID = 0; /* we'll aways increment this the first time (host uses ASID 0) */
80	pCpu->cTLBFlushes = 0;
81	return VINF_SUCCESS;
82	}
83
84	/**
85	* Deactivates AMD-V on the current CPU
86	*
87	* @returns VBox status code.
88	* @param pCpu CPU info struct
89	* @param pvPageCpu Pointer to the global cpu page
90	* @param pPageCpuPhys Physical address of the global cpu page
91	*/
92	HWACCMR0DECL(int) SVMR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
93	{
94	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
95	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
96
97	#ifdef LOG_ENABLED
98	SUPR0Printf("SVMR0DisableCpu cpu %d\n", pCpu->idCpu);
99	#endif
100
101	/* Turn off AMD-V in the EFER MSR. */
102	uint64_t val = ASMRdMsr(MSR_K6_EFER);
103	ASMWrMsr(MSR_K6_EFER, val & ~MSR_K6_EFER_SVME);
104
105	/* Invalidate host state physical address. */
106	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
107	pCpu->uCurrentASID = 0;
108
109	return VINF_SUCCESS;
110	}
111
112	/**
113	* Does Ring-0 per VM AMD-V init.
114	*
115	* @returns VBox status code.
116	* @param pVM The VM to operate on.
117	*/
118	HWACCMR0DECL(int) SVMR0InitVM(PVM pVM)
119	{
120	int rc;
121
122	pVM->hwaccm.s.svm.pMemObjVMCB = NIL_RTR0MEMOBJ;
123	pVM->hwaccm.s.svm.pMemObjVMCBHost = NIL_RTR0MEMOBJ;
124	pVM->hwaccm.s.svm.pMemObjIOBitmap = NIL_RTR0MEMOBJ;
125	pVM->hwaccm.s.svm.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
126
127
128	/* Allocate one page for the VM control block (VMCB). */
129	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjVMCB, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
130	if (RT_FAILURE(rc))
131	return rc;
132
133	pVM->hwaccm.s.svm.pVMCB = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjVMCB);
134	pVM->hwaccm.s.svm.pVMCBPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjVMCB, 0);
135	ASMMemZeroPage(pVM->hwaccm.s.svm.pVMCB);
136
137	/* Allocate one page for the host context */
138	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjVMCBHost, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
139	if (RT_FAILURE(rc))
140	return rc;
141
142	pVM->hwaccm.s.svm.pVMCBHost = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjVMCBHost);
143	pVM->hwaccm.s.svm.pVMCBHostPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjVMCBHost, 0);
144	ASMMemZeroPage(pVM->hwaccm.s.svm.pVMCBHost);
145
146	/* Allocate 12 KB for the IO bitmap (doesn't seem to be a way to convince SVM not to use it) */
147	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjIOBitmap, 3 << PAGE_SHIFT, true /* executable R0 mapping */);
148	if (RT_FAILURE(rc))
149	return rc;
150
151	pVM->hwaccm.s.svm.pIOBitmap = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjIOBitmap);
152	pVM->hwaccm.s.svm.pIOBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjIOBitmap, 0);
153	/* Set all bits to intercept all IO accesses. */
154	ASMMemFill32(pVM->hwaccm.s.svm.pIOBitmap, PAGE_SIZE*3, 0xffffffff);
155
156	/* Allocate 8 KB for the MSR bitmap (doesn't seem to be a way to convince SVM not to use it) */
157	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjMSRBitmap, 2 << PAGE_SHIFT, true /* executable R0 mapping */);
158	if (RT_FAILURE(rc))
159	return rc;
160
161	pVM->hwaccm.s.svm.pMSRBitmap = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjMSRBitmap);
162	pVM->hwaccm.s.svm.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjMSRBitmap, 0);
163	/* Set all bits to intercept all MSR accesses. */
164	ASMMemFill32(pVM->hwaccm.s.svm.pMSRBitmap, PAGE_SIZE*2, 0xffffffff);
165
166	/* Erratum 170 which requires a forced TLB flush for each world switch:
167	* See http://www.amd.com/us-en/assets/content_type/white_papers_and_tech_docs/33610.pdf
168	*
169	* All BH-G1/2 and DH-G1/2 models include a fix:
170	* Athlon X2: 0x6b 1/2
171	* 0x68 1/2
172	* Athlon 64: 0x7f 1
173	* 0x6f 2
174	* Sempron: 0x7f 1/2
175	* 0x6f 2
176	* 0x6c 2
177	* 0x7c 2
178	* Turion 64: 0x68 2
179	*
180	*/
181	uint32_t u32Dummy;
182	uint32_t u32Version, u32Family, u32Model, u32Stepping, u32BaseFamily;
183	ASMCpuId(1, &u32Version, &u32Dummy, &u32Dummy, &u32Dummy);
184	u32BaseFamily= (u32Version >> 8) & 0xf;
185	u32Family = u32BaseFamily + (u32BaseFamily == 0xf ? ((u32Version >> 20) & 0x7f) : 0);
186	u32Model = ((u32Version >> 4) & 0xf);
187	u32Model = u32Model \| ((u32BaseFamily == 0xf ? (u32Version >> 16) & 0x0f : 0) << 4);
188	u32Stepping = u32Version & 0xf;
189	if ( u32Family == 0xf
190	&& !((u32Model == 0x68 \|\| u32Model == 0x6b \|\| u32Model == 0x7f) && u32Stepping >= 1)
191	&& !((u32Model == 0x6f \|\| u32Model == 0x6c \|\| u32Model == 0x7c) && u32Stepping >= 2))
192	{
193	Log(("SVMR0InitVM: AMD cpu with erratum 170 family %x model %x stepping %x\n", u32Family, u32Model, u32Stepping));
194	pVM->hwaccm.s.svm.fAlwaysFlushTLB = true;
195	}
196
197	/* Invalidate the last cpu we were running on. */
198	pVM->hwaccm.s.svm.idLastCpu = NIL_RTCPUID;
199
200	/* we'll aways increment this the first time (host uses ASID 0) */
201	pVM->hwaccm.s.svm.uCurrentASID = 0;
202	return VINF_SUCCESS;
203	}
204
205	/**
206	* Does Ring-0 per VM AMD-V termination.
207	*
208	* @returns VBox status code.
209	* @param pVM The VM to operate on.
210	*/
211	HWACCMR0DECL(int) SVMR0TermVM(PVM pVM)
212	{
213	if (pVM->hwaccm.s.svm.pMemObjVMCB != NIL_RTR0MEMOBJ)
214	{
215	RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjVMCB, false);
216	pVM->hwaccm.s.svm.pVMCB = 0;
217	pVM->hwaccm.s.svm.pVMCBPhys = 0;
218	pVM->hwaccm.s.svm.pMemObjVMCB = NIL_RTR0MEMOBJ;
219	}
220	if (pVM->hwaccm.s.svm.pMemObjVMCBHost != NIL_RTR0MEMOBJ)
221	{
222	RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjVMCBHost, false);
223	pVM->hwaccm.s.svm.pVMCBHost = 0;
224	pVM->hwaccm.s.svm.pVMCBHostPhys = 0;
225	pVM->hwaccm.s.svm.pMemObjVMCBHost = NIL_RTR0MEMOBJ;
226	}
227	if (pVM->hwaccm.s.svm.pMemObjIOBitmap != NIL_RTR0MEMOBJ)
228	{
229	RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjIOBitmap, false);
230	pVM->hwaccm.s.svm.pIOBitmap = 0;
231	pVM->hwaccm.s.svm.pIOBitmapPhys = 0;
232	pVM->hwaccm.s.svm.pMemObjIOBitmap = NIL_RTR0MEMOBJ;
233	}
234	if (pVM->hwaccm.s.svm.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
235	{
236	RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjMSRBitmap, false);
237	pVM->hwaccm.s.svm.pMSRBitmap = 0;
238	pVM->hwaccm.s.svm.pMSRBitmapPhys = 0;
239	pVM->hwaccm.s.svm.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
240	}
241	return VINF_SUCCESS;
242	}
243
244	/**
245	* Sets up AMD-V for the specified VM
246	*
247	* @returns VBox status code.
248	* @param pVM The VM to operate on.
249	*/
250	HWACCMR0DECL(int) SVMR0SetupVM(PVM pVM)
251	{
252	int rc = VINF_SUCCESS;
253	SVM_VMCB *pVMCB;
254
255	AssertReturn(pVM, VERR_INVALID_PARAMETER);
256
257	Assert(pVM->hwaccm.s.svm.fSupported);
258
259	pVMCB = (SVM_VMCB *)pVM->hwaccm.s.svm.pVMCB;
260	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
261
262	/* Program the control fields. Most of them never have to be changed again. */
263	/* CR0/3/4 reads must be intercepted, our shadow values are not necessarily the same as the guest's. */
264	/** @note CR0 & CR4 can be safely read when guest and shadow copies are identical. */
265	if (!pVM->hwaccm.s.fNestedPaging)
266	pVMCB->ctrl.u16InterceptRdCRx = RT_BIT(0) \| RT_BIT(3) \| RT_BIT(4);
267	else
268	pVMCB->ctrl.u16InterceptRdCRx = RT_BIT(0) \| RT_BIT(4);
269
270	/*
271	* CR0/3/4 writes must be intercepted for obvious reasons.
272	*/
273	if (!pVM->hwaccm.s.fNestedPaging)
274	pVMCB->ctrl.u16InterceptWrCRx = RT_BIT(0) \| RT_BIT(3) \| RT_BIT(4);
275	else
276	pVMCB->ctrl.u16InterceptWrCRx = RT_BIT(0) \| RT_BIT(4) \| RT_BIT(8);
277
278	/* Intercept all DRx reads and writes. */
279	pVMCB->ctrl.u16InterceptRdDRx = RT_BIT(0) \| RT_BIT(1) \| RT_BIT(2) \| RT_BIT(3) \| RT_BIT(4) \| RT_BIT(5) \| RT_BIT(6) \| RT_BIT(7);
280	pVMCB->ctrl.u16InterceptWrDRx = RT_BIT(0) \| RT_BIT(1) \| RT_BIT(2) \| RT_BIT(3) \| RT_BIT(4) \| RT_BIT(5) \| RT_BIT(6) \| RT_BIT(7);
281
282	/* Currently we don't care about DRx reads or writes. DRx registers are trashed.
283	* All breakpoints are automatically cleared when the VM exits.
284	*/
285
286	pVMCB->ctrl.u32InterceptException = HWACCM_SVM_TRAP_MASK;
287	#ifndef DEBUG
288	if (pVM->hwaccm.s.fNestedPaging)
289	pVMCB->ctrl.u32InterceptException &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
290	#endif
291
292	pVMCB->ctrl.u32InterceptCtrl1 = SVM_CTRL1_INTERCEPT_INTR
293	\| SVM_CTRL1_INTERCEPT_VINTR
294	\| SVM_CTRL1_INTERCEPT_NMI
295	\| SVM_CTRL1_INTERCEPT_SMI
296	\| SVM_CTRL1_INTERCEPT_INIT
297	\| SVM_CTRL1_INTERCEPT_RDPMC
298	\| SVM_CTRL1_INTERCEPT_CPUID
299	\| SVM_CTRL1_INTERCEPT_RSM
300	\| SVM_CTRL1_INTERCEPT_HLT
301	\| SVM_CTRL1_INTERCEPT_INOUT_BITMAP
302	\| SVM_CTRL1_INTERCEPT_MSR_SHADOW
303	\| SVM_CTRL1_INTERCEPT_INVLPG
304	\| SVM_CTRL1_INTERCEPT_INVLPGA /* AMD only */
305	\| SVM_CTRL1_INTERCEPT_TASK_SWITCH
306	\| SVM_CTRL1_INTERCEPT_SHUTDOWN /* fatal */
307	\| SVM_CTRL1_INTERCEPT_FERR_FREEZE; /* Legacy FPU FERR handling. */
308	;
309	/* With nested paging we don't care about invlpg anymore. */
310	if (pVM->hwaccm.s.fNestedPaging)
311	pVMCB->ctrl.u32InterceptCtrl1 &= ~SVM_CTRL1_INTERCEPT_INVLPG;
312
313	pVMCB->ctrl.u32InterceptCtrl2 = SVM_CTRL2_INTERCEPT_VMRUN /* required */
314	\| SVM_CTRL2_INTERCEPT_VMMCALL
315	\| SVM_CTRL2_INTERCEPT_VMLOAD
316	\| SVM_CTRL2_INTERCEPT_VMSAVE
317	\| SVM_CTRL2_INTERCEPT_STGI
318	\| SVM_CTRL2_INTERCEPT_CLGI
319	\| SVM_CTRL2_INTERCEPT_SKINIT
320	\| SVM_CTRL2_INTERCEPT_RDTSCP /* AMD only; we don't support this one */
321	\| SVM_CTRL2_INTERCEPT_WBINVD
322	\| SVM_CTRL2_INTERCEPT_MWAIT_UNCOND; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
323	;
324	Log(("pVMCB->ctrl.u32InterceptException = %x\n", pVMCB->ctrl.u32InterceptException));
325	Log(("pVMCB->ctrl.u32InterceptCtrl1 = %x\n", pVMCB->ctrl.u32InterceptCtrl1));
326	Log(("pVMCB->ctrl.u32InterceptCtrl2 = %x\n", pVMCB->ctrl.u32InterceptCtrl2));
327
328	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
329	pVMCB->ctrl.IntCtrl.n.u1VIrqMasking = 1;
330	/* Ignore the priority in the TPR; just deliver it when we tell it to. */
331	pVMCB->ctrl.IntCtrl.n.u1IgnoreTPR = 1;
332
333	/* Set IO and MSR bitmap addresses. */
334	pVMCB->ctrl.u64IOPMPhysAddr = pVM->hwaccm.s.svm.pIOBitmapPhys;
335	pVMCB->ctrl.u64MSRPMPhysAddr = pVM->hwaccm.s.svm.pMSRBitmapPhys;
336
337	/* No LBR virtualization. */
338	pVMCB->ctrl.u64LBRVirt = 0;
339
340	/** The ASID must start at 1; the host uses 0. */
341	pVMCB->ctrl.TLBCtrl.n.u32ASID = 1;
342
343	/** Setup the PAT msr (nested paging only) */
344	pVMCB->guest.u64GPAT = 0x0007040600070406ULL;
345	return rc;
346	}
347
348
349	/**
350	* Injects an event (trap or external interrupt)
351	*
352	* @param pVM The VM to operate on.
353	* @param pVMCB SVM control block
354	* @param pCtx CPU Context
355	* @param pIntInfo SVM interrupt info
356	*/
357	inline void SVMR0InjectEvent(PVM pVM, SVM_VMCB pVMCB, CPUMCTX pCtx, SVM_EVENT* pEvent)
358	{
359	#ifdef VBOX_STRICT
360	if (pEvent->n.u8Vector == 0xE)
361	Log(("SVM: Inject int %d at %VGv error code=%02x CR2=%VGv intInfo=%08x\n", pEvent->n.u8Vector, pCtx->rip, pEvent->n.u32ErrorCode, pCtx->cr2, pEvent->au64[0]));
362	else
363	if (pEvent->n.u8Vector < 0x20)
364	Log(("SVM: Inject int %d at %VGv error code=%08x\n", pEvent->n.u8Vector, pCtx->rip, pEvent->n.u32ErrorCode));
365	else
366	{
367	Log(("INJ-EI: %x at %VGv\n", pEvent->n.u8Vector, pCtx->rip));
368	Assert(!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS));
369	Assert(pCtx->eflags.u32 & X86_EFL_IF);
370	}
371	#endif
372
373	/* Set event injection state. */
374	pVMCB->ctrl.EventInject.au64[0] = pEvent->au64[0];
375	}
376
377
378	/**
379	* Checks for pending guest interrupts and injects them
380	*
381	* @returns VBox status code.
382	* @param pVM The VM to operate on.
383	* @param pVMCB SVM control block
384	* @param pCtx CPU Context
385	*/
386	static int SVMR0CheckPendingInterrupt(PVM pVM, SVM_VMCB pVMCB, CPUMCTX pCtx)
387	{
388	int rc;
389
390	/* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
391	if (pVM->hwaccm.s.Event.fPending)
392	{
393	SVM_EVENT Event;
394
395	Log(("Reinjecting event %08x %08x at %VGv\n", pVM->hwaccm.s.Event.intInfo, pVM->hwaccm.s.Event.errCode, pCtx->rip));
396	STAM_COUNTER_INC(&pVM->hwaccm.s.StatIntReinject);
397	Event.au64[0] = pVM->hwaccm.s.Event.intInfo;
398	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
399
400	pVM->hwaccm.s.Event.fPending = false;
401	return VINF_SUCCESS;
402	}
403
404	/* When external interrupts are pending, we should exit the VM when IF is set. */
405	if ( !TRPMHasTrap(pVM)
406	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
407	{
408	if ( !(pCtx->eflags.u32 & X86_EFL_IF)
409	\|\| VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
410	{
411	if (!pVMCB->ctrl.IntCtrl.n.u1VIrqValid)
412	{
413	if (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
414	LogFlow(("Enable irq window exit!\n"));
415	else
416	Log(("Pending interrupt blocked at %VGv by VM_FF_INHIBIT_INTERRUPTS -> irq window exit\n", pCtx->rip));
417
418	/** @todo use virtual interrupt method to inject a pending irq; dispatched as soon as guest.IF is set. */
419	pVMCB->ctrl.u32InterceptCtrl1 \|= SVM_CTRL1_INTERCEPT_VINTR;
420	pVMCB->ctrl.IntCtrl.n.u1VIrqValid = 1;
421	pVMCB->ctrl.IntCtrl.n.u8VIrqVector = 0; /* don't care */
422	}
423	}
424	else
425	{
426	uint8_t u8Interrupt;
427
428	rc = PDMGetInterrupt(pVM, &u8Interrupt);
429	Log(("Dispatch interrupt: u8Interrupt=%x (%d) rc=%Vrc\n", u8Interrupt, u8Interrupt, rc));
430	if (VBOX_SUCCESS(rc))
431	{
432	rc = TRPMAssertTrap(pVM, u8Interrupt, TRPM_HARDWARE_INT);
433	AssertRC(rc);
434	}
435	else
436	{
437	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
438	Assert(!VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)));
439	STAM_COUNTER_INC(&pVM->hwaccm.s.StatSwitchGuestIrq);
440	/* Just continue */
441	}
442	}
443	}
444
445	#ifdef VBOX_STRICT
446	if (TRPMHasTrap(pVM))
447	{
448	uint8_t u8Vector;
449	rc = TRPMQueryTrapAll(pVM, &u8Vector, 0, 0, 0);
450	AssertRC(rc);
451	}
452	#endif
453
454	if ( pCtx->eflags.u32 & X86_EFL_IF
455	&& (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
456	&& TRPMHasTrap(pVM)
457	)
458	{
459	uint8_t u8Vector;
460	int rc;
461	TRPMEVENT enmType;
462	SVM_EVENT Event;
463	RTGCUINT u32ErrorCode;
464
465	Event.au64[0] = 0;
466
467	/* If a new event is pending, then dispatch it now. */
468	rc = TRPMQueryTrapAll(pVM, &u8Vector, &enmType, &u32ErrorCode, 0);
469	AssertRC(rc);
470	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
471	Assert(enmType != TRPM_SOFTWARE_INT);
472
473	/* Clear the pending trap. */
474	rc = TRPMResetTrap(pVM);
475	AssertRC(rc);
476
477	Event.n.u8Vector = u8Vector;
478	Event.n.u1Valid = 1;
479	Event.n.u32ErrorCode = u32ErrorCode;
480
481	if (enmType == TRPM_TRAP)
482	{
483	switch (u8Vector) {
484	case 8:
485	case 10:
486	case 11:
487	case 12:
488	case 13:
489	case 14:
490	case 17:
491	/* Valid error codes. */
492	Event.n.u1ErrorCodeValid = 1;
493	break;
494	default:
495	break;
496	}
497	if (u8Vector == X86_XCPT_NMI)
498	Event.n.u3Type = SVM_EVENT_NMI;
499	else
500	Event.n.u3Type = SVM_EVENT_EXCEPTION;
501	}
502	else
503	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
504
505	STAM_COUNTER_INC(&pVM->hwaccm.s.StatIntInject);
506	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
507	} /* if (interrupts can be dispatched) */
508
509	return VINF_SUCCESS;
510	}
511
512	/**
513	* Save the host state
514	*
515	* @returns VBox status code.
516	* @param pVM The VM to operate on.
517	*/
518	HWACCMR0DECL(int) SVMR0SaveHostState(PVM pVM)
519	{
520	/* Nothing to do here. */
521	return VINF_SUCCESS;
522	}
523
524	/**
525	* Loads the guest state
526	*
527	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
528	*
529	* @returns VBox status code.
530	* @param pVM The VM to operate on.
531	* @param pCtx Guest context
532	*/
533	HWACCMR0DECL(int) SVMR0LoadGuestState(PVM pVM, CPUMCTX *pCtx)
534	{
535	RTGCUINTPTR val;
536	SVM_VMCB *pVMCB;
537
538	if (pVM == NULL)
539	return VERR_INVALID_PARAMETER;
540
541	/* Setup AMD SVM. */
542	Assert(pVM->hwaccm.s.svm.fSupported);
543
544	pVMCB = (SVM_VMCB *)pVM->hwaccm.s.svm.pVMCB;
545	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
546
547	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
548	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
549	{
550	SVM_WRITE_SELREG(CS, cs);
551	SVM_WRITE_SELREG(SS, ss);
552	SVM_WRITE_SELREG(DS, ds);
553	SVM_WRITE_SELREG(ES, es);
554	SVM_WRITE_SELREG(FS, fs);
555	SVM_WRITE_SELREG(GS, gs);
556	}
557
558	/* Guest CPU context: LDTR. */
559	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
560	{
561	SVM_WRITE_SELREG(LDTR, ldtr);
562	}
563
564	/* Guest CPU context: TR. */
565	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
566	{
567	SVM_WRITE_SELREG(TR, tr);
568	}
569
570	/* Guest CPU context: GDTR. */
571	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
572	{
573	pVMCB->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
574	pVMCB->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
575	}
576
577	/* Guest CPU context: IDTR. */
578	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
579	{
580	pVMCB->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
581	pVMCB->guest.IDTR.u64Base = pCtx->idtr.pIdt;
582	}
583
584	/*
585	* Sysenter MSRs (unconditional)
586	*/
587	pVMCB->guest.u64SysEnterCS = pCtx->SysEnter.cs;
588	pVMCB->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
589	pVMCB->guest.u64SysEnterESP = pCtx->SysEnter.esp;
590
591	/* Control registers */
592	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
593	{
594	val = pCtx->cr0;
595	if (!CPUMIsGuestFPUStateActive(pVM))
596	{
597	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
598	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
599	}
600	else
601	{
602	/** @todo check if we support the old style mess correctly. */
603	if (!(val & X86_CR0_NE))
604	{
605	Log(("Forcing X86_CR0_NE!!!\n"));
606
607	/* Also catch floating point exceptions as we need to report them to the guest in a different way. */
608	if (!pVM->hwaccm.s.fFPUOldStyleOverride)
609	{
610	pVMCB->ctrl.u32InterceptException \|= RT_BIT(X86_XCPT_MF);
611	pVM->hwaccm.s.fFPUOldStyleOverride = true;
612	}
613	}
614	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
615	}
616	/* Always enable caching. */
617	val &= ~(X86_CR0_CD\|X86_CR0_NW);
618
619	/* Note: WP is not relevant in nested paging mode as we catch accesses on the (guest) physical level. */
620	/* Note: In nested paging mode the guest is allowed to run with paging disabled; the guest physical to host physical translation will remain active. */
621	if (!pVM->hwaccm.s.fNestedPaging)
622	{
623	val \|= X86_CR0_PG; /* Paging is always enabled; even when the guest is running in real mode or PE without paging. */
624	val \|= X86_CR0_WP; /* Must set this as we rely on protect various pages and supervisor writes must be caught. */
625	}
626	pVMCB->guest.u64CR0 = val;
627	}
628	/* CR2 as well */
629	pVMCB->guest.u64CR2 = pCtx->cr2;
630
631	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
632	{
633	/* Save our shadow CR3 register. */
634	if (pVM->hwaccm.s.fNestedPaging)
635	{
636	pVMCB->ctrl.u64NestedPagingCR3 = PGMGetNestedCR3(pVM, PGMGetHostMode(pVM));
637	Assert(pVMCB->ctrl.u64NestedPagingCR3);
638	pVMCB->guest.u64CR3 = pCtx->cr3;
639	}
640	else
641	{
642	pVMCB->guest.u64CR3 = PGMGetHyperCR3(pVM);
643	Assert(pVMCB->guest.u64CR3);
644	}
645	}
646
647	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
648	{
649	val = pCtx->cr4;
650	if (!pVM->hwaccm.s.fNestedPaging)
651	{
652	switch(pVM->hwaccm.s.enmShadowMode)
653	{
654	case PGMMODE_REAL:
655	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
656	AssertFailed();
657	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
658
659	case PGMMODE_32_BIT: /* 32-bit paging. */
660	break;
661
662	case PGMMODE_PAE: /* PAE paging. */
663	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
664	/** @todo use normal 32 bits paging */
665	val \|= X86_CR4_PAE;
666	break;
667
668	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
669	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
670	#ifdef VBOX_ENABLE_64_BITS_GUESTS
671	break;
672	#else
673	AssertFailed();
674	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
675	#endif
676
677	default: /* shut up gcc */
678	AssertFailed();
679	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
680	}
681	}
682	pVMCB->guest.u64CR4 = val;
683	}
684
685	/* Debug registers. */
686	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
687	{
688	/** @todo DR0-6 */
689	val = pCtx->dr7;
690	val &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
691	val \|= 0x400; /* must be one */
692	#ifdef VBOX_STRICT
693	val = 0x400;
694	#endif
695	pVMCB->guest.u64DR7 = val;
696
697	pVMCB->guest.u64DR6 = pCtx->dr6;
698	}
699
700	/* EIP, ESP and EFLAGS */
701	pVMCB->guest.u64RIP = pCtx->rip;
702	pVMCB->guest.u64RSP = pCtx->rsp;
703	pVMCB->guest.u64RFlags = pCtx->eflags.u32;
704
705	/* Set CPL */
706	pVMCB->guest.u8CPL = pCtx->csHid.Attr.n.u2Dpl;
707
708	/* RAX/EAX too, as VMRUN uses RAX as an implicit parameter. */
709	pVMCB->guest.u64RAX = pCtx->rax;
710
711	/* vmrun will fail without MSR_K6_EFER_SVME. */
712	pVMCB->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
713
714	/* 64 bits guest mode? */
715	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
716	{
717	#if !defined(VBOX_WITH_64_BITS_GUESTS) \|\| HC_ARCH_BITS != 64
718	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
719	#else
720	pVM->hwaccm.s.svm.pfnVMRun = SVMVMRun64;
721	#endif
722	/* Unconditionally update these as wrmsr might have changed them. (HWACCM_CHANGED_GUEST_SEGMENT_REGS will not be set) */
723	pVMCB->guest.FS.u64Base = pCtx->fsHid.u64Base;
724	pVMCB->guest.GS.u64Base = pCtx->gsHid.u64Base;
725	}
726	else
727	{
728	/* Filter out the MSR_K6_LME bit or else AMD-V expects amd64 shadow paging. */
729	pVMCB->guest.u64EFER &= ~MSR_K6_EFER_LME;
730
731	pVM->hwaccm.s.svm.pfnVMRun = SVMVMRun;
732	}
733
734	/** TSC offset. */
735	if (TMCpuTickCanUseRealTSC(pVM, &pVMCB->ctrl.u64TSCOffset))
736	{
737	pVMCB->ctrl.u32InterceptCtrl1 &= ~SVM_CTRL1_INTERCEPT_RDTSC;
738	STAM_COUNTER_INC(&pVM->hwaccm.s.StatTSCOffset);
739	}
740	else
741	{
742	pVMCB->ctrl.u32InterceptCtrl1 \|= SVM_CTRL1_INTERCEPT_RDTSC;
743	STAM_COUNTER_INC(&pVM->hwaccm.s.StatTSCIntercept);
744	}
745
746	/* Sync the various msrs for 64 bits mode. */
747	pVMCB->guest.u64STAR = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
748	pVMCB->guest.u64LSTAR = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
749	pVMCB->guest.u64CSTAR = pCtx->msrCSTAR; /* compatibility mode syscall rip */
750	pVMCB->guest.u64SFMASK = pCtx->msrSFMASK; /* syscall flag mask */
751	pVMCB->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
752
753	#ifdef DEBUG
754	/* Intercept X86_XCPT_DB if stepping is enabled */
755	if (DBGFIsStepping(pVM))
756	pVMCB->ctrl.u32InterceptException \|= RT_BIT(X86_XCPT_DB);
757	else
758	pVMCB->ctrl.u32InterceptException &= ~RT_BIT(X86_XCPT_DB);
759	#endif
760
761	/* Done. */
762	pVM->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
763
764	return VINF_SUCCESS;
765	}
766
767
768	/**
769	* Runs guest code in an SVM VM.
770	*
771	* @todo This can be much more efficient, when we only sync that which has actually changed. (this is the first attempt only)
772	*
773	* @returns VBox status code.
774	* @param pVM The VM to operate on.
775	* @param pCtx Guest context
776	*/
777	HWACCMR0DECL(int) SVMR0RunGuestCode(PVM pVM, CPUMCTX *pCtx)
778	{
779	int rc = VINF_SUCCESS;
780	uint64_t exitCode = (uint64_t)SVM_EXIT_INVALID;
781	SVM_VMCB *pVMCB;
782	bool fGuestStateSynced = false;
783	bool fSyncTPR = false;
784	unsigned cResume = 0;
785	uint8_t u8LastVTPR;
786	PHWACCM_CPUINFO pCpu = 0;
787	#ifdef VBOX_STRICT
788	RTCPUID idCpuCheck;
789	#endif
790
791	STAM_PROFILE_ADV_START(&pVM->hwaccm.s.StatEntry, x);
792
793	pVMCB = (SVM_VMCB *)pVM->hwaccm.s.svm.pVMCB;
794	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
795
796	/* We can jump to this point to resume execution after determining that a VM-exit is innocent.
797	*/
798	ResumeExecution:
799	/* Safety precaution; looping for too long here can have a very bad effect on the host */
800	if (++cResume > HWACCM_MAX_RESUME_LOOPS)
801	{
802	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitMaxResume);
803	rc = VINF_EM_RAW_INTERRUPT;
804	goto end;
805	}
806
807	/* Check for irq inhibition due to instruction fusing (sti, mov ss). */
808	if (VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
809	{
810	Log(("VM_FF_INHIBIT_INTERRUPTS at %VGv successor %VGv\n", pCtx->rip, EMGetInhibitInterruptsPC(pVM)));
811	if (pCtx->rip != EMGetInhibitInterruptsPC(pVM))
812	{
813	/** @note we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
814	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
815	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
816	* break the guest. Sounds very unlikely, but such timing sensitive problem are not as rare as you might think.
817	*/
818	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
819	/* Irq inhibition is no longer active; clear the corresponding SVM state. */
820	pVMCB->ctrl.u64IntShadow = 0;
821	}
822	}
823	else
824	{
825	/* Irq inhibition is no longer active; clear the corresponding SVM state. */
826	pVMCB->ctrl.u64IntShadow = 0;
827	}
828
829	/* Check for pending actions that force us to go back to ring 3. */
830	#ifdef DEBUG
831	/* Intercept X86_XCPT_DB if stepping is enabled */
832	if (!DBGFIsStepping(pVM))
833	#endif
834	{
835	if (VM_FF_ISPENDING(pVM, VM_FF_TO_R3 \| VM_FF_TIMER))
836	{
837	VM_FF_CLEAR(pVM, VM_FF_TO_R3);
838	STAM_COUNTER_INC(&pVM->hwaccm.s.StatSwitchToR3);
839	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatEntry, x);
840	rc = VINF_EM_RAW_TO_R3;
841	goto end;
842	}
843	}
844
845	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
846	if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
847	{
848	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatEntry, x);
849	rc = VINF_EM_PENDING_REQUEST;
850	goto end;
851	}
852
853	/* When external interrupts are pending, we should exit the VM when IF is set. */
854	/** @note after VM_FF_INHIBIT_INTERRUPTS check!!! */
855	rc = SVMR0CheckPendingInterrupt(pVM, pVMCB, pCtx);
856	if (VBOX_FAILURE(rc))
857	{
858	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatEntry, x);
859	goto end;
860	}
861
862	/* TPR caching using CR8 is only available in 64 bits mode */
863	/* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
864	/* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!!!!! */
865	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
866	{
867	bool fPending;
868
869	/* TPR caching in CR8 */
870	int rc = PDMApicGetTPR(pVM, &u8LastVTPR, &fPending);
871	AssertRC(rc);
872	pVMCB->ctrl.IntCtrl.n.u8VTPR = u8LastVTPR;
873
874	if (fPending)
875	{
876	/* A TPR change could activate a pending interrupt, so catch cr8 writes. */
877	pVMCB->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
878	}
879	else
880	/* No interrupts are pending, so we don't need to be explicitely notified.
881	* There are enough world switches for detecting pending interrupts.
882	*/
883	pVMCB->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
884
885	fSyncTPR = !fPending;
886	}
887
888	/* All done! Let's start VM execution. */
889	STAM_PROFILE_ADV_START(&pVM->hwaccm.s.StatInGC, x);
890
891	/* Enable nested paging if necessary (disabled each time after #VMEXIT). */
892	pVMCB->ctrl.NestedPaging.n.u1NestedPaging = pVM->hwaccm.s.fNestedPaging;
893
894	#ifdef LOG_ENABLED
895	pCpu = HWACCMR0GetCurrentCpu();
896	if ( pVM->hwaccm.s.svm.idLastCpu != pCpu->idCpu
897	\|\| pVM->hwaccm.s.svm.cTLBFlushes != pCpu->cTLBFlushes)
898	{
899	if (pVM->hwaccm.s.svm.idLastCpu != pCpu->idCpu)
900	Log(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVM->hwaccm.s.svm.idLastCpu, pCpu->idCpu));
901	else
902	Log(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVM->hwaccm.s.svm.cTLBFlushes, pCpu->cTLBFlushes));
903	}
904	if (pCpu->fFlushTLB)
905	Log(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
906	#endif
907
908	/*
909	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
910	* (until the actual world switch)
911	*/
912
913	#ifdef VBOX_STRICT
914	idCpuCheck = RTMpCpuId();
915	#endif
916
917	/* Load the guest state; must be here as it sets up the shadow cr0 for lazy fpu syncing! */
918	rc = SVMR0LoadGuestState(pVM, pCtx);
919	if (rc != VINF_SUCCESS)
920	{
921	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatEntry, x);
922	goto end;
923	}
924	fGuestStateSynced = true;
925
926	pCpu = HWACCMR0GetCurrentCpu();
927	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
928	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
929	if ( pVM->hwaccm.s.svm.idLastCpu != pCpu->idCpu
930	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
931	\|\| pVM->hwaccm.s.svm.cTLBFlushes != pCpu->cTLBFlushes)
932	{
933	/* Force a TLB flush on VM entry. */
934	pVM->hwaccm.s.svm.fForceTLBFlush = true;
935	}
936	else
937	Assert(!pCpu->fFlushTLB);
938
939	pVM->hwaccm.s.svm.idLastCpu = pCpu->idCpu;
940
941	/* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
942	if ( pVM->hwaccm.s.svm.fForceTLBFlush
943	&& !pVM->hwaccm.s.svm.fAlwaysFlushTLB)
944	{
945	if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.svm.u32MaxASID
946	\|\| pCpu->fFlushTLB)
947	{
948	pCpu->fFlushTLB = false;
949	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
950	pVMCB->ctrl.TLBCtrl.n.u1TLBFlush = 1; /* wrap around; flush TLB */
951	pCpu->cTLBFlushes++;
952	}
953	else
954	STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushASID);
955
956	pVM->hwaccm.s.svm.cTLBFlushes = pCpu->cTLBFlushes;
957	pVM->hwaccm.s.svm.uCurrentASID = pCpu->uCurrentASID;
958	}
959	else
960	{
961	Assert(!pCpu->fFlushTLB \|\| pVM->hwaccm.s.svm.fAlwaysFlushTLB);
962
963	/* We never increase uCurrentASID in the fAlwaysFlushTLB (erratum 170) case. */
964	if (!pCpu->uCurrentASID)
965	pVM->hwaccm.s.svm.uCurrentASID = pCpu->uCurrentASID = 1;
966
967	pVMCB->ctrl.TLBCtrl.n.u1TLBFlush = pVM->hwaccm.s.svm.fForceTLBFlush;
968	}
969	AssertMsg(pVM->hwaccm.s.svm.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVM->hwaccm.s.svm.cTLBFlushes, pCpu->cTLBFlushes));
970	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.svm.u32MaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
971	AssertMsg(pVM->hwaccm.s.svm.uCurrentASID >= 1 && pVM->hwaccm.s.svm.uCurrentASID < pVM->hwaccm.s.svm.u32MaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVM->hwaccm.s.svm.uCurrentASID));
972	pVMCB->ctrl.TLBCtrl.n.u32ASID = pVM->hwaccm.s.svm.uCurrentASID;
973
974	#ifdef VBOX_WITH_STATISTICS
975	if (pVMCB->ctrl.TLBCtrl.n.u1TLBFlush)
976	STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushTLBWorldSwitch);
977	else
978	STAM_COUNTER_INC(&pVM->hwaccm.s.StatNoFlushTLBWorldSwitch);
979	#endif
980
981	/* In case we execute a goto ResumeExecution later on. */
982	pVM->hwaccm.s.svm.fResumeVM = true;
983	pVM->hwaccm.s.svm.fForceTLBFlush = pVM->hwaccm.s.svm.fAlwaysFlushTLB;
984
985	Assert(sizeof(pVM->hwaccm.s.svm.pVMCBPhys) == 8);
986	Assert(pVMCB->ctrl.u32InterceptCtrl2 == ( SVM_CTRL2_INTERCEPT_VMRUN /* required */
987	\| SVM_CTRL2_INTERCEPT_VMMCALL
988	\| SVM_CTRL2_INTERCEPT_VMLOAD
989	\| SVM_CTRL2_INTERCEPT_VMSAVE
990	\| SVM_CTRL2_INTERCEPT_STGI
991	\| SVM_CTRL2_INTERCEPT_CLGI
992	\| SVM_CTRL2_INTERCEPT_SKINIT
993	\| SVM_CTRL2_INTERCEPT_RDTSCP /* AMD only; we don't support this one */
994	\| SVM_CTRL2_INTERCEPT_WBINVD
995	\| SVM_CTRL2_INTERCEPT_MWAIT_UNCOND /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
996	));
997	Assert(pVMCB->ctrl.IntCtrl.n.u1VIrqMasking);
998	Assert(pVMCB->ctrl.u64IOPMPhysAddr == pVM->hwaccm.s.svm.pIOBitmapPhys);
999	Assert(pVMCB->ctrl.u64MSRPMPhysAddr == pVM->hwaccm.s.svm.pMSRBitmapPhys);
1000	Assert(pVMCB->ctrl.u64LBRVirt == 0);
1001
1002	#ifdef VBOX_STRICT
1003	Assert(idCpuCheck == RTMpCpuId());
1004	#endif
1005	pVM->hwaccm.s.svm.pfnVMRun(pVM->hwaccm.s.svm.pVMCBHostPhys, pVM->hwaccm.s.svm.pVMCBPhys, pCtx);
1006	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatInGC, x);
1007
1008	/**
1009	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1010	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
1011	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1012	*/
1013
1014	STAM_PROFILE_ADV_START(&pVM->hwaccm.s.StatExit, x);
1015
1016	/* Reason for the VM exit */
1017	exitCode = pVMCB->ctrl.u64ExitCode;
1018
1019	if (exitCode == (uint64_t)SVM_EXIT_INVALID) /* Invalid guest state. */
1020	{
1021	HWACCMDumpRegs(pVM, pCtx);
1022	#ifdef DEBUG
1023	Log(("ctrl.u16InterceptRdCRx %x\n", pVMCB->ctrl.u16InterceptRdCRx));
1024	Log(("ctrl.u16InterceptWrCRx %x\n", pVMCB->ctrl.u16InterceptWrCRx));
1025	Log(("ctrl.u16InterceptRdDRx %x\n", pVMCB->ctrl.u16InterceptRdDRx));
1026	Log(("ctrl.u16InterceptWrDRx %x\n", pVMCB->ctrl.u16InterceptWrDRx));
1027	Log(("ctrl.u32InterceptException %x\n", pVMCB->ctrl.u32InterceptException));
1028	Log(("ctrl.u32InterceptCtrl1 %x\n", pVMCB->ctrl.u32InterceptCtrl1));
1029	Log(("ctrl.u32InterceptCtrl2 %x\n", pVMCB->ctrl.u32InterceptCtrl2));
1030	Log(("ctrl.u64IOPMPhysAddr %VX64\n", pVMCB->ctrl.u64IOPMPhysAddr));
1031	Log(("ctrl.u64MSRPMPhysAddr %VX64\n", pVMCB->ctrl.u64MSRPMPhysAddr));
1032	Log(("ctrl.u64TSCOffset %VX64\n", pVMCB->ctrl.u64TSCOffset));
1033
1034	Log(("ctrl.TLBCtrl.u32ASID %x\n", pVMCB->ctrl.TLBCtrl.n.u32ASID));
1035	Log(("ctrl.TLBCtrl.u1TLBFlush %x\n", pVMCB->ctrl.TLBCtrl.n.u1TLBFlush));
1036	Log(("ctrl.TLBCtrl.u7Reserved %x\n", pVMCB->ctrl.TLBCtrl.n.u7Reserved));
1037	Log(("ctrl.TLBCtrl.u24Reserved %x\n", pVMCB->ctrl.TLBCtrl.n.u24Reserved));
1038
1039	Log(("ctrl.IntCtrl.u8VTPR %x\n", pVMCB->ctrl.IntCtrl.n.u8VTPR));
1040	Log(("ctrl.IntCtrl.u1VIrqValid %x\n", pVMCB->ctrl.IntCtrl.n.u1VIrqValid));
1041	Log(("ctrl.IntCtrl.u7Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u7Reserved));
1042	Log(("ctrl.IntCtrl.u4VIrqPriority %x\n", pVMCB->ctrl.IntCtrl.n.u4VIrqPriority));
1043	Log(("ctrl.IntCtrl.u1IgnoreTPR %x\n", pVMCB->ctrl.IntCtrl.n.u1IgnoreTPR));
1044	Log(("ctrl.IntCtrl.u3Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u3Reserved));
1045	Log(("ctrl.IntCtrl.u1VIrqMasking %x\n", pVMCB->ctrl.IntCtrl.n.u1VIrqMasking));
1046	Log(("ctrl.IntCtrl.u7Reserved2 %x\n", pVMCB->ctrl.IntCtrl.n.u7Reserved2));
1047	Log(("ctrl.IntCtrl.u8VIrqVector %x\n", pVMCB->ctrl.IntCtrl.n.u8VIrqVector));
1048	Log(("ctrl.IntCtrl.u24Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u24Reserved));
1049
1050	Log(("ctrl.u64IntShadow %VX64\n", pVMCB->ctrl.u64IntShadow));
1051	Log(("ctrl.u64ExitCode %VX64\n", pVMCB->ctrl.u64ExitCode));
1052	Log(("ctrl.u64ExitInfo1 %VX64\n", pVMCB->ctrl.u64ExitInfo1));
1053	Log(("ctrl.u64ExitInfo2 %VX64\n", pVMCB->ctrl.u64ExitInfo2));
1054	Log(("ctrl.ExitIntInfo.u8Vector %x\n", pVMCB->ctrl.ExitIntInfo.n.u8Vector));
1055	Log(("ctrl.ExitIntInfo.u3Type %x\n", pVMCB->ctrl.ExitIntInfo.n.u3Type));
1056	Log(("ctrl.ExitIntInfo.u1ErrorCodeValid %x\n", pVMCB->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
1057	Log(("ctrl.ExitIntInfo.u19Reserved %x\n", pVMCB->ctrl.ExitIntInfo.n.u19Reserved));
1058	Log(("ctrl.ExitIntInfo.u1Valid %x\n", pVMCB->ctrl.ExitIntInfo.n.u1Valid));
1059	Log(("ctrl.ExitIntInfo.u32ErrorCode %x\n", pVMCB->ctrl.ExitIntInfo.n.u32ErrorCode));
1060	Log(("ctrl.NestedPaging %VX64\n", pVMCB->ctrl.NestedPaging.au64));
1061	Log(("ctrl.EventInject.u8Vector %x\n", pVMCB->ctrl.EventInject.n.u8Vector));
1062	Log(("ctrl.EventInject.u3Type %x\n", pVMCB->ctrl.EventInject.n.u3Type));
1063	Log(("ctrl.EventInject.u1ErrorCodeValid %x\n", pVMCB->ctrl.EventInject.n.u1ErrorCodeValid));
1064	Log(("ctrl.EventInject.u19Reserved %x\n", pVMCB->ctrl.EventInject.n.u19Reserved));
1065	Log(("ctrl.EventInject.u1Valid %x\n", pVMCB->ctrl.EventInject.n.u1Valid));
1066	Log(("ctrl.EventInject.u32ErrorCode %x\n", pVMCB->ctrl.EventInject.n.u32ErrorCode));
1067
1068	Log(("ctrl.u64NestedPagingCR3 %VX64\n", pVMCB->ctrl.u64NestedPagingCR3));
1069	Log(("ctrl.u64LBRVirt %VX64\n", pVMCB->ctrl.u64LBRVirt));
1070
1071	Log(("guest.CS.u16Sel %04X\n", pVMCB->guest.CS.u16Sel));
1072	Log(("guest.CS.u16Attr %04X\n", pVMCB->guest.CS.u16Attr));
1073	Log(("guest.CS.u32Limit %X\n", pVMCB->guest.CS.u32Limit));
1074	Log(("guest.CS.u64Base %VX64\n", pVMCB->guest.CS.u64Base));
1075	Log(("guest.DS.u16Sel %04X\n", pVMCB->guest.DS.u16Sel));
1076	Log(("guest.DS.u16Attr %04X\n", pVMCB->guest.DS.u16Attr));
1077	Log(("guest.DS.u32Limit %X\n", pVMCB->guest.DS.u32Limit));
1078	Log(("guest.DS.u64Base %VX64\n", pVMCB->guest.DS.u64Base));
1079	Log(("guest.ES.u16Sel %04X\n", pVMCB->guest.ES.u16Sel));
1080	Log(("guest.ES.u16Attr %04X\n", pVMCB->guest.ES.u16Attr));
1081	Log(("guest.ES.u32Limit %X\n", pVMCB->guest.ES.u32Limit));
1082	Log(("guest.ES.u64Base %VX64\n", pVMCB->guest.ES.u64Base));
1083	Log(("guest.FS.u16Sel %04X\n", pVMCB->guest.FS.u16Sel));
1084	Log(("guest.FS.u16Attr %04X\n", pVMCB->guest.FS.u16Attr));
1085	Log(("guest.FS.u32Limit %X\n", pVMCB->guest.FS.u32Limit));
1086	Log(("guest.FS.u64Base %VX64\n", pVMCB->guest.FS.u64Base));
1087	Log(("guest.GS.u16Sel %04X\n", pVMCB->guest.GS.u16Sel));
1088	Log(("guest.GS.u16Attr %04X\n", pVMCB->guest.GS.u16Attr));
1089	Log(("guest.GS.u32Limit %X\n", pVMCB->guest.GS.u32Limit));
1090	Log(("guest.GS.u64Base %VX64\n", pVMCB->guest.GS.u64Base));
1091
1092	Log(("guest.GDTR.u32Limit %X\n", pVMCB->guest.GDTR.u32Limit));
1093	Log(("guest.GDTR.u64Base %VX64\n", pVMCB->guest.GDTR.u64Base));
1094
1095	Log(("guest.LDTR.u16Sel %04X\n", pVMCB->guest.LDTR.u16Sel));
1096	Log(("guest.LDTR.u16Attr %04X\n", pVMCB->guest.LDTR.u16Attr));
1097	Log(("guest.LDTR.u32Limit %X\n", pVMCB->guest.LDTR.u32Limit));
1098	Log(("guest.LDTR.u64Base %VX64\n", pVMCB->guest.LDTR.u64Base));
1099
1100	Log(("guest.IDTR.u32Limit %X\n", pVMCB->guest.IDTR.u32Limit));
1101	Log(("guest.IDTR.u64Base %VX64\n", pVMCB->guest.IDTR.u64Base));
1102
1103	Log(("guest.TR.u16Sel %04X\n", pVMCB->guest.TR.u16Sel));
1104	Log(("guest.TR.u16Attr %04X\n", pVMCB->guest.TR.u16Attr));
1105	Log(("guest.TR.u32Limit %X\n", pVMCB->guest.TR.u32Limit));
1106	Log(("guest.TR.u64Base %VX64\n", pVMCB->guest.TR.u64Base));
1107
1108	Log(("guest.u8CPL %X\n", pVMCB->guest.u8CPL));
1109	Log(("guest.u64CR0 %VX64\n", pVMCB->guest.u64CR0));
1110	Log(("guest.u64CR2 %VX64\n", pVMCB->guest.u64CR2));
1111	Log(("guest.u64CR3 %VX64\n", pVMCB->guest.u64CR3));
1112	Log(("guest.u64CR4 %VX64\n", pVMCB->guest.u64CR4));
1113	Log(("guest.u64DR6 %VX64\n", pVMCB->guest.u64DR6));
1114	Log(("guest.u64DR7 %VX64\n", pVMCB->guest.u64DR7));
1115
1116	Log(("guest.u64RIP %VX64\n", pVMCB->guest.u64RIP));
1117	Log(("guest.u64RSP %VX64\n", pVMCB->guest.u64RSP));
1118	Log(("guest.u64RAX %VX64\n", pVMCB->guest.u64RAX));
1119	Log(("guest.u64RFlags %VX64\n", pVMCB->guest.u64RFlags));
1120
1121	Log(("guest.u64SysEnterCS %VX64\n", pVMCB->guest.u64SysEnterCS));
1122	Log(("guest.u64SysEnterEIP %VX64\n", pVMCB->guest.u64SysEnterEIP));
1123	Log(("guest.u64SysEnterESP %VX64\n", pVMCB->guest.u64SysEnterESP));
1124
1125	Log(("guest.u64EFER %VX64\n", pVMCB->guest.u64EFER));
1126	Log(("guest.u64STAR %VX64\n", pVMCB->guest.u64STAR));
1127	Log(("guest.u64LSTAR %VX64\n", pVMCB->guest.u64LSTAR));
1128	Log(("guest.u64CSTAR %VX64\n", pVMCB->guest.u64CSTAR));
1129	Log(("guest.u64SFMASK %VX64\n", pVMCB->guest.u64SFMASK));
1130	Log(("guest.u64KernelGSBase %VX64\n", pVMCB->guest.u64KernelGSBase));
1131	Log(("guest.u64GPAT %VX64\n", pVMCB->guest.u64GPAT));
1132	Log(("guest.u64DBGCTL %VX64\n", pVMCB->guest.u64DBGCTL));
1133	Log(("guest.u64BR_FROM %VX64\n", pVMCB->guest.u64BR_FROM));
1134	Log(("guest.u64BR_TO %VX64\n", pVMCB->guest.u64BR_TO));
1135	Log(("guest.u64LASTEXCPFROM %VX64\n", pVMCB->guest.u64LASTEXCPFROM));
1136	Log(("guest.u64LASTEXCPTO %VX64\n", pVMCB->guest.u64LASTEXCPTO));
1137
1138	#endif
1139	rc = VERR_SVM_UNABLE_TO_START_VM;
1140	goto end;
1141	}
1142
1143	/* Let's first sync back eip, esp, and eflags. */
1144	pCtx->rip = pVMCB->guest.u64RIP;
1145	pCtx->rsp = pVMCB->guest.u64RSP;
1146	pCtx->eflags.u32 = pVMCB->guest.u64RFlags;
1147	/* eax is saved/restore across the vmrun instruction */
1148	pCtx->rax = pVMCB->guest.u64RAX;
1149
1150	pCtx->msrKERNELGSBASE = pVMCB->guest.u64KernelGSBase; /* swapgs exchange value */
1151
1152	/* Can be updated behind our back in the nested paging case. */
1153	pCtx->cr2 = pVMCB->guest.u64CR2;
1154
1155	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1156	SVM_READ_SELREG(SS, ss);
1157	SVM_READ_SELREG(CS, cs);
1158	SVM_READ_SELREG(DS, ds);
1159	SVM_READ_SELREG(ES, es);
1160	SVM_READ_SELREG(FS, fs);
1161	SVM_READ_SELREG(GS, gs);
1162
1163	/*
1164	* System MSRs
1165	*/
1166	pCtx->SysEnter.cs = pVMCB->guest.u64SysEnterCS;
1167	pCtx->SysEnter.eip = pVMCB->guest.u64SysEnterEIP;
1168	pCtx->SysEnter.esp = pVMCB->guest.u64SysEnterESP;
1169
1170	/* Note: no reason to sync back the CRx and DRx registers. They can't be changed by the guest. */
1171	/* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1172	if ( pVM->hwaccm.s.fNestedPaging
1173	&& pCtx->cr3 != pVMCB->guest.u64CR3)
1174	{
1175	CPUMSetGuestCR3(pVM, pVMCB->guest.u64CR3);
1176	PGMUpdateCR3(pVM, pVMCB->guest.u64CR3);
1177	}
1178
1179	/** @note NOW IT'S SAFE FOR LOGGING! */
1180
1181	/* Take care of instruction fusing (sti, mov ss) (see 15.20.5 Interrupt Shadows) */
1182	if (pVMCB->ctrl.u64IntShadow & SVM_INTERRUPT_SHADOW_ACTIVE)
1183	{
1184	Log(("uInterruptState %x eip=%VGv\n", pVMCB->ctrl.u64IntShadow, pCtx->rip));
1185	EMSetInhibitInterruptsPC(pVM, pCtx->rip);
1186	}
1187	else
1188	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
1189
1190	Log2(("exitCode = %x\n", exitCode));
1191
1192	/* Sync back the debug registers. */
1193	/** @todo Implement debug registers correctly. */
1194	pCtx->dr6 = pVMCB->guest.u64DR6;
1195	pCtx->dr7 = pVMCB->guest.u64DR7;
1196
1197	/* Check if an injected event was interrupted prematurely. */
1198	pVM->hwaccm.s.Event.intInfo = pVMCB->ctrl.ExitIntInfo.au64[0];
1199	if ( pVMCB->ctrl.ExitIntInfo.n.u1Valid
1200	&& pVMCB->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT /* we don't care about 'int xx' as the instruction will be restarted. */)
1201	{
1202	Log(("Pending inject %VX64 at %VGv exit=%08x\n", pVM->hwaccm.s.Event.intInfo, pCtx->rip, exitCode));
1203
1204	#ifdef LOG_ENABLED
1205	SVM_EVENT Event;
1206	Event.au64[0] = pVM->hwaccm.s.Event.intInfo;
1207
1208	if ( exitCode == SVM_EXIT_EXCEPTION_E
1209	&& Event.n.u8Vector == 0xE)
1210	{
1211	Log(("Double fault!\n"));
1212	}
1213	#endif
1214
1215	pVM->hwaccm.s.Event.fPending = true;
1216	/* Error code present? (redundant) */
1217	if (pVMCB->ctrl.ExitIntInfo.n.u1ErrorCodeValid)
1218	{
1219	pVM->hwaccm.s.Event.errCode = pVMCB->ctrl.ExitIntInfo.n.u32ErrorCode;
1220	}
1221	else
1222	pVM->hwaccm.s.Event.errCode = 0;
1223	}
1224	#ifdef VBOX_WITH_STATISTICS
1225	if (exitCode == SVM_EXIT_NPF)
1226	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitReasonNPF);
1227	else
1228	STAM_COUNTER_INC(&pVM->hwaccm.s.pStatExitReasonR0[exitCode & MASK_EXITREASON_STAT]);
1229	#endif
1230
1231	if (fSyncTPR)
1232	{
1233	rc = PDMApicSetTPR(pVM, pVMCB->ctrl.IntCtrl.n.u8VTPR);
1234	AssertRC(rc);
1235	}
1236
1237	/* Deal with the reason of the VM-exit. */
1238	switch (exitCode)
1239	{
1240	case SVM_EXIT_EXCEPTION_0: case SVM_EXIT_EXCEPTION_1: case SVM_EXIT_EXCEPTION_2: case SVM_EXIT_EXCEPTION_3:
1241	case SVM_EXIT_EXCEPTION_4: case SVM_EXIT_EXCEPTION_5: case SVM_EXIT_EXCEPTION_6: case SVM_EXIT_EXCEPTION_7:
1242	case SVM_EXIT_EXCEPTION_8: case SVM_EXIT_EXCEPTION_9: case SVM_EXIT_EXCEPTION_A: case SVM_EXIT_EXCEPTION_B:
1243	case SVM_EXIT_EXCEPTION_C: case SVM_EXIT_EXCEPTION_D: case SVM_EXIT_EXCEPTION_E: case SVM_EXIT_EXCEPTION_F:
1244	case SVM_EXIT_EXCEPTION_10: case SVM_EXIT_EXCEPTION_11: case SVM_EXIT_EXCEPTION_12: case SVM_EXIT_EXCEPTION_13:
1245	case SVM_EXIT_EXCEPTION_14: case SVM_EXIT_EXCEPTION_15: case SVM_EXIT_EXCEPTION_16: case SVM_EXIT_EXCEPTION_17:
1246	case SVM_EXIT_EXCEPTION_18: case SVM_EXIT_EXCEPTION_19: case SVM_EXIT_EXCEPTION_1A: case SVM_EXIT_EXCEPTION_1B:
1247	case SVM_EXIT_EXCEPTION_1C: case SVM_EXIT_EXCEPTION_1D: case SVM_EXIT_EXCEPTION_1E: case SVM_EXIT_EXCEPTION_1F:
1248	{
1249	/* Pending trap. */
1250	SVM_EVENT Event;
1251	uint32_t vector = exitCode - SVM_EXIT_EXCEPTION_0;
1252
1253	Log2(("Hardware/software interrupt %d\n", vector));
1254	switch (vector)
1255	{
1256	#ifdef DEBUG
1257	case X86_XCPT_DB:
1258	rc = DBGFR0Trap01Handler(pVM, CPUMCTX2CORE(pCtx), pVMCB->guest.u64DR6);
1259	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
1260	break;
1261	#endif
1262
1263	case X86_XCPT_NM:
1264	{
1265	Log(("#NM fault at %VGv\n", pCtx->rip));
1266
1267	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
1268	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
1269	rc = CPUMR0LoadGuestFPU(pVM, pCtx);
1270	if (rc == VINF_SUCCESS)
1271	{
1272	Assert(CPUMIsGuestFPUStateActive(pVM));
1273	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitShadowNM);
1274
1275	/* Continue execution. */
1276	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1277	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
1278
1279	goto ResumeExecution;
1280	}
1281
1282	Log(("Forward #NM fault to the guest\n"));
1283	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestNM);
1284
1285	Event.au64[0] = 0;
1286	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1287	Event.n.u1Valid = 1;
1288	Event.n.u8Vector = X86_XCPT_NM;
1289
1290	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1291	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1292	goto ResumeExecution;
1293	}
1294
1295	case X86_XCPT_PF: /* Page fault */
1296	{
1297	uint32_t errCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1298	RTGCUINTPTR uFaultAddress = pVMCB->ctrl.u64ExitInfo2; /* EXITINFO2 = fault address */
1299
1300	#ifdef DEBUG
1301	if (pVM->hwaccm.s.fNestedPaging)
1302	{ /* A genuine pagefault.
1303	* Forward the trap to the guest by injecting the exception and resuming execution.
1304	*/
1305	Log(("Guest page fault at %VGv cr2=%VGv error code %x rsp=%VGv\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode, (RTGCPTR)pCtx->rsp));
1306	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestPF);
1307
1308	/* Now we must update CR2. */
1309	pCtx->cr2 = uFaultAddress;
1310
1311	Event.au64[0] = 0;
1312	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1313	Event.n.u1Valid = 1;
1314	Event.n.u8Vector = X86_XCPT_PF;
1315	Event.n.u1ErrorCodeValid = 1;
1316	Event.n.u32ErrorCode = errCode;
1317
1318	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1319
1320	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1321	goto ResumeExecution;
1322	}
1323	#endif
1324	Assert(!pVM->hwaccm.s.fNestedPaging);
1325
1326	Log2(("Page fault at %VGv cr2=%VGv error code %x\n", pCtx->rip, uFaultAddress, errCode));
1327	/* Exit qualification contains the linear address of the page fault. */
1328	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
1329	TRPMSetErrorCode(pVM, errCode);
1330	TRPMSetFaultAddress(pVM, uFaultAddress);
1331
1332	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
1333	rc = PGMTrap0eHandler(pVM, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
1334	Log2(("PGMTrap0eHandler %VGv returned %Vrc\n", pCtx->rip, rc));
1335	if (rc == VINF_SUCCESS)
1336	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
1337	Log2(("Shadow page fault at %VGv cr2=%VGv error code %x\n", pCtx->rip, uFaultAddress, errCode));
1338	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitShadowPF);
1339
1340	TRPMResetTrap(pVM);
1341
1342	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1343	goto ResumeExecution;
1344	}
1345	else
1346	if (rc == VINF_EM_RAW_GUEST_TRAP)
1347	{ /* A genuine pagefault.
1348	* Forward the trap to the guest by injecting the exception and resuming execution.
1349	*/
1350	Log2(("Forward page fault to the guest\n"));
1351	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestPF);
1352	/* The error code might have been changed. */
1353	errCode = TRPMGetErrorCode(pVM);
1354
1355	TRPMResetTrap(pVM);
1356
1357	/* Now we must update CR2. */
1358	pCtx->cr2 = uFaultAddress;
1359
1360	Event.au64[0] = 0;
1361	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1362	Event.n.u1Valid = 1;
1363	Event.n.u8Vector = X86_XCPT_PF;
1364	Event.n.u1ErrorCodeValid = 1;
1365	Event.n.u32ErrorCode = errCode;
1366
1367	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1368
1369	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1370	goto ResumeExecution;
1371	}
1372	#ifdef VBOX_STRICT
1373	if (rc != VINF_EM_RAW_EMULATE_INSTR)
1374	LogFlow(("PGMTrap0eHandler failed with %d\n", rc));
1375	#endif
1376	/* Need to go back to the recompiler to emulate the instruction. */
1377	TRPMResetTrap(pVM);
1378	break;
1379	}
1380
1381	case X86_XCPT_MF: /* Floating point exception. */
1382	{
1383	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestMF);
1384	if (!(pCtx->cr0 & X86_CR0_NE))
1385	{
1386	/* old style FPU error reporting needs some extra work. */
1387	/** @todo don't fall back to the recompiler, but do it manually. */
1388	rc = VINF_EM_RAW_EMULATE_INSTR;
1389	break;
1390	}
1391	Log(("Trap %x at %VGv\n", vector, pCtx->rip));
1392
1393	Event.au64[0] = 0;
1394	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1395	Event.n.u1Valid = 1;
1396	Event.n.u8Vector = X86_XCPT_MF;
1397
1398	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1399
1400	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1401	goto ResumeExecution;
1402	}
1403
1404	#ifdef VBOX_STRICT
1405	case X86_XCPT_GP: /* General protection failure exception.*/
1406	case X86_XCPT_UD: /* Unknown opcode exception. */
1407	case X86_XCPT_DE: /* Debug exception. */
1408	case X86_XCPT_SS: /* Stack segment exception. */
1409	case X86_XCPT_NP: /* Segment not present exception. */
1410	{
1411	Event.au64[0] = 0;
1412	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1413	Event.n.u1Valid = 1;
1414	Event.n.u8Vector = vector;
1415
1416	switch(vector)
1417	{
1418	case X86_XCPT_GP:
1419	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestGP);
1420	Event.n.u1ErrorCodeValid = 1;
1421	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1422	break;
1423	case X86_XCPT_DE:
1424	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestDE);
1425	break;
1426	case X86_XCPT_UD:
1427	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestUD);
1428	break;
1429	case X86_XCPT_SS:
1430	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestSS);
1431	Event.n.u1ErrorCodeValid = 1;
1432	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1433	break;
1434	case X86_XCPT_NP:
1435	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestNP);
1436	Event.n.u1ErrorCodeValid = 1;
1437	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1438	break;
1439	}
1440	Log(("Trap %x at %VGv esi=%x\n", vector, pCtx->rip, pCtx->esi));
1441	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1442
1443	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1444	goto ResumeExecution;
1445	}
1446	#endif
1447	default:
1448	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
1449	rc = VERR_EM_INTERNAL_ERROR;
1450	break;
1451
1452	} /* switch (vector) */
1453	break;
1454	}
1455
1456	case SVM_EXIT_NPF:
1457	{
1458	/* EXITINFO1 contains fault errorcode; EXITINFO2 contains the guest physical address causing the fault. */
1459	uint32_t errCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1460	RTGCPHYS uFaultAddress = pVMCB->ctrl.u64ExitInfo2; /* EXITINFO2 = fault address */
1461
1462	Assert(pVM->hwaccm.s.fNestedPaging);
1463	Log(("Nested page fault at %VGv cr2=%VGp error code %x\n", pCtx->rip, uFaultAddress, errCode));
1464	/* Exit qualification contains the linear address of the page fault. */
1465	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
1466	TRPMSetErrorCode(pVM, errCode);
1467	TRPMSetFaultAddress(pVM, uFaultAddress);
1468
1469	/* Handle the pagefault trap for the nested shadow table. */
1470	rc = PGMR0Trap0eHandlerNestedPaging(pVM, PGMGetHostMode(pVM), errCode, CPUMCTX2CORE(pCtx), uFaultAddress);
1471	Log2(("PGMR0Trap0eHandlerNestedPaging %VGv returned %Vrc\n", pCtx->rip, rc));
1472	if (rc == VINF_SUCCESS)
1473	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
1474	Log2(("Shadow page fault at %VGv cr2=%VGp error code %x\n", pCtx->rip, uFaultAddress, errCode));
1475	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitShadowPF);
1476
1477	TRPMResetTrap(pVM);
1478
1479	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1480	goto ResumeExecution;
1481	}
1482
1483	#ifdef VBOX_STRICT
1484	if (rc != VINF_EM_RAW_EMULATE_INSTR)
1485	LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", rc));
1486	#endif
1487	/* Need to go back to the recompiler to emulate the instruction. */
1488	TRPMResetTrap(pVM);
1489	break;
1490	}
1491
1492	case SVM_EXIT_VINTR:
1493	/* A virtual interrupt is about to be delivered, which means IF=1. */
1494	Log(("SVM_EXIT_VINTR IF=%d\n", pCtx->eflags.Bits.u1IF));
1495	pVMCB->ctrl.IntCtrl.n.u1VIrqValid = 0;
1496	pVMCB->ctrl.IntCtrl.n.u8VIrqVector = 0;
1497	goto ResumeExecution;
1498
1499	case SVM_EXIT_FERR_FREEZE:
1500	case SVM_EXIT_INTR:
1501	case SVM_EXIT_NMI:
1502	case SVM_EXIT_SMI:
1503	case SVM_EXIT_INIT:
1504	/* External interrupt; leave to allow it to be dispatched again. */
1505	rc = VINF_EM_RAW_INTERRUPT;
1506	break;
1507
1508	case SVM_EXIT_WBINVD:
1509	case SVM_EXIT_INVD: /* Guest software attempted to execute INVD. */
1510	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitInvd);
1511	/* Skip instruction and continue directly. */
1512	pCtx->rip += 2; /** @note hardcoded opcode size! */
1513	/* Continue execution.*/
1514	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1515	goto ResumeExecution;
1516
1517	case SVM_EXIT_CPUID: /* Guest software attempted to execute CPUID. */
1518	{
1519	Log2(("SVM: Cpuid at %VGv for %x\n", pCtx->rip, pCtx->eax));
1520	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitCpuid);
1521	rc = EMInterpretCpuId(pVM, CPUMCTX2CORE(pCtx));
1522	if (rc == VINF_SUCCESS)
1523	{
1524	/* Update EIP and continue execution. */
1525	pCtx->rip += 2; /** @note hardcoded opcode size! */
1526	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1527	goto ResumeExecution;
1528	}
1529	AssertMsgFailed(("EMU: cpuid failed with %Vrc\n", rc));
1530	rc = VINF_EM_RAW_EMULATE_INSTR;
1531	break;
1532	}
1533
1534	case SVM_EXIT_RDTSC: /* Guest software attempted to execute RDTSC. */
1535	{
1536	Log2(("SVM: Rdtsc\n"));
1537	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitRdtsc);
1538	rc = EMInterpretRdtsc(pVM, CPUMCTX2CORE(pCtx));
1539	if (rc == VINF_SUCCESS)
1540	{
1541	/* Update EIP and continue execution. */
1542	pCtx->rip += 2; /** @note hardcoded opcode size! */
1543	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1544	goto ResumeExecution;
1545	}
1546	AssertMsgFailed(("EMU: rdtsc failed with %Vrc\n", rc));
1547	rc = VINF_EM_RAW_EMULATE_INSTR;
1548	break;
1549	}
1550
1551	case SVM_EXIT_INVLPG: /* Guest software attempted to execute INVPG. */
1552	{
1553	Log2(("SVM: invlpg\n"));
1554	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitInvpg);
1555
1556	Assert(!pVM->hwaccm.s.fNestedPaging);
1557
1558	/* Truly a pita. Why can't SVM give the same information as VT-x? */
1559	rc = SVMR0InterpretInvpg(pVM, CPUMCTX2CORE(pCtx), pVMCB->ctrl.TLBCtrl.n.u32ASID);
1560	if (rc == VINF_SUCCESS)
1561	{
1562	STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushPageInvlpg);
1563	goto ResumeExecution; /* eip already updated */
1564	}
1565	break;
1566	}
1567
1568	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
1569	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
1570	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
1571	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
1572	{
1573	uint32_t cbSize;
1574
1575	Log2(("SVM: %VGv mov cr%d, \n", pCtx->rip, exitCode - SVM_EXIT_WRITE_CR0));
1576	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitCRxWrite);
1577	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1578
1579	switch (exitCode - SVM_EXIT_WRITE_CR0)
1580	{
1581	case 0:
1582	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
1583	break;
1584	case 2:
1585	break;
1586	case 3:
1587	Assert(!pVM->hwaccm.s.fNestedPaging);
1588	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
1589	break;
1590	case 4:
1591	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
1592	break;
1593	case 8:
1594	break;
1595	default:
1596	AssertFailed();
1597	}
1598	/* Check if a sync operation is pending. */
1599	if ( rc == VINF_SUCCESS /* don't bother if we are going to ring 3 anyway */
1600	&& VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL))
1601	{
1602	rc = PGMSyncCR3(pVM, CPUMGetGuestCR0(pVM), CPUMGetGuestCR3(pVM), CPUMGetGuestCR4(pVM), VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3));
1603	AssertRC(rc);
1604
1605	STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushTLBCRxChange);
1606
1607	/* Must be set by PGMSyncCR3 */
1608	Assert(PGMGetGuestMode(pVM) <= PGMMODE_PROTECTED \|\| pVM->hwaccm.s.svm.fForceTLBFlush);
1609	}
1610	if (rc == VINF_SUCCESS)
1611	{
1612	/* EIP has been updated already. */
1613
1614	/* Only resume if successful. */
1615	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1616	goto ResumeExecution;
1617	}
1618	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1619	break;
1620	}
1621
1622	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
1623	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
1624	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
1625	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
1626	{
1627	uint32_t cbSize;
1628
1629	Log2(("SVM: %VGv mov x, cr%d\n", pCtx->rip, exitCode - SVM_EXIT_READ_CR0));
1630	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitCRxRead);
1631	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1632	if (rc == VINF_SUCCESS)
1633	{
1634	/* EIP has been updated already. */
1635
1636	/* Only resume if successful. */
1637	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1638	goto ResumeExecution;
1639	}
1640	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1641	break;
1642	}
1643
1644	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
1645	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
1646	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
1647	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
1648	{
1649	uint32_t cbSize;
1650
1651	Log2(("SVM: %VGv mov dr%d, x\n", pCtx->rip, exitCode - SVM_EXIT_WRITE_DR0));
1652	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitDRxRead);
1653	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1654	if (rc == VINF_SUCCESS)
1655	{
1656	/* EIP has been updated already. */
1657
1658	/* Only resume if successful. */
1659	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1660	goto ResumeExecution;
1661	}
1662	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1663	break;
1664	}
1665
1666	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
1667	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
1668	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
1669	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
1670	{
1671	uint32_t cbSize;
1672
1673	Log2(("SVM: %VGv mov dr%d, x\n", pCtx->rip, exitCode - SVM_EXIT_READ_DR0));
1674	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitDRxRead);
1675	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1676	if (rc == VINF_SUCCESS)
1677	{
1678	/* EIP has been updated already. */
1679
1680	/* Only resume if successful. */
1681	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1682	goto ResumeExecution;
1683	}
1684	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1685	break;
1686	}
1687
1688	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
1689	case SVM_EXIT_IOIO: /* I/O instruction. */
1690	{
1691	SVM_IOIO_EXIT IoExitInfo;
1692	uint32_t uIOSize, uAndVal;
1693
1694	IoExitInfo.au32[0] = pVMCB->ctrl.u64ExitInfo1;
1695
1696	/** @todo could use a lookup table here */
1697	if (IoExitInfo.n.u1OP8)
1698	{
1699	uIOSize = 1;
1700	uAndVal = 0xff;
1701	}
1702	else
1703	if (IoExitInfo.n.u1OP16)
1704	{
1705	uIOSize = 2;
1706	uAndVal = 0xffff;
1707	}
1708	else
1709	if (IoExitInfo.n.u1OP32)
1710	{
1711	uIOSize = 4;
1712	uAndVal = 0xffffffff;
1713	}
1714	else
1715	{
1716	AssertFailed(); /* should be fatal. */
1717	rc = VINF_EM_RAW_EMULATE_INSTR;
1718	break;
1719	}
1720
1721	if (IoExitInfo.n.u1STR)
1722	{
1723	/* ins/outs */
1724	uint32_t prefix = 0;
1725	if (IoExitInfo.n.u1REP)
1726	prefix \|= PREFIX_REP;
1727
1728	if (IoExitInfo.n.u1Type == 0)
1729	{
1730	Log2(("IOMInterpretOUTSEx %VGv %x size=%d\n", pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
1731	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitIOStringWrite);
1732	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, prefix, uIOSize);
1733	}
1734	else
1735	{
1736	Log2(("IOMInterpretINSEx %VGv %x size=%d\n", pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
1737	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitIOStringRead);
1738	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, prefix, uIOSize);
1739	}
1740	}
1741	else
1742	{
1743	/* normal in/out */
1744	Assert(!IoExitInfo.n.u1REP);
1745
1746	if (IoExitInfo.n.u1Type == 0)
1747	{
1748	Log2(("IOMIOPortWrite %VGv %x %x size=%d\n", pCtx->rip, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, uIOSize));
1749	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitIOWrite);
1750	rc = IOMIOPortWrite(pVM, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, uIOSize);
1751	}
1752	else
1753	{
1754	uint32_t u32Val = 0;
1755
1756	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitIORead);
1757	rc = IOMIOPortRead(pVM, IoExitInfo.n.u16Port, &u32Val, uIOSize);
1758	if (IOM_SUCCESS(rc))
1759	{
1760	/* Write back to the EAX register. */
1761	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
1762	Log2(("IOMIOPortRead %VGv %x %x size=%d\n", pCtx->rip, IoExitInfo.n.u16Port, u32Val & uAndVal, uIOSize));
1763	}
1764	}
1765	}
1766	/*
1767	* Handled the I/O return codes.
1768	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
1769	*/
1770	if (IOM_SUCCESS(rc))
1771	{
1772	/* Update EIP and continue execution. */
1773	pCtx->rip = pVMCB->ctrl.u64ExitInfo2; /* RIP/EIP of the next instruction is saved in EXITINFO2. */
1774	if (RT_LIKELY(rc == VINF_SUCCESS))
1775	{
1776	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1777	goto ResumeExecution;
1778	}
1779	Log2(("EM status from IO at %VGv %x size %d: %Vrc\n", pCtx->rip, IoExitInfo.n.u16Port, uIOSize, rc));
1780	break;
1781	}
1782
1783	#ifdef VBOX_STRICT
1784	if (rc == VINF_IOM_HC_IOPORT_READ)
1785	Assert(IoExitInfo.n.u1Type != 0);
1786	else if (rc == VINF_IOM_HC_IOPORT_WRITE)
1787	Assert(IoExitInfo.n.u1Type == 0);
1788	else
1789	AssertMsg(VBOX_FAILURE(rc) \|\| rc == VINF_EM_RAW_EMULATE_INSTR \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Vrc\n", rc));
1790	#endif
1791	Log2(("Failed IO at %VGv %x size %d\n", pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
1792	break;
1793	}
1794
1795	case SVM_EXIT_HLT:
1796	/** Check if external interrupts are pending; if so, don't switch back. */
1797	pCtx->rip++; /* skip hlt */
1798	if ( pCtx->eflags.Bits.u1IF
1799	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
1800	goto ResumeExecution;
1801
1802	rc = VINF_EM_HALT;
1803	break;
1804
1805	case SVM_EXIT_RSM:
1806	case SVM_EXIT_INVLPGA:
1807	case SVM_EXIT_VMRUN:
1808	case SVM_EXIT_VMMCALL:
1809	case SVM_EXIT_VMLOAD:
1810	case SVM_EXIT_VMSAVE:
1811	case SVM_EXIT_STGI:
1812	case SVM_EXIT_CLGI:
1813	case SVM_EXIT_SKINIT:
1814	case SVM_EXIT_RDTSCP:
1815	{
1816	/* Unsupported instructions. */
1817	SVM_EVENT Event;
1818
1819	Event.au64[0] = 0;
1820	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1821	Event.n.u1Valid = 1;
1822	Event.n.u8Vector = X86_XCPT_UD;
1823
1824	Log(("Forced #UD trap at %VGv\n", pCtx->rip));
1825	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1826
1827	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1828	goto ResumeExecution;
1829	}
1830
1831	/* Emulate in ring 3. */
1832	case SVM_EXIT_MSR:
1833	{
1834	uint32_t cbSize;
1835
1836	/* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
1837	Log(("SVM: %s\n", (pVMCB->ctrl.u64ExitInfo1 == 0) ? "rdmsr" : "wrmsr"));
1838	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1839	if (rc == VINF_SUCCESS)
1840	{
1841	/* EIP has been updated already. */
1842
1843	/* Only resume if successful. */
1844	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1845	goto ResumeExecution;
1846	}
1847	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Vrc\n", (pVMCB->ctrl.u64ExitInfo1 == 0) ? "rdmsr" : "wrmsr", rc));
1848	break;
1849	}
1850
1851	case SVM_EXIT_MONITOR:
1852	case SVM_EXIT_RDPMC:
1853	case SVM_EXIT_PAUSE:
1854	case SVM_EXIT_MWAIT_UNCOND:
1855	case SVM_EXIT_MWAIT_ARMED:
1856	case SVM_EXIT_TASK_SWITCH: /* can change CR3; emulate */
1857	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
1858	break;
1859
1860	case SVM_EXIT_SHUTDOWN:
1861	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
1862	break;
1863
1864	case SVM_EXIT_IDTR_READ:
1865	case SVM_EXIT_GDTR_READ:
1866	case SVM_EXIT_LDTR_READ:
1867	case SVM_EXIT_TR_READ:
1868	case SVM_EXIT_IDTR_WRITE:
1869	case SVM_EXIT_GDTR_WRITE:
1870	case SVM_EXIT_LDTR_WRITE:
1871	case SVM_EXIT_TR_WRITE:
1872	case SVM_EXIT_CR0_SEL_WRITE:
1873	default:
1874	/* Unexpected exit codes. */
1875	rc = VERR_EM_INTERNAL_ERROR;
1876	AssertMsgFailed(("Unexpected exit code %x\n", exitCode)); /* Can't happen. */
1877	break;
1878	}
1879
1880	end:
1881	if (fGuestStateSynced)
1882	{
1883	/* Remaining guest CPU context: TR, IDTR, GDTR, LDTR. */
1884	SVM_READ_SELREG(LDTR, ldtr);
1885	SVM_READ_SELREG(TR, tr);
1886
1887	pCtx->gdtr.cbGdt = pVMCB->guest.GDTR.u32Limit;
1888	pCtx->gdtr.pGdt = pVMCB->guest.GDTR.u64Base;
1889
1890	pCtx->idtr.cbIdt = pVMCB->guest.IDTR.u32Limit;
1891	pCtx->idtr.pIdt = pVMCB->guest.IDTR.u64Base;
1892	}
1893
1894	/* Signal changes for the recompiler. */
1895	CPUMSetChangedFlags(pVM, CPUM_CHANGED_SYSENTER_MSR \| CPUM_CHANGED_LDTR \| CPUM_CHANGED_GDTR \| CPUM_CHANGED_IDTR \| CPUM_CHANGED_TR \| CPUM_CHANGED_HIDDEN_SEL_REGS);
1896
1897	/* If we executed vmrun and an external irq was pending, then we don't have to do a full sync the next time. */
1898	if (exitCode == SVM_EXIT_INTR)
1899	{
1900	STAM_COUNTER_INC(&pVM->hwaccm.s.StatPendingHostIrq);
1901	/* On the next entry we'll only sync the host context. */
1902	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
1903	}
1904	else
1905	{
1906	/* On the next entry we'll sync everything. */
1907	/** @todo we can do better than this */
1908	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
1909	}
1910
1911	/* translate into a less severe return code */
1912	if (rc == VERR_EM_INTERPRETER)
1913	rc = VINF_EM_RAW_EMULATE_INSTR;
1914
1915	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1916	return rc;
1917	}
1918
1919	/**
1920	* Enters the AMD-V session
1921	*
1922	* @returns VBox status code.
1923	* @param pVM The VM to operate on.
1924	* @param pCpu CPU info struct
1925	*/
1926	HWACCMR0DECL(int) SVMR0Enter(PVM pVM, PHWACCM_CPUINFO pCpu)
1927	{
1928	Assert(pVM->hwaccm.s.svm.fSupported);
1929
1930	LogFlow(("SVMR0Enter cpu%d last=%d asid=%d\n", pCpu->idCpu, pVM->hwaccm.s.svm.idLastCpu, pVM->hwaccm.s.svm.uCurrentASID));
1931	pVM->hwaccm.s.svm.fResumeVM = false;
1932
1933	/* Force to reload LDTR, so we'll execute VMLoad to load additional guest state. */
1934	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_LDTR;
1935
1936	return VINF_SUCCESS;
1937	}
1938
1939
1940	/**
1941	* Leaves the AMD-V session
1942	*
1943	* @returns VBox status code.
1944	* @param pVM The VM to operate on.
1945	*/
1946	HWACCMR0DECL(int) SVMR0Leave(PVM pVM)
1947	{
1948	Assert(pVM->hwaccm.s.svm.fSupported);
1949	return VINF_SUCCESS;
1950	}
1951
1952
1953	static int svmInterpretInvlPg(PVM pVM, PDISCPUSTATE pCpu, PCPUMCTXCORE pRegFrame, uint32_t uASID)
1954	{
1955	OP_PARAMVAL param1;
1956	RTGCPTR addr;
1957
1958	int rc = DISQueryParamVal(pRegFrame, pCpu, &pCpu->param1, &param1, PARAM_SOURCE);
1959	if(VBOX_FAILURE(rc))
1960	return VERR_EM_INTERPRETER;
1961
1962	switch(param1.type)
1963	{
1964	case PARMTYPE_IMMEDIATE:
1965	case PARMTYPE_ADDRESS:
1966	if(!(param1.flags & (PARAM_VAL32\|PARAM_VAL64)))
1967	return VERR_EM_INTERPRETER;
1968	addr = param1.val.val64;
1969	break;
1970
1971	default:
1972	return VERR_EM_INTERPRETER;
1973	}
1974
1975	/** @todo is addr always a flat linear address or ds based
1976	* (in absence of segment override prefixes)????
1977	*/
1978	rc = PGMInvalidatePage(pVM, addr);
1979	if (VBOX_SUCCESS(rc))
1980	{
1981	/* Manually invalidate the page for the VM's TLB. */
1982	Log(("SVMInvlpgA %VGv ASID=%d\n", addr, uASID));
1983	SVMInvlpgA(addr, uASID);
1984	return VINF_SUCCESS;
1985	}
1986	Assert(rc == VERR_REM_FLUSHED_PAGES_OVERFLOW);
1987	return rc;
1988	}
1989
1990	/**
1991	* Interprets INVLPG
1992	*
1993	* @returns VBox status code.
1994	* @retval VINF_* Scheduling instructions.
1995	* @retval VERR_EM_INTERPRETER Something we can't cope with.
1996	* @retval VERR_* Fatal errors.
1997	*
1998	* @param pVM The VM handle.
1999	* @param pRegFrame The register frame.
2000	* @param ASID Tagged TLB id for the guest
2001	*
2002	* Updates the EIP if an instruction was executed successfully.
2003	*/
2004	static int SVMR0InterpretInvpg(PVM pVM, PCPUMCTXCORE pRegFrame, uint32_t uASID)
2005	{
2006	/*
2007	* Only allow 32 & 64 bits code.
2008	*/
2009	DISCPUMODE enmMode = SELMGetCpuModeFromSelector(pVM, pRegFrame->eflags, pRegFrame->cs, &pRegFrame->csHid);
2010	if (enmMode != CPUMODE_16BIT)
2011	{
2012	RTGCPTR pbCode;
2013	int rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->rip, &pbCode);
2014	if (VBOX_SUCCESS(rc))
2015	{
2016	uint32_t cbOp;
2017	DISCPUSTATE Cpu;
2018
2019	Cpu.mode = enmMode;
2020	rc = EMInterpretDisasOneEx(pVM, pbCode, pRegFrame, &Cpu, &cbOp);
2021	Assert(VBOX_FAILURE(rc) \|\| Cpu.pCurInstr->opcode == OP_INVLPG);
2022	if (VBOX_SUCCESS(rc) && Cpu.pCurInstr->opcode == OP_INVLPG)
2023	{
2024	Assert(cbOp == Cpu.opsize);
2025	rc = svmInterpretInvlPg(pVM, &Cpu, pRegFrame, uASID);
2026	if (VBOX_SUCCESS(rc))
2027	{
2028	pRegFrame->rip += cbOp; /* Move on to the next instruction. */
2029	}
2030	return rc;
2031	}
2032	}
2033	}
2034	return VERR_EM_INTERPRETER;
2035	}
2036
2037
2038	/**
2039	* Invalidates a guest page
2040	*
2041	* @returns VBox status code.
2042	* @param pVM The VM to operate on.
2043	* @param GCVirt Page to invalidate
2044	*/
2045	HWACCMR0DECL(int) SVMR0InvalidatePage(PVM pVM, RTGCPTR GCVirt)
2046	{
2047	bool fFlushPending = pVM->hwaccm.s.svm.fAlwaysFlushTLB \| pVM->hwaccm.s.svm.fForceTLBFlush;
2048
2049	/* Skip it if a TLB flush is already pending. */
2050	if (!fFlushPending)
2051	{
2052	SVM_VMCB *pVMCB;
2053
2054	Log2(("SVMR0InvalidatePage %VGv\n", GCVirt));
2055	AssertReturn(pVM, VERR_INVALID_PARAMETER);
2056	Assert(pVM->hwaccm.s.svm.fSupported);
2057
2058	pVMCB = (SVM_VMCB *)pVM->hwaccm.s.svm.pVMCB;
2059	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
2060
2061	STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushPageManual);
2062	SVMInvlpgA(GCVirt, pVMCB->ctrl.TLBCtrl.n.u32ASID);
2063	}
2064	return VINF_SUCCESS;
2065	}
2066
2067
2068	/**
2069	* Invalidates a guest page by physical address
2070	*
2071	* NOTE: Assumes the current instruction references this physical page though a virtual address!!
2072	*
2073	* @returns VBox status code.
2074	* @param pVM The VM to operate on.
2075	* @param GCPhys Page to invalidate
2076	*/
2077	HWACCMR0DECL(int) SVMR0InvalidatePhysPage(PVM pVM, RTGCPHYS GCPhys)
2078	{
2079	bool fFlushPending = pVM->hwaccm.s.svm.fAlwaysFlushTLB \| pVM->hwaccm.s.svm.fForceTLBFlush;
2080
2081	Assert(pVM->hwaccm.s.fNestedPaging);
2082
2083	/* Skip it if a TLB flush is already pending. */
2084	if (!fFlushPending)
2085	{
2086	CPUMCTX *pCtx;
2087	int rc;
2088	SVM_VMCB *pVMCB;
2089
2090	rc = CPUMQueryGuestCtxPtr(pVM, &pCtx);
2091	AssertRCReturn(rc, rc);
2092
2093	Log2(("SVMR0InvalidatePhysPage %VGp\n", GCPhys));
2094	AssertReturn(pVM, VERR_INVALID_PARAMETER);
2095	Assert(pVM->hwaccm.s.svm.fSupported);
2096
2097	pVMCB = (SVM_VMCB *)pVM->hwaccm.s.svm.pVMCB;
2098	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
2099
2100	/*
2101	* Only allow 32 & 64 bits code.
2102	*/
2103	DISCPUMODE enmMode = SELMGetCpuModeFromSelector(pVM, pCtx->eflags, pCtx->cs, &pCtx->csHid);
2104	if (enmMode != CPUMODE_16BIT)
2105	{
2106	RTGCPTR pbCode;
2107	int rc = SELMValidateAndConvertCSAddr(pVM, pCtx->eflags, pCtx->ss, pCtx->cs, &pCtx->csHid, (RTGCPTR)pCtx->rip, &pbCode);
2108	if (VBOX_SUCCESS(rc))
2109	{
2110	uint32_t cbOp;
2111	DISCPUSTATE Cpu;
2112	OP_PARAMVAL param1;
2113	RTGCPTR addr;
2114
2115	Cpu.mode = enmMode;
2116	rc = EMInterpretDisasOneEx(pVM, pbCode, CPUMCTX2CORE(pCtx), &Cpu, &cbOp);
2117	AssertRCReturn(rc, rc);
2118	Assert(cbOp == Cpu.opsize);
2119
2120	int rc = DISQueryParamVal(CPUMCTX2CORE(pCtx), &Cpu, &Cpu.param1, &param1, PARAM_SOURCE);
2121	AssertRCReturn(rc, VERR_EM_INTERPRETER);
2122
2123	switch(param1.type)
2124	{
2125	case PARMTYPE_IMMEDIATE:
2126	case PARMTYPE_ADDRESS:
2127	AssertReturn((param1.flags & (PARAM_VAL32\|PARAM_VAL64)), VERR_EM_INTERPRETER);
2128
2129	addr = param1.val.val64;
2130	break;
2131
2132	default:
2133	AssertFailed();
2134	return VERR_EM_INTERPRETER;
2135	}
2136
2137	/* Manually invalidate the page for the VM's TLB. */
2138	Log(("SVMR0InvalidatePhysPage Phys=%VGp Virt=%VGv ASID=%d\n", GCPhys, addr, pVMCB->ctrl.TLBCtrl.n.u32ASID));
2139	SVMInvlpgA(addr, pVMCB->ctrl.TLBCtrl.n.u32ASID);
2140	STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushPhysPageManual);
2141
2142	return VINF_SUCCESS;
2143	}
2144	}
2145	AssertFailed();
2146	return VERR_EM_INTERPRETER;
2147	}
2148	return VINF_SUCCESS;
2149	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWSVMR0.cpp@ 11964

Download in other formats: