HWSVMR0.cpp@ 10206

Last change on this file since 10206 was 10206, checked in by vboxsync, 17 years ago
Fixed regression introduced by TPR caching. (never execute code that can jump back to ring 3 in *LoadGuestState)
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 81.7 KB

Line
1	/* $Id: HWSVMR0.cpp 10206 2008-07-04 08:36:48Z vboxsync $ */
2	/** @file
3	* HWACCM SVM - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/*******************************************************************************
24	* Header Files *
25	*******************************************************************************/
26	#define LOG_GROUP LOG_GROUP_HWACCM
27	#include <VBox/hwaccm.h>
28	#include "HWACCMInternal.h"
29	#include <VBox/vm.h>
30	#include <VBox/x86.h>
31	#include <VBox/hwacc_svm.h>
32	#include <VBox/pgm.h>
33	#include <VBox/pdm.h>
34	#include <VBox/err.h>
35	#include <VBox/log.h>
36	#include <VBox/selm.h>
37	#include <VBox/iom.h>
38	#include <VBox/dis.h>
39	#include <VBox/dbgf.h>
40	#include <VBox/disopcode.h>
41	#include <iprt/param.h>
42	#include <iprt/assert.h>
43	#include <iprt/asm.h>
44	#include <iprt/cpuset.h>
45	#include <iprt/mp.h>
46	#include "HWSVMR0.h"
47
48	static int SVMR0InterpretInvpg(PVM pVM, PCPUMCTXCORE pRegFrame, uint32_t uASID);
49
50	/**
51	* Sets up and activates AMD-V on the current CPU
52	*
53	* @returns VBox status code.
54	* @param pCpu CPU info struct
55	* @param pVM The VM to operate on.
56	* @param pvPageCpu Pointer to the global cpu page
57	* @param pPageCpuPhys Physical address of the global cpu page
58	*/
59	HWACCMR0DECL(int) SVMR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
60	{
61	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
62	AssertReturn(pVM, VERR_INVALID_PARAMETER);
63	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
64
65	/* We must turn on AMD-V and setup the host state physical address, as those MSRs are per-cpu/core. */
66
67	#ifdef LOG_ENABLED
68	SUPR0Printf("SVMR0EnableCpu cpu %d page (%x) %x\n", pCpu->idCpu, pvPageCpu, (uint32_t)pPageCpuPhys);
69	#endif
70
71	/* Turn on AMD-V in the EFER MSR. */
72	uint64_t val = ASMRdMsr(MSR_K6_EFER);
73	if (!(val & MSR_K6_EFER_SVME))
74	ASMWrMsr(MSR_K6_EFER, val \| MSR_K6_EFER_SVME);
75
76	/* Write the physical page address where the CPU will store the host state while executing the VM. */
77	ASMWrMsr(MSR_K8_VM_HSAVE_PA, pPageCpuPhys);
78
79	pCpu->uCurrentASID = 0; /* we'll aways increment this the first time (host uses ASID 0) */
80	pCpu->cTLBFlushes = 0;
81	return VINF_SUCCESS;
82	}
83
84	/**
85	* Deactivates AMD-V on the current CPU
86	*
87	* @returns VBox status code.
88	* @param pCpu CPU info struct
89	* @param pvPageCpu Pointer to the global cpu page
90	* @param pPageCpuPhys Physical address of the global cpu page
91	*/
92	HWACCMR0DECL(int) SVMR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
93	{
94	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
95	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
96
97	#ifdef LOG_ENABLED
98	SUPR0Printf("SVMR0DisableCpu cpu %d\n", pCpu->idCpu);
99	#endif
100
101	/* Turn off AMD-V in the EFER MSR. */
102	uint64_t val = ASMRdMsr(MSR_K6_EFER);
103	ASMWrMsr(MSR_K6_EFER, val & ~MSR_K6_EFER_SVME);
104
105	/* Invalidate host state physical address. */
106	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
107	pCpu->uCurrentASID = 0;
108
109	return VINF_SUCCESS;
110	}
111
112	/**
113	* Does Ring-0 per VM AMD-V init.
114	*
115	* @returns VBox status code.
116	* @param pVM The VM to operate on.
117	*/
118	HWACCMR0DECL(int) SVMR0InitVM(PVM pVM)
119	{
120	int rc;
121
122	/* Allocate one page for the VM control block (VMCB). */
123	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjVMCB, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
124	if (RT_FAILURE(rc))
125	return rc;
126
127	pVM->hwaccm.s.svm.pVMCB = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjVMCB);
128	pVM->hwaccm.s.svm.pVMCBPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjVMCB, 0);
129	ASMMemZero32(pVM->hwaccm.s.svm.pVMCB, PAGE_SIZE);
130
131	/* Allocate one page for the host context */
132	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjVMCBHost, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
133	if (RT_FAILURE(rc))
134	return rc;
135
136	pVM->hwaccm.s.svm.pVMCBHost = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjVMCBHost);
137	pVM->hwaccm.s.svm.pVMCBHostPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjVMCBHost, 0);
138	ASMMemZero32(pVM->hwaccm.s.svm.pVMCBHost, PAGE_SIZE);
139
140	/* Allocate 12 KB for the IO bitmap (doesn't seem to be a way to convince SVM not to use it) */
141	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjIOBitmap, 3 << PAGE_SHIFT, true /* executable R0 mapping */);
142	if (RT_FAILURE(rc))
143	return rc;
144
145	pVM->hwaccm.s.svm.pIOBitmap = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjIOBitmap);
146	pVM->hwaccm.s.svm.pIOBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjIOBitmap, 0);
147	/* Set all bits to intercept all IO accesses. */
148	ASMMemFill32(pVM->hwaccm.s.svm.pIOBitmap, PAGE_SIZE*3, 0xffffffff);
149
150	/* Allocate 8 KB for the MSR bitmap (doesn't seem to be a way to convince SVM not to use it) */
151	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjMSRBitmap, 2 << PAGE_SHIFT, true /* executable R0 mapping */);
152	if (RT_FAILURE(rc))
153	return rc;
154
155	pVM->hwaccm.s.svm.pMSRBitmap = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjMSRBitmap);
156	pVM->hwaccm.s.svm.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjMSRBitmap, 0);
157	/* Set all bits to intercept all MSR accesses. */
158	ASMMemFill32(pVM->hwaccm.s.svm.pMSRBitmap, PAGE_SIZE*2, 0xffffffff);
159
160	/* Erratum 170 which requires a forced TLB flush for each world switch:
161	* See http://www.amd.com/us-en/assets/content_type/white_papers_and_tech_docs/33610.pdf
162	*
163	* All BH-G1/2 and DH-G1/2 models include a fix:
164	* Athlon X2: 0x6b 1/2
165	* 0x68 1/2
166	* Athlon 64: 0x7f 1
167	* 0x6f 2
168	* Sempron: 0x7f 1/2
169	* 0x6f 2
170	* 0x6c 2
171	* 0x7c 2
172	* Turion 64: 0x68 2
173	*
174	*/
175	uint32_t u32Dummy;
176	uint32_t u32Version, u32Family, u32Model, u32Stepping, u32BaseFamily;
177	ASMCpuId(1, &u32Version, &u32Dummy, &u32Dummy, &u32Dummy);
178	u32BaseFamily= (u32Version >> 8) & 0xf;
179	u32Family = u32BaseFamily + (u32BaseFamily == 0xf ? ((u32Version >> 20) & 0x7f) : 0);
180	u32Model = ((u32Version >> 4) & 0xf);
181	u32Model = u32Model \| ((u32BaseFamily == 0xf ? (u32Version >> 16) & 0x0f : 0) << 4);
182	u32Stepping = u32Version & 0xf;
183	if ( u32Family == 0xf
184	&& !((u32Model == 0x68 \|\| u32Model == 0x6b \|\| u32Model == 0x7f) && u32Stepping >= 1)
185	&& !((u32Model == 0x6f \|\| u32Model == 0x6c \|\| u32Model == 0x7c) && u32Stepping >= 2))
186	{
187	Log(("SVMR0InitVM: AMD cpu with erratum 170 family %x model %x stepping %x\n", u32Family, u32Model, u32Stepping));
188	pVM->hwaccm.s.svm.fAlwaysFlushTLB = true;
189	}
190
191	/* Invalidate the last cpu we were running on. */
192	pVM->hwaccm.s.svm.idLastCpu = NIL_RTCPUID;
193	return VINF_SUCCESS;
194	}
195
196	/**
197	* Does Ring-0 per VM AMD-V termination.
198	*
199	* @returns VBox status code.
200	* @param pVM The VM to operate on.
201	*/
202	HWACCMR0DECL(int) SVMR0TermVM(PVM pVM)
203	{
204	if (pVM->hwaccm.s.svm.pMemObjVMCB)
205	{
206	RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjVMCB, false);
207	pVM->hwaccm.s.svm.pVMCB = 0;
208	pVM->hwaccm.s.svm.pVMCBPhys = 0;
209	pVM->hwaccm.s.svm.pMemObjVMCB = 0;
210	}
211	if (pVM->hwaccm.s.svm.pMemObjVMCBHost)
212	{
213	RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjVMCBHost, false);
214	pVM->hwaccm.s.svm.pVMCBHost = 0;
215	pVM->hwaccm.s.svm.pVMCBHostPhys = 0;
216	pVM->hwaccm.s.svm.pMemObjVMCBHost = 0;
217	}
218	if (pVM->hwaccm.s.svm.pMemObjIOBitmap)
219	{
220	RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjIOBitmap, false);
221	pVM->hwaccm.s.svm.pIOBitmap = 0;
222	pVM->hwaccm.s.svm.pIOBitmapPhys = 0;
223	pVM->hwaccm.s.svm.pMemObjIOBitmap = 0;
224	}
225	if (pVM->hwaccm.s.svm.pMemObjMSRBitmap)
226	{
227	RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjMSRBitmap, false);
228	pVM->hwaccm.s.svm.pMSRBitmap = 0;
229	pVM->hwaccm.s.svm.pMSRBitmapPhys = 0;
230	pVM->hwaccm.s.svm.pMemObjMSRBitmap = 0;
231	}
232	return VINF_SUCCESS;
233	}
234
235	/**
236	* Sets up AMD-V for the specified VM
237	*
238	* @returns VBox status code.
239	* @param pVM The VM to operate on.
240	*/
241	HWACCMR0DECL(int) SVMR0SetupVM(PVM pVM)
242	{
243	int rc = VINF_SUCCESS;
244	SVM_VMCB *pVMCB;
245
246	AssertReturn(pVM, VERR_INVALID_PARAMETER);
247
248	Assert(pVM->hwaccm.s.svm.fSupported);
249
250	pVMCB = (SVM_VMCB *)pVM->hwaccm.s.svm.pVMCB;
251	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
252
253	/* Program the control fields. Most of them never have to be changed again. */
254	/* CR0/3/4 reads must be intercepted, our shadow values are not necessarily the same as the guest's. */
255	/** @note CR0 & CR4 can be safely read when guest and shadow copies are identical. */
256	if (!pVM->hwaccm.s.fNestedPaging)
257	pVMCB->ctrl.u16InterceptRdCRx = RT_BIT(0) \| RT_BIT(3) \| RT_BIT(4);
258	else
259	pVMCB->ctrl.u16InterceptRdCRx = RT_BIT(0) \| RT_BIT(4);
260
261	/*
262	* CR0/3/4 writes must be intercepted for obvious reasons.
263	*/
264	if (!pVM->hwaccm.s.fNestedPaging)
265	pVMCB->ctrl.u16InterceptWrCRx = RT_BIT(0) \| RT_BIT(3) \| RT_BIT(4);
266	else
267	pVMCB->ctrl.u16InterceptWrCRx = RT_BIT(0) \| RT_BIT(4);
268
269	/* Intercept all DRx reads and writes. */
270	pVMCB->ctrl.u16InterceptRdDRx = RT_BIT(0) \| RT_BIT(1) \| RT_BIT(2) \| RT_BIT(3) \| RT_BIT(4) \| RT_BIT(5) \| RT_BIT(6) \| RT_BIT(7);
271	pVMCB->ctrl.u16InterceptWrDRx = RT_BIT(0) \| RT_BIT(1) \| RT_BIT(2) \| RT_BIT(3) \| RT_BIT(4) \| RT_BIT(5) \| RT_BIT(6) \| RT_BIT(7);
272
273	/* Currently we don't care about DRx reads or writes. DRx registers are trashed.
274	* All breakpoints are automatically cleared when the VM exits.
275	*/
276
277	pVMCB->ctrl.u32InterceptException = HWACCM_SVM_TRAP_MASK;
278	#ifndef DEBUG
279	if (pVM->hwaccm.s.fNestedPaging)
280	pVMCB->ctrl.u32InterceptException &= ~RT_BIT(14); /* no longer need to intercept #PF. */
281	#endif
282
283	pVMCB->ctrl.u32InterceptCtrl1 = SVM_CTRL1_INTERCEPT_INTR
284	\| SVM_CTRL1_INTERCEPT_VINTR
285	\| SVM_CTRL1_INTERCEPT_NMI
286	\| SVM_CTRL1_INTERCEPT_SMI
287	\| SVM_CTRL1_INTERCEPT_INIT
288	\| SVM_CTRL1_INTERCEPT_RDPMC
289	\| SVM_CTRL1_INTERCEPT_CPUID
290	\| SVM_CTRL1_INTERCEPT_RSM
291	\| SVM_CTRL1_INTERCEPT_HLT
292	\| SVM_CTRL1_INTERCEPT_INOUT_BITMAP
293	\| SVM_CTRL1_INTERCEPT_MSR_SHADOW
294	\| SVM_CTRL1_INTERCEPT_INVLPG
295	\| SVM_CTRL1_INTERCEPT_INVLPGA /* AMD only */
296	\| SVM_CTRL1_INTERCEPT_TASK_SWITCH
297	\| SVM_CTRL1_INTERCEPT_SHUTDOWN /* fatal */
298	\| SVM_CTRL1_INTERCEPT_FERR_FREEZE; /* Legacy FPU FERR handling. */
299	;
300	/* With nested paging we don't care about invlpg anymore. */
301	if (pVM->hwaccm.s.fNestedPaging)
302	pVMCB->ctrl.u32InterceptCtrl1 &= ~SVM_CTRL1_INTERCEPT_INVLPG;
303
304	pVMCB->ctrl.u32InterceptCtrl2 = SVM_CTRL2_INTERCEPT_VMRUN /* required */
305	\| SVM_CTRL2_INTERCEPT_VMMCALL
306	\| SVM_CTRL2_INTERCEPT_VMLOAD
307	\| SVM_CTRL2_INTERCEPT_VMSAVE
308	\| SVM_CTRL2_INTERCEPT_STGI
309	\| SVM_CTRL2_INTERCEPT_CLGI
310	\| SVM_CTRL2_INTERCEPT_SKINIT
311	\| SVM_CTRL2_INTERCEPT_RDTSCP /* AMD only; we don't support this one */
312	\| SVM_CTRL2_INTERCEPT_WBINVD
313	\| SVM_CTRL2_INTERCEPT_MWAIT_UNCOND; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
314	;
315	Log(("pVMCB->ctrl.u32InterceptException = %x\n", pVMCB->ctrl.u32InterceptException));
316	Log(("pVMCB->ctrl.u32InterceptCtrl1 = %x\n", pVMCB->ctrl.u32InterceptCtrl1));
317	Log(("pVMCB->ctrl.u32InterceptCtrl2 = %x\n", pVMCB->ctrl.u32InterceptCtrl2));
318
319	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
320	pVMCB->ctrl.IntCtrl.n.u1VIrqMasking = 1;
321
322	/* Set IO and MSR bitmap addresses. */
323	pVMCB->ctrl.u64IOPMPhysAddr = pVM->hwaccm.s.svm.pIOBitmapPhys;
324	pVMCB->ctrl.u64MSRPMPhysAddr = pVM->hwaccm.s.svm.pMSRBitmapPhys;
325
326	/* No LBR virtualization. */
327	pVMCB->ctrl.u64LBRVirt = 0;
328
329	/** The ASID must start at 1; the host uses 0. */
330	pVMCB->ctrl.TLBCtrl.n.u32ASID = 1;
331
332	/** Setup the PAT msr (nested paging only) */
333	pVMCB->guest.u64GPAT = 0x0007040600070406ULL;
334	return rc;
335	}
336
337
338	/**
339	* Injects an event (trap or external interrupt)
340	*
341	* @param pVM The VM to operate on.
342	* @param pVMCB SVM control block
343	* @param pCtx CPU Context
344	* @param pIntInfo SVM interrupt info
345	*/
346	inline void SVMR0InjectEvent(PVM pVM, SVM_VMCB pVMCB, CPUMCTX pCtx, SVM_EVENT* pEvent)
347	{
348	#ifdef VBOX_STRICT
349	if (pEvent->n.u8Vector == 0xE)
350	Log(("SVM: Inject int %d at %VGv error code=%02x CR2=%VGv intInfo=%08x\n", pEvent->n.u8Vector, pCtx->rip, pEvent->n.u32ErrorCode, pCtx->cr2, pEvent->au64[0]));
351	else
352	if (pEvent->n.u8Vector < 0x20)
353	Log(("SVM: Inject int %d at %VGv error code=%08x\n", pEvent->n.u8Vector, pCtx->rip, pEvent->n.u32ErrorCode));
354	else
355	{
356	Log(("INJ-EI: %x at %VGv\n", pEvent->n.u8Vector, pCtx->rip));
357	Assert(!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS));
358	Assert(pCtx->eflags.u32 & X86_EFL_IF);
359	}
360	#endif
361
362	/* Set event injection state. */
363	pVMCB->ctrl.EventInject.au64[0] = pEvent->au64[0];
364	}
365
366
367	/**
368	* Checks for pending guest interrupts and injects them
369	*
370	* @returns VBox status code.
371	* @param pVM The VM to operate on.
372	* @param pVMCB SVM control block
373	* @param pCtx CPU Context
374	*/
375	static int SVMR0CheckPendingInterrupt(PVM pVM, SVM_VMCB pVMCB, CPUMCTX pCtx)
376	{
377	int rc;
378
379	/* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
380	if (pVM->hwaccm.s.Event.fPending)
381	{
382	SVM_EVENT Event;
383
384	Log(("Reinjecting event %08x %08x at %VGv\n", pVM->hwaccm.s.Event.intInfo, pVM->hwaccm.s.Event.errCode, pCtx->rip));
385	STAM_COUNTER_INC(&pVM->hwaccm.s.StatIntReinject);
386	Event.au64[0] = pVM->hwaccm.s.Event.intInfo;
387	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
388
389	pVM->hwaccm.s.Event.fPending = false;
390	return VINF_SUCCESS;
391	}
392
393	/* When external interrupts are pending, we should exit the VM when IF is set. */
394	if ( !TRPMHasTrap(pVM)
395	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
396	{
397	if (!(pCtx->eflags.u32 & X86_EFL_IF))
398	{
399	if (!pVMCB->ctrl.IntCtrl.n.u1VIrqValid)
400	{
401	LogFlow(("Enable irq window exit!\n"));
402	/** @todo use virtual interrupt method to inject a pending irq; dispatched as soon as guest.IF is set. */
403	pVMCB->ctrl.u32InterceptCtrl1 \|= SVM_CTRL1_INTERCEPT_VINTR;
404	pVMCB->ctrl.IntCtrl.n.u1VIrqValid = 1;
405	pVMCB->ctrl.IntCtrl.n.u1IgnoreTPR = 1; /* ignore the priority in the TPR; just deliver it */
406	pVMCB->ctrl.IntCtrl.n.u8VIrqVector = 0; /* don't care */
407	}
408	}
409	else
410	if (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
411	{
412	uint8_t u8Interrupt;
413
414	rc = PDMGetInterrupt(pVM, &u8Interrupt);
415	Log(("Dispatch interrupt: u8Interrupt=%x (%d) rc=%Vrc\n", u8Interrupt, u8Interrupt, rc));
416	if (VBOX_SUCCESS(rc))
417	{
418	rc = TRPMAssertTrap(pVM, u8Interrupt, TRPM_HARDWARE_INT);
419	AssertRC(rc);
420	}
421	else
422	{
423	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
424	Assert(!VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)));
425	STAM_COUNTER_INC(&pVM->hwaccm.s.StatSwitchGuestIrq);
426	/* Just continue */
427	}
428	}
429	else
430	Log(("Pending interrupt blocked at %VGv by VM_FF_INHIBIT_INTERRUPTS!!\n", pCtx->rip));
431	}
432
433	#ifdef VBOX_STRICT
434	if (TRPMHasTrap(pVM))
435	{
436	uint8_t u8Vector;
437	rc = TRPMQueryTrapAll(pVM, &u8Vector, 0, 0, 0);
438	AssertRC(rc);
439	}
440	#endif
441
442	if ( pCtx->eflags.u32 & X86_EFL_IF
443	&& (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
444	&& TRPMHasTrap(pVM)
445	)
446	{
447	uint8_t u8Vector;
448	int rc;
449	TRPMEVENT enmType;
450	SVM_EVENT Event;
451	RTGCUINT u32ErrorCode;
452
453	Event.au64[0] = 0;
454
455	/* If a new event is pending, then dispatch it now. */
456	rc = TRPMQueryTrapAll(pVM, &u8Vector, &enmType, &u32ErrorCode, 0);
457	AssertRC(rc);
458	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
459	Assert(enmType != TRPM_SOFTWARE_INT);
460
461	/* Clear the pending trap. */
462	rc = TRPMResetTrap(pVM);
463	AssertRC(rc);
464
465	Event.n.u8Vector = u8Vector;
466	Event.n.u1Valid = 1;
467	Event.n.u32ErrorCode = u32ErrorCode;
468
469	if (enmType == TRPM_TRAP)
470	{
471	switch (u8Vector) {
472	case 8:
473	case 10:
474	case 11:
475	case 12:
476	case 13:
477	case 14:
478	case 17:
479	/* Valid error codes. */
480	Event.n.u1ErrorCodeValid = 1;
481	break;
482	default:
483	break;
484	}
485	if (u8Vector == X86_XCPT_NMI)
486	Event.n.u3Type = SVM_EVENT_NMI;
487	else
488	Event.n.u3Type = SVM_EVENT_EXCEPTION;
489	}
490	else
491	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
492
493	STAM_COUNTER_INC(&pVM->hwaccm.s.StatIntInject);
494	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
495	} /* if (interrupts can be dispatched) */
496
497	return VINF_SUCCESS;
498	}
499
500	/**
501	* Save the host state
502	*
503	* @returns VBox status code.
504	* @param pVM The VM to operate on.
505	*/
506	HWACCMR0DECL(int) SVMR0SaveHostState(PVM pVM)
507	{
508	/* Nothing to do here. */
509	return VINF_SUCCESS;
510	}
511
512	/**
513	* Loads the guest state
514	*
515	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
516	*
517	* @returns VBox status code.
518	* @param pVM The VM to operate on.
519	* @param pCtx Guest context
520	*/
521	HWACCMR0DECL(int) SVMR0LoadGuestState(PVM pVM, CPUMCTX *pCtx)
522	{
523	RTGCUINTPTR val;
524	SVM_VMCB *pVMCB;
525
526	if (pVM == NULL)
527	return VERR_INVALID_PARAMETER;
528
529	/* Setup AMD SVM. */
530	Assert(pVM->hwaccm.s.svm.fSupported);
531
532	pVMCB = (SVM_VMCB *)pVM->hwaccm.s.svm.pVMCB;
533	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
534
535	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
536	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
537	{
538	SVM_WRITE_SELREG(CS, cs);
539	SVM_WRITE_SELREG(SS, ss);
540	SVM_WRITE_SELREG(DS, ds);
541	SVM_WRITE_SELREG(ES, es);
542	SVM_WRITE_SELREG(FS, fs);
543	SVM_WRITE_SELREG(GS, gs);
544	}
545
546	/* Guest CPU context: LDTR. */
547	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
548	{
549	SVM_WRITE_SELREG(LDTR, ldtr);
550	}
551
552	/* Guest CPU context: TR. */
553	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
554	{
555	SVM_WRITE_SELREG(TR, tr);
556	}
557
558	/* Guest CPU context: GDTR. */
559	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
560	{
561	pVMCB->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
562	pVMCB->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
563	}
564
565	/* Guest CPU context: IDTR. */
566	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
567	{
568	pVMCB->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
569	pVMCB->guest.IDTR.u64Base = pCtx->idtr.pIdt;
570	}
571
572	/*
573	* Sysenter MSRs (unconditional)
574	*/
575	pVMCB->guest.u64SysEnterCS = pCtx->SysEnter.cs;
576	pVMCB->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
577	pVMCB->guest.u64SysEnterESP = pCtx->SysEnter.esp;
578
579	/* Control registers */
580	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
581	{
582	val = pCtx->cr0;
583	if (CPUMIsGuestFPUStateActive(pVM) == false)
584	{
585	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
586	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
587	}
588	else
589	{
590	Assert(pVM->hwaccm.s.svm.fResumeVM == true);
591	/** @todo check if we support the old style mess correctly. */
592	if (!(val & X86_CR0_NE))
593	{
594	Log(("Forcing X86_CR0_NE!!!\n"));
595
596	/* Also catch floating point exceptions as we need to report them to the guest in a different way. */
597	if (!pVM->hwaccm.s.fFPUOldStyleOverride)
598	{
599	pVMCB->ctrl.u32InterceptException \|= RT_BIT(16);
600	pVM->hwaccm.s.fFPUOldStyleOverride = true;
601	}
602	}
603	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
604	}
605	/* Always enable caching. */
606	val &= ~(X86_CR0_CD\|X86_CR0_NW);
607
608	/* Note: WP is not relevant in nested paging mode as we catch accesses on the (guest) physical level. */
609	/* Note: In nested paging mode the guest is allowed to run with paging disabled; the guest physical to host physical translation will remain active. */
610	if (!pVM->hwaccm.s.fNestedPaging)
611	{
612	val \|= X86_CR0_PG; /* Paging is always enabled; even when the guest is running in real mode or PE without paging. */
613	val \|= X86_CR0_WP; /* Must set this as we rely on protect various pages and supervisor writes must be caught. */
614	}
615	pVMCB->guest.u64CR0 = val;
616	}
617	/* CR2 as well */
618	pVMCB->guest.u64CR2 = pCtx->cr2;
619
620	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
621	{
622	/* Save our shadow CR3 register. */
623	if (pVM->hwaccm.s.fNestedPaging)
624	{
625	pVMCB->ctrl.u64NestedPagingCR3 = PGMGetNestedCR3(pVM, PGMGetHostMode(pVM));
626	Assert(pVMCB->ctrl.u64NestedPagingCR3);
627	pVMCB->guest.u64CR3 = pCtx->cr3;
628	}
629	else
630	pVMCB->guest.u64CR3 = PGMGetHyperCR3(pVM);
631	}
632
633	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
634	{
635	val = pCtx->cr4;
636	if (!pVM->hwaccm.s.fNestedPaging)
637	{
638	switch(pVM->hwaccm.s.enmShadowMode)
639	{
640	case PGMMODE_REAL:
641	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
642	AssertFailed();
643	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
644
645	case PGMMODE_32_BIT: /* 32-bit paging. */
646	break;
647
648	case PGMMODE_PAE: /* PAE paging. */
649	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
650	/** @todo use normal 32 bits paging */
651	val \|= X86_CR4_PAE;
652	break;
653
654	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
655	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
656	#ifdef VBOX_ENABLE_64_BITS_GUESTS
657	break;
658	#else
659	AssertFailed();
660	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
661	#endif
662
663	default: /* shut up gcc */
664	AssertFailed();
665	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
666	}
667	}
668	pVMCB->guest.u64CR4 = val;
669	}
670
671	/* Debug registers. */
672	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
673	{
674	/** @todo DR0-6 */
675	val = pCtx->dr7;
676	val &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
677	val \|= 0x400; /* must be one */
678	#ifdef VBOX_STRICT
679	val = 0x400;
680	#endif
681	pVMCB->guest.u64DR7 = val;
682
683	pVMCB->guest.u64DR6 = pCtx->dr6;
684	}
685
686	/* EIP, ESP and EFLAGS */
687	pVMCB->guest.u64RIP = pCtx->rip;
688	pVMCB->guest.u64RSP = pCtx->rsp;
689	pVMCB->guest.u64RFlags = pCtx->eflags.u32;
690
691	/* Set CPL */
692	pVMCB->guest.u8CPL = pCtx->csHid.Attr.n.u2Dpl;
693
694	/* RAX/EAX too, as VMRUN uses RAX as an implicit parameter. */
695	pVMCB->guest.u64RAX = pCtx->rax;
696
697	/* vmrun will fail without MSR_K6_EFER_SVME. */
698	pVMCB->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
699
700	/* 64 bits guest mode? */
701	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
702	{
703	#if !defined(VBOX_WITH_64_BITS_GUESTS) \|\| HC_ARCH_BITS != 64
704	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
705	#else
706	pVM->hwaccm.s.svm.pfnVMRun = SVMVMRun64;
707	#endif
708	/* Unconditionally update these as wrmsr might have changed them. (HWACCM_CHANGED_GUEST_SEGMENT_REGS will not be set) */
709	pVMCB->guest.FS.u64Base = pCtx->fsHid.u64Base;
710	pVMCB->guest.GS.u64Base = pCtx->gsHid.u64Base;
711	}
712	else
713	{
714	/* Filter out the MSR_K6_LME bit or else AMD-V expects amd64 shadow paging. */
715	pVMCB->guest.u64EFER &= ~MSR_K6_EFER_LME;
716
717	pVM->hwaccm.s.svm.pfnVMRun = SVMVMRun;
718	}
719
720	/** TSC offset. */
721	if (TMCpuTickCanUseRealTSC(pVM, &pVMCB->ctrl.u64TSCOffset))
722	{
723	pVMCB->ctrl.u32InterceptCtrl1 &= ~SVM_CTRL1_INTERCEPT_RDTSC;
724	STAM_COUNTER_INC(&pVM->hwaccm.s.StatTSCOffset);
725	}
726	else
727	{
728	pVMCB->ctrl.u32InterceptCtrl1 \|= SVM_CTRL1_INTERCEPT_RDTSC;
729	STAM_COUNTER_INC(&pVM->hwaccm.s.StatTSCIntercept);
730	}
731
732	/* Sync the various msrs for 64 bits mode. */
733	pVMCB->guest.u64STAR = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
734	pVMCB->guest.u64LSTAR = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
735	pVMCB->guest.u64CSTAR = pCtx->msrCSTAR; /* compatibility mode syscall rip */
736	pVMCB->guest.u64SFMASK = pCtx->msrSFMASK; /* syscall flag mask */
737	pVMCB->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
738
739	#ifdef DEBUG
740	/* Intercept X86_XCPT_DB if stepping is enabled */
741	if (DBGFIsStepping(pVM))
742	pVMCB->ctrl.u32InterceptException \|= RT_BIT(1);
743	else
744	pVMCB->ctrl.u32InterceptException &= ~RT_BIT(1);
745	#endif
746
747	/* Done. */
748	pVM->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
749
750	return VINF_SUCCESS;
751	}
752
753
754	/**
755	* Runs guest code in an SVM VM.
756	*
757	* @todo This can be much more efficient, when we only sync that which has actually changed. (this is the first attempt only)
758	*
759	* @returns VBox status code.
760	* @param pVM The VM to operate on.
761	* @param pCtx Guest context
762	* @param pCpu CPU info struct
763	*/
764	HWACCMR0DECL(int) SVMR0RunGuestCode(PVM pVM, CPUMCTX *pCtx, PHWACCM_CPUINFO pCpu)
765	{
766	int rc = VINF_SUCCESS;
767	uint64_t exitCode = (uint64_t)SVM_EXIT_INVALID;
768	SVM_VMCB *pVMCB;
769	bool fGuestStateSynced = false;
770	unsigned cResume = 0;
771
772	STAM_PROFILE_ADV_START(&pVM->hwaccm.s.StatEntry, x);
773
774	AssertReturn(pCpu->fConfigured, VERR_EM_INTERNAL_ERROR);
775
776	pVMCB = (SVM_VMCB *)pVM->hwaccm.s.svm.pVMCB;
777	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
778
779	/* We can jump to this point to resume execution after determining that a VM-exit is innocent.
780	*/
781	ResumeExecution:
782	/* Safety precaution; looping for too long here can have a very bad effect on the host */
783	if (++cResume > HWACCM_MAX_RESUME_LOOPS)
784	{
785	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitMaxResume);
786	rc = VINF_EM_RAW_INTERRUPT;
787	goto end;
788	}
789
790	/* Check for irq inhibition due to instruction fusing (sti, mov ss). */
791	if (VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
792	{
793	Log(("VM_FF_INHIBIT_INTERRUPTS at %VGv successor %VGv\n", pCtx->rip, EMGetInhibitInterruptsPC(pVM)));
794	if (pCtx->rip != EMGetInhibitInterruptsPC(pVM))
795	{
796	/** @note we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
797	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
798	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
799	* break the guest. Sounds very unlikely, but such timing sensitive problem are not as rare as you might think.
800	*/
801	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
802	/* Irq inhibition is no longer active; clear the corresponding SVM state. */
803	pVMCB->ctrl.u64IntShadow = 0;
804	}
805	}
806	else
807	{
808	/* Irq inhibition is no longer active; clear the corresponding SVM state. */
809	pVMCB->ctrl.u64IntShadow = 0;
810	}
811
812	/* Check for pending actions that force us to go back to ring 3. */
813	#ifdef DEBUG
814	/* Intercept X86_XCPT_DB if stepping is enabled */
815	if (!DBGFIsStepping(pVM))
816	#endif
817	{
818	if (VM_FF_ISPENDING(pVM, VM_FF_TO_R3 \| VM_FF_TIMER))
819	{
820	VM_FF_CLEAR(pVM, VM_FF_TO_R3);
821	STAM_COUNTER_INC(&pVM->hwaccm.s.StatSwitchToR3);
822	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatEntry, x);
823	rc = VINF_EM_RAW_TO_R3;
824	goto end;
825	}
826	}
827
828	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
829	if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
830	{
831	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatEntry, x);
832	rc = VINF_EM_PENDING_REQUEST;
833	goto end;
834	}
835
836	/* When external interrupts are pending, we should exit the VM when IF is set. */
837	/** @note after VM_FF_INHIBIT_INTERRUPTS check!!! */
838	rc = SVMR0CheckPendingInterrupt(pVM, pVMCB, pCtx);
839	if (VBOX_FAILURE(rc))
840	{
841	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatEntry, x);
842	goto end;
843	}
844
845	/* Load the guest state */
846	rc = SVMR0LoadGuestState(pVM, pCtx);
847	if (rc != VINF_SUCCESS)
848	{
849	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatEntry, x);
850	goto end;
851	}
852	fGuestStateSynced = true;
853
854	/* TPR caching using CR8 is only available in 64 bits mode */
855	/* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
856	/* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock). */
857	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
858	{
859	/* TPR caching in CR8 */
860	uint8_t u8TPR;
861	int rc = PDMApicGetTPR(pVM, &u8TPR);
862	AssertRC(rc);
863	pCtx->cr8 = u8TPR;
864	pVMCB->ctrl.IntCtrl.n.u8VTPR = u8TPR;
865	}
866
867	/* All done! Let's start VM execution. */
868	STAM_PROFILE_ADV_START(&pVM->hwaccm.s.StatInGC, x);
869
870	/* Enable nested paging if necessary (disabled each time after #VMEXIT). */
871	pVMCB->ctrl.NestedPaging.n.u1NestedPaging = pVM->hwaccm.s.fNestedPaging;
872
873	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
874	if (!pVM->hwaccm.s.svm.fResumeVM)
875	{
876	if ( pVM->hwaccm.s.svm.idLastCpu != pCpu->idCpu
877	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
878	\|\| pVM->hwaccm.s.svm.cTLBFlushes != pCpu->cTLBFlushes)
879	{
880	/* Force a TLB flush on VM entry. */
881	pVM->hwaccm.s.svm.fForceTLBFlush = true;
882	}
883	pVM->hwaccm.s.svm.idLastCpu = pCpu->idCpu;
884	}
885
886	/* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
887	if ( pVM->hwaccm.s.svm.fForceTLBFlush
888	&& !pVM->hwaccm.s.svm.fAlwaysFlushTLB)
889	{
890	if (++pCpu->uCurrentASID >= pVM->hwaccm.s.svm.u32MaxASID)
891	{
892	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
893	pVMCB->ctrl.TLBCtrl.n.u1TLBFlush = 1; /* wrap around; flush TLB */
894	pCpu->cTLBFlushes++;
895	}
896	else
897	STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushASID);
898
899	pVM->hwaccm.s.svm.cTLBFlushes = pCpu->cTLBFlushes;
900	}
901	else
902	{
903	/* We never increase uCurrentASID in the fAlwaysFlushTLB (erratum 170) case. */
904	if (!pCpu->uCurrentASID)
905	pCpu->uCurrentASID = 1;
906
907	pVMCB->ctrl.TLBCtrl.n.u1TLBFlush = pVM->hwaccm.s.svm.fForceTLBFlush;
908	}
909
910	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.svm.u32MaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
911	pVMCB->ctrl.TLBCtrl.n.u32ASID = pCpu->uCurrentASID;
912
913	#ifdef VBOX_WITH_STATISTICS
914	if (pVMCB->ctrl.TLBCtrl.n.u1TLBFlush)
915	STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushTLBWorldSwitch);
916	else
917	STAM_COUNTER_INC(&pVM->hwaccm.s.StatNoFlushTLBWorldSwitch);
918	#endif
919
920	/* In case we execute a goto ResumeExecution later on. */
921	pVM->hwaccm.s.svm.fResumeVM = true;
922	pVM->hwaccm.s.svm.fForceTLBFlush = pVM->hwaccm.s.svm.fAlwaysFlushTLB;
923
924	Assert(sizeof(pVM->hwaccm.s.svm.pVMCBPhys) == 8);
925	Assert(pVMCB->ctrl.u32InterceptCtrl2 == ( SVM_CTRL2_INTERCEPT_VMRUN /* required */
926	\| SVM_CTRL2_INTERCEPT_VMMCALL
927	\| SVM_CTRL2_INTERCEPT_VMLOAD
928	\| SVM_CTRL2_INTERCEPT_VMSAVE
929	\| SVM_CTRL2_INTERCEPT_STGI
930	\| SVM_CTRL2_INTERCEPT_CLGI
931	\| SVM_CTRL2_INTERCEPT_SKINIT
932	\| SVM_CTRL2_INTERCEPT_RDTSCP /* AMD only; we don't support this one */
933	\| SVM_CTRL2_INTERCEPT_WBINVD
934	\| SVM_CTRL2_INTERCEPT_MWAIT_UNCOND /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
935	));
936	Assert(pVMCB->ctrl.IntCtrl.n.u1VIrqMasking);
937	Assert(pVMCB->ctrl.u64IOPMPhysAddr == pVM->hwaccm.s.svm.pIOBitmapPhys);
938	Assert(pVMCB->ctrl.u64MSRPMPhysAddr == pVM->hwaccm.s.svm.pMSRBitmapPhys);
939	Assert(pVMCB->ctrl.u64LBRVirt == 0);
940
941	pVM->hwaccm.s.svm.pfnVMRun(pVM->hwaccm.s.svm.pVMCBHostPhys, pVM->hwaccm.s.svm.pVMCBPhys, pCtx);
942	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatInGC, x);
943
944	/**
945	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
946	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
947	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
948	*/
949
950	STAM_PROFILE_ADV_START(&pVM->hwaccm.s.StatExit, x);
951
952	/* Reason for the VM exit */
953	exitCode = pVMCB->ctrl.u64ExitCode;
954
955	if (exitCode == (uint64_t)SVM_EXIT_INVALID) /* Invalid guest state. */
956	{
957	HWACCMDumpRegs(pCtx);
958	#ifdef DEBUG
959	Log(("ctrl.u16InterceptRdCRx %x\n", pVMCB->ctrl.u16InterceptRdCRx));
960	Log(("ctrl.u16InterceptWrCRx %x\n", pVMCB->ctrl.u16InterceptWrCRx));
961	Log(("ctrl.u16InterceptRdDRx %x\n", pVMCB->ctrl.u16InterceptRdDRx));
962	Log(("ctrl.u16InterceptWrDRx %x\n", pVMCB->ctrl.u16InterceptWrDRx));
963	Log(("ctrl.u32InterceptException %x\n", pVMCB->ctrl.u32InterceptException));
964	Log(("ctrl.u32InterceptCtrl1 %x\n", pVMCB->ctrl.u32InterceptCtrl1));
965	Log(("ctrl.u32InterceptCtrl2 %x\n", pVMCB->ctrl.u32InterceptCtrl2));
966	Log(("ctrl.u64IOPMPhysAddr %VX64\n", pVMCB->ctrl.u64IOPMPhysAddr));
967	Log(("ctrl.u64MSRPMPhysAddr %VX64\n", pVMCB->ctrl.u64MSRPMPhysAddr));
968	Log(("ctrl.u64TSCOffset %VX64\n", pVMCB->ctrl.u64TSCOffset));
969
970	Log(("ctrl.TLBCtrl.u32ASID %x\n", pVMCB->ctrl.TLBCtrl.n.u32ASID));
971	Log(("ctrl.TLBCtrl.u1TLBFlush %x\n", pVMCB->ctrl.TLBCtrl.n.u1TLBFlush));
972	Log(("ctrl.TLBCtrl.u7Reserved %x\n", pVMCB->ctrl.TLBCtrl.n.u7Reserved));
973	Log(("ctrl.TLBCtrl.u24Reserved %x\n", pVMCB->ctrl.TLBCtrl.n.u24Reserved));
974
975	Log(("ctrl.IntCtrl.u8VTPR %x\n", pVMCB->ctrl.IntCtrl.n.u8VTPR));
976	Log(("ctrl.IntCtrl.u1VIrqValid %x\n", pVMCB->ctrl.IntCtrl.n.u1VIrqValid));
977	Log(("ctrl.IntCtrl.u7Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u7Reserved));
978	Log(("ctrl.IntCtrl.u4VIrqPriority %x\n", pVMCB->ctrl.IntCtrl.n.u4VIrqPriority));
979	Log(("ctrl.IntCtrl.u1IgnoreTPR %x\n", pVMCB->ctrl.IntCtrl.n.u1IgnoreTPR));
980	Log(("ctrl.IntCtrl.u3Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u3Reserved));
981	Log(("ctrl.IntCtrl.u1VIrqMasking %x\n", pVMCB->ctrl.IntCtrl.n.u1VIrqMasking));
982	Log(("ctrl.IntCtrl.u7Reserved2 %x\n", pVMCB->ctrl.IntCtrl.n.u7Reserved2));
983	Log(("ctrl.IntCtrl.u8VIrqVector %x\n", pVMCB->ctrl.IntCtrl.n.u8VIrqVector));
984	Log(("ctrl.IntCtrl.u24Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u24Reserved));
985
986	Log(("ctrl.u64IntShadow %VX64\n", pVMCB->ctrl.u64IntShadow));
987	Log(("ctrl.u64ExitCode %VX64\n", pVMCB->ctrl.u64ExitCode));
988	Log(("ctrl.u64ExitInfo1 %VX64\n", pVMCB->ctrl.u64ExitInfo1));
989	Log(("ctrl.u64ExitInfo2 %VX64\n", pVMCB->ctrl.u64ExitInfo2));
990	Log(("ctrl.ExitIntInfo.u8Vector %x\n", pVMCB->ctrl.ExitIntInfo.n.u8Vector));
991	Log(("ctrl.ExitIntInfo.u3Type %x\n", pVMCB->ctrl.ExitIntInfo.n.u3Type));
992	Log(("ctrl.ExitIntInfo.u1ErrorCodeValid %x\n", pVMCB->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
993	Log(("ctrl.ExitIntInfo.u19Reserved %x\n", pVMCB->ctrl.ExitIntInfo.n.u19Reserved));
994	Log(("ctrl.ExitIntInfo.u1Valid %x\n", pVMCB->ctrl.ExitIntInfo.n.u1Valid));
995	Log(("ctrl.ExitIntInfo.u32ErrorCode %x\n", pVMCB->ctrl.ExitIntInfo.n.u32ErrorCode));
996	Log(("ctrl.NestedPaging %VX64\n", pVMCB->ctrl.NestedPaging.au64));
997	Log(("ctrl.EventInject.u8Vector %x\n", pVMCB->ctrl.EventInject.n.u8Vector));
998	Log(("ctrl.EventInject.u3Type %x\n", pVMCB->ctrl.EventInject.n.u3Type));
999	Log(("ctrl.EventInject.u1ErrorCodeValid %x\n", pVMCB->ctrl.EventInject.n.u1ErrorCodeValid));
1000	Log(("ctrl.EventInject.u19Reserved %x\n", pVMCB->ctrl.EventInject.n.u19Reserved));
1001	Log(("ctrl.EventInject.u1Valid %x\n", pVMCB->ctrl.EventInject.n.u1Valid));
1002	Log(("ctrl.EventInject.u32ErrorCode %x\n", pVMCB->ctrl.EventInject.n.u32ErrorCode));
1003
1004	Log(("ctrl.u64NestedPagingCR3 %VX64\n", pVMCB->ctrl.u64NestedPagingCR3));
1005	Log(("ctrl.u64LBRVirt %VX64\n", pVMCB->ctrl.u64LBRVirt));
1006
1007	Log(("guest.CS.u16Sel %04X\n", pVMCB->guest.CS.u16Sel));
1008	Log(("guest.CS.u16Attr %04X\n", pVMCB->guest.CS.u16Attr));
1009	Log(("guest.CS.u32Limit %X\n", pVMCB->guest.CS.u32Limit));
1010	Log(("guest.CS.u64Base %VX64\n", pVMCB->guest.CS.u64Base));
1011	Log(("guest.DS.u16Sel %04X\n", pVMCB->guest.DS.u16Sel));
1012	Log(("guest.DS.u16Attr %04X\n", pVMCB->guest.DS.u16Attr));
1013	Log(("guest.DS.u32Limit %X\n", pVMCB->guest.DS.u32Limit));
1014	Log(("guest.DS.u64Base %VX64\n", pVMCB->guest.DS.u64Base));
1015	Log(("guest.ES.u16Sel %04X\n", pVMCB->guest.ES.u16Sel));
1016	Log(("guest.ES.u16Attr %04X\n", pVMCB->guest.ES.u16Attr));
1017	Log(("guest.ES.u32Limit %X\n", pVMCB->guest.ES.u32Limit));
1018	Log(("guest.ES.u64Base %VX64\n", pVMCB->guest.ES.u64Base));
1019	Log(("guest.FS.u16Sel %04X\n", pVMCB->guest.FS.u16Sel));
1020	Log(("guest.FS.u16Attr %04X\n", pVMCB->guest.FS.u16Attr));
1021	Log(("guest.FS.u32Limit %X\n", pVMCB->guest.FS.u32Limit));
1022	Log(("guest.FS.u64Base %VX64\n", pVMCB->guest.FS.u64Base));
1023	Log(("guest.GS.u16Sel %04X\n", pVMCB->guest.GS.u16Sel));
1024	Log(("guest.GS.u16Attr %04X\n", pVMCB->guest.GS.u16Attr));
1025	Log(("guest.GS.u32Limit %X\n", pVMCB->guest.GS.u32Limit));
1026	Log(("guest.GS.u64Base %VX64\n", pVMCB->guest.GS.u64Base));
1027
1028	Log(("guest.GDTR.u32Limit %X\n", pVMCB->guest.GDTR.u32Limit));
1029	Log(("guest.GDTR.u64Base %VX64\n", pVMCB->guest.GDTR.u64Base));
1030
1031	Log(("guest.LDTR.u16Sel %04X\n", pVMCB->guest.LDTR.u16Sel));
1032	Log(("guest.LDTR.u16Attr %04X\n", pVMCB->guest.LDTR.u16Attr));
1033	Log(("guest.LDTR.u32Limit %X\n", pVMCB->guest.LDTR.u32Limit));
1034	Log(("guest.LDTR.u64Base %VX64\n", pVMCB->guest.LDTR.u64Base));
1035
1036	Log(("guest.IDTR.u32Limit %X\n", pVMCB->guest.IDTR.u32Limit));
1037	Log(("guest.IDTR.u64Base %VX64\n", pVMCB->guest.IDTR.u64Base));
1038
1039	Log(("guest.TR.u16Sel %04X\n", pVMCB->guest.TR.u16Sel));
1040	Log(("guest.TR.u16Attr %04X\n", pVMCB->guest.TR.u16Attr));
1041	Log(("guest.TR.u32Limit %X\n", pVMCB->guest.TR.u32Limit));
1042	Log(("guest.TR.u64Base %VX64\n", pVMCB->guest.TR.u64Base));
1043
1044	Log(("guest.u8CPL %X\n", pVMCB->guest.u8CPL));
1045	Log(("guest.u64CR0 %VX64\n", pVMCB->guest.u64CR0));
1046	Log(("guest.u64CR2 %VX64\n", pVMCB->guest.u64CR2));
1047	Log(("guest.u64CR3 %VX64\n", pVMCB->guest.u64CR3));
1048	Log(("guest.u64CR4 %VX64\n", pVMCB->guest.u64CR4));
1049	Log(("guest.u64DR6 %VX64\n", pVMCB->guest.u64DR6));
1050	Log(("guest.u64DR7 %VX64\n", pVMCB->guest.u64DR7));
1051
1052	Log(("guest.u64RIP %VX64\n", pVMCB->guest.u64RIP));
1053	Log(("guest.u64RSP %VX64\n", pVMCB->guest.u64RSP));
1054	Log(("guest.u64RAX %VX64\n", pVMCB->guest.u64RAX));
1055	Log(("guest.u64RFlags %VX64\n", pVMCB->guest.u64RFlags));
1056
1057	Log(("guest.u64SysEnterCS %VX64\n", pVMCB->guest.u64SysEnterCS));
1058	Log(("guest.u64SysEnterEIP %VX64\n", pVMCB->guest.u64SysEnterEIP));
1059	Log(("guest.u64SysEnterESP %VX64\n", pVMCB->guest.u64SysEnterESP));
1060
1061	Log(("guest.u64EFER %VX64\n", pVMCB->guest.u64EFER));
1062	Log(("guest.u64STAR %VX64\n", pVMCB->guest.u64STAR));
1063	Log(("guest.u64LSTAR %VX64\n", pVMCB->guest.u64LSTAR));
1064	Log(("guest.u64CSTAR %VX64\n", pVMCB->guest.u64CSTAR));
1065	Log(("guest.u64SFMASK %VX64\n", pVMCB->guest.u64SFMASK));
1066	Log(("guest.u64KernelGSBase %VX64\n", pVMCB->guest.u64KernelGSBase));
1067	Log(("guest.u64GPAT %VX64\n", pVMCB->guest.u64GPAT));
1068	Log(("guest.u64DBGCTL %VX64\n", pVMCB->guest.u64DBGCTL));
1069	Log(("guest.u64BR_FROM %VX64\n", pVMCB->guest.u64BR_FROM));
1070	Log(("guest.u64BR_TO %VX64\n", pVMCB->guest.u64BR_TO));
1071	Log(("guest.u64LASTEXCPFROM %VX64\n", pVMCB->guest.u64LASTEXCPFROM));
1072	Log(("guest.u64LASTEXCPTO %VX64\n", pVMCB->guest.u64LASTEXCPTO));
1073
1074	#endif
1075	rc = VERR_SVM_UNABLE_TO_START_VM;
1076	goto end;
1077	}
1078
1079	/* Let's first sync back eip, esp, and eflags. */
1080	pCtx->rip = pVMCB->guest.u64RIP;
1081	pCtx->rsp = pVMCB->guest.u64RSP;
1082	pCtx->eflags.u32 = pVMCB->guest.u64RFlags;
1083	/* eax is saved/restore across the vmrun instruction */
1084	pCtx->rax = pVMCB->guest.u64RAX;
1085
1086	pCtx->msrKERNELGSBASE = pVMCB->guest.u64KernelGSBase; /* swapgs exchange value */
1087
1088	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1089	SVM_READ_SELREG(SS, ss);
1090	SVM_READ_SELREG(CS, cs);
1091	SVM_READ_SELREG(DS, ds);
1092	SVM_READ_SELREG(ES, es);
1093	SVM_READ_SELREG(FS, fs);
1094	SVM_READ_SELREG(GS, gs);
1095
1096	/* Note: no reason to sync back the CRx and DRx registers. They can't be changed by the guest. */
1097	/* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1098	if ( pVM->hwaccm.s.fNestedPaging
1099	&& pCtx->cr3 != pVMCB->guest.u64CR3)
1100	{
1101	CPUMSetGuestCR3(pVM, pVMCB->guest.u64CR3);
1102	PGMUpdateCR3(pVM, pVMCB->guest.u64CR3);
1103	}
1104
1105	/** @note NOW IT'S SAFE FOR LOGGING! */
1106
1107	/* Take care of instruction fusing (sti, mov ss) */
1108	if (pVMCB->ctrl.u64IntShadow & SVM_INTERRUPT_SHADOW_ACTIVE)
1109	{
1110	Log(("uInterruptState %x eip=%VGv\n", pVMCB->ctrl.u64IntShadow, pCtx->rip));
1111	EMSetInhibitInterruptsPC(pVM, pCtx->rip);
1112	}
1113	else
1114	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
1115
1116	Log2(("exitCode = %x\n", exitCode));
1117
1118	/* Sync back the debug registers. */
1119	/** @todo Implement debug registers correctly. */
1120	pCtx->dr6 = pVMCB->guest.u64DR6;
1121	pCtx->dr7 = pVMCB->guest.u64DR7;
1122
1123	/* Update the APIC if the cached TPR value has changed. */
1124	if (pVMCB->ctrl.IntCtrl.n.u8VTPR != pCtx->cr8)
1125	{
1126	rc = PDMApicSetTPR(pVM, pVMCB->ctrl.IntCtrl.n.u8VTPR);
1127	AssertRC(rc);
1128	pCtx->cr8 = pVMCB->ctrl.IntCtrl.n.u8VTPR;
1129	}
1130	pVMCB->ctrl.IntCtrl.n.u8VTPR = pCtx->cr8;
1131
1132	/* Check if an injected event was interrupted prematurely. */
1133	pVM->hwaccm.s.Event.intInfo = pVMCB->ctrl.ExitIntInfo.au64[0];
1134	if ( pVMCB->ctrl.ExitIntInfo.n.u1Valid
1135	&& pVMCB->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT /* we don't care about 'int xx' as the instruction will be restarted. */)
1136	{
1137	Log(("Pending inject %VX64 at %VGv exit=%08x\n", pVM->hwaccm.s.Event.intInfo, pCtx->rip, exitCode));
1138	pVM->hwaccm.s.Event.fPending = true;
1139	/* Error code present? (redundant) */
1140	if (pVMCB->ctrl.ExitIntInfo.n.u1ErrorCodeValid)
1141	{
1142	pVM->hwaccm.s.Event.errCode = pVMCB->ctrl.ExitIntInfo.n.u32ErrorCode;
1143	}
1144	else
1145	pVM->hwaccm.s.Event.errCode = 0;
1146	}
1147	#ifdef VBOX_WITH_STATISTICS
1148	if (exitCode == SVM_EXIT_NPF)
1149	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitReasonNPF);
1150	else
1151	STAM_COUNTER_INC(&pVM->hwaccm.s.pStatExitReasonR0[exitCode & MASK_EXITREASON_STAT]);
1152	#endif
1153
1154	/* Deal with the reason of the VM-exit. */
1155	switch (exitCode)
1156	{
1157	case SVM_EXIT_EXCEPTION_0: case SVM_EXIT_EXCEPTION_1: case SVM_EXIT_EXCEPTION_2: case SVM_EXIT_EXCEPTION_3:
1158	case SVM_EXIT_EXCEPTION_4: case SVM_EXIT_EXCEPTION_5: case SVM_EXIT_EXCEPTION_6: case SVM_EXIT_EXCEPTION_7:
1159	case SVM_EXIT_EXCEPTION_8: case SVM_EXIT_EXCEPTION_9: case SVM_EXIT_EXCEPTION_A: case SVM_EXIT_EXCEPTION_B:
1160	case SVM_EXIT_EXCEPTION_C: case SVM_EXIT_EXCEPTION_D: case SVM_EXIT_EXCEPTION_E: case SVM_EXIT_EXCEPTION_F:
1161	case SVM_EXIT_EXCEPTION_10: case SVM_EXIT_EXCEPTION_11: case SVM_EXIT_EXCEPTION_12: case SVM_EXIT_EXCEPTION_13:
1162	case SVM_EXIT_EXCEPTION_14: case SVM_EXIT_EXCEPTION_15: case SVM_EXIT_EXCEPTION_16: case SVM_EXIT_EXCEPTION_17:
1163	case SVM_EXIT_EXCEPTION_18: case SVM_EXIT_EXCEPTION_19: case SVM_EXIT_EXCEPTION_1A: case SVM_EXIT_EXCEPTION_1B:
1164	case SVM_EXIT_EXCEPTION_1C: case SVM_EXIT_EXCEPTION_1D: case SVM_EXIT_EXCEPTION_1E: case SVM_EXIT_EXCEPTION_1F:
1165	{
1166	/* Pending trap. */
1167	SVM_EVENT Event;
1168	uint32_t vector = exitCode - SVM_EXIT_EXCEPTION_0;
1169
1170	Log2(("Hardware/software interrupt %d\n", vector));
1171	switch (vector)
1172	{
1173	#ifdef DEBUG
1174	case X86_XCPT_DB:
1175	rc = DBGFR0Trap01Handler(pVM, CPUMCTX2CORE(pCtx), pVMCB->guest.u64DR6);
1176	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
1177	break;
1178	#endif
1179
1180	case X86_XCPT_NM:
1181	{
1182	uint32_t oldCR0;
1183
1184	Log(("#NM fault at %VGv\n", pCtx->rip));
1185
1186	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
1187	oldCR0 = ASMGetCR0();
1188	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
1189	rc = CPUMHandleLazyFPU(pVM);
1190	if (rc == VINF_SUCCESS)
1191	{
1192	Assert(CPUMIsGuestFPUStateActive(pVM));
1193
1194	/* CPUMHandleLazyFPU could have changed CR0; restore it. */
1195	ASMSetCR0(oldCR0);
1196
1197	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitShadowNM);
1198
1199	/* Continue execution. */
1200	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1201	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
1202
1203	goto ResumeExecution;
1204	}
1205
1206	Log(("Forward #NM fault to the guest\n"));
1207	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestNM);
1208
1209	Event.au64[0] = 0;
1210	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1211	Event.n.u1Valid = 1;
1212	Event.n.u8Vector = X86_XCPT_NM;
1213
1214	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1215	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1216	goto ResumeExecution;
1217	}
1218
1219	case X86_XCPT_PF: /* Page fault */
1220	{
1221	uint32_t errCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1222	RTGCUINTPTR uFaultAddress = pVMCB->ctrl.u64ExitInfo2; /* EXITINFO2 = fault address */
1223
1224	#ifdef DEBUG
1225	if (pVM->hwaccm.s.fNestedPaging)
1226	{ /* A genuine pagefault.
1227	* Forward the trap to the guest by injecting the exception and resuming execution.
1228	*/
1229	Log(("Guest page fault at %VGv cr2=%VGv error code %x rsp=%RX64\n", pCtx->rip, uFaultAddress, errCode, pCtx->rsp));
1230	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestPF);
1231
1232	/* Now we must update CR2. */
1233	pCtx->cr2 = uFaultAddress;
1234
1235	Event.au64[0] = 0;
1236	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1237	Event.n.u1Valid = 1;
1238	Event.n.u8Vector = X86_XCPT_PF;
1239	Event.n.u1ErrorCodeValid = 1;
1240	Event.n.u32ErrorCode = errCode;
1241
1242	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1243
1244	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1245	goto ResumeExecution;
1246	}
1247	#endif
1248	Assert(!pVM->hwaccm.s.fNestedPaging);
1249
1250	Log2(("Page fault at %VGv cr2=%VGv error code %x\n", pCtx->rip, uFaultAddress, errCode));
1251	/* Exit qualification contains the linear address of the page fault. */
1252	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
1253	TRPMSetErrorCode(pVM, errCode);
1254	TRPMSetFaultAddress(pVM, uFaultAddress);
1255
1256	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
1257	rc = PGMTrap0eHandler(pVM, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
1258	Log2(("PGMTrap0eHandler %VGv returned %Vrc\n", pCtx->rip, rc));
1259	if (rc == VINF_SUCCESS)
1260	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
1261	Log2(("Shadow page fault at %VGv cr2=%VGv error code %x\n", pCtx->rip, uFaultAddress, errCode));
1262	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitShadowPF);
1263
1264	TRPMResetTrap(pVM);
1265
1266	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1267	goto ResumeExecution;
1268	}
1269	else
1270	if (rc == VINF_EM_RAW_GUEST_TRAP)
1271	{ /* A genuine pagefault.
1272	* Forward the trap to the guest by injecting the exception and resuming execution.
1273	*/
1274	Log2(("Forward page fault to the guest\n"));
1275	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestPF);
1276	/* The error code might have been changed. */
1277	errCode = TRPMGetErrorCode(pVM);
1278
1279	TRPMResetTrap(pVM);
1280
1281	/* Now we must update CR2. */
1282	pCtx->cr2 = uFaultAddress;
1283
1284	Event.au64[0] = 0;
1285	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1286	Event.n.u1Valid = 1;
1287	Event.n.u8Vector = X86_XCPT_PF;
1288	Event.n.u1ErrorCodeValid = 1;
1289	Event.n.u32ErrorCode = errCode;
1290
1291	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1292
1293	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1294	goto ResumeExecution;
1295	}
1296	#ifdef VBOX_STRICT
1297	if (rc != VINF_EM_RAW_EMULATE_INSTR)
1298	LogFlow(("PGMTrap0eHandler failed with %d\n", rc));
1299	#endif
1300	/* Need to go back to the recompiler to emulate the instruction. */
1301	TRPMResetTrap(pVM);
1302	break;
1303	}
1304
1305	case X86_XCPT_MF: /* Floating point exception. */
1306	{
1307	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestMF);
1308	if (!(pCtx->cr0 & X86_CR0_NE))
1309	{
1310	/* old style FPU error reporting needs some extra work. */
1311	/** @todo don't fall back to the recompiler, but do it manually. */
1312	rc = VINF_EM_RAW_EMULATE_INSTR;
1313	break;
1314	}
1315	Log(("Trap %x at %VGv\n", vector, pCtx->rip));
1316
1317	Event.au64[0] = 0;
1318	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1319	Event.n.u1Valid = 1;
1320	Event.n.u8Vector = X86_XCPT_MF;
1321
1322	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1323
1324	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1325	goto ResumeExecution;
1326	}
1327
1328	#ifdef VBOX_STRICT
1329	case X86_XCPT_GP: /* General protection failure exception.*/
1330	case X86_XCPT_UD: /* Unknown opcode exception. */
1331	case X86_XCPT_DE: /* Debug exception. */
1332	case X86_XCPT_SS: /* Stack segment exception. */
1333	case X86_XCPT_NP: /* Segment not present exception. */
1334	{
1335	Event.au64[0] = 0;
1336	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1337	Event.n.u1Valid = 1;
1338	Event.n.u8Vector = vector;
1339
1340	switch(vector)
1341	{
1342	case X86_XCPT_GP:
1343	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestGP);
1344	Event.n.u1ErrorCodeValid = 1;
1345	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1346	break;
1347	case X86_XCPT_DE:
1348	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestDE);
1349	break;
1350	case X86_XCPT_UD:
1351	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestUD);
1352	break;
1353	case X86_XCPT_SS:
1354	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestSS);
1355	Event.n.u1ErrorCodeValid = 1;
1356	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1357	break;
1358	case X86_XCPT_NP:
1359	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestNP);
1360	Event.n.u1ErrorCodeValid = 1;
1361	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1362	break;
1363	}
1364	Log(("Trap %x at %VGv esi=%x\n", vector, pCtx->rip, pCtx->esi));
1365	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1366
1367	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1368	goto ResumeExecution;
1369	}
1370	#endif
1371	default:
1372	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
1373	rc = VERR_EM_INTERNAL_ERROR;
1374	break;
1375
1376	} /* switch (vector) */
1377	break;
1378	}
1379
1380	case SVM_EXIT_NPF:
1381	{
1382	/* EXITINFO1 contains fault errorcode; EXITINFO2 contains the guest physical address causing the fault. */
1383	uint32_t errCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1384	RTGCPHYS uFaultAddress = pVMCB->ctrl.u64ExitInfo2; /* EXITINFO2 = fault address */
1385
1386	Assert(pVM->hwaccm.s.fNestedPaging);
1387	Log(("Nested page fault at %VGv cr2=%VGp error code %x\n", pCtx->rip, uFaultAddress, errCode));
1388	/* Exit qualification contains the linear address of the page fault. */
1389	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
1390	TRPMSetErrorCode(pVM, errCode);
1391	TRPMSetFaultAddress(pVM, uFaultAddress);
1392
1393	/* Handle the pagefault trap for the nested shadow table. */
1394	rc = PGMR0Trap0eHandlerNestedPaging(pVM, PGMGetHostMode(pVM), errCode, CPUMCTX2CORE(pCtx), uFaultAddress);
1395	Log2(("PGMR0Trap0eHandlerNestedPaging %VGv returned %Vrc\n", pCtx->rip, rc));
1396	if (rc == VINF_SUCCESS)
1397	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
1398	Log2(("Shadow page fault at %VGv cr2=%VGp error code %x\n", pCtx->rip, uFaultAddress, errCode));
1399	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitShadowPF);
1400
1401	TRPMResetTrap(pVM);
1402
1403	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1404	goto ResumeExecution;
1405	}
1406
1407	#ifdef VBOX_STRICT
1408	if (rc != VINF_EM_RAW_EMULATE_INSTR)
1409	LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", rc));
1410	#endif
1411	/* Need to go back to the recompiler to emulate the instruction. */
1412	TRPMResetTrap(pVM);
1413	break;
1414	}
1415
1416	case SVM_EXIT_VINTR:
1417	/* A virtual interrupt is about to be delivered, which means IF=1. */
1418	Log(("SVM_EXIT_VINTR IF=%d\n", pCtx->eflags.Bits.u1IF));
1419	pVMCB->ctrl.IntCtrl.n.u1VIrqValid = 0;
1420	pVMCB->ctrl.IntCtrl.n.u1IgnoreTPR = 0;
1421	pVMCB->ctrl.IntCtrl.n.u8VIrqVector = 0;
1422	goto ResumeExecution;
1423
1424	case SVM_EXIT_FERR_FREEZE:
1425	case SVM_EXIT_INTR:
1426	case SVM_EXIT_NMI:
1427	case SVM_EXIT_SMI:
1428	case SVM_EXIT_INIT:
1429	/* External interrupt; leave to allow it to be dispatched again. */
1430	rc = VINF_EM_RAW_INTERRUPT;
1431	break;
1432
1433	case SVM_EXIT_WBINVD:
1434	case SVM_EXIT_INVD: /* Guest software attempted to execute INVD. */
1435	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitInvd);
1436	/* Skip instruction and continue directly. */
1437	pCtx->rip += 2; /** @note hardcoded opcode size! */
1438	/* Continue execution.*/
1439	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1440	goto ResumeExecution;
1441
1442	case SVM_EXIT_CPUID: /* Guest software attempted to execute CPUID. */
1443	{
1444	Log2(("SVM: Cpuid %x\n", pCtx->eax));
1445	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitCpuid);
1446	rc = EMInterpretCpuId(pVM, CPUMCTX2CORE(pCtx));
1447	if (rc == VINF_SUCCESS)
1448	{
1449	/* Update EIP and continue execution. */
1450	pCtx->rip += 2; /** @note hardcoded opcode size! */
1451	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1452	goto ResumeExecution;
1453	}
1454	AssertMsgFailed(("EMU: cpuid failed with %Vrc\n", rc));
1455	rc = VINF_EM_RAW_EMULATE_INSTR;
1456	break;
1457	}
1458
1459	case SVM_EXIT_RDTSC: /* Guest software attempted to execute RDTSC. */
1460	{
1461	Log2(("SVM: Rdtsc\n"));
1462	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitRdtsc);
1463	rc = EMInterpretRdtsc(pVM, CPUMCTX2CORE(pCtx));
1464	if (rc == VINF_SUCCESS)
1465	{
1466	/* Update EIP and continue execution. */
1467	pCtx->rip += 2; /** @note hardcoded opcode size! */
1468	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1469	goto ResumeExecution;
1470	}
1471	AssertMsgFailed(("EMU: rdtsc failed with %Vrc\n", rc));
1472	rc = VINF_EM_RAW_EMULATE_INSTR;
1473	break;
1474	}
1475
1476	case SVM_EXIT_INVLPG: /* Guest software attempted to execute INVPG. */
1477	{
1478	Log2(("SVM: invlpg\n"));
1479	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitInvpg);
1480
1481	Assert(!pVM->hwaccm.s.fNestedPaging);
1482
1483	/* Truly a pita. Why can't SVM give the same information as VT-x? */
1484	rc = SVMR0InterpretInvpg(pVM, CPUMCTX2CORE(pCtx), pVMCB->ctrl.TLBCtrl.n.u32ASID);
1485	if (rc == VINF_SUCCESS)
1486	{
1487	STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushPageInvlpg);
1488	goto ResumeExecution; /* eip already updated */
1489	}
1490	break;
1491	}
1492
1493	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
1494	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
1495	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
1496	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
1497	{
1498	uint32_t cbSize;
1499
1500	Log2(("SVM: %VGv mov cr%d, \n", pCtx->rip, exitCode - SVM_EXIT_WRITE_CR0));
1501	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitCRxWrite);
1502	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1503
1504	switch (exitCode - SVM_EXIT_WRITE_CR0)
1505	{
1506	case 0:
1507	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
1508	break;
1509	case 2:
1510	break;
1511	case 3:
1512	Assert(!pVM->hwaccm.s.fNestedPaging);
1513	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
1514	break;
1515	case 4:
1516	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
1517	break;
1518	case 8:
1519	AssertFailed(); /* shouldn't come here anymore */
1520	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR8;
1521	break;
1522	default:
1523	AssertFailed();
1524	}
1525	/* Check if a sync operation is pending. */
1526	if ( rc == VINF_SUCCESS /* don't bother if we are going to ring 3 anyway */
1527	&& VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL))
1528	{
1529	rc = PGMSyncCR3(pVM, CPUMGetGuestCR0(pVM), CPUMGetGuestCR3(pVM), CPUMGetGuestCR4(pVM), VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3));
1530	AssertRC(rc);
1531
1532	STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushTLBCRxChange);
1533
1534	/** @note Force a TLB flush. SVM requires us to do it manually. */
1535	pVM->hwaccm.s.svm.fForceTLBFlush = true;
1536	}
1537	if (rc == VINF_SUCCESS)
1538	{
1539	/* EIP has been updated already. */
1540
1541	/* Only resume if successful. */
1542	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1543	goto ResumeExecution;
1544	}
1545	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1546	break;
1547	}
1548
1549	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
1550	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
1551	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
1552	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
1553	{
1554	uint32_t cbSize;
1555
1556	Log2(("SVM: %VGv mov x, cr%d\n", pCtx->rip, exitCode - SVM_EXIT_READ_CR0));
1557	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitCRxRead);
1558	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1559	if (rc == VINF_SUCCESS)
1560	{
1561	/* EIP has been updated already. */
1562
1563	/* Only resume if successful. */
1564	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1565	goto ResumeExecution;
1566	}
1567	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1568	break;
1569	}
1570
1571	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
1572	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
1573	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
1574	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
1575	{
1576	uint32_t cbSize;
1577
1578	Log2(("SVM: %VGv mov dr%d, x\n", pCtx->rip, exitCode - SVM_EXIT_WRITE_DR0));
1579	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitDRxRead);
1580	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1581	if (rc == VINF_SUCCESS)
1582	{
1583	/* EIP has been updated already. */
1584
1585	/* Only resume if successful. */
1586	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1587	goto ResumeExecution;
1588	}
1589	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1590	break;
1591	}
1592
1593	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
1594	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
1595	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
1596	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
1597	{
1598	uint32_t cbSize;
1599
1600	Log2(("SVM: %VGv mov dr%d, x\n", pCtx->rip, exitCode - SVM_EXIT_READ_DR0));
1601	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitDRxRead);
1602	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1603	if (rc == VINF_SUCCESS)
1604	{
1605	/* EIP has been updated already. */
1606
1607	/* Only resume if successful. */
1608	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1609	goto ResumeExecution;
1610	}
1611	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1612	break;
1613	}
1614
1615	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
1616	case SVM_EXIT_IOIO: /* I/O instruction. */
1617	{
1618	SVM_IOIO_EXIT IoExitInfo;
1619	uint32_t uIOSize, uAndVal;
1620
1621	IoExitInfo.au32[0] = pVMCB->ctrl.u64ExitInfo1;
1622
1623	/** @todo could use a lookup table here */
1624	if (IoExitInfo.n.u1OP8)
1625	{
1626	uIOSize = 1;
1627	uAndVal = 0xff;
1628	}
1629	else
1630	if (IoExitInfo.n.u1OP16)
1631	{
1632	uIOSize = 2;
1633	uAndVal = 0xffff;
1634	}
1635	else
1636	if (IoExitInfo.n.u1OP32)
1637	{
1638	uIOSize = 4;
1639	uAndVal = 0xffffffff;
1640	}
1641	else
1642	{
1643	AssertFailed(); /* should be fatal. */
1644	rc = VINF_EM_RAW_EMULATE_INSTR;
1645	break;
1646	}
1647
1648	if (IoExitInfo.n.u1STR)
1649	{
1650	/* ins/outs */
1651	uint32_t prefix = 0;
1652	if (IoExitInfo.n.u1REP)
1653	prefix \|= PREFIX_REP;
1654
1655	if (IoExitInfo.n.u1Type == 0)
1656	{
1657	Log2(("IOMInterpretOUTSEx %VGv %x size=%d\n", pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
1658	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitIOStringWrite);
1659	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, prefix, uIOSize);
1660	}
1661	else
1662	{
1663	Log2(("IOMInterpretINSEx %VGv %x size=%d\n", pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
1664	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitIOStringRead);
1665	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, prefix, uIOSize);
1666	}
1667	}
1668	else
1669	{
1670	/* normal in/out */
1671	Assert(!IoExitInfo.n.u1REP);
1672
1673	if (IoExitInfo.n.u1Type == 0)
1674	{
1675	Log2(("IOMIOPortWrite %VGv %x %x size=%d\n", pCtx->rip, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, uIOSize));
1676	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitIOWrite);
1677	rc = IOMIOPortWrite(pVM, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, uIOSize);
1678	}
1679	else
1680	{
1681	uint32_t u32Val = 0;
1682
1683	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitIORead);
1684	rc = IOMIOPortRead(pVM, IoExitInfo.n.u16Port, &u32Val, uIOSize);
1685	if (IOM_SUCCESS(rc))
1686	{
1687	/* Write back to the EAX register. */
1688	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
1689	Log2(("IOMIOPortRead %VGv %x %x size=%d\n", pCtx->rip, IoExitInfo.n.u16Port, u32Val & uAndVal, uIOSize));
1690	}
1691	}
1692	}
1693	/*
1694	* Handled the I/O return codes.
1695	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
1696	*/
1697	if (IOM_SUCCESS(rc))
1698	{
1699	/* Update EIP and continue execution. */
1700	pCtx->rip = pVMCB->ctrl.u64ExitInfo2; /* RIP/EIP of the next instruction is saved in EXITINFO2. */
1701	if (RT_LIKELY(rc == VINF_SUCCESS))
1702	{
1703	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1704	goto ResumeExecution;
1705	}
1706	Log2(("EM status from IO at %VGv %x size %d: %Vrc\n", pCtx->rip, IoExitInfo.n.u16Port, uIOSize, rc));
1707	break;
1708	}
1709
1710	#ifdef VBOX_STRICT
1711	if (rc == VINF_IOM_HC_IOPORT_READ)
1712	Assert(IoExitInfo.n.u1Type != 0);
1713	else if (rc == VINF_IOM_HC_IOPORT_WRITE)
1714	Assert(IoExitInfo.n.u1Type == 0);
1715	else
1716	AssertMsg(VBOX_FAILURE(rc) \|\| rc == VINF_EM_RAW_EMULATE_INSTR \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Vrc\n", rc));
1717	#endif
1718	Log2(("Failed IO at %VGv %x size %d\n", pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
1719	break;
1720	}
1721
1722	case SVM_EXIT_HLT:
1723	/** Check if external interrupts are pending; if so, don't switch back. */
1724	if ( pCtx->eflags.Bits.u1IF
1725	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
1726	{
1727	pCtx->rip++; /* skip hlt */
1728	goto ResumeExecution;
1729	}
1730
1731	rc = VINF_EM_RAW_EMULATE_INSTR_HLT;
1732	break;
1733
1734	case SVM_EXIT_RSM:
1735	case SVM_EXIT_INVLPGA:
1736	case SVM_EXIT_VMRUN:
1737	case SVM_EXIT_VMMCALL:
1738	case SVM_EXIT_VMLOAD:
1739	case SVM_EXIT_VMSAVE:
1740	case SVM_EXIT_STGI:
1741	case SVM_EXIT_CLGI:
1742	case SVM_EXIT_SKINIT:
1743	case SVM_EXIT_RDTSCP:
1744	{
1745	/* Unsupported instructions. */
1746	SVM_EVENT Event;
1747
1748	Event.au64[0] = 0;
1749	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1750	Event.n.u1Valid = 1;
1751	Event.n.u8Vector = X86_XCPT_UD;
1752
1753	Log(("Forced #UD trap at %VGv\n", pCtx->rip));
1754	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1755
1756	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1757	goto ResumeExecution;
1758	}
1759
1760	/* Emulate in ring 3. */
1761	case SVM_EXIT_MSR:
1762	{
1763	uint32_t cbSize;
1764
1765	/* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
1766	Log(("SVM: %s\n", (pVMCB->ctrl.u64ExitInfo1 == 0) ? "rdmsr" : "wrmsr"));
1767	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1768	if (rc == VINF_SUCCESS)
1769	{
1770	/* EIP has been updated already. */
1771
1772	/* Only resume if successful. */
1773	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1774	goto ResumeExecution;
1775	}
1776	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Vrc\n", (pVMCB->ctrl.u64ExitInfo1 == 0) ? "rdmsr" : "wrmsr", rc));
1777	break;
1778	}
1779
1780	case SVM_EXIT_MONITOR:
1781	case SVM_EXIT_RDPMC:
1782	case SVM_EXIT_PAUSE:
1783	case SVM_EXIT_MWAIT_UNCOND:
1784	case SVM_EXIT_MWAIT_ARMED:
1785	case SVM_EXIT_TASK_SWITCH: /* can change CR3; emulate */
1786	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
1787	break;
1788
1789	case SVM_EXIT_SHUTDOWN:
1790	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
1791	break;
1792
1793	case SVM_EXIT_IDTR_READ:
1794	case SVM_EXIT_GDTR_READ:
1795	case SVM_EXIT_LDTR_READ:
1796	case SVM_EXIT_TR_READ:
1797	case SVM_EXIT_IDTR_WRITE:
1798	case SVM_EXIT_GDTR_WRITE:
1799	case SVM_EXIT_LDTR_WRITE:
1800	case SVM_EXIT_TR_WRITE:
1801	case SVM_EXIT_CR0_SEL_WRITE:
1802	default:
1803	/* Unexpected exit codes. */
1804	rc = VERR_EM_INTERNAL_ERROR;
1805	AssertMsgFailed(("Unexpected exit code %x\n", exitCode)); /* Can't happen. */
1806	break;
1807	}
1808
1809	end:
1810	if (fGuestStateSynced)
1811	{
1812	/* Remaining guest CPU context: TR, IDTR, GDTR, LDTR. */
1813	SVM_READ_SELREG(LDTR, ldtr);
1814	SVM_READ_SELREG(TR, tr);
1815
1816	pCtx->gdtr.cbGdt = pVMCB->guest.GDTR.u32Limit;
1817	pCtx->gdtr.pGdt = pVMCB->guest.GDTR.u64Base;
1818
1819	pCtx->idtr.cbIdt = pVMCB->guest.IDTR.u32Limit;
1820	pCtx->idtr.pIdt = pVMCB->guest.IDTR.u64Base;
1821
1822	/*
1823	* System MSRs
1824	*/
1825	pCtx->SysEnter.cs = pVMCB->guest.u64SysEnterCS;
1826	pCtx->SysEnter.eip = pVMCB->guest.u64SysEnterEIP;
1827	pCtx->SysEnter.esp = pVMCB->guest.u64SysEnterESP;
1828	}
1829
1830	/* Signal changes for the recompiler. */
1831	CPUMSetChangedFlags(pVM, CPUM_CHANGED_SYSENTER_MSR \| CPUM_CHANGED_LDTR \| CPUM_CHANGED_GDTR \| CPUM_CHANGED_IDTR \| CPUM_CHANGED_TR \| CPUM_CHANGED_HIDDEN_SEL_REGS);
1832
1833	/* If we executed vmrun and an external irq was pending, then we don't have to do a full sync the next time. */
1834	if (exitCode == SVM_EXIT_INTR)
1835	{
1836	STAM_COUNTER_INC(&pVM->hwaccm.s.StatPendingHostIrq);
1837	/* On the next entry we'll only sync the host context. */
1838	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
1839	}
1840	else
1841	{
1842	/* On the next entry we'll sync everything. */
1843	/** @todo we can do better than this */
1844	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
1845	}
1846
1847	/* translate into a less severe return code */
1848	if (rc == VERR_EM_INTERPRETER)
1849	rc = VINF_EM_RAW_EMULATE_INSTR;
1850
1851	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1852	return rc;
1853	}
1854
1855	/**
1856	* Enters the AMD-V session
1857	*
1858	* @returns VBox status code.
1859	* @param pVM The VM to operate on.
1860	* @param pCpu CPU info struct
1861	*/
1862	HWACCMR0DECL(int) SVMR0Enter(PVM pVM, PHWACCM_CPUINFO pCpu)
1863	{
1864	Assert(pVM->hwaccm.s.svm.fSupported);
1865
1866	LogFlow(("SVMR0Enter cpu%d last=%d asid=%d\n", pCpu->idCpu, pVM->hwaccm.s.svm.idLastCpu, pCpu->uCurrentASID));
1867	pVM->hwaccm.s.svm.fResumeVM = false;
1868
1869	/* Force to reload LDTR, so we'll execute VMLoad to load additional guest state. */
1870	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_LDTR;
1871
1872	return VINF_SUCCESS;
1873	}
1874
1875
1876	/**
1877	* Leaves the AMD-V session
1878	*
1879	* @returns VBox status code.
1880	* @param pVM The VM to operate on.
1881	*/
1882	HWACCMR0DECL(int) SVMR0Leave(PVM pVM)
1883	{
1884	Assert(pVM->hwaccm.s.svm.fSupported);
1885	return VINF_SUCCESS;
1886	}
1887
1888
1889	static int svmInterpretInvlPg(PVM pVM, PDISCPUSTATE pCpu, PCPUMCTXCORE pRegFrame, uint32_t uASID)
1890	{
1891	OP_PARAMVAL param1;
1892	RTGCPTR addr;
1893
1894	int rc = DISQueryParamVal(pRegFrame, pCpu, &pCpu->param1, &param1, PARAM_SOURCE);
1895	if(VBOX_FAILURE(rc))
1896	return VERR_EM_INTERPRETER;
1897
1898	switch(param1.type)
1899	{
1900	case PARMTYPE_IMMEDIATE:
1901	case PARMTYPE_ADDRESS:
1902	if(!(param1.flags & (PARAM_VAL32\|PARAM_VAL64)))
1903	return VERR_EM_INTERPRETER;
1904	addr = param1.val.val64;
1905	break;
1906
1907	default:
1908	return VERR_EM_INTERPRETER;
1909	}
1910
1911	/** @todo is addr always a flat linear address or ds based
1912	* (in absence of segment override prefixes)????
1913	*/
1914	rc = PGMInvalidatePage(pVM, addr);
1915	if (VBOX_SUCCESS(rc))
1916	{
1917	/* Manually invalidate the page for the VM's TLB. */
1918	Log(("SVMInvlpgA %VGv ASID=%d\n", addr, uASID));
1919	SVMInvlpgA(addr, uASID);
1920	return VINF_SUCCESS;
1921	}
1922	Assert(rc == VERR_REM_FLUSHED_PAGES_OVERFLOW);
1923	return rc;
1924	}
1925
1926	/**
1927	* Interprets INVLPG
1928	*
1929	* @returns VBox status code.
1930	* @retval VINF_* Scheduling instructions.
1931	* @retval VERR_EM_INTERPRETER Something we can't cope with.
1932	* @retval VERR_* Fatal errors.
1933	*
1934	* @param pVM The VM handle.
1935	* @param pRegFrame The register frame.
1936	* @param ASID Tagged TLB id for the guest
1937	*
1938	* Updates the EIP if an instruction was executed successfully.
1939	*/
1940	static int SVMR0InterpretInvpg(PVM pVM, PCPUMCTXCORE pRegFrame, uint32_t uASID)
1941	{
1942	/*
1943	* Only allow 32 & 64 bits code.
1944	*/
1945	DISCPUMODE enmMode = SELMGetCpuModeFromSelector(pVM, pRegFrame->eflags, pRegFrame->cs, &pRegFrame->csHid);
1946	if (enmMode != CPUMODE_16BIT)
1947	{
1948	RTGCPTR pbCode;
1949	int rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->rip, &pbCode);
1950	if (VBOX_SUCCESS(rc))
1951	{
1952	uint32_t cbOp;
1953	DISCPUSTATE Cpu;
1954
1955	Cpu.mode = enmMode;
1956	rc = EMInterpretDisasOneEx(pVM, pbCode, pRegFrame, &Cpu, &cbOp);
1957	Assert(VBOX_FAILURE(rc) \|\| Cpu.pCurInstr->opcode == OP_INVLPG);
1958	if (VBOX_SUCCESS(rc) && Cpu.pCurInstr->opcode == OP_INVLPG)
1959	{
1960	Assert(cbOp == Cpu.opsize);
1961	rc = svmInterpretInvlPg(pVM, &Cpu, pRegFrame, uASID);
1962	if (VBOX_SUCCESS(rc))
1963	{
1964	pRegFrame->rip += cbOp; /* Move on to the next instruction. */
1965	}
1966	return rc;
1967	}
1968	}
1969	}
1970	return VERR_EM_INTERPRETER;
1971	}
1972
1973
1974	/**
1975	* Invalidates a guest page
1976	*
1977	* @returns VBox status code.
1978	* @param pVM The VM to operate on.
1979	* @param GCVirt Page to invalidate
1980	*/
1981	HWACCMR0DECL(int) SVMR0InvalidatePage(PVM pVM, RTGCPTR GCVirt)
1982	{
1983	bool fFlushPending = pVM->hwaccm.s.svm.fAlwaysFlushTLB \| pVM->hwaccm.s.svm.fForceTLBFlush;
1984
1985	/* Skip it if a TLB flush is already pending. */
1986	if (!fFlushPending)
1987	{
1988	SVM_VMCB *pVMCB;
1989
1990	Log2(("SVMR0InvalidatePage %VGv\n", GCVirt));
1991	AssertReturn(pVM, VERR_INVALID_PARAMETER);
1992	Assert(pVM->hwaccm.s.svm.fSupported);
1993
1994	pVMCB = (SVM_VMCB *)pVM->hwaccm.s.svm.pVMCB;
1995	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
1996
1997	STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushPageManual);
1998	SVMInvlpgA(GCVirt, pVMCB->ctrl.TLBCtrl.n.u32ASID);
1999	}
2000	return VINF_SUCCESS;
2001	}
2002
2003
2004	/**
2005	* Invalidates a guest page by physical address
2006	*
2007	* NOTE: Assumes the current instruction references this physical page though a virtual address!!
2008	*
2009	* @returns VBox status code.
2010	* @param pVM The VM to operate on.
2011	* @param GCPhys Page to invalidate
2012	*/
2013	HWACCMR0DECL(int) SVMR0InvalidatePhysPage(PVM pVM, RTGCPHYS GCPhys)
2014	{
2015	bool fFlushPending = pVM->hwaccm.s.svm.fAlwaysFlushTLB \| pVM->hwaccm.s.svm.fForceTLBFlush;
2016
2017	Assert(pVM->hwaccm.s.fNestedPaging);
2018
2019	/* Skip it if a TLB flush is already pending. */
2020	if (!fFlushPending)
2021	{
2022	CPUMCTX *pCtx;
2023	int rc;
2024	SVM_VMCB *pVMCB;
2025
2026	rc = CPUMQueryGuestCtxPtr(pVM, &pCtx);
2027	AssertRCReturn(rc, rc);
2028
2029	Log2(("SVMR0InvalidatePhysPage %VGp\n", GCPhys));
2030	AssertReturn(pVM, VERR_INVALID_PARAMETER);
2031	Assert(pVM->hwaccm.s.svm.fSupported);
2032
2033	pVMCB = (SVM_VMCB *)pVM->hwaccm.s.svm.pVMCB;
2034	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
2035
2036	/*
2037	* Only allow 32 & 64 bits code.
2038	*/
2039	DISCPUMODE enmMode = SELMGetCpuModeFromSelector(pVM, pCtx->eflags, pCtx->cs, &pCtx->csHid);
2040	if (enmMode != CPUMODE_16BIT)
2041	{
2042	RTGCPTR pbCode;
2043	int rc = SELMValidateAndConvertCSAddr(pVM, pCtx->eflags, pCtx->ss, pCtx->cs, &pCtx->csHid, (RTGCPTR)pCtx->rip, &pbCode);
2044	if (VBOX_SUCCESS(rc))
2045	{
2046	uint32_t cbOp;
2047	DISCPUSTATE Cpu;
2048	OP_PARAMVAL param1;
2049	RTGCPTR addr;
2050
2051	Cpu.mode = enmMode;
2052	rc = EMInterpretDisasOneEx(pVM, pbCode, CPUMCTX2CORE(pCtx), &Cpu, &cbOp);
2053	AssertRCReturn(rc, rc);
2054	Assert(cbOp == Cpu.opsize);
2055
2056	int rc = DISQueryParamVal(CPUMCTX2CORE(pCtx), &Cpu, &Cpu.param1, &param1, PARAM_SOURCE);
2057	AssertRCReturn(rc, VERR_EM_INTERPRETER);
2058
2059	switch(param1.type)
2060	{
2061	case PARMTYPE_IMMEDIATE:
2062	case PARMTYPE_ADDRESS:
2063	AssertReturn((param1.flags & (PARAM_VAL32\|PARAM_VAL64)), VERR_EM_INTERPRETER);
2064
2065	addr = param1.val.val64;
2066	break;
2067
2068	default:
2069	AssertFailed();
2070	return VERR_EM_INTERPRETER;
2071	}
2072
2073	/* Manually invalidate the page for the VM's TLB. */
2074	Log(("SVMR0InvalidatePhysPage Phys=%VGp Virt=%VGv ASID=%d\n", GCPhys, addr, pVMCB->ctrl.TLBCtrl.n.u32ASID));
2075	SVMInvlpgA(addr, pVMCB->ctrl.TLBCtrl.n.u32ASID);
2076	STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushPhysPageManual);
2077
2078	return VINF_SUCCESS;
2079	}
2080	}
2081	AssertFailed();
2082	return VERR_EM_INTERPRETER;
2083	}
2084	return VINF_SUCCESS;
2085	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWSVMR0.cpp@ 10206

Download in other formats: