HWSVMR0.cpp@ 15189

Last change on this file since 15189 was 15154, checked in by vboxsync, 16 years ago
Always use amd64 paging if the guest is in amd64 mode (nested paging).
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 93.3 KB

Line
1	/* $Id: HWSVMR0.cpp 15154 2008-12-09 10:56:22Z vboxsync $ */
2	/** @file
3	* HWACCM SVM - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/*******************************************************************************
24	* Header Files *
25	*******************************************************************************/
26	#define LOG_GROUP LOG_GROUP_HWACCM
27	#include <VBox/hwaccm.h>
28	#include "HWACCMInternal.h"
29	#include <VBox/vm.h>
30	#include <VBox/x86.h>
31	#include <VBox/hwacc_svm.h>
32	#include <VBox/pgm.h>
33	#include <VBox/pdm.h>
34	#include <VBox/err.h>
35	#include <VBox/log.h>
36	#include <VBox/selm.h>
37	#include <VBox/iom.h>
38	#include <VBox/dis.h>
39	#include <VBox/dbgf.h>
40	#include <VBox/disopcode.h>
41	#include <iprt/param.h>
42	#include <iprt/assert.h>
43	#include <iprt/asm.h>
44	#include <iprt/cpuset.h>
45	#include <iprt/mp.h>
46	#include "HWSVMR0.h"
47
48	/*******************************************************************************
49	* Internal Functions *
50	*******************************************************************************/
51	static int SVMR0InterpretInvpg(PVM pVM, PCPUMCTXCORE pRegFrame, uint32_t uASID);
52
53	/*******************************************************************************
54	* Global Variables *
55	*******************************************************************************/
56	/* IO operation lookup arrays. */
57	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
58
59	/**
60	* Sets up and activates AMD-V on the current CPU
61	*
62	* @returns VBox status code.
63	* @param pCpu CPU info struct
64	* @param pVM The VM to operate on. (can be NULL after a resume!!)
65	* @param pvPageCpu Pointer to the global cpu page
66	* @param pPageCpuPhys Physical address of the global cpu page
67	*/
68	VMMR0DECL(int) SVMR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
69	{
70	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
71	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
72
73	/* We must turn on AMD-V and setup the host state physical address, as those MSRs are per-cpu/core. */
74
75	#ifdef LOG_ENABLED
76	SUPR0Printf("SVMR0EnableCpu cpu %d page (%x) %x\n", pCpu->idCpu, pvPageCpu, (uint32_t)pPageCpuPhys);
77	#endif
78
79	/* Turn on AMD-V in the EFER MSR. */
80	uint64_t val = ASMRdMsr(MSR_K6_EFER);
81	if (!(val & MSR_K6_EFER_SVME))
82	ASMWrMsr(MSR_K6_EFER, val \| MSR_K6_EFER_SVME);
83
84	/* Write the physical page address where the CPU will store the host state while executing the VM. */
85	ASMWrMsr(MSR_K8_VM_HSAVE_PA, pPageCpuPhys);
86
87	return VINF_SUCCESS;
88	}
89
90	/**
91	* Deactivates AMD-V on the current CPU
92	*
93	* @returns VBox status code.
94	* @param pCpu CPU info struct
95	* @param pvPageCpu Pointer to the global cpu page
96	* @param pPageCpuPhys Physical address of the global cpu page
97	*/
98	VMMR0DECL(int) SVMR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
99	{
100	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
101	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
102
103	#ifdef LOG_ENABLED
104	SUPR0Printf("SVMR0DisableCpu cpu %d\n", pCpu->idCpu);
105	#endif
106
107	/* Turn off AMD-V in the EFER MSR. */
108	uint64_t val = ASMRdMsr(MSR_K6_EFER);
109	ASMWrMsr(MSR_K6_EFER, val & ~MSR_K6_EFER_SVME);
110
111	/* Invalidate host state physical address. */
112	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
113
114	return VINF_SUCCESS;
115	}
116
117	/**
118	* Does Ring-0 per VM AMD-V init.
119	*
120	* @returns VBox status code.
121	* @param pVM The VM to operate on.
122	*/
123	VMMR0DECL(int) SVMR0InitVM(PVM pVM)
124	{
125	int rc;
126
127	pVM->hwaccm.s.svm.pMemObjVMCBHost = NIL_RTR0MEMOBJ;
128	pVM->hwaccm.s.svm.pMemObjIOBitmap = NIL_RTR0MEMOBJ;
129	pVM->hwaccm.s.svm.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
130
131	/* Allocate one page for the host context */
132	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjVMCBHost, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
133	if (RT_FAILURE(rc))
134	return rc;
135
136	pVM->hwaccm.s.svm.pVMCBHost = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjVMCBHost);
137	pVM->hwaccm.s.svm.pVMCBHostPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjVMCBHost, 0);
138	ASMMemZeroPage(pVM->hwaccm.s.svm.pVMCBHost);
139
140	/* Allocate 12 KB for the IO bitmap (doesn't seem to be a way to convince SVM not to use it) */
141	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjIOBitmap, 3 << PAGE_SHIFT, true /* executable R0 mapping */);
142	if (RT_FAILURE(rc))
143	return rc;
144
145	pVM->hwaccm.s.svm.pIOBitmap = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjIOBitmap);
146	pVM->hwaccm.s.svm.pIOBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjIOBitmap, 0);
147	/* Set all bits to intercept all IO accesses. */
148	ASMMemFill32(pVM->hwaccm.s.svm.pIOBitmap, PAGE_SIZE*3, 0xffffffff);
149
150	/* Allocate 8 KB for the MSR bitmap (doesn't seem to be a way to convince SVM not to use it) */
151	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjMSRBitmap, 2 << PAGE_SHIFT, true /* executable R0 mapping */);
152	if (RT_FAILURE(rc))
153	return rc;
154
155	pVM->hwaccm.s.svm.pMSRBitmap = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjMSRBitmap);
156	pVM->hwaccm.s.svm.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjMSRBitmap, 0);
157	/* Set all bits to intercept all MSR accesses. */
158	ASMMemFill32(pVM->hwaccm.s.svm.pMSRBitmap, PAGE_SIZE*2, 0xffffffff);
159
160	/* Erratum 170 which requires a forced TLB flush for each world switch:
161	* See http://www.amd.com/us-en/assets/content_type/white_papers_and_tech_docs/33610.pdf
162	*
163	* All BH-G1/2 and DH-G1/2 models include a fix:
164	* Athlon X2: 0x6b 1/2
165	* 0x68 1/2
166	* Athlon 64: 0x7f 1
167	* 0x6f 2
168	* Sempron: 0x7f 1/2
169	* 0x6f 2
170	* 0x6c 2
171	* 0x7c 2
172	* Turion 64: 0x68 2
173	*
174	*/
175	uint32_t u32Dummy;
176	uint32_t u32Version, u32Family, u32Model, u32Stepping, u32BaseFamily;
177	ASMCpuId(1, &u32Version, &u32Dummy, &u32Dummy, &u32Dummy);
178	u32BaseFamily= (u32Version >> 8) & 0xf;
179	u32Family = u32BaseFamily + (u32BaseFamily == 0xf ? ((u32Version >> 20) & 0x7f) : 0);
180	u32Model = ((u32Version >> 4) & 0xf);
181	u32Model = u32Model \| ((u32BaseFamily == 0xf ? (u32Version >> 16) & 0x0f : 0) << 4);
182	u32Stepping = u32Version & 0xf;
183	if ( u32Family == 0xf
184	&& !((u32Model == 0x68 \|\| u32Model == 0x6b \|\| u32Model == 0x7f) && u32Stepping >= 1)
185	&& !((u32Model == 0x6f \|\| u32Model == 0x6c \|\| u32Model == 0x7c) && u32Stepping >= 2))
186	{
187	Log(("SVMR0InitVM: AMD cpu with erratum 170 family %x model %x stepping %x\n", u32Family, u32Model, u32Stepping));
188	pVM->hwaccm.s.svm.fAlwaysFlushTLB = true;
189	}
190
191	/* Allocate VMCBs for all guest CPUs. */
192	for (unsigned i=0;i<pVM->cCPUs;i++)
193	{
194	pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB = NIL_RTR0MEMOBJ;
195
196	/* Allocate one page for the VM control block (VMCB). */
197	rc = RTR0MemObjAllocCont(&pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
198	if (RT_FAILURE(rc))
199	return rc;
200
201	pVM->aCpus[i].hwaccm.s.svm.pVMCB = RTR0MemObjAddress(pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB);
202	pVM->aCpus[i].hwaccm.s.svm.pVMCBPhys = RTR0MemObjGetPagePhysAddr(pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB, 0);
203	ASMMemZeroPage(pVM->aCpus[i].hwaccm.s.svm.pVMCB);
204	}
205
206	return VINF_SUCCESS;
207	}
208
209	/**
210	* Does Ring-0 per VM AMD-V termination.
211	*
212	* @returns VBox status code.
213	* @param pVM The VM to operate on.
214	*/
215	VMMR0DECL(int) SVMR0TermVM(PVM pVM)
216	{
217	for (unsigned i=0;i<pVM->cCPUs;i++)
218	{
219	if (pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB != NIL_RTR0MEMOBJ)
220	{
221	RTR0MemObjFree(pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB, false);
222	pVM->aCpus[i].hwaccm.s.svm.pVMCB = 0;
223	pVM->aCpus[i].hwaccm.s.svm.pVMCBPhys = 0;
224	pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB = NIL_RTR0MEMOBJ;
225	}
226	}
227	if (pVM->hwaccm.s.svm.pMemObjVMCBHost != NIL_RTR0MEMOBJ)
228	{
229	RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjVMCBHost, false);
230	pVM->hwaccm.s.svm.pVMCBHost = 0;
231	pVM->hwaccm.s.svm.pVMCBHostPhys = 0;
232	pVM->hwaccm.s.svm.pMemObjVMCBHost = NIL_RTR0MEMOBJ;
233	}
234	if (pVM->hwaccm.s.svm.pMemObjIOBitmap != NIL_RTR0MEMOBJ)
235	{
236	RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjIOBitmap, false);
237	pVM->hwaccm.s.svm.pIOBitmap = 0;
238	pVM->hwaccm.s.svm.pIOBitmapPhys = 0;
239	pVM->hwaccm.s.svm.pMemObjIOBitmap = NIL_RTR0MEMOBJ;
240	}
241	if (pVM->hwaccm.s.svm.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
242	{
243	RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjMSRBitmap, false);
244	pVM->hwaccm.s.svm.pMSRBitmap = 0;
245	pVM->hwaccm.s.svm.pMSRBitmapPhys = 0;
246	pVM->hwaccm.s.svm.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
247	}
248	return VINF_SUCCESS;
249	}
250
251	/**
252	* Sets up AMD-V for the specified VM
253	*
254	* @returns VBox status code.
255	* @param pVM The VM to operate on.
256	*/
257	VMMR0DECL(int) SVMR0SetupVM(PVM pVM)
258	{
259	int rc = VINF_SUCCESS;
260	SVM_VMCB *pVMCB;
261
262	AssertReturn(pVM, VERR_INVALID_PARAMETER);
263
264	Assert(pVM->hwaccm.s.svm.fSupported);
265
266	for (unsigned i=0;i<pVM->cCPUs;i++)
267	{
268	pVMCB = (SVM_VMCB *)pVM->aCpus[i].hwaccm.s.svm.pVMCB;
269	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
270
271	/* Program the control fields. Most of them never have to be changed again. */
272	/* CR0/3/4 reads must be intercepted, our shadow values are not necessarily the same as the guest's. */
273	/* Note: CR0 & CR4 can be safely read when guest and shadow copies are identical. */
274	if (!pVM->hwaccm.s.fNestedPaging)
275	pVMCB->ctrl.u16InterceptRdCRx = RT_BIT(0) \| RT_BIT(3) \| RT_BIT(4);
276	else
277	pVMCB->ctrl.u16InterceptRdCRx = RT_BIT(0) \| RT_BIT(4);
278
279	/*
280	* CR0/3/4 writes must be intercepted for obvious reasons.
281	*/
282	if (!pVM->hwaccm.s.fNestedPaging)
283	pVMCB->ctrl.u16InterceptWrCRx = RT_BIT(0) \| RT_BIT(3) \| RT_BIT(4);
284	else
285	pVMCB->ctrl.u16InterceptWrCRx = RT_BIT(0) \| RT_BIT(4) \| RT_BIT(8);
286
287	/* Intercept all DRx reads and writes by default. Changed later on. */
288	pVMCB->ctrl.u16InterceptRdDRx = 0xFFFF;
289	pVMCB->ctrl.u16InterceptWrDRx = 0xFFFF;
290
291	/* Currently we don't care about DRx reads or writes. DRx registers are trashed.
292	* All breakpoints are automatically cleared when the VM exits.
293	*/
294
295	pVMCB->ctrl.u32InterceptException = HWACCM_SVM_TRAP_MASK;
296	#ifndef DEBUG
297	if (pVM->hwaccm.s.fNestedPaging)
298	pVMCB->ctrl.u32InterceptException &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
299	#endif
300
301	pVMCB->ctrl.u32InterceptCtrl1 = SVM_CTRL1_INTERCEPT_INTR
302	\| SVM_CTRL1_INTERCEPT_VINTR
303	\| SVM_CTRL1_INTERCEPT_NMI
304	\| SVM_CTRL1_INTERCEPT_SMI
305	\| SVM_CTRL1_INTERCEPT_INIT
306	\| SVM_CTRL1_INTERCEPT_RDPMC
307	\| SVM_CTRL1_INTERCEPT_CPUID
308	\| SVM_CTRL1_INTERCEPT_RSM
309	\| SVM_CTRL1_INTERCEPT_HLT
310	\| SVM_CTRL1_INTERCEPT_INOUT_BITMAP
311	\| SVM_CTRL1_INTERCEPT_MSR_SHADOW
312	\| SVM_CTRL1_INTERCEPT_INVLPG
313	\| SVM_CTRL1_INTERCEPT_INVLPGA /* AMD only */
314	\| SVM_CTRL1_INTERCEPT_TASK_SWITCH
315	\| SVM_CTRL1_INTERCEPT_SHUTDOWN /* fatal */
316	\| SVM_CTRL1_INTERCEPT_FERR_FREEZE; /* Legacy FPU FERR handling. */
317	;
318	/* With nested paging we don't care about invlpg anymore. */
319	if (pVM->hwaccm.s.fNestedPaging)
320	pVMCB->ctrl.u32InterceptCtrl1 &= ~SVM_CTRL1_INTERCEPT_INVLPG;
321
322	pVMCB->ctrl.u32InterceptCtrl2 = SVM_CTRL2_INTERCEPT_VMRUN /* required */
323	\| SVM_CTRL2_INTERCEPT_VMMCALL
324	\| SVM_CTRL2_INTERCEPT_VMLOAD
325	\| SVM_CTRL2_INTERCEPT_VMSAVE
326	\| SVM_CTRL2_INTERCEPT_STGI
327	\| SVM_CTRL2_INTERCEPT_CLGI
328	\| SVM_CTRL2_INTERCEPT_SKINIT
329	\| SVM_CTRL2_INTERCEPT_WBINVD
330	\| SVM_CTRL2_INTERCEPT_MWAIT_UNCOND; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
331	;
332	Log(("pVMCB->ctrl.u32InterceptException = %x\n", pVMCB->ctrl.u32InterceptException));
333	Log(("pVMCB->ctrl.u32InterceptCtrl1 = %x\n", pVMCB->ctrl.u32InterceptCtrl1));
334	Log(("pVMCB->ctrl.u32InterceptCtrl2 = %x\n", pVMCB->ctrl.u32InterceptCtrl2));
335
336	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
337	pVMCB->ctrl.IntCtrl.n.u1VIrqMasking = 1;
338	/* Ignore the priority in the TPR; just deliver it when we tell it to. */
339	pVMCB->ctrl.IntCtrl.n.u1IgnoreTPR = 1;
340
341	/* Set IO and MSR bitmap addresses. */
342	pVMCB->ctrl.u64IOPMPhysAddr = pVM->hwaccm.s.svm.pIOBitmapPhys;
343	pVMCB->ctrl.u64MSRPMPhysAddr = pVM->hwaccm.s.svm.pMSRBitmapPhys;
344
345	/* No LBR virtualization. */
346	pVMCB->ctrl.u64LBRVirt = 0;
347
348	/** The ASID must start at 1; the host uses 0. */
349	pVMCB->ctrl.TLBCtrl.n.u32ASID = 1;
350
351	/** Setup the PAT msr (nested paging only) */
352	pVMCB->guest.u64GPAT = 0x0007040600070406ULL;
353	}
354	return rc;
355	}
356
357
358	/**
359	* Injects an event (trap or external interrupt)
360	*
361	* @param pVM The VM to operate on.
362	* @param pVMCB SVM control block
363	* @param pCtx CPU Context
364	* @param pIntInfo SVM interrupt info
365	*/
366	inline void SVMR0InjectEvent(PVM pVM, SVM_VMCB pVMCB, CPUMCTX pCtx, SVM_EVENT* pEvent)
367	{
368	#ifdef VBOX_STRICT
369	if (pEvent->n.u8Vector == 0xE)
370	Log(("SVM: Inject int %d at %RGv error code=%02x CR2=%RGv intInfo=%08x\n", pEvent->n.u8Vector, (RTGCPTR)pCtx->rip, pEvent->n.u32ErrorCode, (RTGCPTR)pCtx->cr2, pEvent->au64[0]));
371	else
372	if (pEvent->n.u8Vector < 0x20)
373	Log(("SVM: Inject int %d at %RGv error code=%08x\n", pEvent->n.u8Vector, (RTGCPTR)pCtx->rip, pEvent->n.u32ErrorCode));
374	else
375	{
376	Log(("INJ-EI: %x at %RGv\n", pEvent->n.u8Vector, (RTGCPTR)pCtx->rip));
377	Assert(!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS));
378	Assert(pCtx->eflags.u32 & X86_EFL_IF);
379	}
380	#endif
381
382	/* Set event injection state. */
383	pVMCB->ctrl.EventInject.au64[0] = pEvent->au64[0];
384	}
385
386
387	/**
388	* Checks for pending guest interrupts and injects them
389	*
390	* @returns VBox status code.
391	* @param pVM The VM to operate on.
392	* @param pVCpu The VM CPU to operate on.
393	* @param pVMCB SVM control block
394	* @param pCtx CPU Context
395	*/
396	static int SVMR0CheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, SVM_VMCB pVMCB, CPUMCTX pCtx)
397	{
398	int rc;
399
400	/* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
401	if (pVCpu->hwaccm.s.Event.fPending)
402	{
403	SVM_EVENT Event;
404
405	Log(("Reinjecting event %08x %08x at %RGv\n", pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip));
406	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
407	Event.au64[0] = pVCpu->hwaccm.s.Event.intInfo;
408	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
409
410	pVCpu->hwaccm.s.Event.fPending = false;
411	return VINF_SUCCESS;
412	}
413
414	if (pVM->hwaccm.s.fInjectNMI)
415	{
416	SVM_EVENT Event;
417
418	Event.n.u8Vector = X86_XCPT_NMI;
419	Event.n.u1Valid = 1;
420	Event.n.u32ErrorCode = 0;
421	Event.n.u3Type = SVM_EVENT_NMI;
422
423	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
424	pVM->hwaccm.s.fInjectNMI = false;
425	return VINF_SUCCESS;
426	}
427
428	/* When external interrupts are pending, we should exit the VM when IF is set. */
429	if ( !TRPMHasTrap(pVM)
430	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
431	{
432	if ( !(pCtx->eflags.u32 & X86_EFL_IF)
433	\|\| VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
434	{
435	if (!pVMCB->ctrl.IntCtrl.n.u1VIrqValid)
436	{
437	if (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
438	LogFlow(("Enable irq window exit!\n"));
439	else
440	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS -> irq window exit\n", (RTGCPTR)pCtx->rip));
441
442	/** @todo use virtual interrupt method to inject a pending irq; dispatched as soon as guest.IF is set. */
443	pVMCB->ctrl.u32InterceptCtrl1 \|= SVM_CTRL1_INTERCEPT_VINTR;
444	pVMCB->ctrl.IntCtrl.n.u1VIrqValid = 1;
445	pVMCB->ctrl.IntCtrl.n.u8VIrqVector = 0; /* don't care */
446	}
447	}
448	else
449	{
450	uint8_t u8Interrupt;
451
452	rc = PDMGetInterrupt(pVM, &u8Interrupt);
453	Log(("Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc\n", u8Interrupt, u8Interrupt, rc));
454	if (RT_SUCCESS(rc))
455	{
456	rc = TRPMAssertTrap(pVM, u8Interrupt, TRPM_HARDWARE_INT);
457	AssertRC(rc);
458	}
459	else
460	{
461	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
462	Assert(!VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)));
463	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
464	/* Just continue */
465	}
466	}
467	}
468
469	#ifdef VBOX_STRICT
470	if (TRPMHasTrap(pVM))
471	{
472	uint8_t u8Vector;
473	rc = TRPMQueryTrapAll(pVM, &u8Vector, 0, 0, 0);
474	AssertRC(rc);
475	}
476	#endif
477
478	if ( pCtx->eflags.u32 & X86_EFL_IF
479	&& (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
480	&& TRPMHasTrap(pVM)
481	)
482	{
483	uint8_t u8Vector;
484	int rc;
485	TRPMEVENT enmType;
486	SVM_EVENT Event;
487	RTGCUINT u32ErrorCode;
488
489	Event.au64[0] = 0;
490
491	/* If a new event is pending, then dispatch it now. */
492	rc = TRPMQueryTrapAll(pVM, &u8Vector, &enmType, &u32ErrorCode, 0);
493	AssertRC(rc);
494	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
495	Assert(enmType != TRPM_SOFTWARE_INT);
496
497	/* Clear the pending trap. */
498	rc = TRPMResetTrap(pVM);
499	AssertRC(rc);
500
501	Event.n.u8Vector = u8Vector;
502	Event.n.u1Valid = 1;
503	Event.n.u32ErrorCode = u32ErrorCode;
504
505	if (enmType == TRPM_TRAP)
506	{
507	switch (u8Vector) {
508	case 8:
509	case 10:
510	case 11:
511	case 12:
512	case 13:
513	case 14:
514	case 17:
515	/* Valid error codes. */
516	Event.n.u1ErrorCodeValid = 1;
517	break;
518	default:
519	break;
520	}
521	if (u8Vector == X86_XCPT_NMI)
522	Event.n.u3Type = SVM_EVENT_NMI;
523	else
524	Event.n.u3Type = SVM_EVENT_EXCEPTION;
525	}
526	else
527	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
528
529	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
530	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
531	} /* if (interrupts can be dispatched) */
532
533	return VINF_SUCCESS;
534	}
535
536	/**
537	* Save the host state
538	*
539	* @returns VBox status code.
540	* @param pVM The VM to operate on.
541	* @param pVCpu The VM CPU to operate on.
542	*/
543	VMMR0DECL(int) SVMR0SaveHostState(PVM pVM, PVMCPU pVCpu)
544	{
545	NOREF(pVM);
546	NOREF(pVCpu);
547	/* Nothing to do here. */
548	return VINF_SUCCESS;
549	}
550
551	/**
552	* Loads the guest state
553	*
554	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
555	*
556	* @returns VBox status code.
557	* @param pVM The VM to operate on.
558	* @param pVCpu The VM CPU to operate on.
559	* @param pCtx Guest context
560	*/
561	VMMR0DECL(int) SVMR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
562	{
563	RTGCUINTPTR val;
564	SVM_VMCB *pVMCB;
565
566	if (pVM == NULL)
567	return VERR_INVALID_PARAMETER;
568
569	/* Setup AMD SVM. */
570	Assert(pVM->hwaccm.s.svm.fSupported);
571
572	pVMCB = (SVM_VMCB *)pVCpu->hwaccm.s.svm.pVMCB;
573	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
574
575	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
576	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
577	{
578	SVM_WRITE_SELREG(CS, cs);
579	SVM_WRITE_SELREG(SS, ss);
580	SVM_WRITE_SELREG(DS, ds);
581	SVM_WRITE_SELREG(ES, es);
582	SVM_WRITE_SELREG(FS, fs);
583	SVM_WRITE_SELREG(GS, gs);
584	}
585
586	/* Guest CPU context: LDTR. */
587	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
588	{
589	SVM_WRITE_SELREG(LDTR, ldtr);
590	}
591
592	/* Guest CPU context: TR. */
593	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
594	{
595	SVM_WRITE_SELREG(TR, tr);
596	}
597
598	/* Guest CPU context: GDTR. */
599	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
600	{
601	pVMCB->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
602	pVMCB->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
603	}
604
605	/* Guest CPU context: IDTR. */
606	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
607	{
608	pVMCB->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
609	pVMCB->guest.IDTR.u64Base = pCtx->idtr.pIdt;
610	}
611
612	/*
613	* Sysenter MSRs (unconditional)
614	*/
615	pVMCB->guest.u64SysEnterCS = pCtx->SysEnter.cs;
616	pVMCB->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
617	pVMCB->guest.u64SysEnterESP = pCtx->SysEnter.esp;
618
619	/* Control registers */
620	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
621	{
622	val = pCtx->cr0;
623	if (!CPUMIsGuestFPUStateActive(pVCpu))
624	{
625	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
626	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
627	}
628	else
629	{
630	/** @todo check if we support the old style mess correctly. */
631	if (!(val & X86_CR0_NE))
632	{
633	Log(("Forcing X86_CR0_NE!!!\n"));
634
635	/* Also catch floating point exceptions as we need to report them to the guest in a different way. */
636	if (!pVCpu->hwaccm.s.fFPUOldStyleOverride)
637	{
638	pVMCB->ctrl.u32InterceptException \|= RT_BIT(X86_XCPT_MF);
639	pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
640	}
641	}
642	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
643	}
644	/* Always enable caching. */
645	val &= ~(X86_CR0_CD\|X86_CR0_NW);
646
647	/* Note: WP is not relevant in nested paging mode as we catch accesses on the (guest) physical level. */
648	/* Note: In nested paging mode the guest is allowed to run with paging disabled; the guest physical to host physical translation will remain active. */
649	if (!pVM->hwaccm.s.fNestedPaging)
650	{
651	val \|= X86_CR0_PG; /* Paging is always enabled; even when the guest is running in real mode or PE without paging. */
652	val \|= X86_CR0_WP; /* Must set this as we rely on protect various pages and supervisor writes must be caught. */
653	}
654	pVMCB->guest.u64CR0 = val;
655	}
656	/* CR2 as well */
657	pVMCB->guest.u64CR2 = pCtx->cr2;
658
659	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
660	{
661	/* Save our shadow CR3 register. */
662	if (pVM->hwaccm.s.fNestedPaging)
663	{
664	PGMMODE enmShwPagingMode;
665
666	#if HC_ARCH_BITS == 32
667	if (CPUMIsGuestInLongModeEx(pCtx))
668	enmShwPagingMode = PGMMODE_AMD64_NX;
669	else
670	#endif
671	enmShwPagingMode = PGMGetHostMode(pVM);
672
673	pVMCB->ctrl.u64NestedPagingCR3 = PGMGetNestedCR3(pVM, enmShwPagingMode);
674	Assert(pVMCB->ctrl.u64NestedPagingCR3);
675	pVMCB->guest.u64CR3 = pCtx->cr3;
676	}
677	else
678	{
679	pVMCB->guest.u64CR3 = PGMGetHyperCR3(pVM);
680	Assert(pVMCB->guest.u64CR3);
681	}
682	}
683
684	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
685	{
686	val = pCtx->cr4;
687	if (!pVM->hwaccm.s.fNestedPaging)
688	{
689	switch(pVCpu->hwaccm.s.enmShadowMode)
690	{
691	case PGMMODE_REAL:
692	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
693	AssertFailed();
694	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
695
696	case PGMMODE_32_BIT: /* 32-bit paging. */
697	break;
698
699	case PGMMODE_PAE: /* PAE paging. */
700	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
701	/** @todo use normal 32 bits paging */
702	val \|= X86_CR4_PAE;
703	break;
704
705	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
706	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
707	#ifdef VBOX_ENABLE_64_BITS_GUESTS
708	break;
709	#else
710	AssertFailed();
711	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
712	#endif
713
714	default: /* shut up gcc */
715	AssertFailed();
716	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
717	}
718	}
719	pVMCB->guest.u64CR4 = val;
720	}
721
722	/* Debug registers. */
723	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
724	{
725	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
726	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
727
728	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
729	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
730	pCtx->dr[7] \|= 0x400; /* must be one */
731
732	pVMCB->guest.u64DR7 = pCtx->dr[7];
733	pVMCB->guest.u64DR6 = pCtx->dr[6];
734
735	/* Sync the debug state now if any breakpoint is armed. */
736	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
737	&& !CPUMIsGuestDebugStateActive(pVM)
738	&& !DBGFIsStepping(pVM))
739	{
740	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
741
742	/* Disable drx move intercepts. */
743	pVMCB->ctrl.u16InterceptRdDRx = 0;
744	pVMCB->ctrl.u16InterceptWrDRx = 0;
745
746	/* Save the host and load the guest debug state. */
747	int rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
748	AssertRC(rc);
749	}
750	}
751
752	/* EIP, ESP and EFLAGS */
753	pVMCB->guest.u64RIP = pCtx->rip;
754	pVMCB->guest.u64RSP = pCtx->rsp;
755	pVMCB->guest.u64RFlags = pCtx->eflags.u32;
756
757	/* Set CPL */
758	pVMCB->guest.u8CPL = pCtx->csHid.Attr.n.u2Dpl;
759
760	/* RAX/EAX too, as VMRUN uses RAX as an implicit parameter. */
761	pVMCB->guest.u64RAX = pCtx->rax;
762
763	/* vmrun will fail without MSR_K6_EFER_SVME. */
764	pVMCB->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
765
766	/* 64 bits guest mode? */
767	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
768	{
769	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
770	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
771	#elif HC_ARCH_BITS == 32
772	pVCpu->hwaccm.s.svm.pfnVMRun = SVMR0VMSwitcherRun64;
773	#else
774	pVCpu->hwaccm.s.svm.pfnVMRun = SVMR0VMRun64;
775	#endif
776	/* Unconditionally update these as wrmsr might have changed them. (HWACCM_CHANGED_GUEST_SEGMENT_REGS will not be set) */
777	pVMCB->guest.FS.u64Base = pCtx->fsHid.u64Base;
778	pVMCB->guest.GS.u64Base = pCtx->gsHid.u64Base;
779	}
780	else
781	{
782	/* Filter out the MSR_K6_LME bit or else AMD-V expects amd64 shadow paging. */
783	pVMCB->guest.u64EFER &= ~MSR_K6_EFER_LME;
784
785	pVCpu->hwaccm.s.svm.pfnVMRun = SVMR0VMRun;
786	}
787
788	/* TSC offset. */
789	if (TMCpuTickCanUseRealTSC(pVM, &pVMCB->ctrl.u64TSCOffset))
790	{
791	pVMCB->ctrl.u32InterceptCtrl1 &= ~SVM_CTRL1_INTERCEPT_RDTSC;
792	pVMCB->ctrl.u32InterceptCtrl2 &= ~SVM_CTRL2_INTERCEPT_RDTSCP;
793	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
794	}
795	else
796	{
797	pVMCB->ctrl.u32InterceptCtrl1 \|= SVM_CTRL1_INTERCEPT_RDTSC;
798	pVMCB->ctrl.u32InterceptCtrl2 \|= SVM_CTRL2_INTERCEPT_RDTSCP;
799	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
800	}
801
802	/* Sync the various msrs for 64 bits mode. */
803	pVMCB->guest.u64STAR = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
804	pVMCB->guest.u64LSTAR = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
805	pVMCB->guest.u64CSTAR = pCtx->msrCSTAR; /* compatibility mode syscall rip */
806	pVMCB->guest.u64SFMASK = pCtx->msrSFMASK; /* syscall flag mask */
807	pVMCB->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
808
809	#ifdef DEBUG
810	/* Intercept X86_XCPT_DB if stepping is enabled */
811	if (DBGFIsStepping(pVM))
812	pVMCB->ctrl.u32InterceptException \|= RT_BIT(X86_XCPT_DB);
813	else
814	pVMCB->ctrl.u32InterceptException &= ~RT_BIT(X86_XCPT_DB);
815	#endif
816
817	/* Done. */
818	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
819
820	return VINF_SUCCESS;
821	}
822
823
824	/**
825	* Runs guest code in an AMD-V VM.
826	*
827	* @returns VBox status code.
828	* @param pVM The VM to operate on.
829	* @param pVCpu The VM CPU to operate on.
830	* @param pCtx Guest context
831	*/
832	VMMR0DECL(int) SVMR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
833	{
834	int rc = VINF_SUCCESS;
835	uint64_t exitCode = (uint64_t)SVM_EXIT_INVALID;
836	SVM_VMCB *pVMCB;
837	bool fSyncTPR = false;
838	unsigned cResume = 0;
839	uint8_t u8LastVTPR;
840	PHWACCM_CPUINFO pCpu = 0;
841	#ifdef VBOX_STRICT
842	RTCPUID idCpuCheck;
843	#endif
844
845	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
846
847	pVMCB = (SVM_VMCB *)pVCpu->hwaccm.s.svm.pVMCB;
848	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
849
850	/* We can jump to this point to resume execution after determining that a VM-exit is innocent.
851	*/
852	ResumeExecution:
853	Assert(!HWACCMR0SuspendPending());
854
855	/* Safety precaution; looping for too long here can have a very bad effect on the host */
856	if (++cResume > HWACCM_MAX_RESUME_LOOPS)
857	{
858	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
859	rc = VINF_EM_RAW_INTERRUPT;
860	goto end;
861	}
862
863	/* Check for irq inhibition due to instruction fusing (sti, mov ss). */
864	if (VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
865	{
866	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVM)));
867	if (pCtx->rip != EMGetInhibitInterruptsPC(pVM))
868	{
869	/* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
870	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
871	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
872	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
873	*/
874	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
875	/* Irq inhibition is no longer active; clear the corresponding SVM state. */
876	pVMCB->ctrl.u64IntShadow = 0;
877	}
878	}
879	else
880	{
881	/* Irq inhibition is no longer active; clear the corresponding SVM state. */
882	pVMCB->ctrl.u64IntShadow = 0;
883	}
884
885	/* Check for pending actions that force us to go back to ring 3. */
886	#ifdef DEBUG
887	/* Intercept X86_XCPT_DB if stepping is enabled */
888	if (!DBGFIsStepping(pVM))
889	#endif
890	{
891	if (VM_FF_ISPENDING(pVM, VM_FF_TO_R3 \| VM_FF_TIMER))
892	{
893	VM_FF_CLEAR(pVM, VM_FF_TO_R3);
894	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
895	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
896	rc = VINF_EM_RAW_TO_R3;
897	goto end;
898	}
899	}
900
901	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
902	if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
903	{
904	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
905	rc = VINF_EM_PENDING_REQUEST;
906	goto end;
907	}
908
909	/* When external interrupts are pending, we should exit the VM when IF is set. */
910	/* Note! After VM_FF_INHIBIT_INTERRUPTS check!!! */
911	rc = SVMR0CheckPendingInterrupt(pVM, pVCpu, pVMCB, pCtx);
912	if (RT_FAILURE(rc))
913	{
914	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
915	goto end;
916	}
917
918	/* TPR caching using CR8 is only available in 64 bits mode */
919	/* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
920	/* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!!!!! */
921	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
922	{
923	bool fPending;
924
925	/* TPR caching in CR8 */
926	int rc = PDMApicGetTPR(pVM, &u8LastVTPR, &fPending);
927	AssertRC(rc);
928	pVMCB->ctrl.IntCtrl.n.u8VTPR = u8LastVTPR;
929
930	if (fPending)
931	{
932	/* A TPR change could activate a pending interrupt, so catch cr8 writes. */
933	pVMCB->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
934	}
935	else
936	/* No interrupts are pending, so we don't need to be explicitely notified.
937	* There are enough world switches for detecting pending interrupts.
938	*/
939	pVMCB->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
940
941	fSyncTPR = !fPending;
942	}
943
944	/* All done! Let's start VM execution. */
945	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatInGC, x);
946
947	/* Enable nested paging if necessary (disabled each time after #VMEXIT). */
948	pVMCB->ctrl.NestedPaging.n.u1NestedPaging = pVM->hwaccm.s.fNestedPaging;
949
950	#ifdef LOG_ENABLED
951	pCpu = HWACCMR0GetCurrentCpu();
952	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
953	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
954	{
955	if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
956	Log(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
957	else
958	Log(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
959	}
960	if (pCpu->fFlushTLB)
961	Log(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
962	#endif
963
964	/*
965	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
966	* (until the actual world switch)
967	*/
968
969	#ifdef VBOX_STRICT
970	idCpuCheck = RTMpCpuId();
971	#endif
972
973	/* Load the guest state; must be here as it sets up the shadow cr0 for lazy fpu syncing! */
974	rc = SVMR0LoadGuestState(pVM, pVCpu, pCtx);
975	if (rc != VINF_SUCCESS)
976	{
977	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
978	goto end;
979	}
980
981	pCpu = HWACCMR0GetCurrentCpu();
982	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
983	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
984	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
985	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
986	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
987	{
988	/* Force a TLB flush on VM entry. */
989	pVCpu->hwaccm.s.fForceTLBFlush = true;
990	}
991	else
992	Assert(!pCpu->fFlushTLB \|\| pVM->hwaccm.s.svm.fAlwaysFlushTLB);
993
994	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
995
996	/* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
997	if ( pVCpu->hwaccm.s.fForceTLBFlush
998	&& !pVM->hwaccm.s.svm.fAlwaysFlushTLB)
999	{
1000	if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
1001	\|\| pCpu->fFlushTLB)
1002	{
1003	pCpu->fFlushTLB = false;
1004	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
1005	pVMCB->ctrl.TLBCtrl.n.u1TLBFlush = 1; /* wrap around; flush TLB */
1006	pCpu->cTLBFlushes++;
1007	}
1008	else
1009	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
1010
1011	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
1012	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
1013	}
1014	else
1015	{
1016	Assert(!pCpu->fFlushTLB \|\| pVM->hwaccm.s.svm.fAlwaysFlushTLB);
1017
1018	/* We never increase uCurrentASID in the fAlwaysFlushTLB (erratum 170) case. */
1019	if (!pCpu->uCurrentASID \|\| !pVCpu->hwaccm.s.uCurrentASID)
1020	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID = 1;
1021
1022	Assert(!pVM->hwaccm.s.svm.fAlwaysFlushTLB \|\| pVCpu->hwaccm.s.fForceTLBFlush);
1023	pVMCB->ctrl.TLBCtrl.n.u1TLBFlush = pVCpu->hwaccm.s.fForceTLBFlush;
1024	}
1025	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
1026	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
1027	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
1028	pVMCB->ctrl.TLBCtrl.n.u32ASID = pVCpu->hwaccm.s.uCurrentASID;
1029
1030	#ifdef VBOX_WITH_STATISTICS
1031	if (pVMCB->ctrl.TLBCtrl.n.u1TLBFlush)
1032	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
1033	else
1034	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
1035	#endif
1036
1037	/* In case we execute a goto ResumeExecution later on. */
1038	pVCpu->hwaccm.s.fResumeVM = true;
1039	pVCpu->hwaccm.s.fForceTLBFlush = pVM->hwaccm.s.svm.fAlwaysFlushTLB;
1040
1041	Assert(sizeof(pVCpu->hwaccm.s.svm.pVMCBPhys) == 8);
1042	Assert(pVMCB->ctrl.IntCtrl.n.u1VIrqMasking);
1043	Assert(pVMCB->ctrl.u64IOPMPhysAddr == pVM->hwaccm.s.svm.pIOBitmapPhys);
1044	Assert(pVMCB->ctrl.u64MSRPMPhysAddr == pVM->hwaccm.s.svm.pMSRBitmapPhys);
1045	Assert(pVMCB->ctrl.u64LBRVirt == 0);
1046
1047	#ifdef VBOX_STRICT
1048	Assert(idCpuCheck == RTMpCpuId());
1049	#endif
1050	TMNotifyStartOfExecution(pVM);
1051	pVCpu->hwaccm.s.svm.pfnVMRun(pVM->hwaccm.s.svm.pVMCBHostPhys, pVCpu->hwaccm.s.svm.pVMCBPhys, pCtx, pVM, pVCpu);
1052	TMNotifyEndOfExecution(pVM);
1053	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatInGC, x);
1054
1055	/*
1056	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1057	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
1058	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1059	*/
1060
1061	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit, x);
1062
1063	/* Reason for the VM exit */
1064	exitCode = pVMCB->ctrl.u64ExitCode;
1065
1066	if (exitCode == (uint64_t)SVM_EXIT_INVALID) /* Invalid guest state. */
1067	{
1068	HWACCMDumpRegs(pVM, pCtx);
1069	#ifdef DEBUG
1070	Log(("ctrl.u16InterceptRdCRx %x\n", pVMCB->ctrl.u16InterceptRdCRx));
1071	Log(("ctrl.u16InterceptWrCRx %x\n", pVMCB->ctrl.u16InterceptWrCRx));
1072	Log(("ctrl.u16InterceptRdDRx %x\n", pVMCB->ctrl.u16InterceptRdDRx));
1073	Log(("ctrl.u16InterceptWrDRx %x\n", pVMCB->ctrl.u16InterceptWrDRx));
1074	Log(("ctrl.u32InterceptException %x\n", pVMCB->ctrl.u32InterceptException));
1075	Log(("ctrl.u32InterceptCtrl1 %x\n", pVMCB->ctrl.u32InterceptCtrl1));
1076	Log(("ctrl.u32InterceptCtrl2 %x\n", pVMCB->ctrl.u32InterceptCtrl2));
1077	Log(("ctrl.u64IOPMPhysAddr %RX64\n", pVMCB->ctrl.u64IOPMPhysAddr));
1078	Log(("ctrl.u64MSRPMPhysAddr %RX64\n", pVMCB->ctrl.u64MSRPMPhysAddr));
1079	Log(("ctrl.u64TSCOffset %RX64\n", pVMCB->ctrl.u64TSCOffset));
1080
1081	Log(("ctrl.TLBCtrl.u32ASID %x\n", pVMCB->ctrl.TLBCtrl.n.u32ASID));
1082	Log(("ctrl.TLBCtrl.u1TLBFlush %x\n", pVMCB->ctrl.TLBCtrl.n.u1TLBFlush));
1083	Log(("ctrl.TLBCtrl.u7Reserved %x\n", pVMCB->ctrl.TLBCtrl.n.u7Reserved));
1084	Log(("ctrl.TLBCtrl.u24Reserved %x\n", pVMCB->ctrl.TLBCtrl.n.u24Reserved));
1085
1086	Log(("ctrl.IntCtrl.u8VTPR %x\n", pVMCB->ctrl.IntCtrl.n.u8VTPR));
1087	Log(("ctrl.IntCtrl.u1VIrqValid %x\n", pVMCB->ctrl.IntCtrl.n.u1VIrqValid));
1088	Log(("ctrl.IntCtrl.u7Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u7Reserved));
1089	Log(("ctrl.IntCtrl.u4VIrqPriority %x\n", pVMCB->ctrl.IntCtrl.n.u4VIrqPriority));
1090	Log(("ctrl.IntCtrl.u1IgnoreTPR %x\n", pVMCB->ctrl.IntCtrl.n.u1IgnoreTPR));
1091	Log(("ctrl.IntCtrl.u3Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u3Reserved));
1092	Log(("ctrl.IntCtrl.u1VIrqMasking %x\n", pVMCB->ctrl.IntCtrl.n.u1VIrqMasking));
1093	Log(("ctrl.IntCtrl.u7Reserved2 %x\n", pVMCB->ctrl.IntCtrl.n.u7Reserved2));
1094	Log(("ctrl.IntCtrl.u8VIrqVector %x\n", pVMCB->ctrl.IntCtrl.n.u8VIrqVector));
1095	Log(("ctrl.IntCtrl.u24Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u24Reserved));
1096
1097	Log(("ctrl.u64IntShadow %RX64\n", pVMCB->ctrl.u64IntShadow));
1098	Log(("ctrl.u64ExitCode %RX64\n", pVMCB->ctrl.u64ExitCode));
1099	Log(("ctrl.u64ExitInfo1 %RX64\n", pVMCB->ctrl.u64ExitInfo1));
1100	Log(("ctrl.u64ExitInfo2 %RX64\n", pVMCB->ctrl.u64ExitInfo2));
1101	Log(("ctrl.ExitIntInfo.u8Vector %x\n", pVMCB->ctrl.ExitIntInfo.n.u8Vector));
1102	Log(("ctrl.ExitIntInfo.u3Type %x\n", pVMCB->ctrl.ExitIntInfo.n.u3Type));
1103	Log(("ctrl.ExitIntInfo.u1ErrorCodeValid %x\n", pVMCB->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
1104	Log(("ctrl.ExitIntInfo.u19Reserved %x\n", pVMCB->ctrl.ExitIntInfo.n.u19Reserved));
1105	Log(("ctrl.ExitIntInfo.u1Valid %x\n", pVMCB->ctrl.ExitIntInfo.n.u1Valid));
1106	Log(("ctrl.ExitIntInfo.u32ErrorCode %x\n", pVMCB->ctrl.ExitIntInfo.n.u32ErrorCode));
1107	Log(("ctrl.NestedPaging %RX64\n", pVMCB->ctrl.NestedPaging.au64));
1108	Log(("ctrl.EventInject.u8Vector %x\n", pVMCB->ctrl.EventInject.n.u8Vector));
1109	Log(("ctrl.EventInject.u3Type %x\n", pVMCB->ctrl.EventInject.n.u3Type));
1110	Log(("ctrl.EventInject.u1ErrorCodeValid %x\n", pVMCB->ctrl.EventInject.n.u1ErrorCodeValid));
1111	Log(("ctrl.EventInject.u19Reserved %x\n", pVMCB->ctrl.EventInject.n.u19Reserved));
1112	Log(("ctrl.EventInject.u1Valid %x\n", pVMCB->ctrl.EventInject.n.u1Valid));
1113	Log(("ctrl.EventInject.u32ErrorCode %x\n", pVMCB->ctrl.EventInject.n.u32ErrorCode));
1114
1115	Log(("ctrl.u64NestedPagingCR3 %RX64\n", pVMCB->ctrl.u64NestedPagingCR3));
1116	Log(("ctrl.u64LBRVirt %RX64\n", pVMCB->ctrl.u64LBRVirt));
1117
1118	Log(("guest.CS.u16Sel %04X\n", pVMCB->guest.CS.u16Sel));
1119	Log(("guest.CS.u16Attr %04X\n", pVMCB->guest.CS.u16Attr));
1120	Log(("guest.CS.u32Limit %X\n", pVMCB->guest.CS.u32Limit));
1121	Log(("guest.CS.u64Base %RX64\n", pVMCB->guest.CS.u64Base));
1122	Log(("guest.DS.u16Sel %04X\n", pVMCB->guest.DS.u16Sel));
1123	Log(("guest.DS.u16Attr %04X\n", pVMCB->guest.DS.u16Attr));
1124	Log(("guest.DS.u32Limit %X\n", pVMCB->guest.DS.u32Limit));
1125	Log(("guest.DS.u64Base %RX64\n", pVMCB->guest.DS.u64Base));
1126	Log(("guest.ES.u16Sel %04X\n", pVMCB->guest.ES.u16Sel));
1127	Log(("guest.ES.u16Attr %04X\n", pVMCB->guest.ES.u16Attr));
1128	Log(("guest.ES.u32Limit %X\n", pVMCB->guest.ES.u32Limit));
1129	Log(("guest.ES.u64Base %RX64\n", pVMCB->guest.ES.u64Base));
1130	Log(("guest.FS.u16Sel %04X\n", pVMCB->guest.FS.u16Sel));
1131	Log(("guest.FS.u16Attr %04X\n", pVMCB->guest.FS.u16Attr));
1132	Log(("guest.FS.u32Limit %X\n", pVMCB->guest.FS.u32Limit));
1133	Log(("guest.FS.u64Base %RX64\n", pVMCB->guest.FS.u64Base));
1134	Log(("guest.GS.u16Sel %04X\n", pVMCB->guest.GS.u16Sel));
1135	Log(("guest.GS.u16Attr %04X\n", pVMCB->guest.GS.u16Attr));
1136	Log(("guest.GS.u32Limit %X\n", pVMCB->guest.GS.u32Limit));
1137	Log(("guest.GS.u64Base %RX64\n", pVMCB->guest.GS.u64Base));
1138
1139	Log(("guest.GDTR.u32Limit %X\n", pVMCB->guest.GDTR.u32Limit));
1140	Log(("guest.GDTR.u64Base %RX64\n", pVMCB->guest.GDTR.u64Base));
1141
1142	Log(("guest.LDTR.u16Sel %04X\n", pVMCB->guest.LDTR.u16Sel));
1143	Log(("guest.LDTR.u16Attr %04X\n", pVMCB->guest.LDTR.u16Attr));
1144	Log(("guest.LDTR.u32Limit %X\n", pVMCB->guest.LDTR.u32Limit));
1145	Log(("guest.LDTR.u64Base %RX64\n", pVMCB->guest.LDTR.u64Base));
1146
1147	Log(("guest.IDTR.u32Limit %X\n", pVMCB->guest.IDTR.u32Limit));
1148	Log(("guest.IDTR.u64Base %RX64\n", pVMCB->guest.IDTR.u64Base));
1149
1150	Log(("guest.TR.u16Sel %04X\n", pVMCB->guest.TR.u16Sel));
1151	Log(("guest.TR.u16Attr %04X\n", pVMCB->guest.TR.u16Attr));
1152	Log(("guest.TR.u32Limit %X\n", pVMCB->guest.TR.u32Limit));
1153	Log(("guest.TR.u64Base %RX64\n", pVMCB->guest.TR.u64Base));
1154
1155	Log(("guest.u8CPL %X\n", pVMCB->guest.u8CPL));
1156	Log(("guest.u64CR0 %RX64\n", pVMCB->guest.u64CR0));
1157	Log(("guest.u64CR2 %RX64\n", pVMCB->guest.u64CR2));
1158	Log(("guest.u64CR3 %RX64\n", pVMCB->guest.u64CR3));
1159	Log(("guest.u64CR4 %RX64\n", pVMCB->guest.u64CR4));
1160	Log(("guest.u64DR6 %RX64\n", pVMCB->guest.u64DR6));
1161	Log(("guest.u64DR7 %RX64\n", pVMCB->guest.u64DR7));
1162
1163	Log(("guest.u64RIP %RX64\n", pVMCB->guest.u64RIP));
1164	Log(("guest.u64RSP %RX64\n", pVMCB->guest.u64RSP));
1165	Log(("guest.u64RAX %RX64\n", pVMCB->guest.u64RAX));
1166	Log(("guest.u64RFlags %RX64\n", pVMCB->guest.u64RFlags));
1167
1168	Log(("guest.u64SysEnterCS %RX64\n", pVMCB->guest.u64SysEnterCS));
1169	Log(("guest.u64SysEnterEIP %RX64\n", pVMCB->guest.u64SysEnterEIP));
1170	Log(("guest.u64SysEnterESP %RX64\n", pVMCB->guest.u64SysEnterESP));
1171
1172	Log(("guest.u64EFER %RX64\n", pVMCB->guest.u64EFER));
1173	Log(("guest.u64STAR %RX64\n", pVMCB->guest.u64STAR));
1174	Log(("guest.u64LSTAR %RX64\n", pVMCB->guest.u64LSTAR));
1175	Log(("guest.u64CSTAR %RX64\n", pVMCB->guest.u64CSTAR));
1176	Log(("guest.u64SFMASK %RX64\n", pVMCB->guest.u64SFMASK));
1177	Log(("guest.u64KernelGSBase %RX64\n", pVMCB->guest.u64KernelGSBase));
1178	Log(("guest.u64GPAT %RX64\n", pVMCB->guest.u64GPAT));
1179	Log(("guest.u64DBGCTL %RX64\n", pVMCB->guest.u64DBGCTL));
1180	Log(("guest.u64BR_FROM %RX64\n", pVMCB->guest.u64BR_FROM));
1181	Log(("guest.u64BR_TO %RX64\n", pVMCB->guest.u64BR_TO));
1182	Log(("guest.u64LASTEXCPFROM %RX64\n", pVMCB->guest.u64LASTEXCPFROM));
1183	Log(("guest.u64LASTEXCPTO %RX64\n", pVMCB->guest.u64LASTEXCPTO));
1184
1185	#endif
1186	rc = VERR_SVM_UNABLE_TO_START_VM;
1187	goto end;
1188	}
1189
1190	/* Let's first sync back eip, esp, and eflags. */
1191	pCtx->rip = pVMCB->guest.u64RIP;
1192	pCtx->rsp = pVMCB->guest.u64RSP;
1193	pCtx->eflags.u32 = pVMCB->guest.u64RFlags;
1194	/* eax is saved/restore across the vmrun instruction */
1195	pCtx->rax = pVMCB->guest.u64RAX;
1196
1197	pCtx->msrKERNELGSBASE = pVMCB->guest.u64KernelGSBase; /* swapgs exchange value */
1198
1199	/* Can be updated behind our back in the nested paging case. */
1200	pCtx->cr2 = pVMCB->guest.u64CR2;
1201
1202	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1203	SVM_READ_SELREG(SS, ss);
1204	SVM_READ_SELREG(CS, cs);
1205	SVM_READ_SELREG(DS, ds);
1206	SVM_READ_SELREG(ES, es);
1207	SVM_READ_SELREG(FS, fs);
1208	SVM_READ_SELREG(GS, gs);
1209
1210	/*
1211	* System MSRs
1212	*/
1213	pCtx->SysEnter.cs = pVMCB->guest.u64SysEnterCS;
1214	pCtx->SysEnter.eip = pVMCB->guest.u64SysEnterEIP;
1215	pCtx->SysEnter.esp = pVMCB->guest.u64SysEnterESP;
1216
1217	/* Remaining guest CPU context: TR, IDTR, GDTR, LDTR; must sync everything otherwise we can get out of sync when jumping to ring 3. */
1218	SVM_READ_SELREG(LDTR, ldtr);
1219	SVM_READ_SELREG(TR, tr);
1220
1221	pCtx->gdtr.cbGdt = pVMCB->guest.GDTR.u32Limit;
1222	pCtx->gdtr.pGdt = pVMCB->guest.GDTR.u64Base;
1223
1224	pCtx->idtr.cbIdt = pVMCB->guest.IDTR.u32Limit;
1225	pCtx->idtr.pIdt = pVMCB->guest.IDTR.u64Base;
1226
1227	/* Note: no reason to sync back the CRx and DRx registers. They can't be changed by the guest. */
1228	/* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1229	if ( pVM->hwaccm.s.fNestedPaging
1230	&& pCtx->cr3 != pVMCB->guest.u64CR3)
1231	{
1232	CPUMSetGuestCR3(pVM, pVMCB->guest.u64CR3);
1233	PGMUpdateCR3(pVM, pVMCB->guest.u64CR3);
1234	}
1235
1236	/* Note! NOW IT'S SAFE FOR LOGGING! */
1237
1238	/* Take care of instruction fusing (sti, mov ss) (see 15.20.5 Interrupt Shadows) */
1239	if (pVMCB->ctrl.u64IntShadow & SVM_INTERRUPT_SHADOW_ACTIVE)
1240	{
1241	Log(("uInterruptState %x rip=%RGv\n", pVMCB->ctrl.u64IntShadow, (RTGCPTR)pCtx->rip));
1242	EMSetInhibitInterruptsPC(pVM, pCtx->rip);
1243	}
1244	else
1245	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
1246
1247	Log2(("exitCode = %x\n", exitCode));
1248
1249	/* Sync back DR6 as it could have been changed by hitting breakpoints. */
1250	pCtx->dr[6] = pVMCB->guest.u64DR6;
1251	/* DR7.GD can be cleared by debug exceptions, so sync it back as well. */
1252	pCtx->dr[7] = pVMCB->guest.u64DR7;
1253
1254	/* Check if an injected event was interrupted prematurely. */
1255	pVCpu->hwaccm.s.Event.intInfo = pVMCB->ctrl.ExitIntInfo.au64[0];
1256	if ( pVMCB->ctrl.ExitIntInfo.n.u1Valid
1257	&& pVMCB->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT /* we don't care about 'int xx' as the instruction will be restarted. */)
1258	{
1259	Log(("Pending inject %RX64 at %RGv exit=%08x\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitCode));
1260
1261	#ifdef LOG_ENABLED
1262	SVM_EVENT Event;
1263	Event.au64[0] = pVCpu->hwaccm.s.Event.intInfo;
1264
1265	if ( exitCode == SVM_EXIT_EXCEPTION_E
1266	&& Event.n.u8Vector == 0xE)
1267	{
1268	Log(("Double fault!\n"));
1269	}
1270	#endif
1271
1272	pVCpu->hwaccm.s.Event.fPending = true;
1273	/* Error code present? (redundant) */
1274	if (pVMCB->ctrl.ExitIntInfo.n.u1ErrorCodeValid)
1275	{
1276	pVCpu->hwaccm.s.Event.errCode = pVMCB->ctrl.ExitIntInfo.n.u32ErrorCode;
1277	}
1278	else
1279	pVCpu->hwaccm.s.Event.errCode = 0;
1280	}
1281	#ifdef VBOX_WITH_STATISTICS
1282	if (exitCode == SVM_EXIT_NPF)
1283	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
1284	else
1285	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitCode & MASK_EXITREASON_STAT]);
1286	#endif
1287
1288	if (fSyncTPR)
1289	{
1290	rc = PDMApicSetTPR(pVM, pVMCB->ctrl.IntCtrl.n.u8VTPR);
1291	AssertRC(rc);
1292	}
1293
1294	/* Deal with the reason of the VM-exit. */
1295	switch (exitCode)
1296	{
1297	case SVM_EXIT_EXCEPTION_0: case SVM_EXIT_EXCEPTION_1: case SVM_EXIT_EXCEPTION_2: case SVM_EXIT_EXCEPTION_3:
1298	case SVM_EXIT_EXCEPTION_4: case SVM_EXIT_EXCEPTION_5: case SVM_EXIT_EXCEPTION_6: case SVM_EXIT_EXCEPTION_7:
1299	case SVM_EXIT_EXCEPTION_8: case SVM_EXIT_EXCEPTION_9: case SVM_EXIT_EXCEPTION_A: case SVM_EXIT_EXCEPTION_B:
1300	case SVM_EXIT_EXCEPTION_C: case SVM_EXIT_EXCEPTION_D: case SVM_EXIT_EXCEPTION_E: case SVM_EXIT_EXCEPTION_F:
1301	case SVM_EXIT_EXCEPTION_10: case SVM_EXIT_EXCEPTION_11: case SVM_EXIT_EXCEPTION_12: case SVM_EXIT_EXCEPTION_13:
1302	case SVM_EXIT_EXCEPTION_14: case SVM_EXIT_EXCEPTION_15: case SVM_EXIT_EXCEPTION_16: case SVM_EXIT_EXCEPTION_17:
1303	case SVM_EXIT_EXCEPTION_18: case SVM_EXIT_EXCEPTION_19: case SVM_EXIT_EXCEPTION_1A: case SVM_EXIT_EXCEPTION_1B:
1304	case SVM_EXIT_EXCEPTION_1C: case SVM_EXIT_EXCEPTION_1D: case SVM_EXIT_EXCEPTION_1E: case SVM_EXIT_EXCEPTION_1F:
1305	{
1306	/* Pending trap. */
1307	SVM_EVENT Event;
1308	uint32_t vector = exitCode - SVM_EXIT_EXCEPTION_0;
1309
1310	Log2(("Hardware/software interrupt %d\n", vector));
1311	switch (vector)
1312	{
1313	case X86_XCPT_DB:
1314	{
1315	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
1316
1317	/* Note that we don't support guest and host-initiated debugging at the same time. */
1318	Assert(DBGFIsStepping(pVM));
1319
1320	rc = DBGFR0Trap01Handler(pVM, CPUMCTX2CORE(pCtx), pCtx->dr[6]);
1321	if (rc == VINF_EM_RAW_GUEST_TRAP)
1322	{
1323	Log(("Trap %x (debug) at %016RX64\n", vector, pCtx->rip));
1324
1325	/* Reinject the exception. */
1326	Event.au64[0] = 0;
1327	Event.n.u3Type = SVM_EVENT_EXCEPTION; /* trap or fault */
1328	Event.n.u1Valid = 1;
1329	Event.n.u8Vector = X86_XCPT_DB;
1330
1331	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1332
1333	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1334	goto ResumeExecution;
1335	}
1336	/* Return to ring 3 to deal with the debug exit code. */
1337	break;
1338	}
1339
1340	case X86_XCPT_NM:
1341	{
1342	Log(("#NM fault at %RGv\n", (RTGCPTR)pCtx->rip));
1343
1344	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
1345	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
1346	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
1347	if (rc == VINF_SUCCESS)
1348	{
1349	Assert(CPUMIsGuestFPUStateActive(pVCpu));
1350	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
1351
1352	/* Continue execution. */
1353	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1354	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
1355
1356	goto ResumeExecution;
1357	}
1358
1359	Log(("Forward #NM fault to the guest\n"));
1360	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
1361
1362	Event.au64[0] = 0;
1363	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1364	Event.n.u1Valid = 1;
1365	Event.n.u8Vector = X86_XCPT_NM;
1366
1367	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1368	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1369	goto ResumeExecution;
1370	}
1371
1372	case X86_XCPT_PF: /* Page fault */
1373	{
1374	uint32_t errCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1375	RTGCUINTPTR uFaultAddress = pVMCB->ctrl.u64ExitInfo2; /* EXITINFO2 = fault address */
1376
1377	#ifdef DEBUG
1378	if (pVM->hwaccm.s.fNestedPaging)
1379	{ /* A genuine pagefault.
1380	* Forward the trap to the guest by injecting the exception and resuming execution.
1381	*/
1382	Log(("Guest page fault at %RGv cr2=%RGv error code %x rsp=%RGv\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode, (RTGCPTR)pCtx->rsp));
1383	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
1384
1385	/* Now we must update CR2. */
1386	pCtx->cr2 = uFaultAddress;
1387
1388	Event.au64[0] = 0;
1389	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1390	Event.n.u1Valid = 1;
1391	Event.n.u8Vector = X86_XCPT_PF;
1392	Event.n.u1ErrorCodeValid = 1;
1393	Event.n.u32ErrorCode = errCode;
1394
1395	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1396
1397	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1398	goto ResumeExecution;
1399	}
1400	#endif
1401	Assert(!pVM->hwaccm.s.fNestedPaging);
1402
1403	Log2(("Page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1404	/* Exit qualification contains the linear address of the page fault. */
1405	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
1406	TRPMSetErrorCode(pVM, errCode);
1407	TRPMSetFaultAddress(pVM, uFaultAddress);
1408
1409	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
1410	rc = PGMTrap0eHandler(pVM, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
1411	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
1412	if (rc == VINF_SUCCESS)
1413	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
1414	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1415	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
1416
1417	TRPMResetTrap(pVM);
1418
1419	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1420	goto ResumeExecution;
1421	}
1422	else
1423	if (rc == VINF_EM_RAW_GUEST_TRAP)
1424	{ /* A genuine pagefault.
1425	* Forward the trap to the guest by injecting the exception and resuming execution.
1426	*/
1427	Log2(("Forward page fault to the guest\n"));
1428	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
1429	/* The error code might have been changed. */
1430	errCode = TRPMGetErrorCode(pVM);
1431
1432	TRPMResetTrap(pVM);
1433
1434	/* Now we must update CR2. */
1435	pCtx->cr2 = uFaultAddress;
1436
1437	Event.au64[0] = 0;
1438	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1439	Event.n.u1Valid = 1;
1440	Event.n.u8Vector = X86_XCPT_PF;
1441	Event.n.u1ErrorCodeValid = 1;
1442	Event.n.u32ErrorCode = errCode;
1443
1444	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1445
1446	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1447	goto ResumeExecution;
1448	}
1449	#ifdef VBOX_STRICT
1450	if (rc != VINF_EM_RAW_EMULATE_INSTR)
1451	LogFlow(("PGMTrap0eHandler failed with %d\n", rc));
1452	#endif
1453	/* Need to go back to the recompiler to emulate the instruction. */
1454	TRPMResetTrap(pVM);
1455	break;
1456	}
1457
1458	case X86_XCPT_MF: /* Floating point exception. */
1459	{
1460	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
1461	if (!(pCtx->cr0 & X86_CR0_NE))
1462	{
1463	/* old style FPU error reporting needs some extra work. */
1464	/** @todo don't fall back to the recompiler, but do it manually. */
1465	rc = VINF_EM_RAW_EMULATE_INSTR;
1466	break;
1467	}
1468	Log(("Trap %x at %RGv\n", vector, (RTGCPTR)pCtx->rip));
1469
1470	Event.au64[0] = 0;
1471	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1472	Event.n.u1Valid = 1;
1473	Event.n.u8Vector = X86_XCPT_MF;
1474
1475	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1476
1477	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1478	goto ResumeExecution;
1479	}
1480
1481	#ifdef VBOX_STRICT
1482	case X86_XCPT_GP: /* General protection failure exception.*/
1483	case X86_XCPT_UD: /* Unknown opcode exception. */
1484	case X86_XCPT_DE: /* Divide error. */
1485	case X86_XCPT_SS: /* Stack segment exception. */
1486	case X86_XCPT_NP: /* Segment not present exception. */
1487	{
1488	Event.au64[0] = 0;
1489	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1490	Event.n.u1Valid = 1;
1491	Event.n.u8Vector = vector;
1492
1493	switch(vector)
1494	{
1495	case X86_XCPT_GP:
1496	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
1497	Event.n.u1ErrorCodeValid = 1;
1498	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1499	break;
1500	case X86_XCPT_DE:
1501	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
1502	break;
1503	case X86_XCPT_UD:
1504	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
1505	break;
1506	case X86_XCPT_SS:
1507	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
1508	Event.n.u1ErrorCodeValid = 1;
1509	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1510	break;
1511	case X86_XCPT_NP:
1512	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
1513	Event.n.u1ErrorCodeValid = 1;
1514	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1515	break;
1516	}
1517	Log(("Trap %x at %RGv esi=%x\n", vector, (RTGCPTR)pCtx->rip, pCtx->esi));
1518	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1519
1520	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1521	goto ResumeExecution;
1522	}
1523	#endif
1524	default:
1525	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
1526	rc = VERR_EM_INTERNAL_ERROR;
1527	break;
1528
1529	} /* switch (vector) */
1530	break;
1531	}
1532
1533	case SVM_EXIT_NPF:
1534	{
1535	/* EXITINFO1 contains fault errorcode; EXITINFO2 contains the guest physical address causing the fault. */
1536	uint32_t errCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1537	RTGCPHYS uFaultAddress = pVMCB->ctrl.u64ExitInfo2; /* EXITINFO2 = fault address */
1538	PGMMODE enmShwPagingMode;
1539
1540	Assert(pVM->hwaccm.s.fNestedPaging);
1541	Log(("Nested page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1542	/* Exit qualification contains the linear address of the page fault. */
1543	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
1544	TRPMSetErrorCode(pVM, errCode);
1545	TRPMSetFaultAddress(pVM, uFaultAddress);
1546
1547	/* Handle the pagefault trap for the nested shadow table. */
1548	#if HC_ARCH_BITS == 32
1549	if (CPUMIsGuestInLongModeEx(pCtx))
1550	enmShwPagingMode = PGMMODE_AMD64_NX;
1551	else
1552	#endif
1553	enmShwPagingMode = PGMGetHostMode(pVM);
1554
1555	rc = PGMR0Trap0eHandlerNestedPaging(pVM, enmShwPagingMode, errCode, CPUMCTX2CORE(pCtx), uFaultAddress);
1556	Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
1557	if (rc == VINF_SUCCESS)
1558	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
1559	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1560	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
1561
1562	TRPMResetTrap(pVM);
1563
1564	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1565	goto ResumeExecution;
1566	}
1567
1568	#ifdef VBOX_STRICT
1569	if (rc != VINF_EM_RAW_EMULATE_INSTR)
1570	LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", rc));
1571	#endif
1572	/* Need to go back to the recompiler to emulate the instruction. */
1573	TRPMResetTrap(pVM);
1574	break;
1575	}
1576
1577	case SVM_EXIT_VINTR:
1578	/* A virtual interrupt is about to be delivered, which means IF=1. */
1579	Log(("SVM_EXIT_VINTR IF=%d\n", pCtx->eflags.Bits.u1IF));
1580	pVMCB->ctrl.IntCtrl.n.u1VIrqValid = 0;
1581	pVMCB->ctrl.IntCtrl.n.u8VIrqVector = 0;
1582	goto ResumeExecution;
1583
1584	case SVM_EXIT_FERR_FREEZE:
1585	case SVM_EXIT_INTR:
1586	case SVM_EXIT_NMI:
1587	case SVM_EXIT_SMI:
1588	case SVM_EXIT_INIT:
1589	/* External interrupt; leave to allow it to be dispatched again. */
1590	rc = VINF_EM_RAW_INTERRUPT;
1591	break;
1592
1593	case SVM_EXIT_WBINVD:
1594	case SVM_EXIT_INVD: /* Guest software attempted to execute INVD. */
1595	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
1596	/* Skip instruction and continue directly. */
1597	pCtx->rip += 2; /* Note! hardcoded opcode size! */
1598	/* Continue execution.*/
1599	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1600	goto ResumeExecution;
1601
1602	case SVM_EXIT_CPUID: /* Guest software attempted to execute CPUID. */
1603	{
1604	Log2(("SVM: Cpuid at %RGv for %x\n", (RTGCPTR)pCtx->rip, pCtx->eax));
1605	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
1606	rc = EMInterpretCpuId(pVM, CPUMCTX2CORE(pCtx));
1607	if (rc == VINF_SUCCESS)
1608	{
1609	/* Update EIP and continue execution. */
1610	pCtx->rip += 2; /* Note! hardcoded opcode size! */
1611	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1612	goto ResumeExecution;
1613	}
1614	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", rc));
1615	rc = VINF_EM_RAW_EMULATE_INSTR;
1616	break;
1617	}
1618
1619	case SVM_EXIT_RDTSC: /* Guest software attempted to execute RDTSC. */
1620	{
1621	Log2(("SVM: Rdtsc\n"));
1622	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
1623	rc = EMInterpretRdtsc(pVM, CPUMCTX2CORE(pCtx));
1624	if (rc == VINF_SUCCESS)
1625	{
1626	/* Update EIP and continue execution. */
1627	pCtx->rip += 2; /* Note! hardcoded opcode size! */
1628	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1629	goto ResumeExecution;
1630	}
1631	AssertMsgFailed(("EMU: rdtsc failed with %Rrc\n", rc));
1632	rc = VINF_EM_RAW_EMULATE_INSTR;
1633	break;
1634	}
1635
1636	case SVM_EXIT_RDTSCP: /* Guest software attempted to execute RDTSCP. */
1637	{
1638	Log2(("SVM: Rdtscp\n"));
1639	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
1640	rc = EMInterpretRdtscp(pVM, pCtx);
1641	if (rc == VINF_SUCCESS)
1642	{
1643	/* Update EIP and continue execution. */
1644	pCtx->rip += 3; /* Note! hardcoded opcode size! */
1645	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1646	goto ResumeExecution;
1647	}
1648	AssertMsgFailed(("EMU: rdtscp failed with %Rrc\n", rc));
1649	rc = VINF_EM_RAW_EMULATE_INSTR;
1650	break;
1651	}
1652
1653	case SVM_EXIT_INVLPG: /* Guest software attempted to execute INVPG. */
1654	{
1655	Log2(("SVM: invlpg\n"));
1656	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
1657
1658	Assert(!pVM->hwaccm.s.fNestedPaging);
1659
1660	/* Truly a pita. Why can't SVM give the same information as VT-x? */
1661	rc = SVMR0InterpretInvpg(pVM, CPUMCTX2CORE(pCtx), pVMCB->ctrl.TLBCtrl.n.u32ASID);
1662	if (rc == VINF_SUCCESS)
1663	{
1664	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushPageInvlpg);
1665	goto ResumeExecution; /* eip already updated */
1666	}
1667	break;
1668	}
1669
1670	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
1671	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
1672	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
1673	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
1674	{
1675	uint32_t cbSize;
1676
1677	Log2(("SVM: %RGv mov cr%d, \n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_WRITE_CR0));
1678	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[exitCode - SVM_EXIT_WRITE_CR0]);
1679	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1680
1681	switch (exitCode - SVM_EXIT_WRITE_CR0)
1682	{
1683	case 0:
1684	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
1685	break;
1686	case 2:
1687	break;
1688	case 3:
1689	Assert(!pVM->hwaccm.s.fNestedPaging);
1690	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
1691	break;
1692	case 4:
1693	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
1694	break;
1695	case 8:
1696	break;
1697	default:
1698	AssertFailed();
1699	}
1700	/* Check if a sync operation is pending. */
1701	if ( rc == VINF_SUCCESS /* don't bother if we are going to ring 3 anyway */
1702	&& VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL))
1703	{
1704	rc = PGMSyncCR3(pVM, CPUMGetGuestCR0(pVM), CPUMGetGuestCR3(pVM), CPUMGetGuestCR4(pVM), VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3));
1705	AssertRC(rc);
1706
1707	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBCRxChange);
1708
1709	/* Must be set by PGMSyncCR3 */
1710	Assert(PGMGetGuestMode(pVM) <= PGMMODE_PROTECTED \|\| pVCpu->hwaccm.s.fForceTLBFlush);
1711	}
1712	if (rc == VINF_SUCCESS)
1713	{
1714	/* EIP has been updated already. */
1715
1716	/* Only resume if successful. */
1717	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1718	goto ResumeExecution;
1719	}
1720	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1721	break;
1722	}
1723
1724	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
1725	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
1726	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
1727	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
1728	{
1729	uint32_t cbSize;
1730
1731	Log2(("SVM: %RGv mov x, cr%d\n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_READ_CR0));
1732	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[exitCode - SVM_EXIT_WRITE_CR0]);
1733	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1734	if (rc == VINF_SUCCESS)
1735	{
1736	/* EIP has been updated already. */
1737
1738	/* Only resume if successful. */
1739	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1740	goto ResumeExecution;
1741	}
1742	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1743	break;
1744	}
1745
1746	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
1747	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
1748	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
1749	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
1750	{
1751	uint32_t cbSize;
1752
1753	Log2(("SVM: %RGv mov dr%d, x\n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_WRITE_DR0));
1754	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
1755
1756	if (!DBGFIsStepping(pVM))
1757	{
1758	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
1759
1760	/* Disable drx move intercepts. */
1761	pVMCB->ctrl.u16InterceptRdDRx = 0;
1762	pVMCB->ctrl.u16InterceptWrDRx = 0;
1763
1764	/* Save the host and load the guest debug state. */
1765	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
1766	AssertRC(rc);
1767
1768	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1769	goto ResumeExecution;
1770	}
1771
1772	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1773	if (rc == VINF_SUCCESS)
1774	{
1775	/* EIP has been updated already. */
1776	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
1777
1778	/* Only resume if successful. */
1779	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1780	goto ResumeExecution;
1781	}
1782	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1783	break;
1784	}
1785
1786	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
1787	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
1788	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
1789	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
1790	{
1791	uint32_t cbSize;
1792
1793	Log2(("SVM: %RGv mov dr%d, x\n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_READ_DR0));
1794	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
1795
1796	if (!DBGFIsStepping(pVM))
1797	{
1798	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
1799
1800	/* Disable drx move intercepts. */
1801	pVMCB->ctrl.u16InterceptRdDRx = 0;
1802	pVMCB->ctrl.u16InterceptWrDRx = 0;
1803
1804	/* Save the host and load the guest debug state. */
1805	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
1806	AssertRC(rc);
1807
1808	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1809	goto ResumeExecution;
1810	}
1811
1812	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1813	if (rc == VINF_SUCCESS)
1814	{
1815	/* EIP has been updated already. */
1816
1817	/* Only resume if successful. */
1818	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1819	goto ResumeExecution;
1820	}
1821	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1822	break;
1823	}
1824
1825	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
1826	case SVM_EXIT_IOIO: /* I/O instruction. */
1827	{
1828	SVM_IOIO_EXIT IoExitInfo;
1829	uint32_t uIOSize, uAndVal;
1830
1831	IoExitInfo.au32[0] = pVMCB->ctrl.u64ExitInfo1;
1832
1833	/** @todo could use a lookup table here */
1834	if (IoExitInfo.n.u1OP8)
1835	{
1836	uIOSize = 1;
1837	uAndVal = 0xff;
1838	}
1839	else
1840	if (IoExitInfo.n.u1OP16)
1841	{
1842	uIOSize = 2;
1843	uAndVal = 0xffff;
1844	}
1845	else
1846	if (IoExitInfo.n.u1OP32)
1847	{
1848	uIOSize = 4;
1849	uAndVal = 0xffffffff;
1850	}
1851	else
1852	{
1853	AssertFailed(); /* should be fatal. */
1854	rc = VINF_EM_RAW_EMULATE_INSTR;
1855	break;
1856	}
1857
1858	if (IoExitInfo.n.u1STR)
1859	{
1860	/* ins/outs */
1861	DISCPUSTATE Cpu;
1862
1863	/* Disassemble manually to deal with segment prefixes. */
1864	rc = EMInterpretDisasOne(pVM, CPUMCTX2CORE(pCtx), &Cpu, NULL);
1865	if (rc == VINF_SUCCESS)
1866	{
1867	if (IoExitInfo.n.u1Type == 0)
1868	{
1869	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
1870	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
1871	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, Cpu.prefix, uIOSize);
1872	}
1873	else
1874	{
1875	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
1876	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
1877	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, Cpu.prefix, uIOSize);
1878	}
1879	}
1880	else
1881	rc = VINF_EM_RAW_EMULATE_INSTR;
1882	}
1883	else
1884	{
1885	/* normal in/out */
1886	Assert(!IoExitInfo.n.u1REP);
1887
1888	if (IoExitInfo.n.u1Type == 0)
1889	{
1890	Log2(("IOMIOPortWrite %RGv %x %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, uIOSize));
1891	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
1892	rc = IOMIOPortWrite(pVM, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, uIOSize);
1893	}
1894	else
1895	{
1896	uint32_t u32Val = 0;
1897
1898	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
1899	rc = IOMIOPortRead(pVM, IoExitInfo.n.u16Port, &u32Val, uIOSize);
1900	if (IOM_SUCCESS(rc))
1901	{
1902	/* Write back to the EAX register. */
1903	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
1904	Log2(("IOMIOPortRead %RGv %x %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, u32Val & uAndVal, uIOSize));
1905	}
1906	}
1907	}
1908	/*
1909	* Handled the I/O return codes.
1910	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
1911	*/
1912	if (IOM_SUCCESS(rc))
1913	{
1914	/* Update EIP and continue execution. */
1915	pCtx->rip = pVMCB->ctrl.u64ExitInfo2; /* RIP/EIP of the next instruction is saved in EXITINFO2. */
1916	if (RT_LIKELY(rc == VINF_SUCCESS))
1917	{
1918	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
1919	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
1920	{
1921	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
1922	for (unsigned i=0;i<4;i++)
1923	{
1924	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
1925
1926	if ( (IoExitInfo.n.u16Port >= pCtx->dr[i] && IoExitInfo.n.u16Port < pCtx->dr[i] + uBPLen)
1927	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
1928	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
1929	{
1930	SVM_EVENT Event;
1931
1932	Assert(CPUMIsGuestDebugStateActive(pVM));
1933
1934	/* Clear all breakpoint status flags and set the one we just hit. */
1935	pCtx->dr[6] &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
1936	pCtx->dr[6] \|= (uint64_t)RT_BIT(i);
1937
1938	/* Note: AMD64 Architecture Programmer's Manual 13.1:
1939	* Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
1940	* the contents have been read.
1941	*/
1942	pVMCB->guest.u64DR6 = pCtx->dr[6];
1943
1944	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
1945	pCtx->dr[7] &= ~X86_DR7_GD;
1946
1947	/* Paranoia. */
1948	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1949	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
1950	pCtx->dr[7] \|= 0x400; /* must be one */
1951
1952	pVMCB->guest.u64DR7 = pCtx->dr[7];
1953
1954	/* Inject the exception. */
1955	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
1956
1957	Event.au64[0] = 0;
1958	Event.n.u3Type = SVM_EVENT_EXCEPTION; /* trap or fault */
1959	Event.n.u1Valid = 1;
1960	Event.n.u8Vector = X86_XCPT_DB;
1961
1962	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1963
1964	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1965	goto ResumeExecution;
1966	}
1967	}
1968	}
1969
1970	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
1971	goto ResumeExecution;
1972	}
1973	Log2(("EM status from IO at %RGv %x size %d: %Rrc\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize, rc));
1974	break;
1975	}
1976
1977	#ifdef VBOX_STRICT
1978	if (rc == VINF_IOM_HC_IOPORT_READ)
1979	Assert(IoExitInfo.n.u1Type != 0);
1980	else if (rc == VINF_IOM_HC_IOPORT_WRITE)
1981	Assert(IoExitInfo.n.u1Type == 0);
1982	else
1983	AssertMsg(RT_FAILURE(rc) \|\| rc == VINF_EM_RAW_EMULATE_INSTR \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
1984	#endif
1985	Log2(("Failed IO at %RGv %x size %d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
1986	break;
1987	}
1988
1989	case SVM_EXIT_HLT:
1990	/** Check if external interrupts are pending; if so, don't switch back. */
1991	pCtx->rip++; /* skip hlt */
1992	if ( pCtx->eflags.Bits.u1IF
1993	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
1994	goto ResumeExecution;
1995
1996	rc = VINF_EM_HALT;
1997	break;
1998
1999	case SVM_EXIT_RSM:
2000	case SVM_EXIT_INVLPGA:
2001	case SVM_EXIT_VMRUN:
2002	case SVM_EXIT_VMMCALL:
2003	case SVM_EXIT_VMLOAD:
2004	case SVM_EXIT_VMSAVE:
2005	case SVM_EXIT_STGI:
2006	case SVM_EXIT_CLGI:
2007	case SVM_EXIT_SKINIT:
2008	{
2009	/* Unsupported instructions. */
2010	SVM_EVENT Event;
2011
2012	Event.au64[0] = 0;
2013	Event.n.u3Type = SVM_EVENT_EXCEPTION;
2014	Event.n.u1Valid = 1;
2015	Event.n.u8Vector = X86_XCPT_UD;
2016
2017	Log(("Forced #UD trap at %RGv\n", (RTGCPTR)pCtx->rip));
2018	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
2019
2020	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
2021	goto ResumeExecution;
2022	}
2023
2024	/* Emulate in ring 3. */
2025	case SVM_EXIT_MSR:
2026	{
2027	uint32_t cbSize;
2028
2029	/* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
2030	Log(("SVM: %s\n", (pVMCB->ctrl.u64ExitInfo1 == 0) ? "rdmsr" : "wrmsr"));
2031	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
2032	if (rc == VINF_SUCCESS)
2033	{
2034	/* EIP has been updated already. */
2035
2036	/* Only resume if successful. */
2037	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
2038	goto ResumeExecution;
2039	}
2040	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (pVMCB->ctrl.u64ExitInfo1 == 0) ? "rdmsr" : "wrmsr", rc));
2041	break;
2042	}
2043
2044	case SVM_EXIT_MONITOR:
2045	case SVM_EXIT_RDPMC:
2046	case SVM_EXIT_PAUSE:
2047	case SVM_EXIT_MWAIT_UNCOND:
2048	case SVM_EXIT_MWAIT_ARMED:
2049	case SVM_EXIT_TASK_SWITCH: /* can change CR3; emulate */
2050	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
2051	break;
2052
2053	case SVM_EXIT_SHUTDOWN:
2054	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
2055	break;
2056
2057	case SVM_EXIT_IDTR_READ:
2058	case SVM_EXIT_GDTR_READ:
2059	case SVM_EXIT_LDTR_READ:
2060	case SVM_EXIT_TR_READ:
2061	case SVM_EXIT_IDTR_WRITE:
2062	case SVM_EXIT_GDTR_WRITE:
2063	case SVM_EXIT_LDTR_WRITE:
2064	case SVM_EXIT_TR_WRITE:
2065	case SVM_EXIT_CR0_SEL_WRITE:
2066	default:
2067	/* Unexpected exit codes. */
2068	rc = VERR_EM_INTERNAL_ERROR;
2069	AssertMsgFailed(("Unexpected exit code %x\n", exitCode)); /* Can't happen. */
2070	break;
2071	}
2072
2073	end:
2074
2075	/* Signal changes for the recompiler. */
2076	CPUMSetChangedFlags(pVM, CPUM_CHANGED_SYSENTER_MSR \| CPUM_CHANGED_LDTR \| CPUM_CHANGED_GDTR \| CPUM_CHANGED_IDTR \| CPUM_CHANGED_TR \| CPUM_CHANGED_HIDDEN_SEL_REGS);
2077
2078	/* If we executed vmrun and an external irq was pending, then we don't have to do a full sync the next time. */
2079	if (exitCode == SVM_EXIT_INTR)
2080	{
2081	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
2082	/* On the next entry we'll only sync the host context. */
2083	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
2084	}
2085	else
2086	{
2087	/* On the next entry we'll sync everything. */
2088	/** @todo we can do better than this */
2089	/* Not in the VINF_PGM_CHANGE_MODE though! */
2090	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
2091	}
2092
2093	/* translate into a less severe return code */
2094	if (rc == VERR_EM_INTERPRETER)
2095	rc = VINF_EM_RAW_EMULATE_INSTR;
2096
2097	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit, x);
2098	return rc;
2099	}
2100
2101	/**
2102	* Enters the AMD-V session
2103	*
2104	* @returns VBox status code.
2105	* @param pVM The VM to operate on.
2106	* @param pVCpu The VM CPU to operate on.
2107	* @param pCpu CPU info struct
2108	*/
2109	VMMR0DECL(int) SVMR0Enter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
2110	{
2111	Assert(pVM->hwaccm.s.svm.fSupported);
2112
2113	LogFlow(("SVMR0Enter cpu%d last=%d asid=%d\n", pCpu->idCpu, pVCpu->hwaccm.s.idLastCpu, pVCpu->hwaccm.s.uCurrentASID));
2114	pVCpu->hwaccm.s.fResumeVM = false;
2115
2116	/* Force to reload LDTR, so we'll execute VMLoad to load additional guest state. */
2117	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_LDTR;
2118
2119	return VINF_SUCCESS;
2120	}
2121
2122
2123	/**
2124	* Leaves the AMD-V session
2125	*
2126	* @returns VBox status code.
2127	* @param pVM The VM to operate on.
2128	* @param pVCpu The VM CPU to operate on.
2129	* @param pCtx CPU context
2130	*/
2131	VMMR0DECL(int) SVMR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2132	{
2133	SVM_VMCB pVMCB = (SVM_VMCB )pVCpu->hwaccm.s.svm.pVMCB;
2134
2135	Assert(pVM->hwaccm.s.svm.fSupported);
2136
2137	/* Save the guest debug state if necessary. */
2138	if (CPUMIsGuestDebugStateActive(pVM))
2139	{
2140	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, false /* skip DR6 */);
2141
2142	/* Intercept all DRx reads and writes again. Changed later on. */
2143	pVMCB->ctrl.u16InterceptRdDRx = 0xFFFF;
2144	pVMCB->ctrl.u16InterceptWrDRx = 0xFFFF;
2145
2146	/* Resync the debug registers the next time. */
2147	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
2148	}
2149	else
2150	Assert(pVMCB->ctrl.u16InterceptRdDRx == 0xFFFF && pVMCB->ctrl.u16InterceptWrDRx == 0xFFFF);
2151
2152	return VINF_SUCCESS;
2153	}
2154
2155
2156	static int svmR0InterpretInvlPg(PVM pVM, PDISCPUSTATE pCpu, PCPUMCTXCORE pRegFrame, uint32_t uASID)
2157	{
2158	OP_PARAMVAL param1;
2159	RTGCPTR addr;
2160
2161	int rc = DISQueryParamVal(pRegFrame, pCpu, &pCpu->param1, &param1, PARAM_SOURCE);
2162	if(RT_FAILURE(rc))
2163	return VERR_EM_INTERPRETER;
2164
2165	switch(param1.type)
2166	{
2167	case PARMTYPE_IMMEDIATE:
2168	case PARMTYPE_ADDRESS:
2169	if(!(param1.flags & (PARAM_VAL32\|PARAM_VAL64)))
2170	return VERR_EM_INTERPRETER;
2171	addr = param1.val.val64;
2172	break;
2173
2174	default:
2175	return VERR_EM_INTERPRETER;
2176	}
2177
2178	/** @todo is addr always a flat linear address or ds based
2179	* (in absence of segment override prefixes)????
2180	*/
2181	rc = PGMInvalidatePage(pVM, addr);
2182	if (RT_SUCCESS(rc))
2183	{
2184	/* Manually invalidate the page for the VM's TLB. */
2185	Log(("SVMR0InvlpgA %RGv ASID=%d\n", addr, uASID));
2186	SVMR0InvlpgA(addr, uASID);
2187	return VINF_SUCCESS;
2188	}
2189	Assert(rc == VERR_REM_FLUSHED_PAGES_OVERFLOW);
2190	return rc;
2191	}
2192
2193	/**
2194	* Interprets INVLPG
2195	*
2196	* @returns VBox status code.
2197	* @retval VINF_* Scheduling instructions.
2198	* @retval VERR_EM_INTERPRETER Something we can't cope with.
2199	* @retval VERR_* Fatal errors.
2200	*
2201	* @param pVM The VM handle.
2202	* @param pRegFrame The register frame.
2203	* @param ASID Tagged TLB id for the guest
2204	*
2205	* Updates the EIP if an instruction was executed successfully.
2206	*/
2207	static int SVMR0InterpretInvpg(PVM pVM, PCPUMCTXCORE pRegFrame, uint32_t uASID)
2208	{
2209	/*
2210	* Only allow 32 & 64 bits code.
2211	*/
2212	DISCPUMODE enmMode = SELMGetCpuModeFromSelector(pVM, pRegFrame->eflags, pRegFrame->cs, &pRegFrame->csHid);
2213	if (enmMode != CPUMODE_16BIT)
2214	{
2215	RTGCPTR pbCode;
2216	int rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->rip, &pbCode);
2217	if (RT_SUCCESS(rc))
2218	{
2219	uint32_t cbOp;
2220	DISCPUSTATE Cpu;
2221
2222	Cpu.mode = enmMode;
2223	rc = EMInterpretDisasOneEx(pVM, pbCode, pRegFrame, &Cpu, &cbOp);
2224	Assert(RT_FAILURE(rc) \|\| Cpu.pCurInstr->opcode == OP_INVLPG);
2225	if (RT_SUCCESS(rc) && Cpu.pCurInstr->opcode == OP_INVLPG)
2226	{
2227	Assert(cbOp == Cpu.opsize);
2228	rc = svmR0InterpretInvlPg(pVM, &Cpu, pRegFrame, uASID);
2229	if (RT_SUCCESS(rc))
2230	{
2231	pRegFrame->rip += cbOp; /* Move on to the next instruction. */
2232	}
2233	return rc;
2234	}
2235	}
2236	}
2237	return VERR_EM_INTERPRETER;
2238	}
2239
2240
2241	/**
2242	* Invalidates a guest page
2243	*
2244	* @returns VBox status code.
2245	* @param pVM The VM to operate on.
2246	* @param pVCpu The VM CPU to operate on.
2247	* @param GCVirt Page to invalidate
2248	*/
2249	VMMR0DECL(int) SVMR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
2250	{
2251	bool fFlushPending = pVM->hwaccm.s.svm.fAlwaysFlushTLB \| pVCpu->hwaccm.s.fForceTLBFlush;
2252
2253	/* Skip it if a TLB flush is already pending. */
2254	if (!fFlushPending)
2255	{
2256	SVM_VMCB *pVMCB;
2257
2258	Log2(("SVMR0InvalidatePage %RGv\n", GCVirt));
2259	AssertReturn(pVM, VERR_INVALID_PARAMETER);
2260	Assert(pVM->hwaccm.s.svm.fSupported);
2261
2262	/* @todo SMP */
2263	pVMCB = (SVM_VMCB *)pVM->aCpus[0].hwaccm.s.svm.pVMCB;
2264	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
2265
2266	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushPageManual);
2267	#if HC_ARCH_BITS == 32
2268	/* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invlpga takes only 32 bits addresses. */
2269	if (CPUMIsGuestInLongMode(pVM))
2270	pVCpu->hwaccm.s.fForceTLBFlush = true;
2271	else
2272	#endif
2273	SVMR0InvlpgA(GCVirt, pVMCB->ctrl.TLBCtrl.n.u32ASID);
2274	}
2275	return VINF_SUCCESS;
2276	}
2277
2278
2279	/**
2280	* Invalidates a guest page by physical address
2281	*
2282	* @returns VBox status code.
2283	* @param pVM The VM to operate on.
2284	* @param pVCpu The VM CPU to operate on.
2285	* @param GCPhys Page to invalidate
2286	*/
2287	VMMR0DECL(int) SVMR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
2288	{
2289	Assert(pVM->hwaccm.s.fNestedPaging);
2290	/* invlpga only invalidates TLB entries for guest virtual addresses; we have no choice but to force a TLB flush here. */
2291	pVCpu->hwaccm.s.fForceTLBFlush = true;
2292	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBInvlpga);
2293	return VINF_SUCCESS;
2294	}
2295
2296	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
2297	/**
2298	* Prepares for and executes VMRUN (64 bits guests from a 32 bits hosts).
2299	*
2300	* @returns VBox status code.
2301	* @param pVMCBHostPhys Physical address of host VMCB.
2302	* @param pVMCBPhys Physical address of the VMCB.
2303	* @param pCtx Guest context.
2304	* @param pVM The VM to operate on.
2305	* @param pVCpu The VMCPU to operate on.
2306	*/
2307	DECLASM(int) SVMR0VMSwitcherRun64(RTHCPHYS pVMCBHostPhys, RTHCPHYS pVMCBPhys, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu)
2308	{
2309	uint32_t aParam[4];
2310
2311	aParam[0] = (uint32_t)(pVMCBHostPhys); /* Param 1: pVMCBHostPhys - Lo. */
2312	aParam[1] = (uint32_t)(pVMCBHostPhys >> 32); /* Param 1: pVMCBHostPhys - Hi. */
2313	aParam[2] = (uint32_t)(pVMCBPhys); /* Param 2: pVMCBPhys - Lo. */
2314	aParam[3] = (uint32_t)(pVMCBPhys >> 32); /* Param 2: pVMCBPhys - Hi. */
2315
2316	return SVMR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnSVMGCVMRun64, 4, &aParam[0]);
2317	}
2318
2319	/**
2320	* Executes the specified handler in 64 mode
2321	*
2322	* @returns VBox status code.
2323	* @param pVM The VM to operate on.
2324	* @param pVCpu The VMCPU to operate on.
2325	* @param pCtx Guest context
2326	* @param pfnHandler RC handler
2327	* @param cbParam Number of parameters
2328	* @param paParam Array of 32 bits parameters
2329	*/
2330	VMMR0DECL(int) SVMR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
2331	{
2332	int rc;
2333	RTCCUINTREG uFlags;
2334
2335	/* @todo This code is not guest SMP safe (hyper context) */
2336	AssertReturn(pVM->cCPUs == 1, VERR_ACCESS_DENIED);
2337	Assert(pfnHandler);
2338
2339	uFlags = ASMIntDisableFlags();
2340
2341	CPUMSetHyperESP(pVM, VMMGetStackRC(pVM));
2342	CPUMSetHyperEIP(pVM, pfnHandler);
2343	for (int i=(int)cbParam-1;i>=0;i--)
2344	CPUMPushHyper(pVM, paParam[i]);
2345
2346	/* Call switcher. */
2347	rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM);
2348
2349	ASMSetFlags(uFlags);
2350	return rc;
2351	}
2352
2353	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWSVMR0.cpp@ 15189

Download in other formats: