HWSVMR0.cpp@ 20876

Last change on this file since 20876 was 20846, checked in by vboxsync, 16 years ago
If an active trap is already pending, then we must forward it first!
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 112.1 KB

Line
1	/* $Id: HWSVMR0.cpp 20846 2009-06-23 14:57:46Z vboxsync $ */
2	/** @file
3	* HWACCM SVM - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/*******************************************************************************
24	* Header Files *
25	*******************************************************************************/
26	#define LOG_GROUP LOG_GROUP_HWACCM
27	#include <VBox/hwaccm.h>
28	#include "HWACCMInternal.h"
29	#include <VBox/vm.h>
30	#include <VBox/x86.h>
31	#include <VBox/hwacc_svm.h>
32	#include <VBox/pgm.h>
33	#include <VBox/pdm.h>
34	#include <VBox/err.h>
35	#include <VBox/log.h>
36	#include <VBox/selm.h>
37	#include <VBox/iom.h>
38	#include <VBox/dis.h>
39	#include <VBox/dbgf.h>
40	#include <VBox/disopcode.h>
41	#include <iprt/param.h>
42	#include <iprt/assert.h>
43	#include <iprt/asm.h>
44	#include <iprt/cpuset.h>
45	#include <iprt/mp.h>
46	#include <iprt/time.h>
47	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
48	# include <iprt/thread.h>
49	#endif
50	#include "HWSVMR0.h"
51
52	/*******************************************************************************
53	* Internal Functions *
54	*******************************************************************************/
55	static int svmR0InterpretInvpg(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame, uint32_t uASID);
56	static int svmR0ReplaceTprInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
57
58	/*******************************************************************************
59	* Global Variables *
60	*******************************************************************************/
61	/* IO operation lookup arrays. */
62	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
63
64	/**
65	* Sets up and activates AMD-V on the current CPU
66	*
67	* @returns VBox status code.
68	* @param pCpu CPU info struct
69	* @param pVM The VM to operate on. (can be NULL after a resume!!)
70	* @param pvPageCpu Pointer to the global cpu page
71	* @param pPageCpuPhys Physical address of the global cpu page
72	*/
73	VMMR0DECL(int) SVMR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
74	{
75	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
76	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
77
78	/* We must turn on AMD-V and setup the host state physical address, as those MSRs are per-cpu/core. */
79
80	#if defined(LOG_ENABLED) && !defined(DEBUG_bird)
81	SUPR0Printf("SVMR0EnableCpu cpu %d page (%x) %x\n", pCpu->idCpu, pvPageCpu, (uint32_t)pPageCpuPhys);
82	#endif
83
84	/* Turn on AMD-V in the EFER MSR. */
85	uint64_t val = ASMRdMsr(MSR_K6_EFER);
86	if (!(val & MSR_K6_EFER_SVME))
87	ASMWrMsr(MSR_K6_EFER, val \| MSR_K6_EFER_SVME);
88
89	/* Write the physical page address where the CPU will store the host state while executing the VM. */
90	ASMWrMsr(MSR_K8_VM_HSAVE_PA, pPageCpuPhys);
91
92	return VINF_SUCCESS;
93	}
94
95	/**
96	* Deactivates AMD-V on the current CPU
97	*
98	* @returns VBox status code.
99	* @param pCpu CPU info struct
100	* @param pvPageCpu Pointer to the global cpu page
101	* @param pPageCpuPhys Physical address of the global cpu page
102	*/
103	VMMR0DECL(int) SVMR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
104	{
105	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
106	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
107
108	#if defined(LOG_ENABLED) && !defined(DEBUG_bird)
109	SUPR0Printf("SVMR0DisableCpu cpu %d\n", pCpu->idCpu);
110	#endif
111
112	/* Turn off AMD-V in the EFER MSR. */
113	uint64_t val = ASMRdMsr(MSR_K6_EFER);
114	ASMWrMsr(MSR_K6_EFER, val & ~MSR_K6_EFER_SVME);
115
116	/* Invalidate host state physical address. */
117	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
118
119	return VINF_SUCCESS;
120	}
121
122	/**
123	* Does Ring-0 per VM AMD-V init.
124	*
125	* @returns VBox status code.
126	* @param pVM The VM to operate on.
127	*/
128	VMMR0DECL(int) SVMR0InitVM(PVM pVM)
129	{
130	int rc;
131
132	pVM->hwaccm.s.svm.pMemObjIOBitmap = NIL_RTR0MEMOBJ;
133	pVM->hwaccm.s.svm.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
134
135	/* Allocate 12 KB for the IO bitmap (doesn't seem to be a way to convince SVM not to use it) */
136	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjIOBitmap, 3 << PAGE_SHIFT, true /* executable R0 mapping */);
137	if (RT_FAILURE(rc))
138	return rc;
139
140	pVM->hwaccm.s.svm.pIOBitmap = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjIOBitmap);
141	pVM->hwaccm.s.svm.pIOBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjIOBitmap, 0);
142	/* Set all bits to intercept all IO accesses. */
143	ASMMemFill32(pVM->hwaccm.s.svm.pIOBitmap, PAGE_SIZE*3, 0xffffffff);
144
145	/* Allocate 8 KB for the MSR bitmap (doesn't seem to be a way to convince SVM not to use it) */
146	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjMSRBitmap, 2 << PAGE_SHIFT, true /* executable R0 mapping */);
147	if (RT_FAILURE(rc))
148	return rc;
149
150	pVM->hwaccm.s.svm.pMSRBitmap = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjMSRBitmap);
151	pVM->hwaccm.s.svm.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjMSRBitmap, 0);
152	/* Set all bits to intercept all MSR accesses. */
153	ASMMemFill32(pVM->hwaccm.s.svm.pMSRBitmap, PAGE_SIZE*2, 0xffffffff);
154
155	/* Erratum 170 which requires a forced TLB flush for each world switch:
156	* See http://www.amd.com/us-en/assets/content_type/white_papers_and_tech_docs/33610.pdf
157	*
158	* All BH-G1/2 and DH-G1/2 models include a fix:
159	* Athlon X2: 0x6b 1/2
160	* 0x68 1/2
161	* Athlon 64: 0x7f 1
162	* 0x6f 2
163	* Sempron: 0x7f 1/2
164	* 0x6f 2
165	* 0x6c 2
166	* 0x7c 2
167	* Turion 64: 0x68 2
168	*
169	*/
170	uint32_t u32Dummy;
171	uint32_t u32Version, u32Family, u32Model, u32Stepping, u32BaseFamily;
172	ASMCpuId(1, &u32Version, &u32Dummy, &u32Dummy, &u32Dummy);
173	u32BaseFamily= (u32Version >> 8) & 0xf;
174	u32Family = u32BaseFamily + (u32BaseFamily == 0xf ? ((u32Version >> 20) & 0x7f) : 0);
175	u32Model = ((u32Version >> 4) & 0xf);
176	u32Model = u32Model \| ((u32BaseFamily == 0xf ? (u32Version >> 16) & 0x0f : 0) << 4);
177	u32Stepping = u32Version & 0xf;
178	if ( u32Family == 0xf
179	&& !((u32Model == 0x68 \|\| u32Model == 0x6b \|\| u32Model == 0x7f) && u32Stepping >= 1)
180	&& !((u32Model == 0x6f \|\| u32Model == 0x6c \|\| u32Model == 0x7c) && u32Stepping >= 2))
181	{
182	Log(("SVMR0InitVM: AMD cpu with erratum 170 family %x model %x stepping %x\n", u32Family, u32Model, u32Stepping));
183	pVM->hwaccm.s.svm.fAlwaysFlushTLB = true;
184	}
185
186	/* Allocate VMCBs for all guest CPUs. */
187	for (unsigned i=0;i<pVM->cCPUs;i++)
188	{
189	PVMCPU pVCpu = &pVM->aCpus[i];
190
191	pVCpu->hwaccm.s.svm.pMemObjVMCBHost = NIL_RTR0MEMOBJ;
192	pVCpu->hwaccm.s.svm.pMemObjVMCB = NIL_RTR0MEMOBJ;
193
194	/* Allocate one page for the host context */
195	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.svm.pMemObjVMCBHost, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
196	if (RT_FAILURE(rc))
197	return rc;
198
199	pVCpu->hwaccm.s.svm.pVMCBHost = RTR0MemObjAddress(pVCpu->hwaccm.s.svm.pMemObjVMCBHost);
200	pVCpu->hwaccm.s.svm.pVMCBHostPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.svm.pMemObjVMCBHost, 0);
201	ASMMemZeroPage(pVCpu->hwaccm.s.svm.pVMCBHost);
202
203	/* Allocate one page for the VM control block (VMCB). */
204	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.svm.pMemObjVMCB, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
205	if (RT_FAILURE(rc))
206	return rc;
207
208	pVCpu->hwaccm.s.svm.pVMCB = RTR0MemObjAddress(pVCpu->hwaccm.s.svm.pMemObjVMCB);
209	pVCpu->hwaccm.s.svm.pVMCBPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.svm.pMemObjVMCB, 0);
210	ASMMemZeroPage(pVCpu->hwaccm.s.svm.pVMCB);
211	}
212
213	return VINF_SUCCESS;
214	}
215
216	/**
217	* Does Ring-0 per VM AMD-V termination.
218	*
219	* @returns VBox status code.
220	* @param pVM The VM to operate on.
221	*/
222	VMMR0DECL(int) SVMR0TermVM(PVM pVM)
223	{
224	for (unsigned i=0;i<pVM->cCPUs;i++)
225	{
226	PVMCPU pVCpu = &pVM->aCpus[i];
227
228	if (pVCpu->hwaccm.s.svm.pMemObjVMCBHost != NIL_RTR0MEMOBJ)
229	{
230	RTR0MemObjFree(pVCpu->hwaccm.s.svm.pMemObjVMCBHost, false);
231	pVCpu->hwaccm.s.svm.pVMCBHost = 0;
232	pVCpu->hwaccm.s.svm.pVMCBHostPhys = 0;
233	pVCpu->hwaccm.s.svm.pMemObjVMCBHost = NIL_RTR0MEMOBJ;
234	}
235
236	if (pVCpu->hwaccm.s.svm.pMemObjVMCB != NIL_RTR0MEMOBJ)
237	{
238	RTR0MemObjFree(pVCpu->hwaccm.s.svm.pMemObjVMCB, false);
239	pVCpu->hwaccm.s.svm.pVMCB = 0;
240	pVCpu->hwaccm.s.svm.pVMCBPhys = 0;
241	pVCpu->hwaccm.s.svm.pMemObjVMCB = NIL_RTR0MEMOBJ;
242	}
243	}
244	if (pVM->hwaccm.s.svm.pMemObjIOBitmap != NIL_RTR0MEMOBJ)
245	{
246	RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjIOBitmap, false);
247	pVM->hwaccm.s.svm.pIOBitmap = 0;
248	pVM->hwaccm.s.svm.pIOBitmapPhys = 0;
249	pVM->hwaccm.s.svm.pMemObjIOBitmap = NIL_RTR0MEMOBJ;
250	}
251	if (pVM->hwaccm.s.svm.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
252	{
253	RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjMSRBitmap, false);
254	pVM->hwaccm.s.svm.pMSRBitmap = 0;
255	pVM->hwaccm.s.svm.pMSRBitmapPhys = 0;
256	pVM->hwaccm.s.svm.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
257	}
258	return VINF_SUCCESS;
259	}
260
261	/**
262	* Sets up AMD-V for the specified VM
263	*
264	* @returns VBox status code.
265	* @param pVM The VM to operate on.
266	*/
267	VMMR0DECL(int) SVMR0SetupVM(PVM pVM)
268	{
269	int rc = VINF_SUCCESS;
270	SVM_VMCB *pVMCB;
271
272	AssertReturn(pVM, VERR_INVALID_PARAMETER);
273
274	Assert(pVM->hwaccm.s.svm.fSupported);
275
276	for (unsigned i=0;i<pVM->cCPUs;i++)
277	{
278	pVMCB = (SVM_VMCB *)pVM->aCpus[i].hwaccm.s.svm.pVMCB;
279	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
280
281	/* Program the control fields. Most of them never have to be changed again. */
282	/* CR0/3/4 reads must be intercepted, our shadow values are not necessarily the same as the guest's. */
283	/* Note: CR0 & CR4 can be safely read when guest and shadow copies are identical. */
284	if (!pVM->hwaccm.s.fNestedPaging)
285	pVMCB->ctrl.u16InterceptRdCRx = RT_BIT(0) \| RT_BIT(3) \| RT_BIT(4);
286	else
287	pVMCB->ctrl.u16InterceptRdCRx = RT_BIT(0) \| RT_BIT(4);
288
289	/*
290	* CR0/3/4 writes must be intercepted for obvious reasons.
291	*/
292	if (!pVM->hwaccm.s.fNestedPaging)
293	pVMCB->ctrl.u16InterceptWrCRx = RT_BIT(0) \| RT_BIT(3) \| RT_BIT(4);
294	else
295	pVMCB->ctrl.u16InterceptWrCRx = RT_BIT(0) \| RT_BIT(4) \| RT_BIT(8);
296
297	/* Intercept all DRx reads and writes by default. Changed later on. */
298	pVMCB->ctrl.u16InterceptRdDRx = 0xFFFF;
299	pVMCB->ctrl.u16InterceptWrDRx = 0xFFFF;
300
301	/* Currently we don't care about DRx reads or writes. DRx registers are trashed.
302	* All breakpoints are automatically cleared when the VM exits.
303	*/
304
305	pVMCB->ctrl.u32InterceptException = HWACCM_SVM_TRAP_MASK;
306	#ifndef DEBUG
307	if (pVM->hwaccm.s.fNestedPaging)
308	pVMCB->ctrl.u32InterceptException &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
309	#endif
310
311	pVMCB->ctrl.u32InterceptCtrl1 = SVM_CTRL1_INTERCEPT_INTR
312	\| SVM_CTRL1_INTERCEPT_VINTR
313	\| SVM_CTRL1_INTERCEPT_NMI
314	\| SVM_CTRL1_INTERCEPT_SMI
315	\| SVM_CTRL1_INTERCEPT_INIT
316	\| SVM_CTRL1_INTERCEPT_RDPMC
317	\| SVM_CTRL1_INTERCEPT_CPUID
318	\| SVM_CTRL1_INTERCEPT_RSM
319	\| SVM_CTRL1_INTERCEPT_HLT
320	\| SVM_CTRL1_INTERCEPT_INOUT_BITMAP
321	\| SVM_CTRL1_INTERCEPT_MSR_SHADOW
322	\| SVM_CTRL1_INTERCEPT_INVLPG
323	\| SVM_CTRL1_INTERCEPT_INVLPGA /* AMD only */
324	\| SVM_CTRL1_INTERCEPT_TASK_SWITCH
325	\| SVM_CTRL1_INTERCEPT_SHUTDOWN /* fatal */
326	\| SVM_CTRL1_INTERCEPT_FERR_FREEZE; /* Legacy FPU FERR handling. */
327	;
328	/* With nested paging we don't care about invlpg anymore. */
329	if (pVM->hwaccm.s.fNestedPaging)
330	pVMCB->ctrl.u32InterceptCtrl1 &= ~SVM_CTRL1_INTERCEPT_INVLPG;
331
332	pVMCB->ctrl.u32InterceptCtrl2 = SVM_CTRL2_INTERCEPT_VMRUN /* required */
333	\| SVM_CTRL2_INTERCEPT_VMMCALL
334	\| SVM_CTRL2_INTERCEPT_VMLOAD
335	\| SVM_CTRL2_INTERCEPT_VMSAVE
336	\| SVM_CTRL2_INTERCEPT_STGI
337	\| SVM_CTRL2_INTERCEPT_CLGI
338	\| SVM_CTRL2_INTERCEPT_SKINIT
339	\| SVM_CTRL2_INTERCEPT_WBINVD
340	\| SVM_CTRL2_INTERCEPT_MWAIT_UNCOND; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
341	;
342	Log(("pVMCB->ctrl.u32InterceptException = %x\n", pVMCB->ctrl.u32InterceptException));
343	Log(("pVMCB->ctrl.u32InterceptCtrl1 = %x\n", pVMCB->ctrl.u32InterceptCtrl1));
344	Log(("pVMCB->ctrl.u32InterceptCtrl2 = %x\n", pVMCB->ctrl.u32InterceptCtrl2));
345
346	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
347	pVMCB->ctrl.IntCtrl.n.u1VIrqMasking = 1;
348	/* Ignore the priority in the TPR; just deliver it when we tell it to. */
349	pVMCB->ctrl.IntCtrl.n.u1IgnoreTPR = 1;
350
351	/* Set IO and MSR bitmap addresses. */
352	pVMCB->ctrl.u64IOPMPhysAddr = pVM->hwaccm.s.svm.pIOBitmapPhys;
353	pVMCB->ctrl.u64MSRPMPhysAddr = pVM->hwaccm.s.svm.pMSRBitmapPhys;
354
355	/* No LBR virtualization. */
356	pVMCB->ctrl.u64LBRVirt = 0;
357
358	/** The ASID must start at 1; the host uses 0. */
359	pVMCB->ctrl.TLBCtrl.n.u32ASID = 1;
360
361	/** Setup the PAT msr (nested paging only) */
362	pVMCB->guest.u64GPAT = 0x0007040600070406ULL;
363	}
364	return rc;
365	}
366
367
368	/**
369	* Injects an event (trap or external interrupt)
370	*
371	* @param pVCpu The VMCPU to operate on.
372	* @param pVMCB SVM control block
373	* @param pCtx CPU Context
374	* @param pIntInfo SVM interrupt info
375	*/
376	inline void SVMR0InjectEvent(PVMCPU pVCpu, SVM_VMCB pVMCB, CPUMCTX pCtx, SVM_EVENT* pEvent)
377	{
378	#ifdef VBOX_WITH_STATISTICS
379	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
380	#endif
381
382	#ifdef VBOX_STRICT
383	if (pEvent->n.u8Vector == 0xE)
384	Log(("SVM: Inject int %d at %RGv error code=%02x CR2=%RGv intInfo=%08x\n", pEvent->n.u8Vector, (RTGCPTR)pCtx->rip, pEvent->n.u32ErrorCode, (RTGCPTR)pCtx->cr2, pEvent->au64[0]));
385	else
386	if (pEvent->n.u8Vector < 0x20)
387	Log(("SVM: Inject int %d at %RGv error code=%08x\n", pEvent->n.u8Vector, (RTGCPTR)pCtx->rip, pEvent->n.u32ErrorCode));
388	else
389	{
390	Log(("INJ-EI: %x at %RGv\n", pEvent->n.u8Vector, (RTGCPTR)pCtx->rip));
391	Assert(!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
392	Assert(pCtx->eflags.u32 & X86_EFL_IF);
393	}
394	#endif
395
396	/* Set event injection state. */
397	pVMCB->ctrl.EventInject.au64[0] = pEvent->au64[0];
398	}
399
400
401	/**
402	* Checks for pending guest interrupts and injects them
403	*
404	* @returns VBox status code.
405	* @param pVM The VM to operate on.
406	* @param pVCpu The VM CPU to operate on.
407	* @param pVMCB SVM control block
408	* @param pCtx CPU Context
409	*/
410	static int SVMR0CheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, SVM_VMCB pVMCB, CPUMCTX pCtx)
411	{
412	int rc;
413
414	/* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
415	if (pVCpu->hwaccm.s.Event.fPending)
416	{
417	SVM_EVENT Event;
418
419	Log(("Reinjecting event %08x %08x at %RGv\n", pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip));
420	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
421	Event.au64[0] = pVCpu->hwaccm.s.Event.intInfo;
422	SVMR0InjectEvent(pVCpu, pVMCB, pCtx, &Event);
423
424	pVCpu->hwaccm.s.Event.fPending = false;
425	return VINF_SUCCESS;
426	}
427
428	/* If an active trap is already pending, then we must forward it first! */
429	if (!TRPMHasTrap(pVCpu))
430	{
431	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI_BIT))
432	{
433	SVM_EVENT Event;
434
435	Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
436	Event.n.u8Vector = X86_XCPT_NMI;
437	Event.n.u1Valid = 1;
438	Event.n.u32ErrorCode = 0;
439	Event.n.u3Type = SVM_EVENT_NMI;
440
441	SVMR0InjectEvent(pVCpu, pVMCB, pCtx, &Event);
442	return VINF_SUCCESS;
443	}
444
445	/* @todo SMI interrupts. */
446
447	/* When external interrupts are pending, we should exit the VM when IF is set. */
448	if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)))
449	{
450	if ( !(pCtx->eflags.u32 & X86_EFL_IF)
451	\|\| VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
452	{
453	if (!pVMCB->ctrl.IntCtrl.n.u1VIrqValid)
454	{
455	if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
456	LogFlow(("Enable irq window exit!\n"));
457	else
458	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS -> irq window exit\n", (RTGCPTR)pCtx->rip));
459
460	/** @todo use virtual interrupt method to inject a pending irq; dispatched as soon as guest.IF is set. */
461	pVMCB->ctrl.u32InterceptCtrl1 \|= SVM_CTRL1_INTERCEPT_VINTR;
462	pVMCB->ctrl.IntCtrl.n.u1VIrqValid = 1;
463	pVMCB->ctrl.IntCtrl.n.u8VIrqVector = 0; /* don't care */
464	}
465	}
466	else
467	{
468	uint8_t u8Interrupt;
469
470	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
471	Log(("Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc\n", u8Interrupt, u8Interrupt, rc));
472	if (RT_SUCCESS(rc))
473	{
474	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
475	AssertRC(rc);
476	}
477	else
478	{
479	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
480	Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)));
481	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
482	/* Just continue */
483	}
484	}
485	}
486	}
487
488	#ifdef VBOX_STRICT
489	if (TRPMHasTrap(pVCpu))
490	{
491	uint8_t u8Vector;
492	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
493	AssertRC(rc);
494	}
495	#endif
496
497	if ( (pCtx->eflags.u32 & X86_EFL_IF)
498	&& (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
499	&& TRPMHasTrap(pVCpu)
500	)
501	{
502	uint8_t u8Vector;
503	int rc;
504	TRPMEVENT enmType;
505	SVM_EVENT Event;
506	RTGCUINT u32ErrorCode;
507
508	Event.au64[0] = 0;
509
510	/* If a new event is pending, then dispatch it now. */
511	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &u32ErrorCode, 0);
512	AssertRC(rc);
513	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
514	Assert(enmType != TRPM_SOFTWARE_INT);
515
516	/* Clear the pending trap. */
517	rc = TRPMResetTrap(pVCpu);
518	AssertRC(rc);
519
520	Event.n.u8Vector = u8Vector;
521	Event.n.u1Valid = 1;
522	Event.n.u32ErrorCode = u32ErrorCode;
523
524	if (enmType == TRPM_TRAP)
525	{
526	switch (u8Vector) {
527	case 8:
528	case 10:
529	case 11:
530	case 12:
531	case 13:
532	case 14:
533	case 17:
534	/* Valid error codes. */
535	Event.n.u1ErrorCodeValid = 1;
536	break;
537	default:
538	break;
539	}
540	if (u8Vector == X86_XCPT_NMI)
541	Event.n.u3Type = SVM_EVENT_NMI;
542	else
543	Event.n.u3Type = SVM_EVENT_EXCEPTION;
544	}
545	else
546	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
547
548	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
549	SVMR0InjectEvent(pVCpu, pVMCB, pCtx, &Event);
550	} /* if (interrupts can be dispatched) */
551
552	return VINF_SUCCESS;
553	}
554
555	/**
556	* Save the host state
557	*
558	* @returns VBox status code.
559	* @param pVM The VM to operate on.
560	* @param pVCpu The VM CPU to operate on.
561	*/
562	VMMR0DECL(int) SVMR0SaveHostState(PVM pVM, PVMCPU pVCpu)
563	{
564	NOREF(pVM);
565	NOREF(pVCpu);
566	/* Nothing to do here. */
567	return VINF_SUCCESS;
568	}
569
570	/**
571	* Loads the guest state
572	*
573	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
574	*
575	* @returns VBox status code.
576	* @param pVM The VM to operate on.
577	* @param pVCpu The VM CPU to operate on.
578	* @param pCtx Guest context
579	*/
580	VMMR0DECL(int) SVMR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
581	{
582	RTGCUINTPTR val;
583	SVM_VMCB *pVMCB;
584
585	if (pVM == NULL)
586	return VERR_INVALID_PARAMETER;
587
588	/* Setup AMD SVM. */
589	Assert(pVM->hwaccm.s.svm.fSupported);
590
591	pVMCB = (SVM_VMCB *)pVCpu->hwaccm.s.svm.pVMCB;
592	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
593
594	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
595	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
596	{
597	SVM_WRITE_SELREG(CS, cs);
598	SVM_WRITE_SELREG(SS, ss);
599	SVM_WRITE_SELREG(DS, ds);
600	SVM_WRITE_SELREG(ES, es);
601	SVM_WRITE_SELREG(FS, fs);
602	SVM_WRITE_SELREG(GS, gs);
603	}
604
605	/* Guest CPU context: LDTR. */
606	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
607	{
608	SVM_WRITE_SELREG(LDTR, ldtr);
609	}
610
611	/* Guest CPU context: TR. */
612	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
613	{
614	SVM_WRITE_SELREG(TR, tr);
615	}
616
617	/* Guest CPU context: GDTR. */
618	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
619	{
620	pVMCB->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
621	pVMCB->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
622	}
623
624	/* Guest CPU context: IDTR. */
625	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
626	{
627	pVMCB->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
628	pVMCB->guest.IDTR.u64Base = pCtx->idtr.pIdt;
629	}
630
631	/*
632	* Sysenter MSRs (unconditional)
633	*/
634	pVMCB->guest.u64SysEnterCS = pCtx->SysEnter.cs;
635	pVMCB->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
636	pVMCB->guest.u64SysEnterESP = pCtx->SysEnter.esp;
637
638	/* Control registers */
639	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
640	{
641	val = pCtx->cr0;
642	if (!CPUMIsGuestFPUStateActive(pVCpu))
643	{
644	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
645	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
646	}
647	else
648	{
649	/** @todo check if we support the old style mess correctly. */
650	if (!(val & X86_CR0_NE))
651	{
652	Log(("Forcing X86_CR0_NE!!!\n"));
653
654	/* Also catch floating point exceptions as we need to report them to the guest in a different way. */
655	if (!pVCpu->hwaccm.s.fFPUOldStyleOverride)
656	{
657	pVMCB->ctrl.u32InterceptException \|= RT_BIT(X86_XCPT_MF);
658	pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
659	}
660	}
661	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
662	}
663	/* Always enable caching. */
664	val &= ~(X86_CR0_CD\|X86_CR0_NW);
665
666	/* Note: WP is not relevant in nested paging mode as we catch accesses on the (guest) physical level. */
667	/* Note: In nested paging mode the guest is allowed to run with paging disabled; the guest physical to host physical translation will remain active. */
668	if (!pVM->hwaccm.s.fNestedPaging)
669	{
670	val \|= X86_CR0_PG; /* Paging is always enabled; even when the guest is running in real mode or PE without paging. */
671	val \|= X86_CR0_WP; /* Must set this as we rely on protect various pages and supervisor writes must be caught. */
672	}
673	pVMCB->guest.u64CR0 = val;
674	}
675	/* CR2 as well */
676	pVMCB->guest.u64CR2 = pCtx->cr2;
677
678	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
679	{
680	/* Save our shadow CR3 register. */
681	if (pVM->hwaccm.s.fNestedPaging)
682	{
683	PGMMODE enmShwPagingMode;
684
685	#if HC_ARCH_BITS == 32
686	if (CPUMIsGuestInLongModeEx(pCtx))
687	enmShwPagingMode = PGMMODE_AMD64_NX;
688	else
689	#endif
690	enmShwPagingMode = PGMGetHostMode(pVM);
691
692	pVMCB->ctrl.u64NestedPagingCR3 = PGMGetNestedCR3(pVCpu, enmShwPagingMode);
693	Assert(pVMCB->ctrl.u64NestedPagingCR3);
694	pVMCB->guest.u64CR3 = pCtx->cr3;
695	}
696	else
697	{
698	pVMCB->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
699	Assert(pVMCB->guest.u64CR3 \|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
700	}
701	}
702
703	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
704	{
705	val = pCtx->cr4;
706	if (!pVM->hwaccm.s.fNestedPaging)
707	{
708	switch(pVCpu->hwaccm.s.enmShadowMode)
709	{
710	case PGMMODE_REAL:
711	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
712	AssertFailed();
713	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
714
715	case PGMMODE_32_BIT: /* 32-bit paging. */
716	val &= ~X86_CR4_PAE;
717	break;
718
719	case PGMMODE_PAE: /* PAE paging. */
720	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
721	/** @todo use normal 32 bits paging */
722	val \|= X86_CR4_PAE;
723	break;
724
725	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
726	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
727	#ifdef VBOX_ENABLE_64_BITS_GUESTS
728	break;
729	#else
730	AssertFailed();
731	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
732	#endif
733
734	default: /* shut up gcc */
735	AssertFailed();
736	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
737	}
738	}
739	pVMCB->guest.u64CR4 = val;
740	}
741
742	/* Debug registers. */
743	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
744	{
745	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
746	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
747
748	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
749	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
750	pCtx->dr[7] \|= 0x400; /* must be one */
751
752	pVMCB->guest.u64DR7 = pCtx->dr[7];
753	pVMCB->guest.u64DR6 = pCtx->dr[6];
754
755	/* Sync the debug state now if any breakpoint is armed. */
756	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
757	&& !CPUMIsGuestDebugStateActive(pVCpu)
758	&& !DBGFIsStepping(pVCpu))
759	{
760	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
761
762	/* Disable drx move intercepts. */
763	pVMCB->ctrl.u16InterceptRdDRx = 0;
764	pVMCB->ctrl.u16InterceptWrDRx = 0;
765
766	/* Save the host and load the guest debug state. */
767	int rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
768	AssertRC(rc);
769	}
770	}
771
772	/* EIP, ESP and EFLAGS */
773	pVMCB->guest.u64RIP = pCtx->rip;
774	pVMCB->guest.u64RSP = pCtx->rsp;
775	pVMCB->guest.u64RFlags = pCtx->eflags.u32;
776
777	/* Set CPL */
778	pVMCB->guest.u8CPL = pCtx->csHid.Attr.n.u2Dpl;
779
780	/* RAX/EAX too, as VMRUN uses RAX as an implicit parameter. */
781	pVMCB->guest.u64RAX = pCtx->rax;
782
783	/* vmrun will fail without MSR_K6_EFER_SVME. */
784	pVMCB->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
785
786	/* 64 bits guest mode? */
787	if (CPUMIsGuestInLongModeEx(pCtx))
788	{
789	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
790	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
791	#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
792	pVCpu->hwaccm.s.svm.pfnVMRun = SVMR0VMSwitcherRun64;
793	#else
794	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
795	if (!pVM->hwaccm.s.fAllow64BitGuests)
796	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
797	# endif
798	pVCpu->hwaccm.s.svm.pfnVMRun = SVMR0VMRun64;
799	#endif
800	/* Unconditionally update these as wrmsr might have changed them. (HWACCM_CHANGED_GUEST_SEGMENT_REGS will not be set) */
801	pVMCB->guest.FS.u64Base = pCtx->fsHid.u64Base;
802	pVMCB->guest.GS.u64Base = pCtx->gsHid.u64Base;
803	}
804	else
805	{
806	/* Filter out the MSR_K6_LME bit or else AMD-V expects amd64 shadow paging. */
807	pVMCB->guest.u64EFER &= ~MSR_K6_EFER_LME;
808
809	pVCpu->hwaccm.s.svm.pfnVMRun = SVMR0VMRun;
810	}
811
812	/* TSC offset. */
813	if (TMCpuTickCanUseRealTSC(pVCpu, &pVMCB->ctrl.u64TSCOffset))
814	{
815	pVMCB->ctrl.u32InterceptCtrl1 &= ~SVM_CTRL1_INTERCEPT_RDTSC;
816	pVMCB->ctrl.u32InterceptCtrl2 &= ~SVM_CTRL2_INTERCEPT_RDTSCP;
817	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
818	}
819	else
820	{
821	pVMCB->ctrl.u32InterceptCtrl1 \|= SVM_CTRL1_INTERCEPT_RDTSC;
822	pVMCB->ctrl.u32InterceptCtrl2 \|= SVM_CTRL2_INTERCEPT_RDTSCP;
823	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
824	}
825
826	/* Sync the various msrs for 64 bits mode. */
827	pVMCB->guest.u64STAR = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
828	pVMCB->guest.u64LSTAR = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
829	pVMCB->guest.u64CSTAR = pCtx->msrCSTAR; /* compatibility mode syscall rip */
830	pVMCB->guest.u64SFMASK = pCtx->msrSFMASK; /* syscall flag mask */
831	pVMCB->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
832
833	#ifdef DEBUG
834	/* Intercept X86_XCPT_DB if stepping is enabled */
835	if (DBGFIsStepping(pVCpu))
836	pVMCB->ctrl.u32InterceptException \|= RT_BIT(X86_XCPT_DB);
837	else
838	pVMCB->ctrl.u32InterceptException &= ~RT_BIT(X86_XCPT_DB);
839	#endif
840
841	/* Done. */
842	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
843
844	return VINF_SUCCESS;
845	}
846
847
848	/**
849	* Runs guest code in an AMD-V VM.
850	*
851	* @returns VBox status code.
852	* @param pVM The VM to operate on.
853	* @param pVCpu The VM CPU to operate on.
854	* @param pCtx Guest context
855	*/
856	VMMR0DECL(int) SVMR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
857	{
858	int rc = VINF_SUCCESS;
859	uint64_t exitCode = (uint64_t)SVM_EXIT_INVALID;
860	SVM_VMCB *pVMCB;
861	bool fSyncTPR = false;
862	unsigned cResume = 0;
863	uint8_t u8LastTPR;
864	PHWACCM_CPUINFO pCpu = 0;
865	RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
866	#ifdef VBOX_STRICT
867	RTCPUID idCpuCheck;
868	#endif
869	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
870	uint64_t u64LastTime = RTTimeMilliTS();
871	#endif
872
873	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
874
875	pVMCB = (SVM_VMCB *)pVCpu->hwaccm.s.svm.pVMCB;
876	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
877
878	/* We can jump to this point to resume execution after determining that a VM-exit is innocent.
879	*/
880	ResumeExecution:
881	Assert(!HWACCMR0SuspendPending());
882
883	/* Safety precaution; looping for too long here can have a very bad effect on the host */
884	if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
885	{
886	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
887	rc = VINF_EM_RAW_INTERRUPT;
888	goto end;
889	}
890
891	/* Check for irq inhibition due to instruction fusing (sti, mov ss). */
892	if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
893	{
894	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
895	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
896	{
897	/* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
898	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
899	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
900	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
901	*/
902	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
903	/* Irq inhibition is no longer active; clear the corresponding SVM state. */
904	pVMCB->ctrl.u64IntShadow = 0;
905	}
906	}
907	else
908	{
909	/* Irq inhibition is no longer active; clear the corresponding SVM state. */
910	pVMCB->ctrl.u64IntShadow = 0;
911	}
912
913	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
914	if (RT_UNLIKELY(cResume & 0xf) == 0)
915	{
916	uint64_t u64CurTime = RTTimeMilliTS();
917
918	if (RT_UNLIKELY(u64CurTime > u64LastTime))
919	{
920	u64LastTime = u64CurTime;
921	TMTimerPollVoid(pVM, pVCpu);
922	}
923	}
924	#endif
925
926	/* Check for pending actions that force us to go back to ring 3. */
927	#ifdef DEBUG
928	/* Intercept X86_XCPT_DB if stepping is enabled */
929	if (!DBGFIsStepping(pVCpu))
930	#endif
931	{
932	if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
933	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
934	{
935	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
936	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
937	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
938	rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
939	goto end;
940	}
941	}
942
943	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
944	if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
945	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
946	{
947	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
948	rc = VINF_EM_PENDING_REQUEST;
949	goto end;
950	}
951
952	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
953	/*
954	* Exit to ring-3 preemption/work is pending.
955	*
956	* Interrupts are disabled before the call to make sure we don't miss any interrupt
957	* that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
958	* further down, but SVMR0CheckPendingInterrupt makes that impossible.)
959	*
960	* Note! Interrupts must be disabled done before we check for TLB flushes; TLB
961	* shootdowns rely on this.
962	*/
963	uOldEFlags = ASMIntDisableFlags();
964	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
965	{
966	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
967	rc = VINF_EM_RAW_INTERRUPT;
968	goto end;
969	}
970	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
971	#endif
972
973	/* When external interrupts are pending, we should exit the VM when IF is set. */
974	/* Note! After VM_FF_INHIBIT_INTERRUPTS check!!! */
975	rc = SVMR0CheckPendingInterrupt(pVM, pVCpu, pVMCB, pCtx);
976	if (RT_FAILURE(rc))
977	{
978	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
979	goto end;
980	}
981
982	/* TPR caching using CR8 is only available in 64 bits mode or with 32 bits guests when X86_CPUID_AMD_FEATURE_ECX_CR8L is supported. */
983	/* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!!!!!
984	* @todo query and update the TPR only when it could have been changed (mmio access)
985	*/
986	if (pVM->hwaccm.s.fHasIoApic)
987	{
988	bool fPending;
989
990	/* TPR caching in CR8 */
991	int rc = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
992	AssertRC(rc);
993	pVMCB->ctrl.IntCtrl.n.u8VTPR = (u8LastTPR >> 4); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
994
995	if (fPending)
996	{
997	/* A TPR change could activate a pending interrupt, so catch cr8 writes. */
998	pVMCB->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
999	}
1000	else
1001	/* No interrupts are pending, so we don't need to be explicitely notified.
1002	* There are enough world switches for detecting pending interrupts.
1003	*/
1004	pVMCB->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
1005
1006	fSyncTPR = !fPending;
1007	}
1008
1009	/* All done! Let's start VM execution. */
1010	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatInGC, x);
1011
1012	/* Enable nested paging if necessary (disabled each time after #VMEXIT). */
1013	pVMCB->ctrl.NestedPaging.n.u1NestedPaging = pVM->hwaccm.s.fNestedPaging;
1014
1015	#ifdef LOG_ENABLED
1016	pCpu = HWACCMR0GetCurrentCpu();
1017	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
1018	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1019	{
1020	if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
1021	Log(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
1022	else
1023	Log(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
1024	}
1025	if (pCpu->fFlushTLB)
1026	Log(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
1027	#endif
1028
1029	/*
1030	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
1031	* (until the actual world switch)
1032	*/
1033	#ifdef VBOX_STRICT
1034	idCpuCheck = RTMpCpuId();
1035	#endif
1036	VMMR0LogFlushDisable(pVCpu);
1037
1038	/* Load the guest state; must be here as it sets up the shadow cr0 for lazy fpu syncing! */
1039	rc = SVMR0LoadGuestState(pVM, pVCpu, pCtx);
1040	if (RT_UNLIKELY(rc != VINF_SUCCESS))
1041	{
1042	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
1043	VMMR0LogFlushEnable(pVCpu);
1044	goto end;
1045	}
1046
1047	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
1048	/* Disable interrupts to make sure a poke will interrupt execution.
1049	* This must be done before we check for TLB flushes; TLB shootdowns rely on this.
1050	*/
1051	uOldEFlags = ASMIntDisableFlags();
1052	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
1053	#endif
1054
1055	pCpu = HWACCMR0GetCurrentCpu();
1056	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
1057	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
1058	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
1059	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
1060	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1061	{
1062	/* Force a TLB flush on VM entry. */
1063	pVCpu->hwaccm.s.fForceTLBFlush = true;
1064	}
1065	else
1066	Assert(!pCpu->fFlushTLB \|\| pVM->hwaccm.s.svm.fAlwaysFlushTLB);
1067
1068	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
1069
1070	/* Check for tlb shootdown flushes. */
1071	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH_BIT))
1072	pVCpu->hwaccm.s.fForceTLBFlush = true;
1073
1074	/* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
1075	if ( pVCpu->hwaccm.s.fForceTLBFlush
1076	&& !pVM->hwaccm.s.svm.fAlwaysFlushTLB)
1077	{
1078	if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
1079	\|\| pCpu->fFlushTLB)
1080	{
1081	pCpu->fFlushTLB = false;
1082	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
1083	pVMCB->ctrl.TLBCtrl.n.u1TLBFlush = 1; /* wrap around; flush TLB */
1084	pCpu->cTLBFlushes++;
1085	}
1086	else
1087	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
1088
1089	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
1090	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
1091	}
1092	else
1093	{
1094	Assert(!pCpu->fFlushTLB \|\| pVM->hwaccm.s.svm.fAlwaysFlushTLB);
1095
1096	/* We never increase uCurrentASID in the fAlwaysFlushTLB (erratum 170) case. */
1097	if (!pCpu->uCurrentASID \|\| !pVCpu->hwaccm.s.uCurrentASID)
1098	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID = 1;
1099
1100	Assert(!pVM->hwaccm.s.svm.fAlwaysFlushTLB \|\| pVCpu->hwaccm.s.fForceTLBFlush);
1101	pVMCB->ctrl.TLBCtrl.n.u1TLBFlush = pVCpu->hwaccm.s.fForceTLBFlush;
1102
1103	if ( !pVM->hwaccm.s.svm.fAlwaysFlushTLB
1104	&& VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1105	{
1106	/* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
1107	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
1108	for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
1109	SVMR0InvlpgA(pVCpu->hwaccm.s.TlbShootdown.aPages[i], pVMCB->ctrl.TLBCtrl.n.u32ASID);
1110	}
1111	}
1112	pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
1113	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1114
1115	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
1116	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
1117	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
1118	pVMCB->ctrl.TLBCtrl.n.u32ASID = pVCpu->hwaccm.s.uCurrentASID;
1119
1120	#ifdef VBOX_WITH_STATISTICS
1121	if (pVMCB->ctrl.TLBCtrl.n.u1TLBFlush)
1122	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
1123	else
1124	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
1125	#endif
1126
1127	/* In case we execute a goto ResumeExecution later on. */
1128	pVCpu->hwaccm.s.fResumeVM = true;
1129	pVCpu->hwaccm.s.fForceTLBFlush = pVM->hwaccm.s.svm.fAlwaysFlushTLB;
1130
1131	Assert(sizeof(pVCpu->hwaccm.s.svm.pVMCBPhys) == 8);
1132	Assert(pVMCB->ctrl.IntCtrl.n.u1VIrqMasking);
1133	Assert(pVMCB->ctrl.u64IOPMPhysAddr == pVM->hwaccm.s.svm.pIOBitmapPhys);
1134	Assert(pVMCB->ctrl.u64MSRPMPhysAddr == pVM->hwaccm.s.svm.pMSRBitmapPhys);
1135	Assert(pVMCB->ctrl.u64LBRVirt == 0);
1136
1137	#ifdef VBOX_STRICT
1138	Assert(idCpuCheck == RTMpCpuId());
1139	#endif
1140	TMNotifyStartOfExecution(pVCpu);
1141	pVCpu->hwaccm.s.svm.pfnVMRun(pVCpu->hwaccm.s.svm.pVMCBHostPhys, pVCpu->hwaccm.s.svm.pVMCBPhys, pCtx, pVM, pVCpu);
1142	TMNotifyEndOfExecution(pVCpu);
1143	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
1144	ASMSetFlags(uOldEFlags);
1145	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
1146	uOldEFlags = ~(RTCCUINTREG)0;
1147	#endif
1148	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatInGC, x);
1149
1150	/*
1151	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1152	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
1153	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1154	*/
1155
1156	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit1, x);
1157
1158	/* Reason for the VM exit */
1159	exitCode = pVMCB->ctrl.u64ExitCode;
1160
1161	if (RT_UNLIKELY(exitCode == (uint64_t)SVM_EXIT_INVALID)) /* Invalid guest state. */
1162	{
1163	HWACCMDumpRegs(pVM, pVCpu, pCtx);
1164	#ifdef DEBUG
1165	Log(("ctrl.u16InterceptRdCRx %x\n", pVMCB->ctrl.u16InterceptRdCRx));
1166	Log(("ctrl.u16InterceptWrCRx %x\n", pVMCB->ctrl.u16InterceptWrCRx));
1167	Log(("ctrl.u16InterceptRdDRx %x\n", pVMCB->ctrl.u16InterceptRdDRx));
1168	Log(("ctrl.u16InterceptWrDRx %x\n", pVMCB->ctrl.u16InterceptWrDRx));
1169	Log(("ctrl.u32InterceptException %x\n", pVMCB->ctrl.u32InterceptException));
1170	Log(("ctrl.u32InterceptCtrl1 %x\n", pVMCB->ctrl.u32InterceptCtrl1));
1171	Log(("ctrl.u32InterceptCtrl2 %x\n", pVMCB->ctrl.u32InterceptCtrl2));
1172	Log(("ctrl.u64IOPMPhysAddr %RX64\n", pVMCB->ctrl.u64IOPMPhysAddr));
1173	Log(("ctrl.u64MSRPMPhysAddr %RX64\n", pVMCB->ctrl.u64MSRPMPhysAddr));
1174	Log(("ctrl.u64TSCOffset %RX64\n", pVMCB->ctrl.u64TSCOffset));
1175
1176	Log(("ctrl.TLBCtrl.u32ASID %x\n", pVMCB->ctrl.TLBCtrl.n.u32ASID));
1177	Log(("ctrl.TLBCtrl.u1TLBFlush %x\n", pVMCB->ctrl.TLBCtrl.n.u1TLBFlush));
1178	Log(("ctrl.TLBCtrl.u7Reserved %x\n", pVMCB->ctrl.TLBCtrl.n.u7Reserved));
1179	Log(("ctrl.TLBCtrl.u24Reserved %x\n", pVMCB->ctrl.TLBCtrl.n.u24Reserved));
1180
1181	Log(("ctrl.IntCtrl.u8VTPR %x\n", pVMCB->ctrl.IntCtrl.n.u8VTPR));
1182	Log(("ctrl.IntCtrl.u1VIrqValid %x\n", pVMCB->ctrl.IntCtrl.n.u1VIrqValid));
1183	Log(("ctrl.IntCtrl.u7Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u7Reserved));
1184	Log(("ctrl.IntCtrl.u4VIrqPriority %x\n", pVMCB->ctrl.IntCtrl.n.u4VIrqPriority));
1185	Log(("ctrl.IntCtrl.u1IgnoreTPR %x\n", pVMCB->ctrl.IntCtrl.n.u1IgnoreTPR));
1186	Log(("ctrl.IntCtrl.u3Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u3Reserved));
1187	Log(("ctrl.IntCtrl.u1VIrqMasking %x\n", pVMCB->ctrl.IntCtrl.n.u1VIrqMasking));
1188	Log(("ctrl.IntCtrl.u7Reserved2 %x\n", pVMCB->ctrl.IntCtrl.n.u7Reserved2));
1189	Log(("ctrl.IntCtrl.u8VIrqVector %x\n", pVMCB->ctrl.IntCtrl.n.u8VIrqVector));
1190	Log(("ctrl.IntCtrl.u24Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u24Reserved));
1191
1192	Log(("ctrl.u64IntShadow %RX64\n", pVMCB->ctrl.u64IntShadow));
1193	Log(("ctrl.u64ExitCode %RX64\n", pVMCB->ctrl.u64ExitCode));
1194	Log(("ctrl.u64ExitInfo1 %RX64\n", pVMCB->ctrl.u64ExitInfo1));
1195	Log(("ctrl.u64ExitInfo2 %RX64\n", pVMCB->ctrl.u64ExitInfo2));
1196	Log(("ctrl.ExitIntInfo.u8Vector %x\n", pVMCB->ctrl.ExitIntInfo.n.u8Vector));
1197	Log(("ctrl.ExitIntInfo.u3Type %x\n", pVMCB->ctrl.ExitIntInfo.n.u3Type));
1198	Log(("ctrl.ExitIntInfo.u1ErrorCodeValid %x\n", pVMCB->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
1199	Log(("ctrl.ExitIntInfo.u19Reserved %x\n", pVMCB->ctrl.ExitIntInfo.n.u19Reserved));
1200	Log(("ctrl.ExitIntInfo.u1Valid %x\n", pVMCB->ctrl.ExitIntInfo.n.u1Valid));
1201	Log(("ctrl.ExitIntInfo.u32ErrorCode %x\n", pVMCB->ctrl.ExitIntInfo.n.u32ErrorCode));
1202	Log(("ctrl.NestedPaging %RX64\n", pVMCB->ctrl.NestedPaging.au64));
1203	Log(("ctrl.EventInject.u8Vector %x\n", pVMCB->ctrl.EventInject.n.u8Vector));
1204	Log(("ctrl.EventInject.u3Type %x\n", pVMCB->ctrl.EventInject.n.u3Type));
1205	Log(("ctrl.EventInject.u1ErrorCodeValid %x\n", pVMCB->ctrl.EventInject.n.u1ErrorCodeValid));
1206	Log(("ctrl.EventInject.u19Reserved %x\n", pVMCB->ctrl.EventInject.n.u19Reserved));
1207	Log(("ctrl.EventInject.u1Valid %x\n", pVMCB->ctrl.EventInject.n.u1Valid));
1208	Log(("ctrl.EventInject.u32ErrorCode %x\n", pVMCB->ctrl.EventInject.n.u32ErrorCode));
1209
1210	Log(("ctrl.u64NestedPagingCR3 %RX64\n", pVMCB->ctrl.u64NestedPagingCR3));
1211	Log(("ctrl.u64LBRVirt %RX64\n", pVMCB->ctrl.u64LBRVirt));
1212
1213	Log(("guest.CS.u16Sel %04X\n", pVMCB->guest.CS.u16Sel));
1214	Log(("guest.CS.u16Attr %04X\n", pVMCB->guest.CS.u16Attr));
1215	Log(("guest.CS.u32Limit %X\n", pVMCB->guest.CS.u32Limit));
1216	Log(("guest.CS.u64Base %RX64\n", pVMCB->guest.CS.u64Base));
1217	Log(("guest.DS.u16Sel %04X\n", pVMCB->guest.DS.u16Sel));
1218	Log(("guest.DS.u16Attr %04X\n", pVMCB->guest.DS.u16Attr));
1219	Log(("guest.DS.u32Limit %X\n", pVMCB->guest.DS.u32Limit));
1220	Log(("guest.DS.u64Base %RX64\n", pVMCB->guest.DS.u64Base));
1221	Log(("guest.ES.u16Sel %04X\n", pVMCB->guest.ES.u16Sel));
1222	Log(("guest.ES.u16Attr %04X\n", pVMCB->guest.ES.u16Attr));
1223	Log(("guest.ES.u32Limit %X\n", pVMCB->guest.ES.u32Limit));
1224	Log(("guest.ES.u64Base %RX64\n", pVMCB->guest.ES.u64Base));
1225	Log(("guest.FS.u16Sel %04X\n", pVMCB->guest.FS.u16Sel));
1226	Log(("guest.FS.u16Attr %04X\n", pVMCB->guest.FS.u16Attr));
1227	Log(("guest.FS.u32Limit %X\n", pVMCB->guest.FS.u32Limit));
1228	Log(("guest.FS.u64Base %RX64\n", pVMCB->guest.FS.u64Base));
1229	Log(("guest.GS.u16Sel %04X\n", pVMCB->guest.GS.u16Sel));
1230	Log(("guest.GS.u16Attr %04X\n", pVMCB->guest.GS.u16Attr));
1231	Log(("guest.GS.u32Limit %X\n", pVMCB->guest.GS.u32Limit));
1232	Log(("guest.GS.u64Base %RX64\n", pVMCB->guest.GS.u64Base));
1233
1234	Log(("guest.GDTR.u32Limit %X\n", pVMCB->guest.GDTR.u32Limit));
1235	Log(("guest.GDTR.u64Base %RX64\n", pVMCB->guest.GDTR.u64Base));
1236
1237	Log(("guest.LDTR.u16Sel %04X\n", pVMCB->guest.LDTR.u16Sel));
1238	Log(("guest.LDTR.u16Attr %04X\n", pVMCB->guest.LDTR.u16Attr));
1239	Log(("guest.LDTR.u32Limit %X\n", pVMCB->guest.LDTR.u32Limit));
1240	Log(("guest.LDTR.u64Base %RX64\n", pVMCB->guest.LDTR.u64Base));
1241
1242	Log(("guest.IDTR.u32Limit %X\n", pVMCB->guest.IDTR.u32Limit));
1243	Log(("guest.IDTR.u64Base %RX64\n", pVMCB->guest.IDTR.u64Base));
1244
1245	Log(("guest.TR.u16Sel %04X\n", pVMCB->guest.TR.u16Sel));
1246	Log(("guest.TR.u16Attr %04X\n", pVMCB->guest.TR.u16Attr));
1247	Log(("guest.TR.u32Limit %X\n", pVMCB->guest.TR.u32Limit));
1248	Log(("guest.TR.u64Base %RX64\n", pVMCB->guest.TR.u64Base));
1249
1250	Log(("guest.u8CPL %X\n", pVMCB->guest.u8CPL));
1251	Log(("guest.u64CR0 %RX64\n", pVMCB->guest.u64CR0));
1252	Log(("guest.u64CR2 %RX64\n", pVMCB->guest.u64CR2));
1253	Log(("guest.u64CR3 %RX64\n", pVMCB->guest.u64CR3));
1254	Log(("guest.u64CR4 %RX64\n", pVMCB->guest.u64CR4));
1255	Log(("guest.u64DR6 %RX64\n", pVMCB->guest.u64DR6));
1256	Log(("guest.u64DR7 %RX64\n", pVMCB->guest.u64DR7));
1257
1258	Log(("guest.u64RIP %RX64\n", pVMCB->guest.u64RIP));
1259	Log(("guest.u64RSP %RX64\n", pVMCB->guest.u64RSP));
1260	Log(("guest.u64RAX %RX64\n", pVMCB->guest.u64RAX));
1261	Log(("guest.u64RFlags %RX64\n", pVMCB->guest.u64RFlags));
1262
1263	Log(("guest.u64SysEnterCS %RX64\n", pVMCB->guest.u64SysEnterCS));
1264	Log(("guest.u64SysEnterEIP %RX64\n", pVMCB->guest.u64SysEnterEIP));
1265	Log(("guest.u64SysEnterESP %RX64\n", pVMCB->guest.u64SysEnterESP));
1266
1267	Log(("guest.u64EFER %RX64\n", pVMCB->guest.u64EFER));
1268	Log(("guest.u64STAR %RX64\n", pVMCB->guest.u64STAR));
1269	Log(("guest.u64LSTAR %RX64\n", pVMCB->guest.u64LSTAR));
1270	Log(("guest.u64CSTAR %RX64\n", pVMCB->guest.u64CSTAR));
1271	Log(("guest.u64SFMASK %RX64\n", pVMCB->guest.u64SFMASK));
1272	Log(("guest.u64KernelGSBase %RX64\n", pVMCB->guest.u64KernelGSBase));
1273	Log(("guest.u64GPAT %RX64\n", pVMCB->guest.u64GPAT));
1274	Log(("guest.u64DBGCTL %RX64\n", pVMCB->guest.u64DBGCTL));
1275	Log(("guest.u64BR_FROM %RX64\n", pVMCB->guest.u64BR_FROM));
1276	Log(("guest.u64BR_TO %RX64\n", pVMCB->guest.u64BR_TO));
1277	Log(("guest.u64LASTEXCPFROM %RX64\n", pVMCB->guest.u64LASTEXCPFROM));
1278	Log(("guest.u64LASTEXCPTO %RX64\n", pVMCB->guest.u64LASTEXCPTO));
1279
1280	#endif
1281	rc = VERR_SVM_UNABLE_TO_START_VM;
1282	VMMR0LogFlushEnable(pVCpu);
1283	goto end;
1284	}
1285
1286	/* Let's first sync back eip, esp, and eflags. */
1287	pCtx->rip = pVMCB->guest.u64RIP;
1288	pCtx->rsp = pVMCB->guest.u64RSP;
1289	pCtx->eflags.u32 = pVMCB->guest.u64RFlags;
1290	/* eax is saved/restore across the vmrun instruction */
1291	pCtx->rax = pVMCB->guest.u64RAX;
1292
1293	pCtx->msrKERNELGSBASE = pVMCB->guest.u64KernelGSBase; /* swapgs exchange value */
1294
1295	/* Can be updated behind our back in the nested paging case. */
1296	pCtx->cr2 = pVMCB->guest.u64CR2;
1297
1298	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1299	SVM_READ_SELREG(SS, ss);
1300	SVM_READ_SELREG(CS, cs);
1301	SVM_READ_SELREG(DS, ds);
1302	SVM_READ_SELREG(ES, es);
1303	SVM_READ_SELREG(FS, fs);
1304	SVM_READ_SELREG(GS, gs);
1305
1306	/*
1307	* System MSRs
1308	*/
1309	pCtx->SysEnter.cs = pVMCB->guest.u64SysEnterCS;
1310	pCtx->SysEnter.eip = pVMCB->guest.u64SysEnterEIP;
1311	pCtx->SysEnter.esp = pVMCB->guest.u64SysEnterESP;
1312
1313	/* Remaining guest CPU context: TR, IDTR, GDTR, LDTR; must sync everything otherwise we can get out of sync when jumping to ring 3. */
1314	SVM_READ_SELREG(LDTR, ldtr);
1315	SVM_READ_SELREG(TR, tr);
1316
1317	pCtx->gdtr.cbGdt = pVMCB->guest.GDTR.u32Limit;
1318	pCtx->gdtr.pGdt = pVMCB->guest.GDTR.u64Base;
1319
1320	pCtx->idtr.cbIdt = pVMCB->guest.IDTR.u32Limit;
1321	pCtx->idtr.pIdt = pVMCB->guest.IDTR.u64Base;
1322
1323	/* Note: no reason to sync back the CRx and DRx registers. They can't be changed by the guest. */
1324	/* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1325	if ( pVM->hwaccm.s.fNestedPaging
1326	&& pCtx->cr3 != pVMCB->guest.u64CR3)
1327	{
1328	CPUMSetGuestCR3(pVCpu, pVMCB->guest.u64CR3);
1329	PGMUpdateCR3(pVCpu, pVMCB->guest.u64CR3);
1330	}
1331
1332	/* Note! NOW IT'S SAFE FOR LOGGING! */
1333	VMMR0LogFlushEnable(pVCpu);
1334
1335	/* Take care of instruction fusing (sti, mov ss) (see 15.20.5 Interrupt Shadows) */
1336	if (pVMCB->ctrl.u64IntShadow & SVM_INTERRUPT_SHADOW_ACTIVE)
1337	{
1338	Log(("uInterruptState %x rip=%RGv\n", pVMCB->ctrl.u64IntShadow, (RTGCPTR)pCtx->rip));
1339	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
1340	}
1341	else
1342	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1343
1344	Log2(("exitCode = %x\n", exitCode));
1345
1346	/* Sync back DR6 as it could have been changed by hitting breakpoints. */
1347	pCtx->dr[6] = pVMCB->guest.u64DR6;
1348	/* DR7.GD can be cleared by debug exceptions, so sync it back as well. */
1349	pCtx->dr[7] = pVMCB->guest.u64DR7;
1350
1351	/* Check if an injected event was interrupted prematurely. */
1352	pVCpu->hwaccm.s.Event.intInfo = pVMCB->ctrl.ExitIntInfo.au64[0];
1353	if ( pVMCB->ctrl.ExitIntInfo.n.u1Valid
1354	&& pVMCB->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT /* we don't care about 'int xx' as the instruction will be restarted. */)
1355	{
1356	Log(("Pending inject %RX64 at %RGv exit=%08x\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitCode));
1357
1358	#ifdef LOG_ENABLED
1359	SVM_EVENT Event;
1360	Event.au64[0] = pVCpu->hwaccm.s.Event.intInfo;
1361
1362	if ( exitCode == SVM_EXIT_EXCEPTION_E
1363	&& Event.n.u8Vector == 0xE)
1364	{
1365	Log(("Double fault!\n"));
1366	}
1367	#endif
1368
1369	pVCpu->hwaccm.s.Event.fPending = true;
1370	/* Error code present? (redundant) */
1371	if (pVMCB->ctrl.ExitIntInfo.n.u1ErrorCodeValid)
1372	pVCpu->hwaccm.s.Event.errCode = pVMCB->ctrl.ExitIntInfo.n.u32ErrorCode;
1373	else
1374	pVCpu->hwaccm.s.Event.errCode = 0;
1375	}
1376	#ifdef VBOX_WITH_STATISTICS
1377	if (exitCode == SVM_EXIT_NPF)
1378	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
1379	else
1380	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitCode & MASK_EXITREASON_STAT]);
1381	#endif
1382
1383	/* Sync back the TPR if it was changed. */
1384	if ( fSyncTPR
1385	&& (u8LastTPR >> 4) != pVMCB->ctrl.IntCtrl.n.u8VTPR)
1386	{
1387	rc = PDMApicSetTPR(pVCpu, pVMCB->ctrl.IntCtrl.n.u8VTPR << 4); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
1388	AssertRC(rc);
1389	}
1390
1391	/* Deal with the reason of the VM-exit. */
1392	switch (exitCode)
1393	{
1394	case SVM_EXIT_EXCEPTION_0: case SVM_EXIT_EXCEPTION_1: case SVM_EXIT_EXCEPTION_2: case SVM_EXIT_EXCEPTION_3:
1395	case SVM_EXIT_EXCEPTION_4: case SVM_EXIT_EXCEPTION_5: case SVM_EXIT_EXCEPTION_6: case SVM_EXIT_EXCEPTION_7:
1396	case SVM_EXIT_EXCEPTION_8: case SVM_EXIT_EXCEPTION_9: case SVM_EXIT_EXCEPTION_A: case SVM_EXIT_EXCEPTION_B:
1397	case SVM_EXIT_EXCEPTION_C: case SVM_EXIT_EXCEPTION_D: case SVM_EXIT_EXCEPTION_E: case SVM_EXIT_EXCEPTION_F:
1398	case SVM_EXIT_EXCEPTION_10: case SVM_EXIT_EXCEPTION_11: case SVM_EXIT_EXCEPTION_12: case SVM_EXIT_EXCEPTION_13:
1399	case SVM_EXIT_EXCEPTION_14: case SVM_EXIT_EXCEPTION_15: case SVM_EXIT_EXCEPTION_16: case SVM_EXIT_EXCEPTION_17:
1400	case SVM_EXIT_EXCEPTION_18: case SVM_EXIT_EXCEPTION_19: case SVM_EXIT_EXCEPTION_1A: case SVM_EXIT_EXCEPTION_1B:
1401	case SVM_EXIT_EXCEPTION_1C: case SVM_EXIT_EXCEPTION_1D: case SVM_EXIT_EXCEPTION_1E: case SVM_EXIT_EXCEPTION_1F:
1402	{
1403	/* Pending trap. */
1404	SVM_EVENT Event;
1405	uint32_t vector = exitCode - SVM_EXIT_EXCEPTION_0;
1406
1407	Log2(("Hardware/software interrupt %d\n", vector));
1408	switch (vector)
1409	{
1410	case X86_XCPT_DB:
1411	{
1412	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
1413
1414	/* Note that we don't support guest and host-initiated debugging at the same time. */
1415	Assert(DBGFIsStepping(pVCpu));
1416
1417	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), pCtx->dr[6]);
1418	if (rc == VINF_EM_RAW_GUEST_TRAP)
1419	{
1420	Log(("Trap %x (debug) at %016RX64\n", vector, pCtx->rip));
1421
1422	/* Reinject the exception. */
1423	Event.au64[0] = 0;
1424	Event.n.u3Type = SVM_EVENT_EXCEPTION; /* trap or fault */
1425	Event.n.u1Valid = 1;
1426	Event.n.u8Vector = X86_XCPT_DB;
1427
1428	SVMR0InjectEvent(pVCpu, pVMCB, pCtx, &Event);
1429
1430	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1431	goto ResumeExecution;
1432	}
1433	/* Return to ring 3 to deal with the debug exit code. */
1434	break;
1435	}
1436
1437	case X86_XCPT_NM:
1438	{
1439	Log(("#NM fault at %RGv\n", (RTGCPTR)pCtx->rip));
1440
1441	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
1442	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
1443	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
1444	if (rc == VINF_SUCCESS)
1445	{
1446	Assert(CPUMIsGuestFPUStateActive(pVCpu));
1447	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
1448
1449	/* Continue execution. */
1450	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1451	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
1452
1453	goto ResumeExecution;
1454	}
1455
1456	Log(("Forward #NM fault to the guest\n"));
1457	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
1458
1459	Event.au64[0] = 0;
1460	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1461	Event.n.u1Valid = 1;
1462	Event.n.u8Vector = X86_XCPT_NM;
1463
1464	SVMR0InjectEvent(pVCpu, pVMCB, pCtx, &Event);
1465	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1466	goto ResumeExecution;
1467	}
1468
1469	case X86_XCPT_PF: /* Page fault */
1470	{
1471	uint32_t errCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1472	RTGCUINTPTR uFaultAddress = pVMCB->ctrl.u64ExitInfo2; /* EXITINFO2 = fault address */
1473
1474	#ifdef DEBUG
1475	if (pVM->hwaccm.s.fNestedPaging)
1476	{ /* A genuine pagefault.
1477	* Forward the trap to the guest by injecting the exception and resuming execution.
1478	*/
1479	Log(("Guest page fault at %04X:%RGv cr2=%RGv error code %x rsp=%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip, uFaultAddress, errCode, (RTGCPTR)pCtx->rsp));
1480	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
1481
1482	/* Now we must update CR2. */
1483	pCtx->cr2 = uFaultAddress;
1484
1485	Event.au64[0] = 0;
1486	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1487	Event.n.u1Valid = 1;
1488	Event.n.u8Vector = X86_XCPT_PF;
1489	Event.n.u1ErrorCodeValid = 1;
1490	Event.n.u32ErrorCode = errCode;
1491
1492	SVMR0InjectEvent(pVCpu, pVMCB, pCtx, &Event);
1493
1494	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1495	goto ResumeExecution;
1496	}
1497	#endif
1498	Assert(!pVM->hwaccm.s.fNestedPaging);
1499
1500	#if 0
1501	/* Shortcut for APIC TPR reads and writes; 32 bits guests only */
1502	if ( (uFaultAddress & 0xfff) == 0x080
1503	&& pVM->hwaccm.s.fHasIoApic
1504	&& !(errCode & X86_TRAP_PF_P) /* not present */
1505	&& !CPUMIsGuestInLongModeEx(pCtx))
1506	{
1507	RTGCPHYS GCPhysApicBase, GCPhys;
1508	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
1509	GCPhysApicBase &= PAGE_BASE_GC_MASK;
1510
1511	rc = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, NULL, &GCPhys);
1512	if ( rc == VINF_SUCCESS
1513	&& GCPhys == GCPhysApicBase)
1514	{
1515	Log(("Replace TPR access at %RGv\n", pCtx->rip));
1516
1517	DISCPUSTATE Cpu;
1518	unsigned cbOp;
1519	rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), &Cpu, &cbOp);
1520	AssertRC(rc);
1521	if ( rc == VINF_SUCCESS
1522	&& Cpu.pCurInstr->opcode == OP_MOV
1523	&& (cbOp == 5 \|\| cbOp == 6))
1524	{
1525	uint8_t szInstr[15];
1526	if ( (errCode & X86_TRAP_PF_RW)
1527	&& Cpu.param1.disp32 == (uint32_t)uFaultAddress
1528	&& Cpu.param2.flags == USE_REG_GEN32)
1529	{
1530	/* 0xF0, 0x0F, 0x22, 0xC0 = mov cr8, eax */
1531	szInstr[0] = 0xF0;
1532	szInstr[1] = 0x0F;
1533	szInstr[2] = 0x22;
1534	szInstr[3] = 0xC0 \| Cpu.param2.base.reg_gen;
1535	for (unsigned i = 4; i < cbOp; i++)
1536	szInstr[i] = 0x90; /* nop */
1537
1538	rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->rip, szInstr, cbOp);
1539	AssertRC(rc);
1540
1541	Log(("Acceptable write candidate!\n"));
1542	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1543	goto ResumeExecution;
1544	}
1545	else
1546	if ( Cpu.param2.disp32 == (uint32_t)uFaultAddress
1547	&& Cpu.param1.flags == USE_REG_GEN32)
1548	{
1549	/* 0xF0, 0x0F, 0x20, 0xC0 = mov eax, cr8 */
1550	szInstr[0] = 0xF0;
1551	szInstr[1] = 0x0F;
1552	szInstr[2] = 0x20;
1553	szInstr[3] = 0xC0 \| Cpu.param1.base.reg_gen;
1554	for (unsigned i = 4; i < cbOp; i++)
1555	szInstr[i] = 0x90; /* nop */
1556
1557	rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->rip, szInstr, cbOp);
1558	AssertRC(rc);
1559
1560	Log(("Acceptable read candidate!\n"));
1561	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1562	goto ResumeExecution;
1563	}
1564	}
1565	}
1566	}
1567	#endif
1568
1569	Log2(("Page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1570	/* Exit qualification contains the linear address of the page fault. */
1571	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
1572	TRPMSetErrorCode(pVCpu, errCode);
1573	TRPMSetFaultAddress(pVCpu, uFaultAddress);
1574
1575	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
1576	rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
1577	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
1578	if (rc == VINF_SUCCESS)
1579	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
1580	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1581	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
1582
1583	TRPMResetTrap(pVCpu);
1584
1585	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1586	goto ResumeExecution;
1587	}
1588	else
1589	if (rc == VINF_EM_RAW_GUEST_TRAP)
1590	{ /* A genuine pagefault.
1591	* Forward the trap to the guest by injecting the exception and resuming execution.
1592	*/
1593	Log2(("Forward page fault to the guest\n"));
1594	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
1595	/* The error code might have been changed. */
1596	errCode = TRPMGetErrorCode(pVCpu);
1597
1598	TRPMResetTrap(pVCpu);
1599
1600	/* Now we must update CR2. */
1601	pCtx->cr2 = uFaultAddress;
1602
1603	Event.au64[0] = 0;
1604	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1605	Event.n.u1Valid = 1;
1606	Event.n.u8Vector = X86_XCPT_PF;
1607	Event.n.u1ErrorCodeValid = 1;
1608	Event.n.u32ErrorCode = errCode;
1609
1610	SVMR0InjectEvent(pVCpu, pVMCB, pCtx, &Event);
1611
1612	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1613	goto ResumeExecution;
1614	}
1615	#ifdef VBOX_STRICT
1616	if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
1617	LogFlow(("PGMTrap0eHandler failed with %d\n", rc));
1618	#endif
1619	/* Need to go back to the recompiler to emulate the instruction. */
1620	TRPMResetTrap(pVCpu);
1621	break;
1622	}
1623
1624	case X86_XCPT_MF: /* Floating point exception. */
1625	{
1626	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
1627	if (!(pCtx->cr0 & X86_CR0_NE))
1628	{
1629	/* old style FPU error reporting needs some extra work. */
1630	/** @todo don't fall back to the recompiler, but do it manually. */
1631	rc = VINF_EM_RAW_EMULATE_INSTR;
1632	break;
1633	}
1634	Log(("Trap %x at %RGv\n", vector, (RTGCPTR)pCtx->rip));
1635
1636	Event.au64[0] = 0;
1637	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1638	Event.n.u1Valid = 1;
1639	Event.n.u8Vector = X86_XCPT_MF;
1640
1641	SVMR0InjectEvent(pVCpu, pVMCB, pCtx, &Event);
1642
1643	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1644	goto ResumeExecution;
1645	}
1646
1647	#ifdef VBOX_STRICT
1648	case X86_XCPT_GP: /* General protection failure exception.*/
1649	case X86_XCPT_UD: /* Unknown opcode exception. */
1650	case X86_XCPT_DE: /* Divide error. */
1651	case X86_XCPT_SS: /* Stack segment exception. */
1652	case X86_XCPT_NP: /* Segment not present exception. */
1653	{
1654	Event.au64[0] = 0;
1655	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1656	Event.n.u1Valid = 1;
1657	Event.n.u8Vector = vector;
1658
1659	switch(vector)
1660	{
1661	case X86_XCPT_GP:
1662	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
1663	Event.n.u1ErrorCodeValid = 1;
1664	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1665	break;
1666	case X86_XCPT_DE:
1667	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
1668	break;
1669	case X86_XCPT_UD:
1670	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
1671	break;
1672	case X86_XCPT_SS:
1673	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
1674	Event.n.u1ErrorCodeValid = 1;
1675	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1676	break;
1677	case X86_XCPT_NP:
1678	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
1679	Event.n.u1ErrorCodeValid = 1;
1680	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1681	break;
1682	}
1683	Log(("Trap %x at %04x:%RGv esi=%x\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, pCtx->esi));
1684	SVMR0InjectEvent(pVCpu, pVMCB, pCtx, &Event);
1685
1686	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1687	goto ResumeExecution;
1688	}
1689	#endif
1690	default:
1691	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
1692	rc = VERR_EM_INTERNAL_ERROR;
1693	break;
1694
1695	} /* switch (vector) */
1696	break;
1697	}
1698
1699	case SVM_EXIT_NPF:
1700	{
1701	/* EXITINFO1 contains fault errorcode; EXITINFO2 contains the guest physical address causing the fault. */
1702	uint32_t errCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1703	RTGCPHYS uFaultAddress = pVMCB->ctrl.u64ExitInfo2; /* EXITINFO2 = fault address */
1704	PGMMODE enmShwPagingMode;
1705
1706	Assert(pVM->hwaccm.s.fNestedPaging);
1707	LogFlow(("Nested page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1708
1709	#if 0
1710	/* Shortcut for APIC TPR reads and writes; 32 bits guests only */
1711	if ( (uFaultAddress & 0xfff) == 0x080
1712	&& pVM->hwaccm.s.fHasIoApic
1713	&& !(errCode & X86_TRAP_PF_P) /* not present */
1714	&& CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx)) == 0
1715	&& !CPUMIsGuestInLongModeEx(pCtx))
1716	{
1717	RTGCPHYS GCPhysApicBase;
1718	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
1719	GCPhysApicBase &= PAGE_BASE_GC_MASK;
1720
1721	if (uFaultAddress == GCPhysApicBase + 0x80)
1722	{
1723	rc = svmR0ReplaceTprInstr(pVM, pVCpu, pCtx);
1724	if (rc == VINF_SUCCESS)
1725	{
1726	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1727	goto ResumeExecution;
1728	}
1729
1730	rc = IOMMMIOPhysHandler(pVM, errCode, CPUMCTX2CORE(pCtx), uFaultAddress);
1731	if (rc == VINF_SUCCESS)
1732	{
1733	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1734	goto ResumeExecution; /* rip already updated */
1735	}
1736	}
1737	}
1738	#endif
1739
1740	/* Exit qualification contains the linear address of the page fault. */
1741	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
1742	TRPMSetErrorCode(pVCpu, errCode);
1743	TRPMSetFaultAddress(pVCpu, uFaultAddress);
1744
1745	/* Handle the pagefault trap for the nested shadow table. */
1746	#if HC_ARCH_BITS == 32
1747	if (CPUMIsGuestInLongModeEx(pCtx))
1748	enmShwPagingMode = PGMMODE_AMD64_NX;
1749	else
1750	#endif
1751	enmShwPagingMode = PGMGetHostMode(pVM);
1752
1753	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmShwPagingMode, errCode, CPUMCTX2CORE(pCtx), uFaultAddress);
1754	Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
1755	if (rc == VINF_SUCCESS)
1756	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
1757	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1758	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
1759
1760	TRPMResetTrap(pVCpu);
1761
1762	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1763	goto ResumeExecution;
1764	}
1765
1766	#ifdef VBOX_STRICT
1767	if (rc != VINF_EM_RAW_EMULATE_INSTR)
1768	LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", rc));
1769	#endif
1770	/* Need to go back to the recompiler to emulate the instruction. */
1771	TRPMResetTrap(pVCpu);
1772	break;
1773	}
1774
1775	case SVM_EXIT_VINTR:
1776	/* A virtual interrupt is about to be delivered, which means IF=1. */
1777	Log(("SVM_EXIT_VINTR IF=%d\n", pCtx->eflags.Bits.u1IF));
1778	pVMCB->ctrl.IntCtrl.n.u1VIrqValid = 0;
1779	pVMCB->ctrl.IntCtrl.n.u8VIrqVector = 0;
1780	goto ResumeExecution;
1781
1782	case SVM_EXIT_FERR_FREEZE:
1783	case SVM_EXIT_INTR:
1784	case SVM_EXIT_NMI:
1785	case SVM_EXIT_SMI:
1786	case SVM_EXIT_INIT:
1787	/* External interrupt; leave to allow it to be dispatched again. */
1788	rc = VINF_EM_RAW_INTERRUPT;
1789	break;
1790
1791	case SVM_EXIT_WBINVD:
1792	case SVM_EXIT_INVD: /* Guest software attempted to execute INVD. */
1793	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
1794	/* Skip instruction and continue directly. */
1795	pCtx->rip += 2; /* Note! hardcoded opcode size! */
1796	/* Continue execution.*/
1797	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1798	goto ResumeExecution;
1799
1800	case SVM_EXIT_CPUID: /* Guest software attempted to execute CPUID. */
1801	{
1802	Log2(("SVM: Cpuid at %RGv for %x\n", (RTGCPTR)pCtx->rip, pCtx->eax));
1803	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
1804	rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
1805	if (rc == VINF_SUCCESS)
1806	{
1807	/* Update EIP and continue execution. */
1808	pCtx->rip += 2; /* Note! hardcoded opcode size! */
1809	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1810	goto ResumeExecution;
1811	}
1812	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", rc));
1813	rc = VINF_EM_RAW_EMULATE_INSTR;
1814	break;
1815	}
1816
1817	case SVM_EXIT_RDTSC: /* Guest software attempted to execute RDTSC. */
1818	{
1819	Log2(("SVM: Rdtsc\n"));
1820	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
1821	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
1822	if (rc == VINF_SUCCESS)
1823	{
1824	/* Update EIP and continue execution. */
1825	pCtx->rip += 2; /* Note! hardcoded opcode size! */
1826	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1827	goto ResumeExecution;
1828	}
1829	rc = VINF_EM_RAW_EMULATE_INSTR;
1830	break;
1831	}
1832
1833	case SVM_EXIT_RDPMC: /* Guest software attempted to execute RDPMC. */
1834	{
1835	Log2(("SVM: Rdpmc %x\n", pCtx->ecx));
1836	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
1837	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
1838	if (rc == VINF_SUCCESS)
1839	{
1840	/* Update EIP and continue execution. */
1841	pCtx->rip += 2; /* Note! hardcoded opcode size! */
1842	goto ResumeExecution;
1843	}
1844	rc = VINF_EM_RAW_EMULATE_INSTR;
1845	break;
1846	}
1847
1848	case SVM_EXIT_RDTSCP: /* Guest software attempted to execute RDTSCP. */
1849	{
1850	Log2(("SVM: Rdtscp\n"));
1851	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
1852	rc = EMInterpretRdtscp(pVM, pVCpu, pCtx);
1853	if (rc == VINF_SUCCESS)
1854	{
1855	/* Update EIP and continue execution. */
1856	pCtx->rip += 3; /* Note! hardcoded opcode size! */
1857	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1858	goto ResumeExecution;
1859	}
1860	AssertMsgFailed(("EMU: rdtscp failed with %Rrc\n", rc));
1861	rc = VINF_EM_RAW_EMULATE_INSTR;
1862	break;
1863	}
1864
1865	case SVM_EXIT_INVLPG: /* Guest software attempted to execute INVPG. */
1866	{
1867	Log2(("SVM: invlpg\n"));
1868	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
1869
1870	Assert(!pVM->hwaccm.s.fNestedPaging);
1871
1872	/* Truly a pita. Why can't SVM give the same information as VT-x? */
1873	rc = svmR0InterpretInvpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), pVMCB->ctrl.TLBCtrl.n.u32ASID);
1874	if (rc == VINF_SUCCESS)
1875	{
1876	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushPageInvlpg);
1877	goto ResumeExecution; /* eip already updated */
1878	}
1879	break;
1880	}
1881
1882	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
1883	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
1884	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
1885	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
1886	{
1887	uint32_t cbSize;
1888
1889	Log2(("SVM: %RGv mov cr%d, \n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_WRITE_CR0));
1890	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[exitCode - SVM_EXIT_WRITE_CR0]);
1891	rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
1892
1893	switch (exitCode - SVM_EXIT_WRITE_CR0)
1894	{
1895	case 0:
1896	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
1897	break;
1898	case 2:
1899	break;
1900	case 3:
1901	Assert(!pVM->hwaccm.s.fNestedPaging);
1902	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
1903	break;
1904	case 4:
1905	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
1906	break;
1907	case 8:
1908	break;
1909	default:
1910	AssertFailed();
1911	}
1912	/* Check if a sync operation is pending. */
1913	if ( rc == VINF_SUCCESS /* don't bother if we are going to ring 3 anyway */
1914	&& VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
1915	{
1916	rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
1917	AssertRC(rc);
1918
1919	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBCRxChange);
1920
1921	/* Must be set by PGMSyncCR3 */
1922	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| PGMGetGuestMode(pVCpu) <= PGMMODE_PROTECTED \|\| pVCpu->hwaccm.s.fForceTLBFlush,
1923	("rc=%Rrc mode=%d fForceTLBFlush=%RTbool\n", rc, PGMGetGuestMode(pVCpu), pVCpu->hwaccm.s.fForceTLBFlush));
1924	}
1925	if (rc == VINF_SUCCESS)
1926	{
1927	/* EIP has been updated already. */
1928
1929	/* Only resume if successful. */
1930	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1931	goto ResumeExecution;
1932	}
1933	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1934	break;
1935	}
1936
1937	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
1938	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
1939	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
1940	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
1941	{
1942	uint32_t cbSize;
1943
1944	Log2(("SVM: %RGv mov x, cr%d\n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_READ_CR0));
1945	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[exitCode - SVM_EXIT_READ_CR0]);
1946	rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
1947	if (rc == VINF_SUCCESS)
1948	{
1949	/* EIP has been updated already. */
1950
1951	/* Only resume if successful. */
1952	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1953	goto ResumeExecution;
1954	}
1955	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1956	break;
1957	}
1958
1959	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
1960	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
1961	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
1962	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
1963	{
1964	uint32_t cbSize;
1965
1966	Log2(("SVM: %RGv mov dr%d, x\n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_WRITE_DR0));
1967	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
1968
1969	if (!DBGFIsStepping(pVCpu))
1970	{
1971	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
1972
1973	/* Disable drx move intercepts. */
1974	pVMCB->ctrl.u16InterceptRdDRx = 0;
1975	pVMCB->ctrl.u16InterceptWrDRx = 0;
1976
1977	/* Save the host and load the guest debug state. */
1978	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
1979	AssertRC(rc);
1980
1981	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1982	goto ResumeExecution;
1983	}
1984
1985	rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
1986	if (rc == VINF_SUCCESS)
1987	{
1988	/* EIP has been updated already. */
1989	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
1990
1991	/* Only resume if successful. */
1992	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1993	goto ResumeExecution;
1994	}
1995	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1996	break;
1997	}
1998
1999	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
2000	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
2001	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
2002	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
2003	{
2004	uint32_t cbSize;
2005
2006	Log2(("SVM: %RGv mov x, dr%d\n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_READ_DR0));
2007	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
2008
2009	if (!DBGFIsStepping(pVCpu))
2010	{
2011	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
2012
2013	/* Disable drx move intercepts. */
2014	pVMCB->ctrl.u16InterceptRdDRx = 0;
2015	pVMCB->ctrl.u16InterceptWrDRx = 0;
2016
2017	/* Save the host and load the guest debug state. */
2018	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
2019	AssertRC(rc);
2020
2021	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
2022	goto ResumeExecution;
2023	}
2024
2025	rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
2026	if (rc == VINF_SUCCESS)
2027	{
2028	/* EIP has been updated already. */
2029
2030	/* Only resume if successful. */
2031	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
2032	goto ResumeExecution;
2033	}
2034	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
2035	break;
2036	}
2037
2038	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
2039	case SVM_EXIT_IOIO: /* I/O instruction. */
2040	{
2041	SVM_IOIO_EXIT IoExitInfo;
2042	uint32_t uIOSize, uAndVal;
2043
2044	IoExitInfo.au32[0] = pVMCB->ctrl.u64ExitInfo1;
2045
2046	/** @todo could use a lookup table here */
2047	if (IoExitInfo.n.u1OP8)
2048	{
2049	uIOSize = 1;
2050	uAndVal = 0xff;
2051	}
2052	else
2053	if (IoExitInfo.n.u1OP16)
2054	{
2055	uIOSize = 2;
2056	uAndVal = 0xffff;
2057	}
2058	else
2059	if (IoExitInfo.n.u1OP32)
2060	{
2061	uIOSize = 4;
2062	uAndVal = 0xffffffff;
2063	}
2064	else
2065	{
2066	AssertFailed(); /* should be fatal. */
2067	rc = VINF_EM_RAW_EMULATE_INSTR;
2068	break;
2069	}
2070
2071	if (IoExitInfo.n.u1STR)
2072	{
2073	/* ins/outs */
2074	PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
2075
2076	/* Disassemble manually to deal with segment prefixes. */
2077	rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
2078	if (rc == VINF_SUCCESS)
2079	{
2080	if (IoExitInfo.n.u1Type == 0)
2081	{
2082	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
2083	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
2084	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, pDis->prefix, uIOSize);
2085	}
2086	else
2087	{
2088	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
2089	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
2090	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, pDis->prefix, uIOSize);
2091	}
2092	}
2093	else
2094	rc = VINF_EM_RAW_EMULATE_INSTR;
2095	}
2096	else
2097	{
2098	/* normal in/out */
2099	Assert(!IoExitInfo.n.u1REP);
2100
2101	if (IoExitInfo.n.u1Type == 0)
2102	{
2103	Log2(("IOMIOPortWrite %RGv %x %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, uIOSize));
2104	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
2105	rc = IOMIOPortWrite(pVM, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, uIOSize);
2106	}
2107	else
2108	{
2109	uint32_t u32Val = 0;
2110
2111	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
2112	rc = IOMIOPortRead(pVM, IoExitInfo.n.u16Port, &u32Val, uIOSize);
2113	if (IOM_SUCCESS(rc))
2114	{
2115	/* Write back to the EAX register. */
2116	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
2117	Log2(("IOMIOPortRead %RGv %x %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, u32Val & uAndVal, uIOSize));
2118	}
2119	}
2120	}
2121	/*
2122	* Handled the I/O return codes.
2123	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
2124	*/
2125	if (IOM_SUCCESS(rc))
2126	{
2127	/* Update EIP and continue execution. */
2128	pCtx->rip = pVMCB->ctrl.u64ExitInfo2; /* RIP/EIP of the next instruction is saved in EXITINFO2. */
2129	if (RT_LIKELY(rc == VINF_SUCCESS))
2130	{
2131	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
2132	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
2133	{
2134	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
2135	for (unsigned i=0;i<4;i++)
2136	{
2137	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
2138
2139	if ( (IoExitInfo.n.u16Port >= pCtx->dr[i] && IoExitInfo.n.u16Port < pCtx->dr[i] + uBPLen)
2140	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
2141	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
2142	{
2143	SVM_EVENT Event;
2144
2145	Assert(CPUMIsGuestDebugStateActive(pVCpu));
2146
2147	/* Clear all breakpoint status flags and set the one we just hit. */
2148	pCtx->dr[6] &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
2149	pCtx->dr[6] \|= (uint64_t)RT_BIT(i);
2150
2151	/* Note: AMD64 Architecture Programmer's Manual 13.1:
2152	* Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
2153	* the contents have been read.
2154	*/
2155	pVMCB->guest.u64DR6 = pCtx->dr[6];
2156
2157	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
2158	pCtx->dr[7] &= ~X86_DR7_GD;
2159
2160	/* Paranoia. */
2161	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2162	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2163	pCtx->dr[7] \|= 0x400; /* must be one */
2164
2165	pVMCB->guest.u64DR7 = pCtx->dr[7];
2166
2167	/* Inject the exception. */
2168	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
2169
2170	Event.au64[0] = 0;
2171	Event.n.u3Type = SVM_EVENT_EXCEPTION; /* trap or fault */
2172	Event.n.u1Valid = 1;
2173	Event.n.u8Vector = X86_XCPT_DB;
2174
2175	SVMR0InjectEvent(pVCpu, pVMCB, pCtx, &Event);
2176
2177	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
2178	goto ResumeExecution;
2179	}
2180	}
2181	}
2182
2183	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
2184	goto ResumeExecution;
2185	}
2186	Log2(("EM status from IO at %RGv %x size %d: %Rrc\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize, rc));
2187	break;
2188	}
2189
2190	#ifdef VBOX_STRICT
2191	if (rc == VINF_IOM_HC_IOPORT_READ)
2192	Assert(IoExitInfo.n.u1Type != 0);
2193	else if (rc == VINF_IOM_HC_IOPORT_WRITE)
2194	Assert(IoExitInfo.n.u1Type == 0);
2195	else
2196	AssertMsg(RT_FAILURE(rc) \|\| rc == VINF_EM_RAW_EMULATE_INSTR \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
2197	#endif
2198	Log2(("Failed IO at %RGv %x size %d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
2199	break;
2200	}
2201
2202	case SVM_EXIT_HLT:
2203	/** Check if external interrupts are pending; if so, don't switch back. */
2204	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
2205	pCtx->rip++; /* skip hlt */
2206	if ( pCtx->eflags.Bits.u1IF
2207	&& VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)))
2208	goto ResumeExecution;
2209
2210	rc = VINF_EM_HALT;
2211	break;
2212
2213	case SVM_EXIT_MWAIT_UNCOND:
2214	Log2(("SVM: mwait\n"));
2215	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
2216	rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
2217	if ( rc == VINF_EM_HALT
2218	\|\| rc == VINF_SUCCESS)
2219	{
2220	/* Update EIP and continue execution. */
2221	pCtx->rip += 3; /* Note: hardcoded opcode size assumption! */
2222
2223	/** Check if external interrupts are pending; if so, don't switch back. */
2224	if ( rc == VINF_SUCCESS
2225	\|\| ( rc == VINF_EM_HALT
2226	&& pCtx->eflags.Bits.u1IF
2227	&& VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)))
2228	)
2229	goto ResumeExecution;
2230	}
2231	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", rc));
2232	break;
2233
2234	case SVM_EXIT_RSM:
2235	case SVM_EXIT_INVLPGA:
2236	case SVM_EXIT_VMRUN:
2237	case SVM_EXIT_VMMCALL:
2238	case SVM_EXIT_VMLOAD:
2239	case SVM_EXIT_VMSAVE:
2240	case SVM_EXIT_STGI:
2241	case SVM_EXIT_CLGI:
2242	case SVM_EXIT_SKINIT:
2243	{
2244	/* Unsupported instructions. */
2245	SVM_EVENT Event;
2246
2247	Event.au64[0] = 0;
2248	Event.n.u3Type = SVM_EVENT_EXCEPTION;
2249	Event.n.u1Valid = 1;
2250	Event.n.u8Vector = X86_XCPT_UD;
2251
2252	Log(("Forced #UD trap at %RGv\n", (RTGCPTR)pCtx->rip));
2253	SVMR0InjectEvent(pVCpu, pVMCB, pCtx, &Event);
2254
2255	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
2256	goto ResumeExecution;
2257	}
2258
2259	/* Emulate in ring 3. */
2260	case SVM_EXIT_MSR:
2261	{
2262	uint32_t cbSize;
2263
2264	/* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
2265	STAM_COUNTER_INC((pVMCB->ctrl.u64ExitInfo1 == 0) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
2266	Log(("SVM: %s\n", (pVMCB->ctrl.u64ExitInfo1 == 0) ? "rdmsr" : "wrmsr"));
2267	rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
2268	if (rc == VINF_SUCCESS)
2269	{
2270	/* EIP has been updated already. */
2271
2272	/* Only resume if successful. */
2273	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
2274	goto ResumeExecution;
2275	}
2276	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (pVMCB->ctrl.u64ExitInfo1 == 0) ? "rdmsr" : "wrmsr", rc));
2277	break;
2278	}
2279
2280	case SVM_EXIT_MONITOR:
2281	case SVM_EXIT_PAUSE:
2282	case SVM_EXIT_MWAIT_ARMED:
2283	case SVM_EXIT_TASK_SWITCH: /* can change CR3; emulate */
2284	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
2285	break;
2286
2287	case SVM_EXIT_SHUTDOWN:
2288	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
2289	break;
2290
2291	case SVM_EXIT_IDTR_READ:
2292	case SVM_EXIT_GDTR_READ:
2293	case SVM_EXIT_LDTR_READ:
2294	case SVM_EXIT_TR_READ:
2295	case SVM_EXIT_IDTR_WRITE:
2296	case SVM_EXIT_GDTR_WRITE:
2297	case SVM_EXIT_LDTR_WRITE:
2298	case SVM_EXIT_TR_WRITE:
2299	case SVM_EXIT_CR0_SEL_WRITE:
2300	default:
2301	/* Unexpected exit codes. */
2302	rc = VERR_EM_INTERNAL_ERROR;
2303	AssertMsgFailed(("Unexpected exit code %x\n", exitCode)); /* Can't happen. */
2304	break;
2305	}
2306
2307	end:
2308
2309	/* Signal changes for the recompiler. */
2310	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR \| CPUM_CHANGED_LDTR \| CPUM_CHANGED_GDTR \| CPUM_CHANGED_IDTR \| CPUM_CHANGED_TR \| CPUM_CHANGED_HIDDEN_SEL_REGS);
2311
2312	/* If we executed vmrun and an external irq was pending, then we don't have to do a full sync the next time. */
2313	if (exitCode == SVM_EXIT_INTR)
2314	{
2315	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
2316	/* On the next entry we'll only sync the host context. */
2317	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
2318	}
2319	else
2320	{
2321	/* On the next entry we'll sync everything. */
2322	/** @todo we can do better than this */
2323	/* Not in the VINF_PGM_CHANGE_MODE though! */
2324	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
2325	}
2326
2327	/* translate into a less severe return code */
2328	if (rc == VERR_EM_INTERPRETER)
2329	rc = VINF_EM_RAW_EMULATE_INSTR;
2330
2331	/* Just set the correct state here instead of trying to catch every goto above. */
2332	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
2333
2334	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2335	/* Restore interrupts if we exitted after disabling them. */
2336	if (uOldEFlags != ~(RTCCUINTREG)0)
2337	ASMSetFlags(uOldEFlags);
2338	#endif
2339
2340	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
2341	return rc;
2342	}
2343
2344	/**
2345	* Emulate simple mov tpr instruction
2346	*
2347	* @returns VBox status code.
2348	* @param pVCpu The VM CPU to operate on.
2349	* @param pDis Disassembly state
2350	* @param pCtx CPU context
2351	* @param cbOp Opcode size
2352	*/
2353	static int svmR0EmulateTprMov(PVMCPU pVCpu, PDISCPUSTATE pDis, PCPUMCTX pCtx, unsigned cbOp)
2354	{
2355	int rc;
2356
2357	if (pDis->param1.flags == USE_DISPLACEMENT32)
2358	{
2359	/* write */
2360	uint8_t u8Tpr;
2361
2362	/* Fetch the new TPR value */
2363	if (pDis->param2.flags == USE_REG_GEN32)
2364	{
2365	uint32_t val;
2366
2367	rc = DISFetchReg32(CPUMCTX2CORE(pCtx), pDis->param2.base.reg_gen, &val);
2368	AssertRC(rc);
2369	u8Tpr = val;
2370	}
2371	else
2372	if (pDis->param2.flags == USE_IMMEDIATE32)
2373	{
2374	u8Tpr = (uint8_t)pDis->param2.parval;
2375	}
2376	else
2377	return VERR_EM_INTERPRETER;
2378
2379	rc = PDMApicSetTPR(pVCpu, u8Tpr);
2380	AssertRC(rc);
2381
2382	Log(("Emulated write successfully\n"));
2383	pCtx->rip += cbOp;
2384	return VINF_SUCCESS;
2385	}
2386	else
2387	if (pDis->param2.flags == USE_DISPLACEMENT32)
2388	{
2389	/* read */
2390	bool fPending;
2391	uint8_t u8Tpr;
2392
2393	/* TPR caching in CR8 */
2394	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPending);
2395	AssertRC(rc);
2396
2397	rc = DISWriteReg32(CPUMCTX2CORE(pCtx), pDis->param1.base.reg_gen, u8Tpr);
2398	AssertRC(rc);
2399
2400	Log(("Emulated read successfully\n"));
2401	pCtx->rip += cbOp;
2402	return VINF_SUCCESS;
2403	}
2404	return VERR_EM_INTERPRETER;
2405	}
2406
2407	/**
2408	* Attempt to patch TPR mmio instructions
2409	*
2410	* @returns VBox status code.
2411	* @param pVM The VM to operate on.
2412	* @param pVCpu The VM CPU to operate on.
2413	* @param pCtx CPU context
2414	*/
2415	static int svmR0ReplaceTprInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2416	{
2417	RTGCPTR oldrip = pCtx->rip;
2418	PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
2419	unsigned cbOp;
2420
2421	Log(("Replace TPR access at %RGv\n", pCtx->rip));
2422
2423	int rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
2424	AssertRC(rc);
2425	if ( rc == VINF_SUCCESS
2426	&& pDis->pCurInstr->opcode == OP_MOV)
2427	{
2428	#if 0
2429	uint8_t szInstr[15];
2430	if ( cbOp == 10
2431	&& pDis->param1.flags == USE_DISPLACEMENT32
2432	&& pDis->param2.flags == USE_IMMEDIATE32)
2433	{
2434	/* Found:
2435	* mov [fffe0080], immediate_dword (10 bytes)
2436	*
2437	* Replace with:
2438	* mov free_register, immediate_dword >> 4 (5 bytes)
2439	* mov cr8, free_register (4 bytes)
2440	* nop (1 byte)
2441	*
2442	*/
2443	uint32_t u32tpr = (uint32_t)pDis->param2.parval;
2444
2445	u32tpr = (u32tpr >> 4) & 0xf;
2446
2447	/* Check if the next instruction overwrites a general purpose register. If
2448	* it does, then we can safely use it ourselves.
2449	*/
2450	pCtx->rip += cbOp;
2451	rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
2452	pCtx->rip = oldrip;
2453	if ( rc == VINF_SUCCESS
2454	&& pDis->pCurInstr->opcode == OP_MOV
2455	&& pDis->param1.flags == USE_REG_GEN32)
2456	{
2457	/* 0xB8, dword immediate = mov eax, dword immediate */
2458	szInstr[0] = 0xB8 + pDis->param1.base.reg_gen;
2459	szInstr[1] = (uint8_t)u32tpr;
2460	szInstr[2] = 0;
2461	szInstr[3] = 0;
2462	szInstr[4] = 0;
2463
2464	/* 0xF0, 0x0F, 0x22, 0xC0 = mov cr8, eax */
2465	szInstr[5] = 0xF0;
2466	szInstr[6] = 0x0F;
2467	szInstr[7] = 0x22;
2468	szInstr[8] = 0xC0 \| pDis->param1.base.reg_gen;
2469	szInstr[9] = 0x90; /* nop */
2470
2471	rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->rip, szInstr, 10);
2472	AssertRC(rc);
2473
2474	Log(("Acceptable write candidate!\n"));
2475	return VINF_SUCCESS;
2476	}
2477	}
2478	else
2479	{
2480	if ( pDis->param2.flags == USE_REG_GEN32
2481	&& cbOp == 6)
2482	{
2483	RTGCPTR GCPtrTpr = (uint32_t)pDis->param1.disp32;
2484	uint32_t uMmioReg = pDis->param2.base.reg_gen;
2485
2486	/* Found:
2487	* mov dword [fffe0080], eax (6 bytes)
2488	* Check if next instruction is a TPR read:
2489	* mov ecx, dword [fffe0080] (5 bytes)
2490	*/
2491	pCtx->rip += cbOp;
2492	rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
2493	pCtx->rip = oldrip;
2494	if ( rc == VINF_SUCCESS
2495	&& pDis->pCurInstr->opcode == OP_MOV
2496	&& pDis->param1.flags == USE_REG_GEN32
2497	&& pDis->param2.flags == USE_DISPLACEMENT32
2498	&& pDis->param2.disp32 == (uint32_t)GCPtrTpr
2499	&& cbOp == 5)
2500	{
2501	/* mov new_reg, uMmioReg */
2502	szInstr[0] = 0x89;
2503	szInstr[1] = MAKE_MODRM(3, uMmioReg, pDis->param1.base.reg_gen);
2504
2505	/* Let's hope the guest won't mind us trashing the source register...
2506	* shr uMmioReg, 4
2507	*/
2508	szInstr[2] = 0xC1;
2509	szInstr[3] = 0xE8 \| uMmioReg;
2510	szInstr[4] = 4;
2511
2512	/* 0xF0, 0x0F, 0x22, 0xC0 = mov cr8, eax */
2513	szInstr[5] = 0xF0;
2514	szInstr[6] = 0x0F;
2515	szInstr[7] = 0x22;
2516	szInstr[8] = 0xC0 \| uMmioReg;
2517
2518	/* Two nop instructions */
2519	szInstr[9] = 0x90;
2520	szInstr[10] = 0x90;
2521
2522	rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->rip, szInstr, 6+cbOp);
2523	AssertRC(rc);
2524
2525	Log(("Acceptable read/write candidate!\n"));
2526	return VINF_SUCCESS;
2527	}
2528	}
2529	else
2530	if ( pDis->param1.flags == USE_REG_GEN32
2531	&& cbOp == 5)
2532	{
2533	uint32_t uMmioReg = pDis->param1.base.reg_gen;
2534
2535	/* Found:
2536	* mov eax, dword [fffe0080] (5 bytes)
2537	* Check if next instruction is:
2538	* shr eax, 4
2539	*/
2540	pCtx->rip += cbOp;
2541	rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
2542	pCtx->rip = oldrip;
2543	if ( rc == VINF_SUCCESS
2544	&& pDis->pCurInstr->opcode == OP_SHR
2545	&& pDis->param1.flags == USE_REG_GEN32
2546	&& pDis->param1.base.reg_gen == uMmioReg
2547	&& pDis->param2.flags == USE_IMMEDIATE8
2548	&& pDis->param2.parval == 4)
2549	{
2550	/* 0xF0, 0x0F, 0x20, 0xC0 = mov eax, cr8 */
2551	szInstr[0] = 0xF0;
2552	szInstr[1] = 0x0F;
2553	szInstr[2] = 0x20;
2554	szInstr[3] = 0xC0 \| pDis->param1.base.reg_gen;
2555	for (unsigned i = 4; i < 5+cbOp; i++)
2556	szInstr[i] = 0x90; /* nop */
2557
2558	rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->rip, szInstr, 5+cbOp);
2559	AssertRC(rc);
2560
2561	Log(("Acceptable read candidate!\n"));
2562	return VINF_SUCCESS;
2563	}
2564	}
2565	}
2566	#endif
2567	rc = svmR0EmulateTprMov(pVCpu, pDis, pCtx, cbOp);
2568	if (rc != VINF_SUCCESS)
2569	return rc;
2570
2571	/* Emulated successfully, so continue. */
2572	return VINF_SUCCESS;
2573	}
2574	return VERR_ACCESS_DENIED;
2575	}
2576
2577	/**
2578	* Enters the AMD-V session
2579	*
2580	* @returns VBox status code.
2581	* @param pVM The VM to operate on.
2582	* @param pVCpu The VM CPU to operate on.
2583	* @param pCpu CPU info struct
2584	*/
2585	VMMR0DECL(int) SVMR0Enter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
2586	{
2587	Assert(pVM->hwaccm.s.svm.fSupported);
2588
2589	LogFlow(("SVMR0Enter cpu%d last=%d asid=%d\n", pCpu->idCpu, pVCpu->hwaccm.s.idLastCpu, pVCpu->hwaccm.s.uCurrentASID));
2590	pVCpu->hwaccm.s.fResumeVM = false;
2591
2592	/* Force to reload LDTR, so we'll execute VMLoad to load additional guest state. */
2593	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_LDTR;
2594
2595	return VINF_SUCCESS;
2596	}
2597
2598
2599	/**
2600	* Leaves the AMD-V session
2601	*
2602	* @returns VBox status code.
2603	* @param pVM The VM to operate on.
2604	* @param pVCpu The VM CPU to operate on.
2605	* @param pCtx CPU context
2606	*/
2607	VMMR0DECL(int) SVMR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2608	{
2609	SVM_VMCB pVMCB = (SVM_VMCB )pVCpu->hwaccm.s.svm.pVMCB;
2610
2611	Assert(pVM->hwaccm.s.svm.fSupported);
2612
2613	/* Save the guest debug state if necessary. */
2614	if (CPUMIsGuestDebugStateActive(pVCpu))
2615	{
2616	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, false /* skip DR6 */);
2617
2618	/* Intercept all DRx reads and writes again. Changed later on. */
2619	pVMCB->ctrl.u16InterceptRdDRx = 0xFFFF;
2620	pVMCB->ctrl.u16InterceptWrDRx = 0xFFFF;
2621
2622	/* Resync the debug registers the next time. */
2623	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
2624	}
2625	else
2626	Assert(pVMCB->ctrl.u16InterceptRdDRx == 0xFFFF && pVMCB->ctrl.u16InterceptWrDRx == 0xFFFF);
2627
2628	return VINF_SUCCESS;
2629	}
2630
2631
2632	static int svmR0InterpretInvlPg(PVMCPU pVCpu, PDISCPUSTATE pCpu, PCPUMCTXCORE pRegFrame, uint32_t uASID)
2633	{
2634	OP_PARAMVAL param1;
2635	RTGCPTR addr;
2636
2637	int rc = DISQueryParamVal(pRegFrame, pCpu, &pCpu->param1, &param1, PARAM_SOURCE);
2638	if(RT_FAILURE(rc))
2639	return VERR_EM_INTERPRETER;
2640
2641	switch(param1.type)
2642	{
2643	case PARMTYPE_IMMEDIATE:
2644	case PARMTYPE_ADDRESS:
2645	if(!(param1.flags & (PARAM_VAL32\|PARAM_VAL64)))
2646	return VERR_EM_INTERPRETER;
2647	addr = param1.val.val64;
2648	break;
2649
2650	default:
2651	return VERR_EM_INTERPRETER;
2652	}
2653
2654	/** @todo is addr always a flat linear address or ds based
2655	* (in absence of segment override prefixes)????
2656	*/
2657	rc = PGMInvalidatePage(pVCpu, addr);
2658	if (RT_SUCCESS(rc))
2659	{
2660	/* Manually invalidate the page for the VM's TLB. */
2661	Log(("SVMR0InvlpgA %RGv ASID=%d\n", addr, uASID));
2662	SVMR0InvlpgA(addr, uASID);
2663	return VINF_SUCCESS;
2664	}
2665	AssertRC(rc);
2666	return rc;
2667	}
2668
2669	/**
2670	* Interprets INVLPG
2671	*
2672	* @returns VBox status code.
2673	* @retval VINF_* Scheduling instructions.
2674	* @retval VERR_EM_INTERPRETER Something we can't cope with.
2675	* @retval VERR_* Fatal errors.
2676	*
2677	* @param pVM The VM handle.
2678	* @param pRegFrame The register frame.
2679	* @param ASID Tagged TLB id for the guest
2680	*
2681	* Updates the EIP if an instruction was executed successfully.
2682	*/
2683	static int svmR0InterpretInvpg(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame, uint32_t uASID)
2684	{
2685	/*
2686	* Only allow 32 & 64 bits code.
2687	*/
2688	DISCPUMODE enmMode = SELMGetCpuModeFromSelector(pVM, pRegFrame->eflags, pRegFrame->cs, &pRegFrame->csHid);
2689	if (enmMode != CPUMODE_16BIT)
2690	{
2691	RTGCPTR pbCode;
2692	int rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->rip, &pbCode);
2693	if (RT_SUCCESS(rc))
2694	{
2695	uint32_t cbOp;
2696	PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
2697
2698	pDis->mode = enmMode;
2699	rc = EMInterpretDisasOneEx(pVM, pVCpu, pbCode, pRegFrame, pDis, &cbOp);
2700	Assert(RT_FAILURE(rc) \|\| pDis->pCurInstr->opcode == OP_INVLPG);
2701	if (RT_SUCCESS(rc) && pDis->pCurInstr->opcode == OP_INVLPG)
2702	{
2703	Assert(cbOp == pDis->opsize);
2704	rc = svmR0InterpretInvlPg(pVCpu, pDis, pRegFrame, uASID);
2705	if (RT_SUCCESS(rc))
2706	{
2707	pRegFrame->rip += cbOp; /* Move on to the next instruction. */
2708	}
2709	return rc;
2710	}
2711	}
2712	}
2713	return VERR_EM_INTERPRETER;
2714	}
2715
2716
2717	/**
2718	* Invalidates a guest page
2719	*
2720	* @returns VBox status code.
2721	* @param pVM The VM to operate on.
2722	* @param pVCpu The VM CPU to operate on.
2723	* @param GCVirt Page to invalidate
2724	*/
2725	VMMR0DECL(int) SVMR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
2726	{
2727	bool fFlushPending = pVM->hwaccm.s.svm.fAlwaysFlushTLB \| pVCpu->hwaccm.s.fForceTLBFlush;
2728
2729	/* Skip it if a TLB flush is already pending. */
2730	if (!fFlushPending)
2731	{
2732	SVM_VMCB *pVMCB;
2733
2734	Log2(("SVMR0InvalidatePage %RGv\n", GCVirt));
2735	AssertReturn(pVM, VERR_INVALID_PARAMETER);
2736	Assert(pVM->hwaccm.s.svm.fSupported);
2737
2738	/* @todo SMP */
2739	pVMCB = (SVM_VMCB *)pVM->aCpus[0].hwaccm.s.svm.pVMCB;
2740	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
2741
2742	#if HC_ARCH_BITS == 32
2743	/* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invlpga takes only 32 bits addresses. */
2744	if (CPUMIsGuestInLongMode(pVCpu))
2745	pVCpu->hwaccm.s.fForceTLBFlush = true;
2746	else
2747	#endif
2748	SVMR0InvlpgA(GCVirt, pVMCB->ctrl.TLBCtrl.n.u32ASID);
2749	}
2750	return VINF_SUCCESS;
2751	}
2752
2753
2754	#if 0 /* obsolete, but left here for clarification. */
2755	/**
2756	* Invalidates a guest page by physical address
2757	*
2758	* @returns VBox status code.
2759	* @param pVM The VM to operate on.
2760	* @param pVCpu The VM CPU to operate on.
2761	* @param GCPhys Page to invalidate
2762	*/
2763	VMMR0DECL(int) SVMR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
2764	{
2765	Assert(pVM->hwaccm.s.fNestedPaging);
2766	/* invlpga only invalidates TLB entries for guest virtual addresses; we have no choice but to force a TLB flush here. */
2767	pVCpu->hwaccm.s.fForceTLBFlush = true;
2768	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBInvlpga);
2769	return VINF_SUCCESS;
2770	}
2771	#endif
2772
2773	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2774	/**
2775	* Prepares for and executes VMRUN (64 bits guests from a 32 bits hosts).
2776	*
2777	* @returns VBox status code.
2778	* @param pVMCBHostPhys Physical address of host VMCB.
2779	* @param pVMCBPhys Physical address of the VMCB.
2780	* @param pCtx Guest context.
2781	* @param pVM The VM to operate on.
2782	* @param pVCpu The VMCPU to operate on.
2783	*/
2784	DECLASM(int) SVMR0VMSwitcherRun64(RTHCPHYS pVMCBHostPhys, RTHCPHYS pVMCBPhys, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu)
2785	{
2786	uint32_t aParam[4];
2787
2788	aParam[0] = (uint32_t)(pVMCBHostPhys); /* Param 1: pVMCBHostPhys - Lo. */
2789	aParam[1] = (uint32_t)(pVMCBHostPhys >> 32); /* Param 1: pVMCBHostPhys - Hi. */
2790	aParam[2] = (uint32_t)(pVMCBPhys); /* Param 2: pVMCBPhys - Lo. */
2791	aParam[3] = (uint32_t)(pVMCBPhys >> 32); /* Param 2: pVMCBPhys - Hi. */
2792
2793	return SVMR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnSVMGCVMRun64, 4, &aParam[0]);
2794	}
2795
2796	/**
2797	* Executes the specified handler in 64 mode
2798	*
2799	* @returns VBox status code.
2800	* @param pVM The VM to operate on.
2801	* @param pVCpu The VMCPU to operate on.
2802	* @param pCtx Guest context
2803	* @param pfnHandler RC handler
2804	* @param cbParam Number of parameters
2805	* @param paParam Array of 32 bits parameters
2806	*/
2807	VMMR0DECL(int) SVMR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
2808	{
2809	int rc;
2810	RTHCUINTREG uOldEFlags;
2811
2812	/* @todo This code is not guest SMP safe (hyper stack and switchers) */
2813	AssertReturn(pVM->cCPUs == 1, VERR_TOO_MANY_CPUS);
2814	Assert(pfnHandler);
2815
2816	uOldEFlags = ASMIntDisableFlags();
2817
2818	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVM));
2819	CPUMSetHyperEIP(pVCpu, pfnHandler);
2820	for (int i=(int)cbParam-1;i>=0;i--)
2821	CPUMPushHyper(pVCpu, paParam[i]);
2822
2823	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
2824	/* Call switcher. */
2825	rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM);
2826	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
2827
2828	ASMSetFlags(uOldEFlags);
2829	return rc;
2830	}
2831
2832	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWSVMR0.cpp@ 20876

Download in other formats: