HWSVMR0.cpp@ 43387

Last change on this file since 43387 was 43387, checked in by vboxsync, 12 years ago
VMM: HM cleanup.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 121.6 KB

Line
1	/* $Id: HWSVMR0.cpp 43387 2012-09-21 09:40:25Z vboxsync $ */
2	/** @file
3	* HM SVM (AMD-V) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <VBox/vmm/hm.h>
23	#include <VBox/vmm/pgm.h>
24	#include <VBox/vmm/selm.h>
25	#include <VBox/vmm/iom.h>
26	#include <VBox/vmm/dbgf.h>
27	#include <VBox/vmm/dbgftrace.h>
28	#include <VBox/vmm/tm.h>
29	#include <VBox/vmm/pdmapi.h>
30	#include "HMInternal.h"
31	#include <VBox/vmm/vm.h>
32	#include <VBox/vmm/hm_svm.h>
33	#include <VBox/err.h>
34	#include <VBox/log.h>
35	#include <VBox/dis.h>
36	#include <VBox/disopcode.h>
37	#include <iprt/param.h>
38	#include <iprt/assert.h>
39	#include <iprt/asm.h>
40	#include <iprt/asm-amd64-x86.h>
41	#include <iprt/cpuset.h>
42	#include <iprt/mp.h>
43	#include <iprt/time.h>
44	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
45	# include <iprt/thread.h>
46	#endif
47	#include <iprt/x86.h>
48	#include "HWSVMR0.h"
49
50	#include "dtrace/VBoxVMM.h"
51
52
53	/*******************************************************************************
54	* Internal Functions *
55	*******************************************************************************/
56	static int hmR0SvmInterpretInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame);
57	static int hmR0SvmEmulateTprVMMCall(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
58	static void hmR0SvmSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
59
60
61	/*******************************************************************************
62	* Global Variables *
63	*******************************************************************************/
64	/* IO operation lookup arrays. */
65	static uint32_t const g_aIOSize[8] = {0, 1, 2, 0, 4, 0, 0, 0};
66	static uint32_t const g_aIOOpAnd[8] = {0, 0xff, 0xffff, 0, 0xffffffff, 0, 0, 0};
67
68
69	/**
70	* Sets up and activates AMD-V on the current CPU.
71	*
72	* @returns VBox status code.
73	* @param pCpu Pointer to the CPU info struct.
74	* @param pVM Pointer to the VM (can be NULL after a resume!).
75	* @param pvCpuPage Pointer to the global CPU page.
76	* @param HCPhysCpuPage Physical address of the global CPU page.
77	*/
78	VMMR0DECL(int) SVMR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
79	{
80	AssertReturn(!fEnabledByHost, VERR_INVALID_PARAMETER);
81	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
82	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
83
84	/*
85	* We must turn on AMD-V and setup the host state physical address, as those MSRs are per cpu/core.
86	*/
87	uint64_t fEfer = ASMRdMsr(MSR_K6_EFER);
88	if (fEfer & MSR_K6_EFER_SVME)
89	{
90	/*
91	* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V.
92	*/
93	if ( pVM
94	&& pVM->hm.s.svm.fIgnoreInUseError)
95	{
96	pCpu->fIgnoreAMDVInUseError = true;
97	}
98
99	if (!pCpu->fIgnoreAMDVInUseError)
100	return VERR_SVM_IN_USE;
101	}
102
103	/* Turn on AMD-V in the EFER MSR. */
104	ASMWrMsr(MSR_K6_EFER, fEfer \| MSR_K6_EFER_SVME);
105
106	/* Write the physical page address where the CPU will store the host state while executing the VM. */
107	ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
108
109	/*
110	* Theoretically, other hypervisors may have used ASIDs, ideally we should flush all non-zero ASIDs
111	* when enabling SVM. AMD doesn't have an SVM instruction to flush all ASIDs (flushing is done
112	* upon VMRUN). Therefore, just set the fFlushASIDBeforeUse flag which instructs hmR0SvmSetupTLB()
113	* to flush the TLB with before using a new ASID.
114	*/
115	pCpu->fFlushASIDBeforeUse = true;
116
117	/*
118	* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}.
119	*/
120	++pCpu->cTLBFlushes;
121
122	return VINF_SUCCESS;
123	}
124
125
126	/**
127	* Deactivates AMD-V on the current CPU.
128	*
129	* @returns VBox status code.
130	* @param pCpu Pointer to the CPU info struct.
131	* @param pvCpuPage Pointer to the global CPU page.
132	* @param HCPhysCpuPage Physical address of the global CPU page.
133	*/
134	VMMR0DECL(int) SVMR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
135	{
136	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
137	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
138	NOREF(pCpu);
139
140	/* Turn off AMD-V in the EFER MSR. */
141	uint64_t fEfer = ASMRdMsr(MSR_K6_EFER);
142	ASMWrMsr(MSR_K6_EFER, fEfer & ~MSR_K6_EFER_SVME);
143
144	/* Invalidate host state physical address. */
145	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
146
147	return VINF_SUCCESS;
148	}
149
150
151	/**
152	* Does Ring-0 per VM AMD-V init.
153	*
154	* @returns VBox status code.
155	* @param pVM Pointer to the VM.
156	*/
157	VMMR0DECL(int) SVMR0InitVM(PVM pVM)
158	{
159	int rc;
160
161	pVM->hm.s.svm.pMemObjIOBitmap = NIL_RTR0MEMOBJ;
162
163	/* Allocate 12 KB for the IO bitmap (doesn't seem to be a way to convince SVM not to use it) */
164	rc = RTR0MemObjAllocCont(&pVM->hm.s.svm.pMemObjIOBitmap, 3 << PAGE_SHIFT, false /* executable R0 mapping */);
165	if (RT_FAILURE(rc))
166	return rc;
167
168	pVM->hm.s.svm.pIOBitmap = RTR0MemObjAddress(pVM->hm.s.svm.pMemObjIOBitmap);
169	pVM->hm.s.svm.pIOBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hm.s.svm.pMemObjIOBitmap, 0);
170	/* Set all bits to intercept all IO accesses. */
171	ASMMemFill32(pVM->hm.s.svm.pIOBitmap, 3 << PAGE_SHIFT, 0xffffffff);
172
173	/*
174	* Erratum 170 which requires a forced TLB flush for each world switch:
175	* See http://www.amd.com/us-en/assets/content_type/white_papers_and_tech_docs/33610.pdf
176	*
177	* All BH-G1/2 and DH-G1/2 models include a fix:
178	* Athlon X2: 0x6b 1/2
179	* 0x68 1/2
180	* Athlon 64: 0x7f 1
181	* 0x6f 2
182	* Sempron: 0x7f 1/2
183	* 0x6f 2
184	* 0x6c 2
185	* 0x7c 2
186	* Turion 64: 0x68 2
187	*/
188	uint32_t u32Dummy;
189	uint32_t u32Version, u32Family, u32Model, u32Stepping, u32BaseFamily;
190	ASMCpuId(1, &u32Version, &u32Dummy, &u32Dummy, &u32Dummy);
191	u32BaseFamily = (u32Version >> 8) & 0xf;
192	u32Family = u32BaseFamily + (u32BaseFamily == 0xf ? ((u32Version >> 20) & 0x7f) : 0);
193	u32Model = ((u32Version >> 4) & 0xf);
194	u32Model = u32Model \| ((u32BaseFamily == 0xf ? (u32Version >> 16) & 0x0f : 0) << 4);
195	u32Stepping = u32Version & 0xf;
196	if ( u32Family == 0xf
197	&& !((u32Model == 0x68 \|\| u32Model == 0x6b \|\| u32Model == 0x7f) && u32Stepping >= 1)
198	&& !((u32Model == 0x6f \|\| u32Model == 0x6c \|\| u32Model == 0x7c) && u32Stepping >= 2))
199	{
200	Log(("SVMR0InitVM: AMD cpu with erratum 170 family %x model %x stepping %x\n", u32Family, u32Model, u32Stepping));
201	pVM->hm.s.svm.fAlwaysFlushTLB = true;
202	}
203
204	/* Allocate VMCBs for all guest CPUs. */
205	for (VMCPUID i = 0; i < pVM->cCpus; i++)
206	{
207	PVMCPU pVCpu = &pVM->aCpus[i];
208
209	pVCpu->hm.s.svm.pMemObjVMCBHost = NIL_RTR0MEMOBJ;
210	pVCpu->hm.s.svm.pMemObjVMCB = NIL_RTR0MEMOBJ;
211	pVCpu->hm.s.svm.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
212
213	/* Allocate one page for the host context */
214	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.pMemObjVMCBHost, 1 << PAGE_SHIFT, false /* executable R0 mapping */);
215	if (RT_FAILURE(rc))
216	return rc;
217
218	pVCpu->hm.s.svm.pVMCBHost = RTR0MemObjAddress(pVCpu->hm.s.svm.pMemObjVMCBHost);
219	pVCpu->hm.s.svm.pVMCBHostPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.pMemObjVMCBHost, 0);
220	Assert(pVCpu->hm.s.svm.pVMCBHostPhys < _4G);
221	ASMMemZeroPage(pVCpu->hm.s.svm.pVMCBHost);
222
223	/* Allocate one page for the VM control block (VMCB). */
224	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.pMemObjVMCB, 1 << PAGE_SHIFT, false /* executable R0 mapping */);
225	if (RT_FAILURE(rc))
226	return rc;
227
228	pVCpu->hm.s.svm.pVMCB = RTR0MemObjAddress(pVCpu->hm.s.svm.pMemObjVMCB);
229	pVCpu->hm.s.svm.pVMCBPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.pMemObjVMCB, 0);
230	Assert(pVCpu->hm.s.svm.pVMCBPhys < _4G);
231	ASMMemZeroPage(pVCpu->hm.s.svm.pVMCB);
232
233	/* Allocate 8 KB for the MSR bitmap (doesn't seem to be a way to convince SVM not to use it) */
234	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.pMemObjMSRBitmap, 2 << PAGE_SHIFT, false /* executable R0 mapping */);
235	if (RT_FAILURE(rc))
236	return rc;
237
238	pVCpu->hm.s.svm.pMSRBitmap = RTR0MemObjAddress(pVCpu->hm.s.svm.pMemObjMSRBitmap);
239	pVCpu->hm.s.svm.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.pMemObjMSRBitmap, 0);
240	/* Set all bits to intercept all MSR accesses. */
241	ASMMemFill32(pVCpu->hm.s.svm.pMSRBitmap, 2 << PAGE_SHIFT, 0xffffffff);
242	}
243
244	return VINF_SUCCESS;
245	}
246
247
248	/**
249	* Does Ring-0 per VM AMD-V termination.
250	*
251	* @returns VBox status code.
252	* @param pVM Pointer to the VM.
253	*/
254	VMMR0DECL(int) SVMR0TermVM(PVM pVM)
255	{
256	for (VMCPUID i = 0; i < pVM->cCpus; i++)
257	{
258	PVMCPU pVCpu = &pVM->aCpus[i];
259
260	if (pVCpu->hm.s.svm.pMemObjVMCBHost != NIL_RTR0MEMOBJ)
261	{
262	RTR0MemObjFree(pVCpu->hm.s.svm.pMemObjVMCBHost, false);
263	pVCpu->hm.s.svm.pVMCBHost = 0;
264	pVCpu->hm.s.svm.pVMCBHostPhys = 0;
265	pVCpu->hm.s.svm.pMemObjVMCBHost = NIL_RTR0MEMOBJ;
266	}
267
268	if (pVCpu->hm.s.svm.pMemObjVMCB != NIL_RTR0MEMOBJ)
269	{
270	RTR0MemObjFree(pVCpu->hm.s.svm.pMemObjVMCB, false);
271	pVCpu->hm.s.svm.pVMCB = 0;
272	pVCpu->hm.s.svm.pVMCBPhys = 0;
273	pVCpu->hm.s.svm.pMemObjVMCB = NIL_RTR0MEMOBJ;
274	}
275	if (pVCpu->hm.s.svm.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
276	{
277	RTR0MemObjFree(pVCpu->hm.s.svm.pMemObjMSRBitmap, false);
278	pVCpu->hm.s.svm.pMSRBitmap = 0;
279	pVCpu->hm.s.svm.pMSRBitmapPhys = 0;
280	pVCpu->hm.s.svm.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
281	}
282	}
283	if (pVM->hm.s.svm.pMemObjIOBitmap != NIL_RTR0MEMOBJ)
284	{
285	RTR0MemObjFree(pVM->hm.s.svm.pMemObjIOBitmap, false);
286	pVM->hm.s.svm.pIOBitmap = 0;
287	pVM->hm.s.svm.pIOBitmapPhys = 0;
288	pVM->hm.s.svm.pMemObjIOBitmap = NIL_RTR0MEMOBJ;
289	}
290	return VINF_SUCCESS;
291	}
292
293
294	/**
295	* Sets up AMD-V for the specified VM.
296	*
297	* @returns VBox status code.
298	* @param pVM Pointer to the VM.
299	*/
300	VMMR0DECL(int) SVMR0SetupVM(PVM pVM)
301	{
302	int rc = VINF_SUCCESS;
303
304	AssertReturn(pVM, VERR_INVALID_PARAMETER);
305	Assert(pVM->hm.s.svm.fSupported);
306
307	for (VMCPUID i = 0; i < pVM->cCpus; i++)
308	{
309	PVMCPU pVCpu = &pVM->aCpus[i];
310	SVM_VMCB pVMCB = (SVM_VMCB )pVM->aCpus[i].hm.s.svm.pVMCB;
311
312	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_HMSVM_INVALID_PVMCB);
313
314	/*
315	* Program the control fields. Most of them never have to be changed again.
316	* CR0/4 reads must be intercepted, our shadow values are not necessarily the same as the guest's.
317	* Note: CR0 & CR4 can be safely read when guest and shadow copies are identical.
318	*/
319	pVMCB->ctrl.u16InterceptRdCRx = RT_BIT(0) \| RT_BIT(4);
320
321	/* CR0/4 writes must be intercepted for obvious reasons. */
322	pVMCB->ctrl.u16InterceptWrCRx = RT_BIT(0) \| RT_BIT(4);
323
324	/* Intercept all DRx reads and writes by default. Changed later on. */
325	pVMCB->ctrl.u16InterceptRdDRx = 0xFFFF;
326	pVMCB->ctrl.u16InterceptWrDRx = 0xFFFF;
327
328	/* Intercept traps; only #NM is always intercepted. */
329	pVMCB->ctrl.u32InterceptException = RT_BIT(X86_XCPT_NM);
330	#ifdef VBOX_ALWAYS_TRAP_PF
331	pVMCB->ctrl.u32InterceptException \|= RT_BIT(X86_XCPT_PF);
332	#endif
333	#ifdef VBOX_STRICT
334	pVMCB->ctrl.u32InterceptException \|= RT_BIT(X86_XCPT_BP)
335	\| RT_BIT(X86_XCPT_DB)
336	\| RT_BIT(X86_XCPT_DE)
337	\| RT_BIT(X86_XCPT_UD)
338	\| RT_BIT(X86_XCPT_NP)
339	\| RT_BIT(X86_XCPT_SS)
340	\| RT_BIT(X86_XCPT_GP)
341	\| RT_BIT(X86_XCPT_MF)
342	;
343	#endif
344
345	/* Set up instruction and miscellaneous intercepts. */
346	pVMCB->ctrl.u32InterceptCtrl1 = SVM_CTRL1_INTERCEPT_INTR
347	\| SVM_CTRL1_INTERCEPT_VINTR
348	\| SVM_CTRL1_INTERCEPT_NMI
349	\| SVM_CTRL1_INTERCEPT_SMI
350	\| SVM_CTRL1_INTERCEPT_INIT
351	\| SVM_CTRL1_INTERCEPT_RDPMC
352	\| SVM_CTRL1_INTERCEPT_CPUID
353	\| SVM_CTRL1_INTERCEPT_RSM
354	\| SVM_CTRL1_INTERCEPT_HLT
355	\| SVM_CTRL1_INTERCEPT_INOUT_BITMAP
356	\| SVM_CTRL1_INTERCEPT_MSR_SHADOW
357	\| SVM_CTRL1_INTERCEPT_INVLPGA /* AMD only */
358	\| SVM_CTRL1_INTERCEPT_SHUTDOWN /* fatal */
359	\| SVM_CTRL1_INTERCEPT_FERR_FREEZE; /* Legacy FPU FERR handling. */
360	;
361	pVMCB->ctrl.u32InterceptCtrl2 = SVM_CTRL2_INTERCEPT_VMRUN /* required */
362	\| SVM_CTRL2_INTERCEPT_VMMCALL
363	\| SVM_CTRL2_INTERCEPT_VMLOAD
364	\| SVM_CTRL2_INTERCEPT_VMSAVE
365	\| SVM_CTRL2_INTERCEPT_STGI
366	\| SVM_CTRL2_INTERCEPT_CLGI
367	\| SVM_CTRL2_INTERCEPT_SKINIT
368	\| SVM_CTRL2_INTERCEPT_WBINVD
369	\| SVM_CTRL2_INTERCEPT_MONITOR
370	\| SVM_CTRL2_INTERCEPT_MWAIT_UNCOND; /* don't execute mwait or else we'll idle inside the
371	guest (host thinks the cpu load is high) */
372
373	Log(("pVMCB->ctrl.u32InterceptException = %x\n", pVMCB->ctrl.u32InterceptException));
374	Log(("pVMCB->ctrl.u32InterceptCtrl1 = %x\n", pVMCB->ctrl.u32InterceptCtrl1));
375	Log(("pVMCB->ctrl.u32InterceptCtrl2 = %x\n", pVMCB->ctrl.u32InterceptCtrl2));
376
377	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
378	pVMCB->ctrl.IntCtrl.n.u1VIrqMasking = 1;
379
380	/* Ignore the priority in the TPR; just deliver it when we tell it to. */
381	pVMCB->ctrl.IntCtrl.n.u1IgnoreTPR = 1;
382
383	/* Set IO and MSR bitmap addresses. */
384	pVMCB->ctrl.u64IOPMPhysAddr = pVM->hm.s.svm.pIOBitmapPhys;
385	pVMCB->ctrl.u64MSRPMPhysAddr = pVCpu->hm.s.svm.pMSRBitmapPhys;
386
387	/* No LBR virtualization. */
388	pVMCB->ctrl.u64LBRVirt = 0;
389
390	/* The ASID must start at 1; the host uses 0. */
391	pVMCB->ctrl.TLBCtrl.n.u32ASID = 1;
392
393	/*
394	* Setup the PAT MSR (nested paging only)
395	* The default value should be 0x0007040600070406ULL, but we want to treat all guest memory as WB,
396	* so choose type 6 for all PAT slots.
397	*/
398	pVMCB->guest.u64GPAT = 0x0006060606060606ULL;
399
400	/* If nested paging is not in use, additional intercepts have to be set up. */
401	if (!pVM->hm.s.fNestedPaging)
402	{
403	/* CR3 reads/writes must be intercepted; our shadow values are different from guest's. */
404	pVMCB->ctrl.u16InterceptRdCRx \|= RT_BIT(3);
405	pVMCB->ctrl.u16InterceptWrCRx \|= RT_BIT(3);
406
407	/*
408	* We must also intercept:
409	* - INVLPG (must go through shadow paging)
410	* - task switches (may change CR3/EFLAGS/LDT)
411	*/
412	pVMCB->ctrl.u32InterceptCtrl1 \|= SVM_CTRL1_INTERCEPT_INVLPG
413	\| SVM_CTRL1_INTERCEPT_TASK_SWITCH;
414
415	/* Page faults must be intercepted to implement shadow paging. */
416	pVMCB->ctrl.u32InterceptException \|= RT_BIT(X86_XCPT_PF);
417	}
418
419	/*
420	* The following MSRs are saved automatically by vmload/vmsave, so we allow the guest
421	* to modify them directly.
422	*/
423	hmR0SvmSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
424	hmR0SvmSetMSRPermission(pVCpu, MSR_K8_CSTAR, true, true);
425	hmR0SvmSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
426	hmR0SvmSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
427	hmR0SvmSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
428	hmR0SvmSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
429	hmR0SvmSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
430	hmR0SvmSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
431	hmR0SvmSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
432	hmR0SvmSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
433	}
434
435	return rc;
436	}
437
438
439	/**
440	* Sets the permission bits for the specified MSR.
441	*
442	* @param pVCpu Pointer to the VMCPU.
443	* @param ulMSR MSR value.
444	* @param fRead Whether reading is allowed.
445	* @param fWrite Whether writing is allowed.
446	*/
447	static void hmR0SvmSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
448	{
449	unsigned ulBit;
450	uint8_t pMSRBitmap = (uint8_t )pVCpu->hm.s.svm.pMSRBitmap;
451
452	if (ulMSR <= 0x00001FFF)
453	{
454	/* Pentium-compatible MSRs */
455	ulBit = ulMSR * 2;
456	}
457	else if ( ulMSR >= 0xC0000000
458	&& ulMSR <= 0xC0001FFF)
459	{
460	/* AMD Sixth Generation x86 Processor MSRs and SYSCALL */
461	ulBit = (ulMSR - 0xC0000000) * 2;
462	pMSRBitmap += 0x800;
463	}
464	else if ( ulMSR >= 0xC0010000
465	&& ulMSR <= 0xC0011FFF)
466	{
467	/* AMD Seventh and Eighth Generation Processor MSRs */
468	ulBit = (ulMSR - 0xC0001000) * 2;
469	pMSRBitmap += 0x1000;
470	}
471	else
472	{
473	AssertFailed();
474	return;
475	}
476	Assert(ulBit < 16 * 1024 - 1);
477	if (fRead)
478	ASMBitClear(pMSRBitmap, ulBit);
479	else
480	ASMBitSet(pMSRBitmap, ulBit);
481
482	if (fWrite)
483	ASMBitClear(pMSRBitmap, ulBit + 1);
484	else
485	ASMBitSet(pMSRBitmap, ulBit + 1);
486	}
487
488
489	/**
490	* Injects an event (trap or external interrupt).
491	*
492	* @param pVCpu Pointer to the VMCPU.
493	* @param pVMCB Pointer to the VMCB.
494	* @param pCtx Pointer to the guest CPU context.
495	* @param pIntInfo Pointer to the SVM interrupt info.
496	*/
497	DECLINLINE(void) hmR0SvmInjectEvent(PVMCPU pVCpu, SVM_VMCB pVMCB, CPUMCTX pCtx, SVM_EVENT *pEvent)
498	{
499	#ifdef VBOX_WITH_STATISTICS
500	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
501	#endif
502
503	#ifdef VBOX_STRICT
504	if (pEvent->n.u8Vector == 0xE)
505	{
506	Log(("SVM: Inject int %d at %RGv error code=%02x CR2=%RGv intInfo=%08x\n", pEvent->n.u8Vector,
507	(RTGCPTR)pCtx->rip, pEvent->n.u32ErrorCode, (RTGCPTR)pCtx->cr2, pEvent->au64[0]));
508	}
509	else if (pEvent->n.u8Vector < 0x20)
510	Log(("SVM: Inject int %d at %RGv error code=%08x\n", pEvent->n.u8Vector, (RTGCPTR)pCtx->rip, pEvent->n.u32ErrorCode));
511	else
512	{
513	Log(("INJ-EI: %x at %RGv\n", pEvent->n.u8Vector, (RTGCPTR)pCtx->rip));
514	Assert(!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
515	Assert(pCtx->eflags.u32 & X86_EFL_IF);
516	}
517	#endif
518
519	/* Set event injection state. */
520	pVMCB->ctrl.EventInject.au64[0] = pEvent->au64[0];
521	}
522
523
524	/**
525	* Checks for pending guest interrupts and injects them.
526	*
527	* @returns VBox status code.
528	* @param pVM Pointer to the VM.
529	* @param pVCpu Pointer to the VMCPU.
530	* @param pVMCB Pointer to the VMCB.
531	* @param pCtx Pointer to the guest CPU Context.
532	*/
533	static int hmR0SvmCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, SVM_VMCB pVMCB, CPUMCTX pCtx)
534	{
535	int rc;
536	NOREF(pVM);
537
538	/*
539	* Dispatch any pending interrupts (injected before, but a VM-exit occurred prematurely).
540	*/
541	if (pVCpu->hm.s.Event.fPending)
542	{
543	SVM_EVENT Event;
544
545	Log(("Reinjecting event %08x %08x at %RGv\n", pVCpu->hm.s.Event.intInfo, pVCpu->hm.s.Event.errCode,
546	(RTGCPTR)pCtx->rip));
547	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
548	Event.au64[0] = pVCpu->hm.s.Event.intInfo;
549	hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
550
551	pVCpu->hm.s.Event.fPending = false;
552	return VINF_SUCCESS;
553	}
554
555	/*
556	* If an active trap is already pending, we must forward it first!
557	*/
558	if (!TRPMHasTrap(pVCpu))
559	{
560	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
561	{
562	SVM_EVENT Event;
563
564	Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
565	Event.n.u8Vector = X86_XCPT_NMI;
566	Event.n.u1Valid = 1;
567	Event.n.u32ErrorCode = 0;
568	Event.n.u3Type = SVM_EVENT_NMI;
569
570	hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
571	return VINF_SUCCESS;
572	}
573
574	/** @todo SMI interrupts. */
575
576	/*
577	* When external interrupts are pending, we should exit the VM when IF is set.
578	*/
579	if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)))
580	{
581	if ( !(pCtx->eflags.u32 & X86_EFL_IF)
582	\|\| VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
583	{
584	if (!pVMCB->ctrl.IntCtrl.n.u1VIrqValid)
585	{
586	if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
587	LogFlow(("Enable irq window exit!\n"));
588	else
589	{
590	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS -> irq window exit\n",
591	(RTGCPTR)pCtx->rip));
592	}
593
594	/** @todo Use virtual interrupt method to inject a pending IRQ; dispatched as
595	* soon as guest.IF is set. */
596	pVMCB->ctrl.u32InterceptCtrl1 \|= SVM_CTRL1_INTERCEPT_VINTR;
597	pVMCB->ctrl.IntCtrl.n.u1VIrqValid = 1;
598	pVMCB->ctrl.IntCtrl.n.u8VIrqVector = 0; /* don't care */
599	}
600	}
601	else
602	{
603	uint8_t u8Interrupt;
604
605	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
606	Log(("Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc\n", u8Interrupt, u8Interrupt, rc));
607	if (RT_SUCCESS(rc))
608	{
609	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
610	AssertRC(rc);
611	}
612	else
613	{
614	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
615	Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)));
616	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
617	/* Just continue */
618	}
619	}
620	}
621	}
622
623	#ifdef VBOX_STRICT
624	if (TRPMHasTrap(pVCpu))
625	{
626	uint8_t u8Vector;
627	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
628	AssertRC(rc);
629	}
630	#endif
631
632	if ( (pCtx->eflags.u32 & X86_EFL_IF)
633	&& (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
634	&& TRPMHasTrap(pVCpu)
635	)
636	{
637	uint8_t u8Vector;
638	TRPMEVENT enmType;
639	SVM_EVENT Event;
640	RTGCUINT u32ErrorCode;
641
642	Event.au64[0] = 0;
643
644	/* If a new event is pending, then dispatch it now. */
645	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &u32ErrorCode, 0);
646	AssertRC(rc);
647	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
648	Assert(enmType != TRPM_SOFTWARE_INT);
649
650	/* Clear the pending trap. */
651	rc = TRPMResetTrap(pVCpu);
652	AssertRC(rc);
653
654	Event.n.u8Vector = u8Vector;
655	Event.n.u1Valid = 1;
656	Event.n.u32ErrorCode = u32ErrorCode;
657
658	if (enmType == TRPM_TRAP)
659	{
660	switch (u8Vector)
661	{
662	case X86_XCPT_DF:
663	case X86_XCPT_TS:
664	case X86_XCPT_NP:
665	case X86_XCPT_SS:
666	case X86_XCPT_GP:
667	case X86_XCPT_PF:
668	case X86_XCPT_AC:
669	/* Valid error codes. */
670	Event.n.u1ErrorCodeValid = 1;
671	break;
672	default:
673	break;
674	}
675	if (u8Vector == X86_XCPT_NMI)
676	Event.n.u3Type = SVM_EVENT_NMI;
677	else
678	Event.n.u3Type = SVM_EVENT_EXCEPTION;
679	}
680	else
681	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
682
683	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
684	hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
685	} /* if (interrupts can be dispatched) */
686
687	return VINF_SUCCESS;
688	}
689
690
691	/**
692	* Save the host state.
693	*
694	* @returns VBox status code.
695	* @param pVM Pointer to the VM.
696	* @param pVCpu Pointer to the VMCPU.
697	*/
698	VMMR0DECL(int) SVMR0SaveHostState(PVM pVM, PVMCPU pVCpu)
699	{
700	NOREF(pVM);
701	NOREF(pVCpu);
702	/* Nothing to do here. */
703	return VINF_SUCCESS;
704	}
705
706
707	/**
708	* Loads the guest state.
709	*
710	* NOTE: Don't do anything here that can cause a jump back to ring-3!!!
711	*
712	* @returns VBox status code.
713	* @param pVM Pointer to the VM.
714	* @param pVCpu Pointer to the VMCPU.
715	* @param pCtx Pointer to the guest CPU context.
716	*/
717	VMMR0DECL(int) SVMR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
718	{
719	RTGCUINTPTR val;
720	SVM_VMCB *pVMCB;
721
722	if (pVM == NULL)
723	return VERR_INVALID_PARAMETER;
724
725	/* Setup AMD SVM. */
726	Assert(pVM->hm.s.svm.fSupported);
727
728	pVMCB = (SVM_VMCB *)pVCpu->hm.s.svm.pVMCB;
729	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_HMSVM_INVALID_PVMCB);
730
731	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
732	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
733	{
734	SVM_WRITE_SELREG(CS, cs);
735	SVM_WRITE_SELREG(SS, ss);
736	SVM_WRITE_SELREG(DS, ds);
737	SVM_WRITE_SELREG(ES, es);
738	SVM_WRITE_SELREG(FS, fs);
739	SVM_WRITE_SELREG(GS, gs);
740	}
741
742	/* Guest CPU context: LDTR. */
743	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
744	{
745	SVM_WRITE_SELREG(LDTR, ldtr);
746	}
747
748	/* Guest CPU context: TR. */
749	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
750	{
751	SVM_WRITE_SELREG(TR, tr);
752	}
753
754	/* Guest CPU context: GDTR. */
755	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
756	{
757	pVMCB->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
758	pVMCB->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
759	}
760
761	/* Guest CPU context: IDTR. */
762	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
763	{
764	pVMCB->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
765	pVMCB->guest.IDTR.u64Base = pCtx->idtr.pIdt;
766	}
767
768	/*
769	* Sysenter MSRs (unconditional)
770	*/
771	pVMCB->guest.u64SysEnterCS = pCtx->SysEnter.cs;
772	pVMCB->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
773	pVMCB->guest.u64SysEnterESP = pCtx->SysEnter.esp;
774
775	/* Control registers */
776	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
777	{
778	val = pCtx->cr0;
779	if (!CPUMIsGuestFPUStateActive(pVCpu))
780	{
781	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
782	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
783	}
784	else
785	{
786	/** @todo check if we support the old style mess correctly. */
787	if (!(val & X86_CR0_NE))
788	{
789	Log(("Forcing X86_CR0_NE!!!\n"));
790
791	/* Also catch floating point exceptions as we need to report them to the guest in a different way. */
792	if (!pVCpu->hm.s.fFPUOldStyleOverride)
793	{
794	pVMCB->ctrl.u32InterceptException \|= RT_BIT(X86_XCPT_MF);
795	pVCpu->hm.s.fFPUOldStyleOverride = true;
796	}
797	}
798	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
799	}
800	/* Always enable caching. */
801	val &= ~(X86_CR0_CD\|X86_CR0_NW);
802
803	/*
804	* Note: WP is not relevant in nested paging mode as we catch accesses on the (guest) physical level.
805	* Note: In nested paging mode, the guest is allowed to run with paging disabled; the guest-physical to host-physical
806	* translation will remain active.
807	*/
808	if (!pVM->hm.s.fNestedPaging)
809	{
810	val \|= X86_CR0_PG; /* Paging is always enabled; even when the guest is running in real mode or PE without paging. */
811	val \|= X86_CR0_WP; /* Must set this as we rely on protecting various pages and supervisor writes must be caught. */
812	}
813	pVMCB->guest.u64CR0 = val;
814	}
815	/* CR2 as well */
816	pVMCB->guest.u64CR2 = pCtx->cr2;
817
818	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
819	{
820	/* Save our shadow CR3 register. */
821	if (pVM->hm.s.fNestedPaging)
822	{
823	PGMMODE enmShwPagingMode;
824
825	#if HC_ARCH_BITS == 32
826	if (CPUMIsGuestInLongModeEx(pCtx))
827	enmShwPagingMode = PGMMODE_AMD64_NX;
828	else
829	#endif
830	enmShwPagingMode = PGMGetHostMode(pVM);
831
832	pVMCB->ctrl.u64NestedPagingCR3 = PGMGetNestedCR3(pVCpu, enmShwPagingMode);
833	Assert(pVMCB->ctrl.u64NestedPagingCR3);
834	pVMCB->guest.u64CR3 = pCtx->cr3;
835	}
836	else
837	{
838	pVMCB->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
839	Assert(pVMCB->guest.u64CR3 \|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
840	}
841	}
842
843	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
844	{
845	val = pCtx->cr4;
846	if (!pVM->hm.s.fNestedPaging)
847	{
848	switch (pVCpu->hm.s.enmShadowMode)
849	{
850	case PGMMODE_REAL:
851	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
852	AssertFailed();
853	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
854
855	case PGMMODE_32_BIT: /* 32-bit paging. */
856	val &= ~X86_CR4_PAE;
857	break;
858
859	case PGMMODE_PAE: /* PAE paging. */
860	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
861	/** Must use PAE paging as we could use physical memory > 4 GB */
862	val \|= X86_CR4_PAE;
863	break;
864
865	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
866	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
867	#ifdef VBOX_ENABLE_64_BITS_GUESTS
868	break;
869	#else
870	AssertFailed();
871	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
872	#endif
873
874	default: /* shut up gcc */
875	AssertFailed();
876	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
877	}
878	}
879	pVMCB->guest.u64CR4 = val;
880	}
881
882	/* Debug registers. */
883	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG)
884	{
885	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
886	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
887
888	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
889	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
890	pCtx->dr[7] \|= 0x400; /* must be one */
891
892	pVMCB->guest.u64DR7 = pCtx->dr[7];
893	pVMCB->guest.u64DR6 = pCtx->dr[6];
894
895	#ifdef DEBUG
896	/* Sync the hypervisor debug state now if any breakpoint is armed. */
897	if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK\|X86_DR7_GD)
898	&& !CPUMIsHyperDebugStateActive(pVCpu)
899	&& !DBGFIsStepping(pVCpu))
900	{
901	/* Save the host and load the hypervisor debug state. */
902	int rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
903	AssertRC(rc);
904
905	/* DRx intercepts remain enabled. */
906
907	/* Override dr6 & dr7 with the hypervisor values. */
908	pVMCB->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
909	pVMCB->guest.u64DR6 = CPUMGetHyperDR6(pVCpu);
910	}
911	else
912	#endif
913	/* Sync the debug state now if any breakpoint is armed. */
914	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
915	&& !CPUMIsGuestDebugStateActive(pVCpu)
916	&& !DBGFIsStepping(pVCpu))
917	{
918	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
919
920	/* Disable drx move intercepts. */
921	pVMCB->ctrl.u16InterceptRdDRx = 0;
922	pVMCB->ctrl.u16InterceptWrDRx = 0;
923
924	/* Save the host and load the guest debug state. */
925	int rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
926	AssertRC(rc);
927	}
928	}
929
930	/* EIP, ESP and EFLAGS */
931	pVMCB->guest.u64RIP = pCtx->rip;
932	pVMCB->guest.u64RSP = pCtx->rsp;
933	pVMCB->guest.u64RFlags = pCtx->eflags.u32;
934
935	/* Set CPL */
936	pVMCB->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
937
938	/* RAX/EAX too, as VMRUN uses RAX as an implicit parameter. */
939	pVMCB->guest.u64RAX = pCtx->rax;
940
941	/* vmrun will fail without MSR_K6_EFER_SVME. */
942	pVMCB->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
943
944	/* 64 bits guest mode? */
945	if (CPUMIsGuestInLongModeEx(pCtx))
946	{
947	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
948	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
949	#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
950	pVCpu->hm.s.svm.pfnVMRun = SVMR0VMSwitcherRun64;
951	#else
952	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
953	if (!pVM->hm.s.fAllow64BitGuests)
954	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
955	# endif
956	pVCpu->hm.s.svm.pfnVMRun = SVMR0VMRun64;
957	#endif
958	/* Unconditionally update these as wrmsr might have changed them. (HM_CHANGED_GUEST_SEGMENT_REGS will not be set) */
959	pVMCB->guest.FS.u64Base = pCtx->fs.u64Base;
960	pVMCB->guest.GS.u64Base = pCtx->gs.u64Base;
961	}
962	else
963	{
964	/* Filter out the MSR_K6_LME bit or else AMD-V expects amd64 shadow paging. */
965	pVMCB->guest.u64EFER &= ~MSR_K6_EFER_LME;
966
967	pVCpu->hm.s.svm.pfnVMRun = SVMR0VMRun;
968	}
969
970	/* TSC offset. */
971	if (TMCpuTickCanUseRealTSC(pVCpu, &pVMCB->ctrl.u64TSCOffset))
972	{
973	uint64_t u64CurTSC = ASMReadTSC();
974	if (u64CurTSC + pVMCB->ctrl.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
975	{
976	pVMCB->ctrl.u32InterceptCtrl1 &= ~SVM_CTRL1_INTERCEPT_RDTSC;
977	pVMCB->ctrl.u32InterceptCtrl2 &= ~SVM_CTRL2_INTERCEPT_RDTSCP;
978	STAM_COUNTER_INC(&pVCpu->hm.s.StatTSCOffset);
979	}
980	else
981	{
982	/* Fall back to rdtsc emulation as we would otherwise pass decreasing tsc values to the guest. */
983	LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC,
984	pVMCB->ctrl.u64TSCOffset, u64CurTSC + pVMCB->ctrl.u64TSCOffset, TMCpuTickGetLastSeen(pVCpu),
985	TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVMCB->ctrl.u64TSCOffset, TMCpuTickGet(pVCpu)));
986	pVMCB->ctrl.u32InterceptCtrl1 \|= SVM_CTRL1_INTERCEPT_RDTSC;
987	pVMCB->ctrl.u32InterceptCtrl2 \|= SVM_CTRL2_INTERCEPT_RDTSCP;
988	STAM_COUNTER_INC(&pVCpu->hm.s.StatTSCInterceptOverFlow);
989	}
990	}
991	else
992	{
993	pVMCB->ctrl.u32InterceptCtrl1 \|= SVM_CTRL1_INTERCEPT_RDTSC;
994	pVMCB->ctrl.u32InterceptCtrl2 \|= SVM_CTRL2_INTERCEPT_RDTSCP;
995	STAM_COUNTER_INC(&pVCpu->hm.s.StatTSCIntercept);
996	}
997
998	/* Sync the various MSRs for 64-bit mode. */
999	pVMCB->guest.u64STAR = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
1000	pVMCB->guest.u64LSTAR = pCtx->msrLSTAR; /* 64-bit mode syscall rip */
1001	pVMCB->guest.u64CSTAR = pCtx->msrCSTAR; /* compatibility mode syscall rip */
1002	pVMCB->guest.u64SFMASK = pCtx->msrSFMASK; /* syscall flag mask */
1003	pVMCB->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE; /* SWAPGS exchange value */
1004
1005	#ifdef DEBUG
1006	/* Intercept X86_XCPT_DB if stepping is enabled */
1007	if ( DBGFIsStepping(pVCpu)
1008	\|\| CPUMIsHyperDebugStateActive(pVCpu))
1009	pVMCB->ctrl.u32InterceptException \|= RT_BIT(X86_XCPT_DB);
1010	else
1011	pVMCB->ctrl.u32InterceptException &= ~RT_BIT(X86_XCPT_DB);
1012	#endif
1013
1014	/* Done. */
1015	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_ALL_GUEST;
1016
1017	return VINF_SUCCESS;
1018	}
1019
1020
1021	/**
1022	* Setup TLB for ASID.
1023	*
1024	* @param pVM Pointer to the VM.
1025	* @param pVCpu Pointer to the VMCPU.
1026	*/
1027	static void hmR0SvmSetupTLB(PVM pVM, PVMCPU pVCpu)
1028	{
1029	PHMGLOBLCPUINFO pCpu;
1030
1031	AssertPtr(pVM);
1032	AssertPtr(pVCpu);
1033
1034	SVM_VMCB pVMCB = (SVM_VMCB )pVCpu->hm.s.svm.pVMCB;
1035	pCpu = HMR0GetCurrentCpu();
1036
1037	/*
1038	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1039	* This can happen both for start & resume due to long jumps back to ring-3.
1040	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB,
1041	* so we cannot reuse the ASIDs without flushing.
1042	*/
1043	bool fNewASID = false;
1044	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1045	\|\| pVCpu->hm.s.cTLBFlushes != pCpu->cTLBFlushes)
1046	{
1047	pVCpu->hm.s.fForceTLBFlush = true;
1048	fNewASID = true;
1049	}
1050
1051	/*
1052	* Set TLB flush state as checked until we return from the world switch.
1053	*/
1054	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1055
1056	/*
1057	* Check for TLB shootdown flushes.
1058	*/
1059	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1060	pVCpu->hm.s.fForceTLBFlush = true;
1061
1062	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1063	pVMCB->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1064
1065	if (RT_UNLIKELY(pVM->hm.s.svm.fAlwaysFlushTLB))
1066	{
1067	/*
1068	* This is the AMD erratum 170. We need to flush the entire TLB for each world switch. Sad.
1069	*/
1070	pCpu->uCurrentASID = 1;
1071	pVCpu->hm.s.uCurrentASID = 1;
1072	pVCpu->hm.s.cTLBFlushes = pCpu->cTLBFlushes;
1073	pVMCB->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1074	}
1075	else if (pVCpu->hm.s.fForceTLBFlush)
1076	{
1077	if (fNewASID)
1078	{
1079	++pCpu->uCurrentASID;
1080	bool fHitASIDLimit = false;
1081	if (pCpu->uCurrentASID >= pVM->hm.s.uMaxASID)
1082	{
1083	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
1084	pCpu->cTLBFlushes++;
1085	fHitASIDLimit = true;
1086
1087	if (pVM->hm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1088	{
1089	pVMCB->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1090	pCpu->fFlushASIDBeforeUse = true;
1091	}
1092	else
1093	{
1094	pVMCB->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1095	pCpu->fFlushASIDBeforeUse = false;
1096	}
1097	}
1098
1099	if ( !fHitASIDLimit
1100	&& pCpu->fFlushASIDBeforeUse)
1101	{
1102	if (pVM->hm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1103	pVMCB->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1104	else
1105	{
1106	pVMCB->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1107	pCpu->fFlushASIDBeforeUse = false;
1108	}
1109	}
1110
1111	pVCpu->hm.s.uCurrentASID = pCpu->uCurrentASID;
1112	pVCpu->hm.s.cTLBFlushes = pCpu->cTLBFlushes;
1113	}
1114	else
1115	{
1116	if (pVM->hm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1117	pVMCB->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1118	else
1119	pVMCB->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1120	}
1121
1122	pVCpu->hm.s.fForceTLBFlush = false;
1123	}
1124	else
1125	{
1126	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1127	* not be executed. See hmQueueInvlPage() where it is commented
1128	* out. Support individual entry flushing someday. */
1129	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1130	{
1131	/* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
1132	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1133	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1134	SVMR0InvlpgA(pVCpu->hm.s.TlbShootdown.aPages[i], pVMCB->ctrl.TLBCtrl.n.u32ASID);
1135	}
1136	}
1137
1138	pVCpu->hm.s.TlbShootdown.cPages = 0;
1139	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1140
1141	/* Update VMCB with the ASID. */
1142	pVMCB->ctrl.TLBCtrl.n.u32ASID = pVCpu->hm.s.uCurrentASID;
1143
1144	AssertMsg(pVCpu->hm.s.cTLBFlushes == pCpu->cTLBFlushes,
1145	("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hm.s.cTLBFlushes, pCpu->cTLBFlushes));
1146	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hm.s.uMaxASID,
1147	("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
1148	AssertMsg(pVCpu->hm.s.uCurrentASID >= 1 && pVCpu->hm.s.uCurrentASID < pVM->hm.s.uMaxASID,
1149	("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hm.s.uCurrentASID));
1150
1151	#ifdef VBOX_WITH_STATISTICS
1152	if (pVMCB->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1153	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTLBWorldSwitch);
1154	else if ( pVMCB->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1155	\|\| pVMCB->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1156	{
1157	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushASID);
1158	}
1159	else
1160	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTLBWorldSwitch);
1161	#endif
1162	}
1163
1164
1165	/**
1166	* Runs guest code in an AMD-V VM.
1167	*
1168	* @returns VBox status code.
1169	* @param pVM Pointer to the VM.
1170	* @param pVCpu Pointer to the VMCPU.
1171	* @param pCtx Pointer to the guest CPU context.
1172	*/
1173	VMMR0DECL(int) SVMR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1174	{
1175	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
1176	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
1177	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
1178
1179	VBOXSTRICTRC rc = VINF_SUCCESS;
1180	int rc2;
1181	uint64_t exitCode = (uint64_t)SVM_EXIT_INVALID;
1182	SVM_VMCB *pVMCB = NULL;
1183	bool fSyncTPR = false;
1184	unsigned cResume = 0;
1185	uint8_t u8LastTPR = 0; /* Initialized for potentially stupid compilers. */
1186	uint32_t u32HostExtFeatures = 0;
1187	PHMGLOBLCPUINFO pCpu = 0;
1188	RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
1189	#ifdef VBOX_STRICT
1190	RTCPUID idCpuCheck;
1191	#endif
1192	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
1193	uint64_t u64LastTime = RTTimeMilliTS();
1194	#endif
1195
1196	pVMCB = (SVM_VMCB *)pVCpu->hm.s.svm.pVMCB;
1197	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_HMSVM_INVALID_PVMCB);
1198
1199	/*
1200	* We can jump to this point to resume execution after determining that a VM-exit is innocent.
1201	*/
1202	ResumeExecution:
1203	if (!STAM_PROFILE_ADV_IS_RUNNING(&pVCpu->hm.s.StatEntry))
1204	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit2, &pVCpu->hm.s.StatEntry, x);
1205	Assert(!HMR0SuspendPending());
1206
1207	/*
1208	* Safety precaution; looping for too long here can have a very bad effect on the host.
1209	*/
1210	if (RT_UNLIKELY(++cResume > pVM->hm.s.cMaxResumeLoops))
1211	{
1212	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
1213	rc = VINF_EM_RAW_INTERRUPT;
1214	goto end;
1215	}
1216
1217	/*
1218	* Check for IRQ inhibition due to instruction fusing (sti, mov ss).
1219	*/
1220	if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1221	{
1222	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
1223	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
1224	{
1225	/*
1226	* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
1227	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
1228	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
1229	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
1230	*/
1231	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1232	/* Irq inhibition is no longer active; clear the corresponding SVM state. */
1233	pVMCB->ctrl.u64IntShadow = 0;
1234	}
1235	}
1236	else
1237	{
1238	/* Irq inhibition is no longer active; clear the corresponding SVM state. */
1239	pVMCB->ctrl.u64IntShadow = 0;
1240	}
1241
1242	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
1243	if (RT_UNLIKELY((cResume & 0xf) == 0))
1244	{
1245	uint64_t u64CurTime = RTTimeMilliTS();
1246
1247	if (RT_UNLIKELY(u64CurTime > u64LastTime))
1248	{
1249	u64LastTime = u64CurTime;
1250	TMTimerPollVoid(pVM, pVCpu);
1251	}
1252	}
1253	#endif
1254
1255	/*
1256	* Check for pending actions that force us to go back to ring-3.
1257	*/
1258	if ( VM_FF_ISPENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
1259	\|\| VMCPU_FF_ISPENDING(pVCpu,
1260	VMCPU_FF_HM_TO_R3_MASK
1261	\| VMCPU_FF_PGM_SYNC_CR3
1262	\| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
1263	\| VMCPU_FF_REQUEST))
1264	{
1265	/* Check if a sync operation is pending. */
1266	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
1267	{
1268	rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
1269	AssertRC(VBOXSTRICTRC_VAL(rc));
1270	if (rc != VINF_SUCCESS)
1271	{
1272	Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
1273	goto end;
1274	}
1275	}
1276
1277	#ifdef DEBUG
1278	/* Intercept X86_XCPT_DB if stepping is enabled */
1279	if (!DBGFIsStepping(pVCpu))
1280	#endif
1281	{
1282	if ( VM_FF_ISPENDING(pVM, VM_FF_HM_TO_R3_MASK)
1283	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
1284	{
1285	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchToR3);
1286	rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
1287	goto end;
1288	}
1289	}
1290
1291	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
1292	if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
1293	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
1294	{
1295	rc = VINF_EM_PENDING_REQUEST;
1296	goto end;
1297	}
1298
1299	/* Check if a pgm pool flush is in progress. */
1300	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
1301	{
1302	rc = VINF_PGM_POOL_FLUSH_PENDING;
1303	goto end;
1304	}
1305
1306	/* Check if DMA work is pending (2nd+ run). */
1307	if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
1308	{
1309	rc = VINF_EM_RAW_TO_R3;
1310	goto end;
1311	}
1312	}
1313
1314	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
1315	/*
1316	* Exit to ring-3 preemption/work is pending.
1317	*
1318	* Interrupts are disabled before the call to make sure we don't miss any interrupt
1319	* that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
1320	* further down, but hmR0SvmCheckPendingInterrupt makes that impossible.)
1321	*
1322	* Note! Interrupts must be disabled done before we check for TLB flushes; TLB
1323	* shootdowns rely on this.
1324	*/
1325	uOldEFlags = ASMIntDisableFlags();
1326	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
1327	{
1328	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptPending);
1329	rc = VINF_EM_RAW_INTERRUPT;
1330	goto end;
1331	}
1332	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
1333	#endif
1334
1335	/*
1336	* When external interrupts are pending, we should exit the VM when IF is set.
1337	* Note: After VM_FF_INHIBIT_INTERRUPTS check!!
1338	*/
1339	rc = hmR0SvmCheckPendingInterrupt(pVM, pVCpu, pVMCB, pCtx);
1340	if (RT_FAILURE(rc))
1341	goto end;
1342
1343	/*
1344	* TPR caching using CR8 is only available in 64-bit mode or with 32-bit guests when X86_CPUID_AMD_FEATURE_ECX_CR8L is
1345	* supported.
1346	* Note: we can't do this in LoddGuestState as PDMApicGetTPR can jump back to ring 3 (lock)! (no longer true)
1347	*/
1348	/** @todo query and update the TPR only when it could have been changed (mmio access)
1349	*/
1350	if (pVM->hm.s.fHasIoApic)
1351	{
1352	/* TPR caching in CR8 */
1353	bool fPending;
1354	rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
1355	AssertRC(rc2);
1356
1357	if (pVM->hm.s.fTPRPatchingActive)
1358	{
1359	/* Our patch code uses LSTAR for TPR caching. */
1360	pCtx->msrLSTAR = u8LastTPR;
1361
1362	if (fPending)
1363	{
1364	/* A TPR change could activate a pending interrupt, so catch lstar writes. */
1365	hmR0SvmSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
1366	}
1367	else
1368	{
1369	/*
1370	* No interrupts are pending, so we don't need to be explicitely notified.
1371	* There are enough world switches for detecting pending interrupts.
1372	*/
1373	hmR0SvmSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
1374	}
1375	}
1376	else
1377	{
1378	/* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
1379	pVMCB->ctrl.IntCtrl.n.u8VTPR = (u8LastTPR >> 4);
1380
1381	if (fPending)
1382	{
1383	/* A TPR change could activate a pending interrupt, so catch cr8 writes. */
1384	pVMCB->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
1385	}
1386	else
1387	{
1388	/*
1389	* No interrupts are pending, so we don't need to be explicitly notified.
1390	* There are enough world switches for detecting pending interrupts.
1391	*/
1392	pVMCB->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
1393	}
1394	}
1395	fSyncTPR = !fPending;
1396	}
1397
1398	/* All done! Let's start VM execution. */
1399
1400	/* Enable nested paging if necessary (disabled each time after #VMEXIT). */
1401	pVMCB->ctrl.NestedPaging.n.u1NestedPaging = pVM->hm.s.fNestedPaging;
1402
1403	#ifdef LOG_ENABLED
1404	pCpu = HMR0GetCurrentCpu();
1405	if (pVCpu->hm.s.idLastCpu != pCpu->idCpu)
1406	LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hm.s.idLastCpu, pCpu->idCpu));
1407	else if (pVCpu->hm.s.cTLBFlushes != pCpu->cTLBFlushes)
1408	LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hm.s.cTLBFlushes, pCpu->cTLBFlushes));
1409	else if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH))
1410	LogFlow(("Manual TLB flush\n"));
1411	#endif
1412
1413	/*
1414	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
1415	* (until the actual world switch)
1416	*/
1417	#ifdef VBOX_STRICT
1418	idCpuCheck = RTMpCpuId();
1419	#endif
1420	VMMR0LogFlushDisable(pVCpu);
1421
1422	/*
1423	* Load the guest state; must be here as it sets up the shadow CR0 for lazy FPU syncing!
1424	*/
1425	rc = SVMR0LoadGuestState(pVM, pVCpu, pCtx);
1426	if (RT_UNLIKELY(rc != VINF_SUCCESS))
1427	{
1428	VMMR0LogFlushEnable(pVCpu);
1429	goto end;
1430	}
1431
1432	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
1433	/*
1434	* Disable interrupts to make sure a poke will interrupt execution.
1435	* This must be done before we check for TLB flushes; TLB shootdowns rely on this.
1436	*/
1437	uOldEFlags = ASMIntDisableFlags();
1438	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
1439	#endif
1440	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
1441
1442	/* Setup TLB control and ASID in the VMCB. */
1443	hmR0SvmSetupTLB(pVM, pVCpu);
1444
1445	/* In case we execute a goto ResumeExecution later on. */
1446	pVCpu->hm.s.fResumeVM = true;
1447	pVCpu->hm.s.fForceTLBFlush = pVM->hm.s.svm.fAlwaysFlushTLB;
1448
1449	Assert(sizeof(pVCpu->hm.s.svm.pVMCBPhys) == 8);
1450	Assert(pVMCB->ctrl.IntCtrl.n.u1VIrqMasking);
1451	Assert(pVMCB->ctrl.u64IOPMPhysAddr == pVM->hm.s.svm.pIOBitmapPhys);
1452	Assert(pVMCB->ctrl.u64MSRPMPhysAddr == pVCpu->hm.s.svm.pMSRBitmapPhys);
1453	Assert(pVMCB->ctrl.u64LBRVirt == 0);
1454
1455	#ifdef VBOX_STRICT
1456	Assert(idCpuCheck == RTMpCpuId());
1457	#endif
1458	TMNotifyStartOfExecution(pVCpu);
1459
1460	/*
1461	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
1462	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
1463	*/
1464	u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
1465	if ( (u32HostExtFeatures & X86_CPUID_EXT_FEATURE_EDX_RDTSCP)
1466	&& !(pVMCB->ctrl.u32InterceptCtrl2 & SVM_CTRL2_INTERCEPT_RDTSCP))
1467	{
1468	pVCpu->hm.s.u64HostTSCAux = ASMRdMsr(MSR_K8_TSC_AUX);
1469	uint64_t u64GuestTSCAux = 0;
1470	rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64GuestTSCAux);
1471	AssertRC(rc2);
1472	ASMWrMsr(MSR_K8_TSC_AUX, u64GuestTSCAux);
1473	}
1474
1475	#ifdef VBOX_WITH_KERNEL_USING_XMM
1476	hmR0SVMRunWrapXMM(pVCpu->hm.s.svm.pVMCBHostPhys, pVCpu->hm.s.svm.pVMCBPhys, pCtx, pVM, pVCpu,
1477	pVCpu->hm.s.svm.pfnVMRun);
1478	#else
1479	pVCpu->hm.s.svm.pfnVMRun(pVCpu->hm.s.svm.pVMCBHostPhys, pVCpu->hm.s.svm.pVMCBPhys, pCtx, pVM, pVCpu);
1480	#endif
1481	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false);
1482	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits);
1483	/* Possibly the last TSC value seen by the guest (too high) (only when we're in TSC offset mode). */
1484	if (!(pVMCB->ctrl.u32InterceptCtrl1 & SVM_CTRL1_INTERCEPT_RDTSC))
1485	{
1486	/* Restore host's TSC_AUX. */
1487	if (u32HostExtFeatures & X86_CPUID_EXT_FEATURE_EDX_RDTSCP)
1488	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTSCAux);
1489
1490	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() +
1491	pVMCB->ctrl.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
1492	}
1493	TMNotifyEndOfExecution(pVCpu);
1494	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
1495	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
1496	ASMSetFlags(uOldEFlags);
1497	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
1498	uOldEFlags = ~(RTCCUINTREG)0;
1499	#endif
1500
1501	/*
1502	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1503	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING-3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
1504	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1505	*/
1506
1507	/* Reason for the VM exit */
1508	exitCode = pVMCB->ctrl.u64ExitCode;
1509
1510	if (RT_UNLIKELY(exitCode == (uint64_t)SVM_EXIT_INVALID)) /* Invalid guest state. */
1511	{
1512	HMDumpRegs(pVM, pVCpu, pCtx);
1513	#ifdef DEBUG
1514	Log(("ctrl.u16InterceptRdCRx %x\n", pVMCB->ctrl.u16InterceptRdCRx));
1515	Log(("ctrl.u16InterceptWrCRx %x\n", pVMCB->ctrl.u16InterceptWrCRx));
1516	Log(("ctrl.u16InterceptRdDRx %x\n", pVMCB->ctrl.u16InterceptRdDRx));
1517	Log(("ctrl.u16InterceptWrDRx %x\n", pVMCB->ctrl.u16InterceptWrDRx));
1518	Log(("ctrl.u32InterceptException %x\n", pVMCB->ctrl.u32InterceptException));
1519	Log(("ctrl.u32InterceptCtrl1 %x\n", pVMCB->ctrl.u32InterceptCtrl1));
1520	Log(("ctrl.u32InterceptCtrl2 %x\n", pVMCB->ctrl.u32InterceptCtrl2));
1521	Log(("ctrl.u64IOPMPhysAddr %RX64\n", pVMCB->ctrl.u64IOPMPhysAddr));
1522	Log(("ctrl.u64MSRPMPhysAddr %RX64\n", pVMCB->ctrl.u64MSRPMPhysAddr));
1523	Log(("ctrl.u64TSCOffset %RX64\n", pVMCB->ctrl.u64TSCOffset));
1524
1525	Log(("ctrl.TLBCtrl.u32ASID %x\n", pVMCB->ctrl.TLBCtrl.n.u32ASID));
1526	Log(("ctrl.TLBCtrl.u8TLBFlush %x\n", pVMCB->ctrl.TLBCtrl.n.u8TLBFlush));
1527	Log(("ctrl.TLBCtrl.u24Reserved %x\n", pVMCB->ctrl.TLBCtrl.n.u24Reserved));
1528
1529	Log(("ctrl.IntCtrl.u8VTPR %x\n", pVMCB->ctrl.IntCtrl.n.u8VTPR));
1530	Log(("ctrl.IntCtrl.u1VIrqValid %x\n", pVMCB->ctrl.IntCtrl.n.u1VIrqValid));
1531	Log(("ctrl.IntCtrl.u7Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u7Reserved));
1532	Log(("ctrl.IntCtrl.u4VIrqPriority %x\n", pVMCB->ctrl.IntCtrl.n.u4VIrqPriority));
1533	Log(("ctrl.IntCtrl.u1IgnoreTPR %x\n", pVMCB->ctrl.IntCtrl.n.u1IgnoreTPR));
1534	Log(("ctrl.IntCtrl.u3Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u3Reserved));
1535	Log(("ctrl.IntCtrl.u1VIrqMasking %x\n", pVMCB->ctrl.IntCtrl.n.u1VIrqMasking));
1536	Log(("ctrl.IntCtrl.u7Reserved2 %x\n", pVMCB->ctrl.IntCtrl.n.u7Reserved2));
1537	Log(("ctrl.IntCtrl.u8VIrqVector %x\n", pVMCB->ctrl.IntCtrl.n.u8VIrqVector));
1538	Log(("ctrl.IntCtrl.u24Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u24Reserved));
1539
1540	Log(("ctrl.u64IntShadow %RX64\n", pVMCB->ctrl.u64IntShadow));
1541	Log(("ctrl.u64ExitCode %RX64\n", pVMCB->ctrl.u64ExitCode));
1542	Log(("ctrl.u64ExitInfo1 %RX64\n", pVMCB->ctrl.u64ExitInfo1));
1543	Log(("ctrl.u64ExitInfo2 %RX64\n", pVMCB->ctrl.u64ExitInfo2));
1544	Log(("ctrl.ExitIntInfo.u8Vector %x\n", pVMCB->ctrl.ExitIntInfo.n.u8Vector));
1545	Log(("ctrl.ExitIntInfo.u3Type %x\n", pVMCB->ctrl.ExitIntInfo.n.u3Type));
1546	Log(("ctrl.ExitIntInfo.u1ErrorCodeValid %x\n", pVMCB->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
1547	Log(("ctrl.ExitIntInfo.u19Reserved %x\n", pVMCB->ctrl.ExitIntInfo.n.u19Reserved));
1548	Log(("ctrl.ExitIntInfo.u1Valid %x\n", pVMCB->ctrl.ExitIntInfo.n.u1Valid));
1549	Log(("ctrl.ExitIntInfo.u32ErrorCode %x\n", pVMCB->ctrl.ExitIntInfo.n.u32ErrorCode));
1550	Log(("ctrl.NestedPaging %RX64\n", pVMCB->ctrl.NestedPaging.au64));
1551	Log(("ctrl.EventInject.u8Vector %x\n", pVMCB->ctrl.EventInject.n.u8Vector));
1552	Log(("ctrl.EventInject.u3Type %x\n", pVMCB->ctrl.EventInject.n.u3Type));
1553	Log(("ctrl.EventInject.u1ErrorCodeValid %x\n", pVMCB->ctrl.EventInject.n.u1ErrorCodeValid));
1554	Log(("ctrl.EventInject.u19Reserved %x\n", pVMCB->ctrl.EventInject.n.u19Reserved));
1555	Log(("ctrl.EventInject.u1Valid %x\n", pVMCB->ctrl.EventInject.n.u1Valid));
1556	Log(("ctrl.EventInject.u32ErrorCode %x\n", pVMCB->ctrl.EventInject.n.u32ErrorCode));
1557
1558	Log(("ctrl.u64NestedPagingCR3 %RX64\n", pVMCB->ctrl.u64NestedPagingCR3));
1559	Log(("ctrl.u64LBRVirt %RX64\n", pVMCB->ctrl.u64LBRVirt));
1560
1561	Log(("guest.CS.u16Sel %04X\n", pVMCB->guest.CS.u16Sel));
1562	Log(("guest.CS.u16Attr %04X\n", pVMCB->guest.CS.u16Attr));
1563	Log(("guest.CS.u32Limit %X\n", pVMCB->guest.CS.u32Limit));
1564	Log(("guest.CS.u64Base %RX64\n", pVMCB->guest.CS.u64Base));
1565	Log(("guest.DS.u16Sel %04X\n", pVMCB->guest.DS.u16Sel));
1566	Log(("guest.DS.u16Attr %04X\n", pVMCB->guest.DS.u16Attr));
1567	Log(("guest.DS.u32Limit %X\n", pVMCB->guest.DS.u32Limit));
1568	Log(("guest.DS.u64Base %RX64\n", pVMCB->guest.DS.u64Base));
1569	Log(("guest.ES.u16Sel %04X\n", pVMCB->guest.ES.u16Sel));
1570	Log(("guest.ES.u16Attr %04X\n", pVMCB->guest.ES.u16Attr));
1571	Log(("guest.ES.u32Limit %X\n", pVMCB->guest.ES.u32Limit));
1572	Log(("guest.ES.u64Base %RX64\n", pVMCB->guest.ES.u64Base));
1573	Log(("guest.FS.u16Sel %04X\n", pVMCB->guest.FS.u16Sel));
1574	Log(("guest.FS.u16Attr %04X\n", pVMCB->guest.FS.u16Attr));
1575	Log(("guest.FS.u32Limit %X\n", pVMCB->guest.FS.u32Limit));
1576	Log(("guest.FS.u64Base %RX64\n", pVMCB->guest.FS.u64Base));
1577	Log(("guest.GS.u16Sel %04X\n", pVMCB->guest.GS.u16Sel));
1578	Log(("guest.GS.u16Attr %04X\n", pVMCB->guest.GS.u16Attr));
1579	Log(("guest.GS.u32Limit %X\n", pVMCB->guest.GS.u32Limit));
1580	Log(("guest.GS.u64Base %RX64\n", pVMCB->guest.GS.u64Base));
1581
1582	Log(("guest.GDTR.u32Limit %X\n", pVMCB->guest.GDTR.u32Limit));
1583	Log(("guest.GDTR.u64Base %RX64\n", pVMCB->guest.GDTR.u64Base));
1584
1585	Log(("guest.LDTR.u16Sel %04X\n", pVMCB->guest.LDTR.u16Sel));
1586	Log(("guest.LDTR.u16Attr %04X\n", pVMCB->guest.LDTR.u16Attr));
1587	Log(("guest.LDTR.u32Limit %X\n", pVMCB->guest.LDTR.u32Limit));
1588	Log(("guest.LDTR.u64Base %RX64\n", pVMCB->guest.LDTR.u64Base));
1589
1590	Log(("guest.IDTR.u32Limit %X\n", pVMCB->guest.IDTR.u32Limit));
1591	Log(("guest.IDTR.u64Base %RX64\n", pVMCB->guest.IDTR.u64Base));
1592
1593	Log(("guest.TR.u16Sel %04X\n", pVMCB->guest.TR.u16Sel));
1594	Log(("guest.TR.u16Attr %04X\n", pVMCB->guest.TR.u16Attr));
1595	Log(("guest.TR.u32Limit %X\n", pVMCB->guest.TR.u32Limit));
1596	Log(("guest.TR.u64Base %RX64\n", pVMCB->guest.TR.u64Base));
1597
1598	Log(("guest.u8CPL %X\n", pVMCB->guest.u8CPL));
1599	Log(("guest.u64CR0 %RX64\n", pVMCB->guest.u64CR0));
1600	Log(("guest.u64CR2 %RX64\n", pVMCB->guest.u64CR2));
1601	Log(("guest.u64CR3 %RX64\n", pVMCB->guest.u64CR3));
1602	Log(("guest.u64CR4 %RX64\n", pVMCB->guest.u64CR4));
1603	Log(("guest.u64DR6 %RX64\n", pVMCB->guest.u64DR6));
1604	Log(("guest.u64DR7 %RX64\n", pVMCB->guest.u64DR7));
1605
1606	Log(("guest.u64RIP %RX64\n", pVMCB->guest.u64RIP));
1607	Log(("guest.u64RSP %RX64\n", pVMCB->guest.u64RSP));
1608	Log(("guest.u64RAX %RX64\n", pVMCB->guest.u64RAX));
1609	Log(("guest.u64RFlags %RX64\n", pVMCB->guest.u64RFlags));
1610
1611	Log(("guest.u64SysEnterCS %RX64\n", pVMCB->guest.u64SysEnterCS));
1612	Log(("guest.u64SysEnterEIP %RX64\n", pVMCB->guest.u64SysEnterEIP));
1613	Log(("guest.u64SysEnterESP %RX64\n", pVMCB->guest.u64SysEnterESP));
1614
1615	Log(("guest.u64EFER %RX64\n", pVMCB->guest.u64EFER));
1616	Log(("guest.u64STAR %RX64\n", pVMCB->guest.u64STAR));
1617	Log(("guest.u64LSTAR %RX64\n", pVMCB->guest.u64LSTAR));
1618	Log(("guest.u64CSTAR %RX64\n", pVMCB->guest.u64CSTAR));
1619	Log(("guest.u64SFMASK %RX64\n", pVMCB->guest.u64SFMASK));
1620	Log(("guest.u64KernelGSBase %RX64\n", pVMCB->guest.u64KernelGSBase));
1621	Log(("guest.u64GPAT %RX64\n", pVMCB->guest.u64GPAT));
1622	Log(("guest.u64DBGCTL %RX64\n", pVMCB->guest.u64DBGCTL));
1623	Log(("guest.u64BR_FROM %RX64\n", pVMCB->guest.u64BR_FROM));
1624	Log(("guest.u64BR_TO %RX64\n", pVMCB->guest.u64BR_TO));
1625	Log(("guest.u64LASTEXCPFROM %RX64\n", pVMCB->guest.u64LASTEXCPFROM));
1626	Log(("guest.u64LASTEXCPTO %RX64\n", pVMCB->guest.u64LASTEXCPTO));
1627	#endif
1628	rc = VERR_SVM_UNABLE_TO_START_VM;
1629	VMMR0LogFlushEnable(pVCpu);
1630	goto end;
1631	}
1632
1633	/* Let's first sync back EIP, ESP, and EFLAGS. */
1634	pCtx->rip = pVMCB->guest.u64RIP;
1635	pCtx->rsp = pVMCB->guest.u64RSP;
1636	pCtx->eflags.u32 = pVMCB->guest.u64RFlags;
1637	/* eax is saved/restore across the vmrun instruction */
1638	pCtx->rax = pVMCB->guest.u64RAX;
1639
1640	/*
1641	* Save all the MSRs that can be changed by the guest without causing a world switch.
1642	* FS & GS base are saved with SVM_READ_SELREG.
1643	*/
1644	pCtx->msrSTAR = pVMCB->guest.u64STAR; /* legacy syscall eip, cs & ss */
1645	pCtx->msrLSTAR = pVMCB->guest.u64LSTAR; /* 64-bit mode syscall rip */
1646	pCtx->msrCSTAR = pVMCB->guest.u64CSTAR; /* compatibility mode syscall rip */
1647	pCtx->msrSFMASK = pVMCB->guest.u64SFMASK; /* syscall flag mask */
1648	pCtx->msrKERNELGSBASE = pVMCB->guest.u64KernelGSBase; /* swapgs exchange value */
1649	pCtx->SysEnter.cs = pVMCB->guest.u64SysEnterCS;
1650	pCtx->SysEnter.eip = pVMCB->guest.u64SysEnterEIP;
1651	pCtx->SysEnter.esp = pVMCB->guest.u64SysEnterESP;
1652
1653	/* Can be updated behind our back in the nested paging case. */
1654	pCtx->cr2 = pVMCB->guest.u64CR2;
1655
1656	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1657	SVM_READ_SELREG(SS, ss);
1658	SVM_READ_SELREG(CS, cs);
1659	SVM_READ_SELREG(DS, ds);
1660	SVM_READ_SELREG(ES, es);
1661	SVM_READ_SELREG(FS, fs);
1662	SVM_READ_SELREG(GS, gs);
1663
1664	/*
1665	* Correct the hidden CS granularity flag. Haven't seen it being wrong in any other
1666	* register (yet).
1667	*/
1668	if ( !pCtx->cs.Attr.n.u1Granularity
1669	&& pCtx->cs.Attr.n.u1Present
1670	&& pCtx->cs.u32Limit > UINT32_C(0xfffff))
1671	{
1672	Assert((pCtx->cs.u32Limit & 0xfff) == 0xfff);
1673	pCtx->cs.Attr.n.u1Granularity = 1;
1674	}
1675	#define SVM_ASSERT_SEL_GRANULARITY(reg) \
1676	AssertMsg( !pCtx->reg.Attr.n.u1Present \
1677	\|\| ( pCtx->reg.Attr.n.u1Granularity \
1678	? (pCtx->reg.u32Limit & 0xfff) == 0xfff \
1679	: pCtx->reg.u32Limit <= 0xfffff), \
1680	("%#x %#x %#llx\n", pCtx->reg.u32Limit, pCtx->reg.Attr.u, pCtx->reg.u64Base))
1681	SVM_ASSERT_SEL_GRANULARITY(ss);
1682	SVM_ASSERT_SEL_GRANULARITY(cs);
1683	SVM_ASSERT_SEL_GRANULARITY(ds);
1684	SVM_ASSERT_SEL_GRANULARITY(es);
1685	SVM_ASSERT_SEL_GRANULARITY(fs);
1686	SVM_ASSERT_SEL_GRANULARITY(gs);
1687	#undef SVM_ASSERT_SEL_GRANULARITY
1688
1689	/*
1690	* Correct the hidden SS DPL field. It can be wrong on certain CPUs
1691	* sometimes (seen it on AMD Fusion CPUs with 64-bit guests). The CPU
1692	* always uses the CPL field in the VMCB instead of the DPL in the hidden
1693	* SS (chapter AMD spec. 15.5.1 Basic operation).
1694	*/
1695	Assert(!(pVMCB->guest.u8CPL & ~0x3));
1696	pCtx->ss.Attr.n.u2Dpl = pVMCB->guest.u8CPL & 0x3;
1697
1698	/*
1699	* Remaining guest CPU context: TR, IDTR, GDTR, LDTR;
1700	* must sync everything otherwise we can get out of sync when jumping back to ring-3.
1701	*/
1702	SVM_READ_SELREG(LDTR, ldtr);
1703	SVM_READ_SELREG(TR, tr);
1704
1705	pCtx->gdtr.cbGdt = pVMCB->guest.GDTR.u32Limit;
1706	pCtx->gdtr.pGdt = pVMCB->guest.GDTR.u64Base;
1707
1708	pCtx->idtr.cbIdt = pVMCB->guest.IDTR.u32Limit;
1709	pCtx->idtr.pIdt = pVMCB->guest.IDTR.u64Base;
1710
1711	/*
1712	* No reason to sync back the CRx and DRx registers as they cannot be changed by the guest
1713	* unless in the nested paging case where CR3 can be changed by the guest.
1714	*/
1715	if ( pVM->hm.s.fNestedPaging
1716	&& pCtx->cr3 != pVMCB->guest.u64CR3)
1717	{
1718	CPUMSetGuestCR3(pVCpu, pVMCB->guest.u64CR3);
1719	PGMUpdateCR3(pVCpu, pVMCB->guest.u64CR3);
1720	}
1721
1722	/* Note! NOW IT'S SAFE FOR LOGGING! */
1723	VMMR0LogFlushEnable(pVCpu);
1724
1725	/* Take care of instruction fusing (sti, mov ss) (see AMD spec. 15.20.5 Interrupt Shadows) */
1726	if (pVMCB->ctrl.u64IntShadow & SVM_INTERRUPT_SHADOW_ACTIVE)
1727	{
1728	Log(("uInterruptState %x rip=%RGv\n", pVMCB->ctrl.u64IntShadow, (RTGCPTR)pCtx->rip));
1729	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
1730	}
1731	else
1732	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1733
1734	Log2(("exitCode = %x\n", exitCode));
1735
1736	/* Sync back DR6 as it could have been changed by hitting breakpoints. */
1737	pCtx->dr[6] = pVMCB->guest.u64DR6;
1738	/* DR7.GD can be cleared by debug exceptions, so sync it back as well. */
1739	pCtx->dr[7] = pVMCB->guest.u64DR7;
1740
1741	/* Check if an injected event was interrupted prematurely. */
1742	pVCpu->hm.s.Event.intInfo = pVMCB->ctrl.ExitIntInfo.au64[0];
1743	if ( pVMCB->ctrl.ExitIntInfo.n.u1Valid
1744	/* we don't care about 'int xx' as the instruction will be restarted. */
1745	&& pVMCB->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT)
1746	{
1747	Log(("Pending inject %RX64 at %RGv exit=%08x\n", pVCpu->hm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitCode));
1748
1749	#ifdef LOG_ENABLED
1750	SVM_EVENT Event;
1751	Event.au64[0] = pVCpu->hm.s.Event.intInfo;
1752
1753	if ( exitCode == SVM_EXIT_EXCEPTION_E
1754	&& Event.n.u8Vector == 0xE)
1755	{
1756	Log(("Double fault!\n"));
1757	}
1758	#endif
1759
1760	pVCpu->hm.s.Event.fPending = true;
1761	/* Error code present? (redundant) */
1762	if (pVMCB->ctrl.ExitIntInfo.n.u1ErrorCodeValid)
1763	pVCpu->hm.s.Event.errCode = pVMCB->ctrl.ExitIntInfo.n.u32ErrorCode;
1764	else
1765	pVCpu->hm.s.Event.errCode = 0;
1766	}
1767	#ifdef VBOX_WITH_STATISTICS
1768	if (exitCode == SVM_EXIT_NPF)
1769	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNPF);
1770	else
1771	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[exitCode & MASK_EXITREASON_STAT]);
1772	#endif
1773
1774	/* Sync back the TPR if it was changed. */
1775	if (fSyncTPR)
1776	{
1777	if (pVM->hm.s.fTPRPatchingActive)
1778	{
1779	if ((pCtx->msrLSTAR & 0xff) != u8LastTPR)
1780	{
1781	/* Our patch code uses LSTAR for TPR caching. */
1782	rc2 = PDMApicSetTPR(pVCpu, pCtx->msrLSTAR & 0xff);
1783	AssertRC(rc2);
1784	}
1785	}
1786	else
1787	{
1788	if ((uint8_t)(u8LastTPR >> 4) != pVMCB->ctrl.IntCtrl.n.u8VTPR)
1789	{
1790	/* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
1791	rc2 = PDMApicSetTPR(pVCpu, pVMCB->ctrl.IntCtrl.n.u8VTPR << 4);
1792	AssertRC(rc2);
1793	}
1794	}
1795	}
1796
1797	#ifdef DBGFTRACE_ENABLED /** @todo DTrace */
1798	RTTraceBufAddMsgF(pVM->CTX_SUFF(hTraceBuf), "vmexit %08x at %04:%08RX64 %RX64 %RX64 %RX64",
1799	exitCode, pCtx->cs.Sel, pCtx->rip,
1800	pVMCB->ctrl.u64ExitInfo1, pVMCB->ctrl.u64ExitInfo2, pVMCB->ctrl.ExitIntInfo.au64[0]);
1801	#endif
1802	#if ARCH_BITS == 64 /* for the time being */
1803	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, exitCode, pVMCB->ctrl.u64ExitInfo1, pVMCB->ctrl.u64ExitInfo2,
1804	pVMCB->ctrl.ExitIntInfo.au64[0], UINT64_MAX);
1805	#endif
1806	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
1807
1808	/* Deal with the reason of the VM-exit. */
1809	switch (exitCode)
1810	{
1811	case SVM_EXIT_EXCEPTION_0: case SVM_EXIT_EXCEPTION_1: case SVM_EXIT_EXCEPTION_2: case SVM_EXIT_EXCEPTION_3:
1812	case SVM_EXIT_EXCEPTION_4: case SVM_EXIT_EXCEPTION_5: case SVM_EXIT_EXCEPTION_6: case SVM_EXIT_EXCEPTION_7:
1813	case SVM_EXIT_EXCEPTION_8: case SVM_EXIT_EXCEPTION_9: case SVM_EXIT_EXCEPTION_A: case SVM_EXIT_EXCEPTION_B:
1814	case SVM_EXIT_EXCEPTION_C: case SVM_EXIT_EXCEPTION_D: case SVM_EXIT_EXCEPTION_E: case SVM_EXIT_EXCEPTION_F:
1815	case SVM_EXIT_EXCEPTION_10: case SVM_EXIT_EXCEPTION_11: case SVM_EXIT_EXCEPTION_12: case SVM_EXIT_EXCEPTION_13:
1816	case SVM_EXIT_EXCEPTION_14: case SVM_EXIT_EXCEPTION_15: case SVM_EXIT_EXCEPTION_16: case SVM_EXIT_EXCEPTION_17:
1817	case SVM_EXIT_EXCEPTION_18: case SVM_EXIT_EXCEPTION_19: case SVM_EXIT_EXCEPTION_1A: case SVM_EXIT_EXCEPTION_1B:
1818	case SVM_EXIT_EXCEPTION_1C: case SVM_EXIT_EXCEPTION_1D: case SVM_EXIT_EXCEPTION_1E: case SVM_EXIT_EXCEPTION_1F:
1819	{
1820	/* Pending trap. */
1821	SVM_EVENT Event;
1822	uint32_t vector = exitCode - SVM_EXIT_EXCEPTION_0;
1823
1824	Log2(("Hardware/software interrupt %d\n", vector));
1825	switch (vector)
1826	{
1827	case X86_XCPT_DB:
1828	{
1829	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
1830
1831	/* Note that we don't support guest and host-initiated debugging at the same time. */
1832	Assert(DBGFIsStepping(pVCpu) \|\| CPUMIsHyperDebugStateActive(pVCpu));
1833
1834	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), pCtx->dr[6]);
1835	if (rc == VINF_EM_RAW_GUEST_TRAP)
1836	{
1837	Log(("Trap %x (debug) at %016RX64\n", vector, pCtx->rip));
1838
1839	/* Reinject the exception. */
1840	Event.au64[0] = 0;
1841	Event.n.u3Type = SVM_EVENT_EXCEPTION; /* trap or fault */
1842	Event.n.u1Valid = 1;
1843	Event.n.u8Vector = X86_XCPT_DB;
1844
1845	hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
1846	goto ResumeExecution;
1847	}
1848	/* Return to ring 3 to deal with the debug exit code. */
1849	Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
1850	break;
1851	}
1852
1853	case X86_XCPT_NM:
1854	{
1855	Log(("#NM fault at %RGv\n", (RTGCPTR)pCtx->rip));
1856
1857	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
1858	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
1859	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
1860	if (rc == VINF_SUCCESS)
1861	{
1862	Assert(CPUMIsGuestFPUStateActive(pVCpu));
1863	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
1864
1865	/* Continue execution. */
1866	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
1867
1868	goto ResumeExecution;
1869	}
1870
1871	Log(("Forward #NM fault to the guest\n"));
1872	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
1873
1874	Event.au64[0] = 0;
1875	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1876	Event.n.u1Valid = 1;
1877	Event.n.u8Vector = X86_XCPT_NM;
1878
1879	hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
1880	goto ResumeExecution;
1881	}
1882
1883	case X86_XCPT_PF: /* Page fault */
1884	{
1885	uint32_t errCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1886	RTGCUINTPTR uFaultAddress = pVMCB->ctrl.u64ExitInfo2; /* EXITINFO2 = fault address */
1887
1888	#ifdef VBOX_ALWAYS_TRAP_PF
1889	if (pVM->hm.s.fNestedPaging)
1890	{
1891	/*
1892	* A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
1893	*/
1894	Log(("Guest page fault at %04X:%RGv cr2=%RGv error code %x rsp=%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip,
1895	uFaultAddress, errCode, (RTGCPTR)pCtx->rsp));
1896	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
1897
1898	/* Now we must update CR2. */
1899	pCtx->cr2 = uFaultAddress;
1900
1901	Event.au64[0] = 0;
1902	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1903	Event.n.u1Valid = 1;
1904	Event.n.u8Vector = X86_XCPT_PF;
1905	Event.n.u1ErrorCodeValid = 1;
1906	Event.n.u32ErrorCode = errCode;
1907
1908	hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
1909	goto ResumeExecution;
1910	}
1911	#endif
1912	Assert(!pVM->hm.s.fNestedPaging);
1913
1914	#ifdef VBOX_HM_WITH_GUEST_PATCHING
1915	/* Shortcut for APIC TPR reads and writes; 32 bits guests only */
1916	if ( pVM->hm.s.fTRPPatchingAllowed
1917	&& (uFaultAddress & 0xfff) == 0x080
1918	&& !(errCode & X86_TRAP_PF_P) /* not present */
1919	&& CPUMGetGuestCPL(pVCpu) == 0
1920	&& !CPUMIsGuestInLongModeEx(pCtx)
1921	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
1922	{
1923	RTGCPHYS GCPhysApicBase, GCPhys;
1924	PDMApicGetBase(pVM, &GCPhysApicBase); /** @todo cache this */
1925	GCPhysApicBase &= PAGE_BASE_GC_MASK;
1926
1927	rc = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, NULL, &GCPhys);
1928	if ( rc == VINF_SUCCESS
1929	&& GCPhys == GCPhysApicBase)
1930	{
1931	/* Only attempt to patch the instruction once. */
1932	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
1933	if (!pPatch)
1934	{
1935	rc = VINF_EM_HM_PATCH_TPR_INSTR;
1936	break;
1937	}
1938	}
1939	}
1940	#endif
1941
1942	Log2(("Page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1943	/* Exit qualification contains the linear address of the page fault. */
1944	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
1945	TRPMSetErrorCode(pVCpu, errCode);
1946	TRPMSetFaultAddress(pVCpu, uFaultAddress);
1947
1948	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
1949	rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
1950	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
1951	if (rc == VINF_SUCCESS)
1952	{
1953	/* We've successfully synced our shadow pages, so let's just continue execution. */
1954	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1955	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
1956
1957	TRPMResetTrap(pVCpu);
1958	goto ResumeExecution;
1959	}
1960	else if (rc == VINF_EM_RAW_GUEST_TRAP)
1961	{
1962	/*
1963	* A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
1964	*/
1965	Log2(("Forward page fault to the guest\n"));
1966	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
1967	/* The error code might have been changed. */
1968	errCode = TRPMGetErrorCode(pVCpu);
1969
1970	TRPMResetTrap(pVCpu);
1971
1972	/* Now we must update CR2. */
1973	pCtx->cr2 = uFaultAddress;
1974
1975	Event.au64[0] = 0;
1976	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1977	Event.n.u1Valid = 1;
1978	Event.n.u8Vector = X86_XCPT_PF;
1979	Event.n.u1ErrorCodeValid = 1;
1980	Event.n.u32ErrorCode = errCode;
1981
1982	hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
1983	goto ResumeExecution;
1984	}
1985	#ifdef VBOX_STRICT
1986	if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
1987	LogFlow(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
1988	#endif
1989	/* Need to go back to the recompiler to emulate the instruction. */
1990	TRPMResetTrap(pVCpu);
1991	break;
1992	}
1993
1994	case X86_XCPT_MF: /* Floating point exception. */
1995	{
1996	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
1997	if (!(pCtx->cr0 & X86_CR0_NE))
1998	{
1999	/* old style FPU error reporting needs some extra work. */
2000	/** @todo don't fall back to the recompiler, but do it manually. */
2001	rc = VINF_EM_RAW_EMULATE_INSTR;
2002	break;
2003	}
2004	Log(("Trap %x at %RGv\n", vector, (RTGCPTR)pCtx->rip));
2005
2006	Event.au64[0] = 0;
2007	Event.n.u3Type = SVM_EVENT_EXCEPTION;
2008	Event.n.u1Valid = 1;
2009	Event.n.u8Vector = X86_XCPT_MF;
2010
2011	hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
2012	goto ResumeExecution;
2013	}
2014
2015	#ifdef VBOX_STRICT
2016	case X86_XCPT_BP: /* Breakpoint. */
2017	case X86_XCPT_GP: /* General protection failure exception.*/
2018	case X86_XCPT_UD: /* Unknown opcode exception. */
2019	case X86_XCPT_DE: /* Divide error. */
2020	case X86_XCPT_SS: /* Stack segment exception. */
2021	case X86_XCPT_NP: /* Segment not present exception. */
2022	{
2023	Event.au64[0] = 0;
2024	Event.n.u3Type = SVM_EVENT_EXCEPTION;
2025	Event.n.u1Valid = 1;
2026	Event.n.u8Vector = vector;
2027
2028	switch (vector)
2029	{
2030	case X86_XCPT_GP:
2031	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
2032	Event.n.u1ErrorCodeValid = 1;
2033	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
2034	break;
2035	case X86_XCPT_BP:
2036	/** Saves the wrong EIP on the stack (pointing to the int3 instead of the next instruction. */
2037	break;
2038	case X86_XCPT_DE:
2039	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
2040	break;
2041	case X86_XCPT_UD:
2042	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
2043	break;
2044	case X86_XCPT_SS:
2045	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
2046	Event.n.u1ErrorCodeValid = 1;
2047	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
2048	break;
2049	case X86_XCPT_NP:
2050	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
2051	Event.n.u1ErrorCodeValid = 1;
2052	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
2053	break;
2054	}
2055	Log(("Trap %x at %04x:%RGv esi=%x\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, pCtx->esi));
2056	hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
2057	goto ResumeExecution;
2058	}
2059	#endif
2060	default:
2061	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
2062	rc = VERR_HMSVM_UNEXPECTED_XCPT_EXIT;
2063	break;
2064
2065	} /* switch (vector) */
2066	break;
2067	}
2068
2069	case SVM_EXIT_NPF:
2070	{
2071	/* EXITINFO1 contains fault errorcode; EXITINFO2 contains the guest physical address causing the fault. */
2072	uint32_t errCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
2073	RTGCPHYS GCPhysFault = pVMCB->ctrl.u64ExitInfo2; /* EXITINFO2 = fault address */
2074	PGMMODE enmShwPagingMode;
2075
2076	Assert(pVM->hm.s.fNestedPaging);
2077	LogFlow(("Nested page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, GCPhysFault, errCode));
2078
2079	#ifdef VBOX_HM_WITH_GUEST_PATCHING
2080	/* Shortcut for APIC TPR reads and writes; 32 bits guests only */
2081	if ( pVM->hm.s.fTRPPatchingAllowed
2082	&& (GCPhysFault & PAGE_OFFSET_MASK) == 0x080
2083	&& ( !(errCode & X86_TRAP_PF_P) /* not present */
2084	\|\| (errCode & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD) /* mmio optimization */)
2085	&& CPUMGetGuestCPL(pVCpu) == 0
2086	&& !CPUMIsGuestInLongModeEx(pCtx)
2087	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
2088	{
2089	RTGCPHYS GCPhysApicBase;
2090	PDMApicGetBase(pVM, &GCPhysApicBase); /** @todo cache this */
2091	GCPhysApicBase &= PAGE_BASE_GC_MASK;
2092
2093	if (GCPhysFault == GCPhysApicBase + 0x80)
2094	{
2095	/* Only attempt to patch the instruction once. */
2096	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
2097	if (!pPatch)
2098	{
2099	rc = VINF_EM_HM_PATCH_TPR_INSTR;
2100	break;
2101	}
2102	}
2103	}
2104	#endif
2105
2106	/* Handle the pagefault trap for the nested shadow table. */
2107	#if HC_ARCH_BITS == 32 /** @todo shadow this in a variable. */
2108	if (CPUMIsGuestInLongModeEx(pCtx))
2109	enmShwPagingMode = PGMMODE_AMD64_NX;
2110	else
2111	#endif
2112	enmShwPagingMode = PGMGetHostMode(pVM);
2113
2114	/* MMIO optimization */
2115	Assert((errCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != X86_TRAP_PF_RSVD);
2116	if ((errCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2117	{
2118	rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmShwPagingMode, CPUMCTX2CORE(pCtx), GCPhysFault, errCode);
2119
2120	/*
2121	* If we succeed, resume execution.
2122	* Or, if fail in interpreting the instruction because we couldn't get the guest physical address
2123	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
2124	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
2125	* weird case. See @bugref{6043}.
2126	*/
2127	if ( rc == VINF_SUCCESS
2128	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
2129	\|\| rc == VERR_PAGE_NOT_PRESENT)
2130	{
2131	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhysFault, (RTGCPTR)pCtx->rip));
2132	goto ResumeExecution;
2133	}
2134	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhysFault, (RTGCPTR)pCtx->rip));
2135	break;
2136	}
2137
2138	/* Exit qualification contains the linear address of the page fault. */
2139	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2140	TRPMSetErrorCode(pVCpu, errCode);
2141	TRPMSetFaultAddress(pVCpu, GCPhysFault);
2142
2143	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmShwPagingMode, errCode, CPUMCTX2CORE(pCtx), GCPhysFault);
2144	Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
2145
2146	/*
2147	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}.
2148	*/
2149	if ( rc == VINF_SUCCESS
2150	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
2151	\|\| rc == VERR_PAGE_NOT_PRESENT)
2152	{
2153	/* We've successfully synced our shadow pages, so let's just continue execution. */
2154	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, GCPhysFault, errCode));
2155	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
2156
2157	TRPMResetTrap(pVCpu);
2158	goto ResumeExecution;
2159	}
2160
2161	#ifdef VBOX_STRICT
2162	if (rc != VINF_EM_RAW_EMULATE_INSTR)
2163	LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", VBOXSTRICTRC_VAL(rc)));
2164	#endif
2165	/* Need to go back to the recompiler to emulate the instruction. */
2166	TRPMResetTrap(pVCpu);
2167	break;
2168	}
2169
2170	case SVM_EXIT_VINTR:
2171	/* A virtual interrupt is about to be delivered, which means IF=1. */
2172	Log(("SVM_EXIT_VINTR IF=%d\n", pCtx->eflags.Bits.u1IF));
2173	pVMCB->ctrl.IntCtrl.n.u1VIrqValid = 0;
2174	pVMCB->ctrl.IntCtrl.n.u8VIrqVector = 0;
2175	goto ResumeExecution;
2176
2177	case SVM_EXIT_FERR_FREEZE:
2178	case SVM_EXIT_INTR:
2179	case SVM_EXIT_NMI:
2180	case SVM_EXIT_SMI:
2181	case SVM_EXIT_INIT:
2182	/* External interrupt; leave to allow it to be dispatched again. */
2183	rc = VINF_EM_RAW_INTERRUPT;
2184	break;
2185
2186	case SVM_EXIT_WBINVD:
2187	case SVM_EXIT_INVD: /* Guest software attempted to execute INVD. */
2188	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
2189	/* Skip instruction and continue directly. */
2190	pCtx->rip += 2; /* Note! hardcoded opcode size! */
2191	/* Continue execution.*/
2192	goto ResumeExecution;
2193
2194	case SVM_EXIT_CPUID: /* Guest software attempted to execute CPUID. */
2195	{
2196	Log2(("SVM: Cpuid at %RGv for %x\n", (RTGCPTR)pCtx->rip, pCtx->eax));
2197	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
2198	rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
2199	if (rc == VINF_SUCCESS)
2200	{
2201	/* Update EIP and continue execution. */
2202	pCtx->rip += 2; /* Note! hardcoded opcode size! */
2203	goto ResumeExecution;
2204	}
2205	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
2206	rc = VINF_EM_RAW_EMULATE_INSTR;
2207	break;
2208	}
2209
2210	case SVM_EXIT_RDTSC: /* Guest software attempted to execute RDTSC. */
2211	{
2212	Log2(("SVM: Rdtsc\n"));
2213	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
2214	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
2215	if (rc == VINF_SUCCESS)
2216	{
2217	/* Update EIP and continue execution. */
2218	pCtx->rip += 2; /* Note! hardcoded opcode size! */
2219	goto ResumeExecution;
2220	}
2221	rc = VINF_EM_RAW_EMULATE_INSTR;
2222	break;
2223	}
2224
2225	case SVM_EXIT_RDPMC: /* Guest software attempted to execute RDPMC. */
2226	{
2227	Log2(("SVM: Rdpmc %x\n", pCtx->ecx));
2228	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
2229	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
2230	if (rc == VINF_SUCCESS)
2231	{
2232	/* Update EIP and continue execution. */
2233	pCtx->rip += 2; /* Note! hardcoded opcode size! */
2234	goto ResumeExecution;
2235	}
2236	rc = VINF_EM_RAW_EMULATE_INSTR;
2237	break;
2238	}
2239
2240	case SVM_EXIT_RDTSCP: /* Guest software attempted to execute RDTSCP. */
2241	{
2242	Log2(("SVM: Rdtscp\n"));
2243	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtscp);
2244	rc = EMInterpretRdtscp(pVM, pVCpu, pCtx);
2245	if (rc == VINF_SUCCESS)
2246	{
2247	/* Update EIP and continue execution. */
2248	pCtx->rip += 3; /* Note! hardcoded opcode size! */
2249	goto ResumeExecution;
2250	}
2251	AssertMsgFailed(("EMU: rdtscp failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
2252	rc = VINF_EM_RAW_EMULATE_INSTR;
2253	break;
2254	}
2255
2256	case SVM_EXIT_INVLPG: /* Guest software attempted to execute INVLPG. */
2257	{
2258	Log2(("SVM: invlpg\n"));
2259	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
2260
2261	Assert(!pVM->hm.s.fNestedPaging);
2262
2263	/* Truly a pita. Why can't SVM give the same information as VT-x? */
2264	rc = hmR0SvmInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx));
2265	if (rc == VINF_SUCCESS)
2266	{
2267	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushPageInvlpg);
2268	goto ResumeExecution; /* eip already updated */
2269	}
2270	break;
2271	}
2272
2273	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
2274	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
2275	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
2276	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
2277	{
2278	Log2(("SVM: %RGv mov cr%d, \n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_WRITE_CR0));
2279	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[exitCode - SVM_EXIT_WRITE_CR0]);
2280	rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
2281
2282	switch (exitCode - SVM_EXIT_WRITE_CR0)
2283	{
2284	case 0:
2285	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
2286	break;
2287	case 2:
2288	break;
2289	case 3:
2290	Assert(!pVM->hm.s.fNestedPaging);
2291	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
2292	break;
2293	case 4:
2294	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
2295	break;
2296	case 8:
2297	break;
2298	default:
2299	AssertFailed();
2300	}
2301	if (rc == VINF_SUCCESS)
2302	{
2303	/* EIP has been updated already. */
2304	/* Only resume if successful. */
2305	goto ResumeExecution;
2306	}
2307	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
2308	break;
2309	}
2310
2311	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
2312	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
2313	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
2314	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
2315	{
2316	Log2(("SVM: %RGv mov x, cr%d\n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_READ_CR0));
2317	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[exitCode - SVM_EXIT_READ_CR0]);
2318	rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
2319	if (rc == VINF_SUCCESS)
2320	{
2321	/* EIP has been updated already. */
2322	/* Only resume if successful. */
2323	goto ResumeExecution;
2324	}
2325	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
2326	break;
2327	}
2328
2329	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
2330	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
2331	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
2332	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
2333	{
2334	Log2(("SVM: %RGv mov dr%d, x\n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_WRITE_DR0));
2335	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
2336
2337	if ( !DBGFIsStepping(pVCpu)
2338	&& !CPUMIsHyperDebugStateActive(pVCpu))
2339	{
2340	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
2341
2342	/* Disable drx move intercepts. */
2343	pVMCB->ctrl.u16InterceptRdDRx = 0;
2344	pVMCB->ctrl.u16InterceptWrDRx = 0;
2345
2346	/* Save the host and load the guest debug state. */
2347	rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
2348	AssertRC(rc2);
2349	goto ResumeExecution;
2350	}
2351
2352	rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
2353	if (rc == VINF_SUCCESS)
2354	{
2355	/* EIP has been updated already. */
2356	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
2357
2358	/* Only resume if successful. */
2359	goto ResumeExecution;
2360	}
2361	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
2362	break;
2363	}
2364
2365	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
2366	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
2367	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
2368	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
2369	{
2370	Log2(("SVM: %RGv mov x, dr%d\n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_READ_DR0));
2371	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
2372
2373	if (!DBGFIsStepping(pVCpu))
2374	{
2375	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
2376
2377	/* Disable DRx move intercepts. */
2378	pVMCB->ctrl.u16InterceptRdDRx = 0;
2379	pVMCB->ctrl.u16InterceptWrDRx = 0;
2380
2381	/* Save the host and load the guest debug state. */
2382	rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
2383	AssertRC(rc2);
2384	goto ResumeExecution;
2385	}
2386
2387	rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
2388	if (rc == VINF_SUCCESS)
2389	{
2390	/* EIP has been updated already. */
2391	/* Only resume if successful. */
2392	goto ResumeExecution;
2393	}
2394	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
2395	break;
2396	}
2397
2398	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
2399	case SVM_EXIT_IOIO: /* I/O instruction. */
2400	{
2401	SVM_IOIO_EXIT IoExitInfo;
2402
2403	IoExitInfo.au32[0] = pVMCB->ctrl.u64ExitInfo1;
2404	unsigned uIdx = (IoExitInfo.au32[0] >> 4) & 0x7;
2405	uint32_t uIOSize = g_aIOSize[uIdx];
2406	uint32_t uAndVal = g_aIOOpAnd[uIdx];
2407	if (RT_UNLIKELY(!uIOSize))
2408	{
2409	AssertFailed(); /* should be fatal. */
2410	rc = VINF_EM_RAW_EMULATE_INSTR; /** @todo r=ramshankar: would this really fall back to the recompiler and work? */
2411	break;
2412	}
2413
2414	if (IoExitInfo.n.u1STR)
2415	{
2416	/* ins/outs */
2417	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
2418
2419	/* Disassemble manually to deal with segment prefixes. */
2420	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
2421	if (rc == VINF_SUCCESS)
2422	{
2423	if (IoExitInfo.n.u1Type == 0)
2424	{
2425	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
2426	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
2427	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, pDis->fPrefix,
2428	(DISCPUMODE)pDis->uAddrMode, uIOSize);
2429	}
2430	else
2431	{
2432	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
2433	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
2434	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, pDis->fPrefix,
2435	(DISCPUMODE)pDis->uAddrMode, uIOSize);
2436	}
2437	}
2438	else
2439	rc = VINF_EM_RAW_EMULATE_INSTR;
2440	}
2441	else
2442	{
2443	/* Normal in/out */
2444	Assert(!IoExitInfo.n.u1REP);
2445
2446	if (IoExitInfo.n.u1Type == 0)
2447	{
2448	Log2(("IOMIOPortWrite %RGv %x %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, pCtx->eax & uAndVal,
2449	uIOSize));
2450	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
2451	rc = IOMIOPortWrite(pVM, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, uIOSize);
2452	if (rc == VINF_IOM_R3_IOPORT_WRITE)
2453	{
2454	HMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pVMCB->ctrl.u64ExitInfo2, IoExitInfo.n.u16Port,
2455	uAndVal, uIOSize);
2456	}
2457	}
2458	else
2459	{
2460	uint32_t u32Val = 0;
2461
2462	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
2463	rc = IOMIOPortRead(pVM, IoExitInfo.n.u16Port, &u32Val, uIOSize);
2464	if (IOM_SUCCESS(rc))
2465	{
2466	/* Write back to the EAX register. */
2467	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
2468	Log2(("IOMIOPortRead %RGv %x %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, u32Val & uAndVal,
2469	uIOSize));
2470	}
2471	else if (rc == VINF_IOM_R3_IOPORT_READ)
2472	{
2473	HMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pVMCB->ctrl.u64ExitInfo2, IoExitInfo.n.u16Port,
2474	uAndVal, uIOSize);
2475	}
2476	}
2477	}
2478
2479	/*
2480	* Handled the I/O return codes.
2481	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
2482	*/
2483	if (IOM_SUCCESS(rc))
2484	{
2485	/* Update EIP and continue execution. */
2486	pCtx->rip = pVMCB->ctrl.u64ExitInfo2; /* RIP/EIP of the next instruction is saved in EXITINFO2. */
2487	if (RT_LIKELY(rc == VINF_SUCCESS))
2488	{
2489	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
2490	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
2491	{
2492	/* IO operation lookup arrays. */
2493	static uint32_t const aIOSize[4] = { 1, 2, 0, 4 };
2494
2495	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIOCheck);
2496	for (unsigned i = 0; i < 4; i++)
2497	{
2498	unsigned uBPLen = aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
2499
2500	if ( (IoExitInfo.n.u16Port >= pCtx->dr[i] && IoExitInfo.n.u16Port < pCtx->dr[i] + uBPLen)
2501	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
2502	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
2503	{
2504	SVM_EVENT Event;
2505
2506	Assert(CPUMIsGuestDebugStateActive(pVCpu));
2507
2508	/* Clear all breakpoint status flags and set the one we just hit. */
2509	pCtx->dr[6] &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
2510	pCtx->dr[6] \|= (uint64_t)RT_BIT(i);
2511
2512	/*
2513	* Note: AMD64 Architecture Programmer's Manual 13.1:
2514	* Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared
2515	* by software after the contents have been read.
2516	*/
2517	pVMCB->guest.u64DR6 = pCtx->dr[6];
2518
2519	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
2520	pCtx->dr[7] &= ~X86_DR7_GD;
2521
2522	/* Paranoia. */
2523	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2524	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2525	pCtx->dr[7] \|= 0x400; /* must be one */
2526
2527	pVMCB->guest.u64DR7 = pCtx->dr[7];
2528
2529	/* Inject the exception. */
2530	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
2531
2532	Event.au64[0] = 0;
2533	Event.n.u3Type = SVM_EVENT_EXCEPTION; /* trap or fault */
2534	Event.n.u1Valid = 1;
2535	Event.n.u8Vector = X86_XCPT_DB;
2536
2537	hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
2538	goto ResumeExecution;
2539	}
2540	}
2541	}
2542	goto ResumeExecution;
2543	}
2544	Log2(("EM status from IO at %RGv %x size %d: %Rrc\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize,
2545	VBOXSTRICTRC_VAL(rc)));
2546	break;
2547	}
2548
2549	#ifdef VBOX_STRICT
2550	if (rc == VINF_IOM_R3_IOPORT_READ)
2551	Assert(IoExitInfo.n.u1Type != 0);
2552	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
2553	Assert(IoExitInfo.n.u1Type == 0);
2554	else
2555	{
2556	AssertMsg( RT_FAILURE(rc)
2557	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
2558	\|\| rc == VINF_EM_RAW_GUEST_TRAP
2559	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
2560	}
2561	#endif
2562	Log2(("Failed IO at %RGv %x size %d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
2563	break;
2564	}
2565
2566	case SVM_EXIT_HLT:
2567	/* Check if external interrupts are pending; if so, don't switch back. */
2568	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
2569	pCtx->rip++; /* skip hlt */
2570	if (EMShouldContinueAfterHalt(pVCpu, pCtx))
2571	goto ResumeExecution;
2572
2573	rc = VINF_EM_HALT;
2574	break;
2575
2576	case SVM_EXIT_MWAIT_UNCOND:
2577	Log2(("SVM: mwait\n"));
2578	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
2579	rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
2580	if ( rc == VINF_EM_HALT
2581	\|\| rc == VINF_SUCCESS)
2582	{
2583	/* Update EIP and continue execution. */
2584	pCtx->rip += 3; /* Note: hardcoded opcode size assumption! */
2585
2586	/* Check if external interrupts are pending; if so, don't switch back. */
2587	if ( rc == VINF_SUCCESS
2588	\|\| ( rc == VINF_EM_HALT
2589	&& EMShouldContinueAfterHalt(pVCpu, pCtx))
2590	)
2591	goto ResumeExecution;
2592	}
2593	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
2594	break;
2595
2596	case SVM_EXIT_MONITOR:
2597	{
2598	Log2(("SVM: monitor\n"));
2599
2600	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
2601	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
2602	if (rc == VINF_SUCCESS)
2603	{
2604	/* Update EIP and continue execution. */
2605	pCtx->rip += 3; /* Note: hardcoded opcode size assumption! */
2606	goto ResumeExecution;
2607	}
2608	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
2609	break;
2610	}
2611
2612	case SVM_EXIT_VMMCALL:
2613	rc = hmR0SvmEmulateTprVMMCall(pVM, pVCpu, pCtx);
2614	if (rc == VINF_SUCCESS)
2615	{
2616	goto ResumeExecution; /* rip already updated. */
2617	}
2618	/* no break */
2619
2620	case SVM_EXIT_RSM:
2621	case SVM_EXIT_INVLPGA:
2622	case SVM_EXIT_VMRUN:
2623	case SVM_EXIT_VMLOAD:
2624	case SVM_EXIT_VMSAVE:
2625	case SVM_EXIT_STGI:
2626	case SVM_EXIT_CLGI:
2627	case SVM_EXIT_SKINIT:
2628	{
2629	/* Unsupported instructions. */
2630	SVM_EVENT Event;
2631
2632	Event.au64[0] = 0;
2633	Event.n.u3Type = SVM_EVENT_EXCEPTION;
2634	Event.n.u1Valid = 1;
2635	Event.n.u8Vector = X86_XCPT_UD;
2636
2637	Log(("Forced #UD trap at %RGv\n", (RTGCPTR)pCtx->rip));
2638	hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
2639	goto ResumeExecution;
2640	}
2641
2642	/* Emulate in ring-3. */
2643	case SVM_EXIT_MSR:
2644	{
2645	/* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
2646	if ( pVM->hm.s.fTPRPatchingActive
2647	&& pCtx->ecx == MSR_K8_LSTAR
2648	&& pVMCB->ctrl.u64ExitInfo1 == 1 /* wrmsr */)
2649	{
2650	if ((pCtx->eax & 0xff) != u8LastTPR)
2651	{
2652	Log(("SVM: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
2653
2654	/* Our patch code uses LSTAR for TPR caching. */
2655	rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
2656	AssertRC(rc2);
2657	}
2658
2659	/* Skip the instruction and continue. */
2660	pCtx->rip += 2; /* wrmsr = [0F 30] */
2661
2662	/* Only resume if successful. */
2663	goto ResumeExecution;
2664	}
2665
2666	/*
2667	* The Intel spec. claims there's an REX version of RDMSR that's slightly different,
2668	* so we play safe by completely disassembling the instruction.
2669	*/
2670	STAM_COUNTER_INC((pVMCB->ctrl.u64ExitInfo1 == 0) ? &pVCpu->hm.s.StatExitRdmsr : &pVCpu->hm.s.StatExitWrmsr);
2671	Log(("SVM: %s\n", (pVMCB->ctrl.u64ExitInfo1 == 0) ? "rdmsr" : "wrmsr"));
2672	rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
2673	if (rc == VINF_SUCCESS)
2674	{
2675	/* EIP has been updated already. */
2676	/* Only resume if successful. */
2677	goto ResumeExecution;
2678	}
2679	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (pVMCB->ctrl.u64ExitInfo1 == 0) ? "rdmsr" : "wrmsr",
2680	VBOXSTRICTRC_VAL(rc)));
2681	break;
2682	}
2683
2684	case SVM_EXIT_TASK_SWITCH: /* too complicated to emulate, so fall back to the recompiler */
2685	Log(("SVM_EXIT_TASK_SWITCH: exit2=%RX64\n", pVMCB->ctrl.u64ExitInfo2));
2686	if ( !(pVMCB->ctrl.u64ExitInfo2 & (SVM_EXIT2_TASK_SWITCH_IRET \| SVM_EXIT2_TASK_SWITCH_JMP))
2687	&& pVCpu->hm.s.Event.fPending)
2688	{
2689	SVM_EVENT Event;
2690	Event.au64[0] = pVCpu->hm.s.Event.intInfo;
2691
2692	/* Caused by an injected interrupt. */
2693	pVCpu->hm.s.Event.fPending = false;
2694	switch (Event.n.u3Type)
2695	{
2696	case SVM_EVENT_EXTERNAL_IRQ:
2697	case SVM_EVENT_NMI:
2698	Log(("SVM_EXIT_TASK_SWITCH: reassert trap %d\n", Event.n.u8Vector));
2699	Assert(!Event.n.u1ErrorCodeValid);
2700	rc2 = TRPMAssertTrap(pVCpu, Event.n.u8Vector, TRPM_HARDWARE_INT);
2701	AssertRC(rc2);
2702	break;
2703
2704	default:
2705	/* Exceptions and software interrupts can just be restarted. */
2706	break;
2707	}
2708	}
2709	rc = VERR_EM_INTERPRETER;
2710	break;
2711
2712	case SVM_EXIT_PAUSE:
2713	case SVM_EXIT_MWAIT_ARMED:
2714	rc = VERR_EM_INTERPRETER;
2715	break;
2716
2717	case SVM_EXIT_SHUTDOWN:
2718	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
2719	break;
2720
2721	case SVM_EXIT_IDTR_READ:
2722	case SVM_EXIT_GDTR_READ:
2723	case SVM_EXIT_LDTR_READ:
2724	case SVM_EXIT_TR_READ:
2725	case SVM_EXIT_IDTR_WRITE:
2726	case SVM_EXIT_GDTR_WRITE:
2727	case SVM_EXIT_LDTR_WRITE:
2728	case SVM_EXIT_TR_WRITE:
2729	case SVM_EXIT_CR0_SEL_WRITE:
2730	default:
2731	/* Unexpected exit codes. */
2732	rc = VERR_HMSVM_UNEXPECTED_EXIT;
2733	AssertMsgFailed(("Unexpected exit code %x\n", exitCode)); /* Can't happen. */
2734	break;
2735	}
2736
2737	end:
2738
2739	/*
2740	* We are now going back to ring-3, so clear the forced action flag.
2741	*/
2742	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
2743
2744	/*
2745	* Signal changes to the recompiler.
2746	*/
2747	CPUMSetChangedFlags(pVCpu,
2748	CPUM_CHANGED_SYSENTER_MSR
2749	\| CPUM_CHANGED_LDTR
2750	\| CPUM_CHANGED_GDTR
2751	\| CPUM_CHANGED_IDTR
2752	\| CPUM_CHANGED_TR
2753	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
2754
2755	/*
2756	* If we executed vmrun and an external IRQ was pending, then we don't have to do a full sync the next time.
2757	*/
2758	if (exitCode == SVM_EXIT_INTR)
2759	{
2760	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
2761	/* On the next entry we'll only sync the host context. */
2762	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
2763	}
2764	else
2765	{
2766	/* On the next entry we'll sync everything. */
2767	/** @todo we can do better than this */
2768	/* Not in the VINF_PGM_CHANGE_MODE though! */
2769	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL;
2770	}
2771
2772	/* Translate into a less severe return code */
2773	if (rc == VERR_EM_INTERPRETER)
2774	rc = VINF_EM_RAW_EMULATE_INSTR;
2775
2776	/* Just set the correct state here instead of trying to catch every goto above. */
2777	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
2778
2779	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2780	/* Restore interrupts if we exitted after disabling them. */
2781	if (uOldEFlags != ~(RTCCUINTREG)0)
2782	ASMSetFlags(uOldEFlags);
2783	#endif
2784
2785	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
2786	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
2787	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
2788	return VBOXSTRICTRC_TODO(rc);
2789	}
2790
2791
2792	/**
2793	* Emulate simple mov tpr instruction.
2794	*
2795	* @returns VBox status code.
2796	* @param pVM Pointer to the VM.
2797	* @param pVCpu Pointer to the VMCPU.
2798	* @param pCtx Pointer to the guest CPU context.
2799	*/
2800	static int hmR0SvmEmulateTprVMMCall(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2801	{
2802	int rc;
2803
2804	LogFlow(("Emulated VMMCall TPR access replacement at %RGv\n", pCtx->rip));
2805
2806	for (;;)
2807	{
2808	bool fPending;
2809	uint8_t u8Tpr;
2810
2811	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
2812	if (!pPatch)
2813	break;
2814
2815	switch (pPatch->enmType)
2816	{
2817	case HMTPRINSTR_READ:
2818	/* TPR caching in CR8 */
2819	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPending);
2820	AssertRC(rc);
2821
2822	rc = DISWriteReg32(CPUMCTX2CORE(pCtx), pPatch->uDstOperand, u8Tpr);
2823	AssertRC(rc);
2824
2825	LogFlow(("Emulated read successfully\n"));
2826	pCtx->rip += pPatch->cbOp;
2827	break;
2828
2829	case HMTPRINSTR_WRITE_REG:
2830	case HMTPRINSTR_WRITE_IMM:
2831	/* Fetch the new TPR value */
2832	if (pPatch->enmType == HMTPRINSTR_WRITE_REG)
2833	{
2834	uint32_t val;
2835
2836	rc = DISFetchReg32(CPUMCTX2CORE(pCtx), pPatch->uSrcOperand, &val);
2837	AssertRC(rc);
2838	u8Tpr = val;
2839	}
2840	else
2841	u8Tpr = (uint8_t)pPatch->uSrcOperand;
2842
2843	rc = PDMApicSetTPR(pVCpu, u8Tpr);
2844	AssertRC(rc);
2845	LogFlow(("Emulated write successfully\n"));
2846	pCtx->rip += pPatch->cbOp;
2847	break;
2848
2849	default:
2850	AssertMsgFailedReturn(("Unexpected type %d\n", pPatch->enmType), VERR_HMSVM_UNEXPECTED_PATCH_TYPE);
2851	}
2852	}
2853	return VINF_SUCCESS;
2854	}
2855
2856
2857	/**
2858	* Enters the AMD-V session.
2859	*
2860	* @returns VBox status code.
2861	* @param pVM Pointer to the VM.
2862	* @param pVCpu Pointer to the VMCPU.
2863	* @param pCpu Pointer to the CPU info struct.
2864	*/
2865	VMMR0DECL(int) SVMR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
2866	{
2867	Assert(pVM->hm.s.svm.fSupported);
2868
2869	LogFlow(("SVMR0Enter cpu%d last=%d asid=%d\n", pCpu->idCpu, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.uCurrentASID));
2870	pVCpu->hm.s.fResumeVM = false;
2871
2872	/* Force to reload LDTR, so we'll execute VMLoad to load additional guest state. */
2873	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_LDTR;
2874
2875	return VINF_SUCCESS;
2876	}
2877
2878
2879	/**
2880	* Leaves the AMD-V session.
2881	*
2882	* @returns VBox status code.
2883	* @param pVM Pointer to the VM.
2884	* @param pVCpu Pointer to the VMCPU.
2885	* @param pCtx Pointer to the guest CPU context.
2886	*/
2887	VMMR0DECL(int) SVMR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2888	{
2889	SVM_VMCB pVMCB = (SVM_VMCB )pVCpu->hm.s.svm.pVMCB;
2890
2891	Assert(pVM->hm.s.svm.fSupported);
2892
2893	#ifdef DEBUG
2894	if (CPUMIsHyperDebugStateActive(pVCpu))
2895	{
2896	CPUMR0LoadHostDebugState(pVM, pVCpu);
2897	}
2898	else
2899	#endif
2900	/* Save the guest debug state if necessary. */
2901	if (CPUMIsGuestDebugStateActive(pVCpu))
2902	{
2903	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, false /* skip DR6 */);
2904
2905	/* Intercept all DRx reads and writes again. Changed later on. */
2906	pVMCB->ctrl.u16InterceptRdDRx = 0xFFFF;
2907	pVMCB->ctrl.u16InterceptWrDRx = 0xFFFF;
2908
2909	/* Resync the debug registers the next time. */
2910	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
2911	}
2912	else
2913	Assert(pVMCB->ctrl.u16InterceptRdDRx == 0xFFFF && pVMCB->ctrl.u16InterceptWrDRx == 0xFFFF);
2914
2915	return VINF_SUCCESS;
2916	}
2917
2918
2919	/**
2920	* Worker for Interprets INVLPG.
2921	*
2922	* @return VBox status code.
2923	* @param pVCpu Pointer to the VMCPU.
2924	* @param pCpu Pointer to the CPU info struct.
2925	* @param pRegFrame Pointer to the register frame.
2926	*/
2927	static int hmR0svmInterpretInvlPgEx(PVMCPU pVCpu, PDISCPUSTATE pCpu, PCPUMCTXCORE pRegFrame)
2928	{
2929	DISQPVPARAMVAL param1;
2930	RTGCPTR addr;
2931
2932	int rc = DISQueryParamVal(pRegFrame, pCpu, &pCpu->Param1, &param1, DISQPVWHICH_SRC);
2933	if (RT_FAILURE(rc))
2934	return VERR_EM_INTERPRETER;
2935
2936	switch (param1.type)
2937	{
2938	case DISQPV_TYPE_IMMEDIATE:
2939	case DISQPV_TYPE_ADDRESS:
2940	if (!(param1.flags & (DISQPV_FLAG_32 \| DISQPV_FLAG_64)))
2941	return VERR_EM_INTERPRETER;
2942	addr = param1.val.val64;
2943	break;
2944
2945	default:
2946	return VERR_EM_INTERPRETER;
2947	}
2948
2949	/** @todo is addr always a flat linear address or ds based
2950	* (in absence of segment override prefixes)????
2951	*/
2952	rc = PGMInvalidatePage(pVCpu, addr);
2953	if (RT_SUCCESS(rc))
2954	return VINF_SUCCESS;
2955
2956	AssertRC(rc);
2957	return rc;
2958	}
2959
2960
2961	/**
2962	* Interprets INVLPG.
2963	*
2964	* @returns VBox status code.
2965	* @retval VINF_* Scheduling instructions.
2966	* @retval VERR_EM_INTERPRETER Something we can't cope with.
2967	* @retval VERR_* Fatal errors.
2968	*
2969	* @param pVM Pointer to the VM.
2970	* @param pRegFrame Pointer to the register frame.
2971	*
2972	* @remarks Updates the EIP if an instruction was executed successfully.
2973	*/
2974	static int hmR0SvmInterpretInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame)
2975	{
2976	/*
2977	* Only allow 32 & 64 bit code.
2978	*/
2979	if (CPUMGetGuestCodeBits(pVCpu) != 16)
2980	{
2981	PDISSTATE pDis = &pVCpu->hm.s.DisState;
2982	int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
2983	if (RT_SUCCESS(rc) && pDis->pCurInstr->uOpcode == OP_INVLPG)
2984	{
2985	rc = hmR0svmInterpretInvlPgEx(pVCpu, pDis, pRegFrame);
2986	if (RT_SUCCESS(rc))
2987	pRegFrame->rip += pDis->cbInstr; /* Move on to the next instruction. */
2988	return rc;
2989	}
2990	}
2991	return VERR_EM_INTERPRETER;
2992	}
2993
2994
2995	/**
2996	* Invalidates a guest page by guest virtual address.
2997	*
2998	* @returns VBox status code.
2999	* @param pVM Pointer to the VM.
3000	* @param pVCpu Pointer to the VMCPU.
3001	* @param GCVirt Guest virtual address of the page to invalidate.
3002	*/
3003	VMMR0DECL(int) SVMR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
3004	{
3005	bool fFlushPending = pVM->hm.s.svm.fAlwaysFlushTLB \| VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
3006
3007	/* Skip it if a TLB flush is already pending. */
3008	if (!fFlushPending)
3009	{
3010	SVM_VMCB *pVMCB;
3011
3012	Log2(("SVMR0InvalidatePage %RGv\n", GCVirt));
3013	AssertReturn(pVM, VERR_INVALID_PARAMETER);
3014	Assert(pVM->hm.s.svm.fSupported);
3015
3016	pVMCB = (SVM_VMCB *)pVCpu->hm.s.svm.pVMCB;
3017	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_HMSVM_INVALID_PVMCB);
3018
3019	#if HC_ARCH_BITS == 32
3020	/* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invlpga takes only 32 bits addresses. */
3021	if (CPUMIsGuestInLongMode(pVCpu))
3022	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3023	else
3024	#endif
3025	SVMR0InvlpgA(GCVirt, pVMCB->ctrl.TLBCtrl.n.u32ASID);
3026	}
3027	return VINF_SUCCESS;
3028	}
3029
3030
3031	#if 0 /* obsolete, but left here for clarification. */
3032	/**
3033	* Invalidates a guest page by physical address.
3034	*
3035	* @returns VBox status code.
3036	* @param pVM Pointer to the VM.
3037	* @param pVCpu Pointer to the VMCPU.
3038	* @param GCPhys Guest physical address of the page to invalidate.
3039	*/
3040	VMMR0DECL(int) SVMR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
3041	{
3042	Assert(pVM->hm.s.fNestedPaging);
3043	/* invlpga only invalidates TLB entries for guest virtual addresses; we have no choice but to force a TLB flush here. */
3044	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3045	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTLBInvlpga);
3046	return VINF_SUCCESS;
3047	}
3048	#endif
3049
3050
3051	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3052	/**
3053	* Prepares for and executes VMRUN (64-bit guests from a 32-bit host).
3054	*
3055	* @returns VBox status code.
3056	* @param pVMCBHostPhys Physical address of host VMCB.
3057	* @param pVMCBPhys Physical address of the VMCB.
3058	* @param pCtx Pointer to the guest CPU context.
3059	* @param pVM Pointer to the VM.
3060	* @param pVCpu Pointer to the VMCPU.
3061	*/
3062	DECLASM(int) SVMR0VMSwitcherRun64(RTHCPHYS pVMCBHostPhys, RTHCPHYS pVMCBPhys, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu)
3063	{
3064	uint32_t aParam[4];
3065
3066	aParam[0] = (uint32_t)(pVMCBHostPhys); /* Param 1: pVMCBHostPhys - Lo. */
3067	aParam[1] = (uint32_t)(pVMCBHostPhys >> 32); /* Param 1: pVMCBHostPhys - Hi. */
3068	aParam[2] = (uint32_t)(pVMCBPhys); /* Param 2: pVMCBPhys - Lo. */
3069	aParam[3] = (uint32_t)(pVMCBPhys >> 32); /* Param 2: pVMCBPhys - Hi. */
3070
3071	return SVMR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnSVMGCVMRun64, 4, &aParam[0]);
3072	}
3073
3074
3075	/**
3076	* Executes the specified handler in 64-bit mode.
3077	*
3078	* @returns VBox status code.
3079	* @param pVM Pointer to the VM.
3080	* @param pVCpu Pointer to the VMCPU.
3081	* @param pCtx Pointer to the guest CPU context.
3082	* @param pfnHandler Pointer to the RC handler function.
3083	* @param cbParam Number of parameters.
3084	* @param paParam Array of 32-bit parameters.
3085	*/
3086	VMMR0DECL(int) SVMR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
3087	uint32_t *paParam)
3088	{
3089	int rc;
3090	RTHCUINTREG uOldEFlags;
3091
3092	Assert(pfnHandler);
3093
3094	/* Disable interrupts. */
3095	uOldEFlags = ASMIntDisableFlags();
3096
3097	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3098	RTCPUID idHostCpu = RTMpCpuId();
3099	CPUMR0SetLApic(pVM, idHostCpu);
3100	#endif
3101
3102	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3103	CPUMSetHyperEIP(pVCpu, pfnHandler);
3104	for (int i = (int)cbParam - 1; i >= 0; i--)
3105	CPUMPushHyper(pVCpu, paParam[i]);
3106
3107	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3108	/* Call switcher. */
3109	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
3110	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3111
3112	ASMSetFlags(uOldEFlags);
3113	return rc;
3114	}
3115
3116	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
3117

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWSVMR0.cpp@ 43387

Download in other formats: