HWACCMR0.cpp@ 32963

Last change on this file since 32963 was 32489, checked in by vboxsync, 14 years ago
VMM: More work on the periodic preemption timer (no actual timers yet).
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 69.1 KB

Line
1	/* $Id: HWACCMR0.cpp 32489 2010-09-14 15:50:31Z vboxsync $ */
2	/** @file
3	* HWACCM - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HWACCM
23	#include <VBox/hwaccm.h>
24	#include <VBox/pgm.h>
25	#include "HWACCMInternal.h"
26	#include <VBox/vm.h>
27	#include <VBox/x86.h>
28	#include <VBox/hwacc_vmx.h>
29	#include <VBox/hwacc_svm.h>
30	#include <VBox/err.h>
31	#include <VBox/log.h>
32	#include <iprt/assert.h>
33	#include <iprt/asm.h>
34	#include <iprt/asm-amd64-x86.h>
35	#include <iprt/cpuset.h>
36	#include <iprt/memobj.h>
37	#include <iprt/param.h>
38	#include <iprt/power.h>
39	#include <iprt/string.h>
40	#include <iprt/thread.h>
41	#include "HWVMXR0.h"
42	#include "HWSVMR0.h"
43
44	/*******************************************************************************
45	* Internal Functions *
46	*******************************************************************************/
47	static DECLCALLBACK(void) hwaccmR0EnableCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2);
48	static DECLCALLBACK(void) hwaccmR0DisableCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2);
49	static DECLCALLBACK(void) HWACCMR0InitCPU(RTCPUID idCpu, void pvUser1, void pvUser2);
50	static int hwaccmR0CheckCpuRcArray(int paRc, unsigned cErrorCodes, RTCPUID pidCpu);
51	static bool hwaccmR0IsSubjectToVmxPreemptionTimerErratum(void);
52	static DECLCALLBACK(void) hwaccmR0PowerCallback(RTPOWEREVENT enmEvent, void *pvUser);
53
54	/*******************************************************************************
55	* Global Variables *
56	*******************************************************************************/
57
58	static struct
59	{
60	HWACCM_CPUINFO aCpuInfo[RTCPUSET_MAX_CPUS];
61
62	/** Ring 0 handlers for VT-x and AMD-V. */
63	DECLR0CALLBACKMEMBER(int, pfnEnterSession,(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu));
64	DECLR0CALLBACKMEMBER(int, pfnLeaveSession,(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx));
65	DECLR0CALLBACKMEMBER(int, pfnSaveHostState,(PVM pVM, PVMCPU pVCpu));
66	DECLR0CALLBACKMEMBER(int, pfnLoadGuestState,(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx));
67	DECLR0CALLBACKMEMBER(int, pfnRunGuestCode,(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx));
68	DECLR0CALLBACKMEMBER(int, pfnEnableCpu, (PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys));
69	DECLR0CALLBACKMEMBER(int, pfnDisableCpu, (PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys));
70	DECLR0CALLBACKMEMBER(int, pfnInitVM, (PVM pVM));
71	DECLR0CALLBACKMEMBER(int, pfnTermVM, (PVM pVM));
72	DECLR0CALLBACKMEMBER(int, pfnSetupVM, (PVM pVM));
73
74	/** Maximum ASID allowed. */
75	uint32_t uMaxASID;
76
77	struct
78	{
79	/** Set by the ring-0 driver to indicate VMX is supported by the CPU. */
80	bool fSupported;
81	/** Whether we're using SUPR0EnableVTx or not. */
82	bool fUsingSUPR0EnableVTx;
83	/** Whether we're using the preemption timer or not. */
84	bool fUsePreemptTimer;
85	/** The shift mask employed by the VMX-Preemption timer. */
86	uint8_t cPreemptTimerShift;
87
88	/** Host CR4 value (set by ring-0 VMX init) */
89	uint64_t hostCR4;
90
91	/** Host EFER value (set by ring-0 VMX init) */
92	uint64_t hostEFER;
93
94	/** VMX MSR values */
95	struct
96	{
97	uint64_t feature_ctrl;
98	uint64_t vmx_basic_info;
99	VMX_CAPABILITY vmx_pin_ctls;
100	VMX_CAPABILITY vmx_proc_ctls;
101	VMX_CAPABILITY vmx_proc_ctls2;
102	VMX_CAPABILITY vmx_exit;
103	VMX_CAPABILITY vmx_entry;
104	uint64_t vmx_misc;
105	uint64_t vmx_cr0_fixed0;
106	uint64_t vmx_cr0_fixed1;
107	uint64_t vmx_cr4_fixed0;
108	uint64_t vmx_cr4_fixed1;
109	uint64_t vmx_vmcs_enum;
110	uint64_t vmx_eptcaps;
111	} msr;
112	/* Last instruction error */
113	uint32_t ulLastInstrError;
114	} vmx;
115	struct
116	{
117	/* HWCR msr (for diagnostics) */
118	uint64_t msrHWCR;
119
120	/** SVM revision. */
121	uint32_t u32Rev;
122
123	/** SVM feature bits from cpuid 0x8000000a */
124	uint32_t u32Features;
125
126	/** Set by the ring-0 driver to indicate SVM is supported by the CPU. */
127	bool fSupported;
128	} svm;
129	/** Saved error from detection */
130	int32_t lLastError;
131
132	struct
133	{
134	uint32_t u32AMDFeatureECX;
135	uint32_t u32AMDFeatureEDX;
136	} cpuid;
137
138	HWACCMSTATE enmHwAccmState;
139
140	bool fGlobalInit;
141	volatile bool fSuspended;
142	} HWACCMR0Globals;
143
144
145
146	/**
147	* Does global Ring-0 HWACCM initialization.
148	*
149	* @returns VBox status code.
150	*/
151	VMMR0DECL(int) HWACCMR0Init(void)
152	{
153	int rc;
154	bool fAMDVPresent = false;
155
156	memset(&HWACCMR0Globals, 0, sizeof(HWACCMR0Globals));
157	HWACCMR0Globals.enmHwAccmState = HWACCMSTATE_UNINITIALIZED;
158	for (unsigned i = 0; i < RT_ELEMENTS(HWACCMR0Globals.aCpuInfo); i++)
159	HWACCMR0Globals.aCpuInfo[i].pMemObj = NIL_RTR0MEMOBJ;
160
161	/* Fill in all callbacks with placeholders. */
162	HWACCMR0Globals.pfnEnterSession = HWACCMR0DummyEnter;
163	HWACCMR0Globals.pfnLeaveSession = HWACCMR0DummyLeave;
164	HWACCMR0Globals.pfnSaveHostState = HWACCMR0DummySaveHostState;
165	HWACCMR0Globals.pfnLoadGuestState = HWACCMR0DummyLoadGuestState;
166	HWACCMR0Globals.pfnRunGuestCode = HWACCMR0DummyRunGuestCode;
167	HWACCMR0Globals.pfnEnableCpu = HWACCMR0DummyEnableCpu;
168	HWACCMR0Globals.pfnDisableCpu = HWACCMR0DummyDisableCpu;
169	HWACCMR0Globals.pfnInitVM = HWACCMR0DummyInitVM;
170	HWACCMR0Globals.pfnTermVM = HWACCMR0DummyTermVM;
171	HWACCMR0Globals.pfnSetupVM = HWACCMR0DummySetupVM;
172
173	/* Default is global VT-x/AMD-V init */
174	HWACCMR0Globals.fGlobalInit = true;
175
176	/*
177	* Check for VT-x and AMD-V capabilities
178	*/
179	if (ASMHasCpuId())
180	{
181	uint32_t u32FeaturesECX;
182	uint32_t u32Dummy;
183	uint32_t u32FeaturesEDX;
184	uint32_t u32VendorEBX, u32VendorECX, u32VendorEDX;
185
186	ASMCpuId(0, &u32Dummy, &u32VendorEBX, &u32VendorECX, &u32VendorEDX);
187	ASMCpuId(1, &u32Dummy, &u32Dummy, &u32FeaturesECX, &u32FeaturesEDX);
188	/* Query AMD features. */
189	ASMCpuId(0x80000001, &u32Dummy, &u32Dummy, &HWACCMR0Globals.cpuid.u32AMDFeatureECX, &HWACCMR0Globals.cpuid.u32AMDFeatureEDX);
190
191	if ( u32VendorEBX == X86_CPUID_VENDOR_INTEL_EBX
192	&& u32VendorECX == X86_CPUID_VENDOR_INTEL_ECX
193	&& u32VendorEDX == X86_CPUID_VENDOR_INTEL_EDX
194	)
195	{
196	/*
197	* Read all VMX MSRs if VMX is available. (same goes for RDMSR/WRMSR)
198	* We also assume all VMX-enabled CPUs support fxsave/fxrstor.
199	*/
200	if ( (u32FeaturesECX & X86_CPUID_FEATURE_ECX_VMX)
201	&& (u32FeaturesEDX & X86_CPUID_FEATURE_EDX_MSR)
202	&& (u32FeaturesEDX & X86_CPUID_FEATURE_EDX_FXSR)
203	)
204	{
205	int aRc[RTCPUSET_MAX_CPUS];
206	RTCPUID idCpu = 0;
207
208	HWACCMR0Globals.vmx.msr.feature_ctrl = ASMRdMsr(MSR_IA32_FEATURE_CONTROL);
209
210	/*
211	* First try use native kernel API for controlling VT-x.
212	* (This is only supported by some Mac OS X kernels atm.)
213	*/
214	HWACCMR0Globals.lLastError = rc = SUPR0EnableVTx(true /* fEnable */);
215	if (rc != VERR_NOT_SUPPORTED)
216	{
217	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_VMX_IN_VMX_ROOT_MODE \|\| rc == VERR_VMX_NO_VMX, ("%Rrc\n", rc));
218	HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx = true;
219	if (RT_SUCCESS(rc))
220	{
221	HWACCMR0Globals.vmx.fSupported = true;
222	rc = SUPR0EnableVTx(false /* fEnable */);
223	AssertRC(rc);
224	}
225	}
226	else
227	{
228	HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx = false;
229
230	/* We need to check if VT-x has been properly initialized on all CPUs. Some BIOSes do a lousy job. */
231	memset(aRc, 0, sizeof(aRc));
232	HWACCMR0Globals.lLastError = RTMpOnAll(HWACCMR0InitCPU, (void *)u32VendorEBX, aRc);
233
234	/* Check the return code of all invocations. */
235	if (RT_SUCCESS(HWACCMR0Globals.lLastError))
236	HWACCMR0Globals.lLastError = hwaccmR0CheckCpuRcArray(aRc, RT_ELEMENTS(aRc), &idCpu);
237	}
238	if (RT_SUCCESS(HWACCMR0Globals.lLastError))
239	{
240	/* Reread in case we've changed it. */
241	HWACCMR0Globals.vmx.msr.feature_ctrl = ASMRdMsr(MSR_IA32_FEATURE_CONTROL);
242
243	if ( (HWACCMR0Globals.vmx.msr.feature_ctrl & (MSR_IA32_FEATURE_CONTROL_VMXON\|MSR_IA32_FEATURE_CONTROL_LOCK))
244	== (MSR_IA32_FEATURE_CONTROL_VMXON\|MSR_IA32_FEATURE_CONTROL_LOCK))
245	{
246	RTR0MEMOBJ pScatchMemObj;
247	void *pvScatchPage;
248	RTHCPHYS pScatchPagePhys;
249
250	HWACCMR0Globals.vmx.msr.vmx_basic_info = ASMRdMsr(MSR_IA32_VMX_BASIC_INFO);
251	HWACCMR0Globals.vmx.msr.vmx_pin_ctls.u = ASMRdMsr(MSR_IA32_VMX_PINBASED_CTLS);
252	HWACCMR0Globals.vmx.msr.vmx_proc_ctls.u = ASMRdMsr(MSR_IA32_VMX_PROCBASED_CTLS);
253	HWACCMR0Globals.vmx.msr.vmx_exit.u = ASMRdMsr(MSR_IA32_VMX_EXIT_CTLS);
254	HWACCMR0Globals.vmx.msr.vmx_entry.u = ASMRdMsr(MSR_IA32_VMX_ENTRY_CTLS);
255	HWACCMR0Globals.vmx.msr.vmx_misc = ASMRdMsr(MSR_IA32_VMX_MISC);
256	HWACCMR0Globals.vmx.msr.vmx_cr0_fixed0 = ASMRdMsr(MSR_IA32_VMX_CR0_FIXED0);
257	HWACCMR0Globals.vmx.msr.vmx_cr0_fixed1 = ASMRdMsr(MSR_IA32_VMX_CR0_FIXED1);
258	HWACCMR0Globals.vmx.msr.vmx_cr4_fixed0 = ASMRdMsr(MSR_IA32_VMX_CR4_FIXED0);
259	HWACCMR0Globals.vmx.msr.vmx_cr4_fixed1 = ASMRdMsr(MSR_IA32_VMX_CR4_FIXED1);
260	HWACCMR0Globals.vmx.msr.vmx_vmcs_enum = ASMRdMsr(MSR_IA32_VMX_VMCS_ENUM);
261	/* VPID 16 bits ASID. */
262	HWACCMR0Globals.uMaxASID = 0x10000; /* exclusive */
263
264	if (HWACCMR0Globals.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
265	{
266	HWACCMR0Globals.vmx.msr.vmx_proc_ctls2.u = ASMRdMsr(MSR_IA32_VMX_PROCBASED_CTLS2);
267	if (HWACCMR0Globals.vmx.msr.vmx_proc_ctls2.n.allowed1 & (VMX_VMCS_CTRL_PROC_EXEC2_EPT\|VMX_VMCS_CTRL_PROC_EXEC2_VPID))
268	HWACCMR0Globals.vmx.msr.vmx_eptcaps = ASMRdMsr(MSR_IA32_VMX_EPT_CAPS);
269	}
270
271	if (!HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx)
272	{
273	HWACCMR0Globals.vmx.hostCR4 = ASMGetCR4();
274	HWACCMR0Globals.vmx.hostEFER = ASMRdMsr(MSR_K6_EFER);
275
276	rc = RTR0MemObjAllocCont(&pScatchMemObj, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
277	if (RT_FAILURE(rc))
278	return rc;
279
280	pvScatchPage = RTR0MemObjAddress(pScatchMemObj);
281	pScatchPagePhys = RTR0MemObjGetPagePhysAddr(pScatchMemObj, 0);
282	memset(pvScatchPage, 0, PAGE_SIZE);
283
284	/* Set revision dword at the beginning of the structure. */
285	(uint32_t )pvScatchPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(HWACCMR0Globals.vmx.msr.vmx_basic_info);
286
287	/* Make sure we don't get rescheduled to another cpu during this probe. */
288	RTCCUINTREG fFlags = ASMIntDisableFlags();
289
290	/*
291	* Check CR4.VMXE
292	*/
293	if (!(HWACCMR0Globals.vmx.hostCR4 & X86_CR4_VMXE))
294	{
295	/* In theory this bit could be cleared behind our back. Which would cause #UD faults when we
296	* try to execute the VMX instructions...
297	*/
298	ASMSetCR4(HWACCMR0Globals.vmx.hostCR4 \| X86_CR4_VMXE);
299	}
300
301	/* Enter VMX Root Mode */
302	rc = VMXEnable(pScatchPagePhys);
303	if (RT_FAILURE(rc))
304	{
305	/* KVM leaves the CPU in VMX root mode. Not only is this not allowed, it will crash the host when we enter raw mode, because
306	* (a) clearing X86_CR4_VMXE in CR4 causes a #GP (we no longer modify this bit)
307	* (b) turning off paging causes a #GP (unavoidable when switching from long to 32 bits mode or 32 bits to PAE)
308	*
309	* They should fix their code, but until they do we simply refuse to run.
310	*/
311	HWACCMR0Globals.lLastError = VERR_VMX_IN_VMX_ROOT_MODE;
312	}
313	else
314	{
315	HWACCMR0Globals.vmx.fSupported = true;
316	VMXDisable();
317
318	/*
319	* Check for the VMX-Preemption Timer and adjust for the
320	* "VMX-Preemption Timer Does Not Count Down at the Rate Specified" erratum.
321	*/
322	if ( HWACCMR0Globals.vmx.msr.vmx_pin_ctls.n.allowed1
323	& VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER)
324	{
325	HWACCMR0Globals.vmx.fUsePreemptTimer = true;
326	HWACCMR0Globals.vmx.cPreemptTimerShift = MSR_IA32_VMX_MISC_PREEMPT_TSC_BIT(HWACCMR0Globals.vmx.msr.vmx_misc);
327	if (hwaccmR0IsSubjectToVmxPreemptionTimerErratum())
328	HWACCMR0Globals.vmx.cPreemptTimerShift = 0; /* This is about right most of the time here. */
329	}
330	}
331
332	/* Restore CR4 again; don't leave the X86_CR4_VMXE flag set if it wasn't so before (some software could incorrectly think it's in VMX mode) */
333	ASMSetCR4(HWACCMR0Globals.vmx.hostCR4);
334	ASMSetFlags(fFlags);
335
336	RTR0MemObjFree(pScatchMemObj, false);
337	if (RT_FAILURE(HWACCMR0Globals.lLastError))
338	return HWACCMR0Globals.lLastError;
339	}
340	}
341	else
342	{
343	AssertFailed(); /* can't hit this case anymore */
344	HWACCMR0Globals.lLastError = VERR_VMX_ILLEGAL_FEATURE_CONTROL_MSR;
345	}
346	}
347	#ifdef LOG_ENABLED
348	else
349	SUPR0Printf("HWACCMR0InitCPU failed with rc=%d\n", HWACCMR0Globals.lLastError);
350	#endif
351	}
352	else
353	HWACCMR0Globals.lLastError = VERR_VMX_NO_VMX;
354	}
355	else
356	if ( u32VendorEBX == X86_CPUID_VENDOR_AMD_EBX
357	&& u32VendorECX == X86_CPUID_VENDOR_AMD_ECX
358	&& u32VendorEDX == X86_CPUID_VENDOR_AMD_EDX
359	)
360	{
361	/*
362	* Read all SVM MSRs if SVM is available. (same goes for RDMSR/WRMSR)
363	* We also assume all SVM-enabled CPUs support fxsave/fxrstor.
364	*/
365	if ( (HWACCMR0Globals.cpuid.u32AMDFeatureECX & X86_CPUID_AMD_FEATURE_ECX_SVM)
366	&& (u32FeaturesEDX & X86_CPUID_FEATURE_EDX_MSR)
367	&& (u32FeaturesEDX & X86_CPUID_FEATURE_EDX_FXSR)
368	)
369	{
370	int aRc[RTCPUSET_MAX_CPUS];
371	RTCPUID idCpu = 0;
372
373	fAMDVPresent = true;
374
375	/* Query AMD features. */
376	ASMCpuId(0x8000000A, &HWACCMR0Globals.svm.u32Rev, &HWACCMR0Globals.uMaxASID, &u32Dummy, &HWACCMR0Globals.svm.u32Features);
377
378	/* We need to check if AMD-V has been properly initialized on all CPUs. Some BIOSes might do a poor job. */
379	memset(aRc, 0, sizeof(aRc));
380	rc = RTMpOnAll(HWACCMR0InitCPU, (void *)u32VendorEBX, aRc);
381	AssertRC(rc);
382
383	/* Check the return code of all invocations. */
384	if (RT_SUCCESS(rc))
385	rc = hwaccmR0CheckCpuRcArray(aRc, RT_ELEMENTS(aRc), &idCpu);
386
387	#ifndef DEBUG_bird
388	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_SVM_IN_USE, ("HWACCMR0InitCPU failed for cpu %d with rc=%d\n", idCpu, rc));
389	#endif
390	if (RT_SUCCESS(rc))
391	{
392	/* Read the HWCR msr for diagnostics. */
393	HWACCMR0Globals.svm.msrHWCR = ASMRdMsr(MSR_K8_HWCR);
394	HWACCMR0Globals.svm.fSupported = true;
395	}
396	else
397	HWACCMR0Globals.lLastError = rc;
398	}
399	else
400	HWACCMR0Globals.lLastError = VERR_SVM_NO_SVM;
401	}
402	else
403	HWACCMR0Globals.lLastError = VERR_HWACCM_UNKNOWN_CPU;
404	}
405	else
406	HWACCMR0Globals.lLastError = VERR_HWACCM_NO_CPUID;
407
408	if (HWACCMR0Globals.vmx.fSupported)
409	{
410	HWACCMR0Globals.pfnEnterSession = VMXR0Enter;
411	HWACCMR0Globals.pfnLeaveSession = VMXR0Leave;
412	HWACCMR0Globals.pfnSaveHostState = VMXR0SaveHostState;
413	HWACCMR0Globals.pfnLoadGuestState = VMXR0LoadGuestState;
414	HWACCMR0Globals.pfnRunGuestCode = VMXR0RunGuestCode;
415	HWACCMR0Globals.pfnEnableCpu = VMXR0EnableCpu;
416	HWACCMR0Globals.pfnDisableCpu = VMXR0DisableCpu;
417	HWACCMR0Globals.pfnInitVM = VMXR0InitVM;
418	HWACCMR0Globals.pfnTermVM = VMXR0TermVM;
419	HWACCMR0Globals.pfnSetupVM = VMXR0SetupVM;
420	}
421	else
422	if (fAMDVPresent)
423	{
424	HWACCMR0Globals.pfnEnterSession = SVMR0Enter;
425	HWACCMR0Globals.pfnLeaveSession = SVMR0Leave;
426	HWACCMR0Globals.pfnSaveHostState = SVMR0SaveHostState;
427	HWACCMR0Globals.pfnLoadGuestState = SVMR0LoadGuestState;
428	HWACCMR0Globals.pfnRunGuestCode = SVMR0RunGuestCode;
429	HWACCMR0Globals.pfnEnableCpu = SVMR0EnableCpu;
430	HWACCMR0Globals.pfnDisableCpu = SVMR0DisableCpu;
431	HWACCMR0Globals.pfnInitVM = SVMR0InitVM;
432	HWACCMR0Globals.pfnTermVM = SVMR0TermVM;
433	HWACCMR0Globals.pfnSetupVM = SVMR0SetupVM;
434	}
435
436	if (!HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx)
437	{
438	rc = RTPowerNotificationRegister(hwaccmR0PowerCallback, 0);
439	AssertRC(rc);
440	}
441
442	return VINF_SUCCESS;
443	}
444
445
446	/**
447	* Checks the error code array filled in for each cpu in the system.
448	*
449	* @returns VBox status code.
450	* @param paRc Error code array
451	* @param cErrorCodes Array size
452	* @param pidCpu Value of the first cpu that set an error (out)
453	*/
454	static int hwaccmR0CheckCpuRcArray(int paRc, unsigned cErrorCodes, RTCPUID pidCpu)
455	{
456	int rc = VINF_SUCCESS;
457
458	Assert(cErrorCodes == RTCPUSET_MAX_CPUS);
459
460	for (unsigned i=0;i<cErrorCodes;i++)
461	{
462	if (RTMpIsCpuOnline(i))
463	{
464	if (RT_FAILURE(paRc[i]))
465	{
466	rc = paRc[i];
467	*pidCpu = i;
468	break;
469	}
470	}
471	}
472	return rc;
473	}
474
475
476	/**
477	* Checks if the CPU is subject to the "VMX-Preemption Timer Does Not Count
478	* Down at the Rate Specified" erratum.
479	*
480	* Errata names and related steppings:
481	* - BA86 - D0.
482	* - AAX65 - C2.
483	* - AAU65 - C2, K0.
484	* - AAO95 - B1.
485	* - AAT59 - C2.
486	* - AAK139 - D0.
487	* - AAM126 - C0, C1, D0.
488	* - AAN92 - B1.
489	* - AAJ124 - C0, D0.
490	*
491	* - AAP86 - B1.
492	*
493	* Steppings: B1, C0, C1, C2, D0, K0.
494	*
495	* @returns true if subject to it, false if not.
496	*/
497	static bool hwaccmR0IsSubjectToVmxPreemptionTimerErratum(void)
498	{
499	uint32_t u = ASMCpuId_EAX(1);
500	u &= ~(RT_BIT_32(14) \| RT_BIT_32(15) \| RT_BIT_32(28) \| RT_BIT_32(29) \| RT_BIT_32(30) \| RT_BIT_32(31));
501	if ( u == UINT32_C(0x000206E6) /* 323344.pdf - BA86 - D0 - Intel Xeon Processor 7500 Series */
502	\|\| u == UINT32_C(0x00020652) /* 323056.pdf - AAX65 - C2 - Intel Xeon Processor L3406 */
503	\|\| u == UINT32_C(0x00020652) /* 322814.pdf - AAT59 - C2 - Intel CoreTM i7-600, i5-500, i5-400 and i3-300 Mobile Processor Series */
504	\|\| u == UINT32_C(0x00020652) /* 322911.pdf - AAU65 - C2 - Intel CoreTM i5-600, i3-500 Desktop Processor Series and Intel Pentium Processor G6950 */
505	\|\| u == UINT32_C(0x00020655) /* 322911.pdf - AAU65 - K0 - Intel CoreTM i5-600, i3-500 Desktop Processor Series and Intel Pentium Processor G6950 */
506	\|\| u == UINT32_C(0x000106E5) /* 322373.pdf - AAO95 - B1 - Intel Xeon Processor 3400 Series */
507	\|\| u == UINT32_C(0x000106E5) /* 322166.pdf - AAN92 - B1 - Intel CoreTM i7-800 and i5-700 Desktop Processor Series */
508	\|\| u == UINT32_C(0x000106E5) /* 320767.pdf - AAP86 - B1 - Intel Core i7-900 Mobile Processor Extreme Edition Series, Intel Core i7-800 and i7-700 Mobile Processor Series */
509	\|\| u == UINT32_C(0x000106A0) /?321333.pdf - AAM126 - C0 - Intel Xeon Processor 3500 Series Specification /
510	\|\| u == UINT32_C(0x000106A1) /?321333.pdf - AAM126 - C1 - Intel Xeon Processor 3500 Series Specification /
511	\|\| u == UINT32_C(0x000106A4) /* 320836.pdf - AAJ124 - C0 - Intel Core i7-900 Desktop Processor Extreme Edition Series and Intel Core i7-900 Desktop Processor Series */
512	\|\| u == UINT32_C(0x000106A5) /* 321333.pdf - AAM126 - D0 - Intel Xeon Processor 3500 Series Specification */
513	\|\| u == UINT32_C(0x000106A5) /* 321324.pdf - AAK139 - D0 - Intel Xeon Processor 5500 Series Specification */
514	\|\| u == UINT32_C(0x000106A5) /* 320836.pdf - AAJ124 - D0 - Intel Core i7-900 Desktop Processor Extreme Edition Series and Intel Core i7-900 Desktop Processor Series */
515	)
516	return true;
517	return false;
518	}
519
520
521	/**
522	* Does global Ring-0 HWACCM termination.
523	*
524	* @returns VBox status code.
525	*/
526	VMMR0DECL(int) HWACCMR0Term(void)
527	{
528	int rc;
529	if ( HWACCMR0Globals.vmx.fSupported
530	&& HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx)
531	{
532	Assert(HWACCMR0Globals.fGlobalInit);
533	rc = SUPR0EnableVTx(false /* fEnable */);
534	for (unsigned iCpu = 0; iCpu < RT_ELEMENTS(HWACCMR0Globals.aCpuInfo); iCpu++)
535	{
536	HWACCMR0Globals.aCpuInfo[iCpu].fConfigured = false;
537	Assert(HWACCMR0Globals.aCpuInfo[iCpu].pMemObj == NIL_RTR0MEMOBJ);
538	}
539	}
540	else
541	{
542	Assert(!HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx);
543	if (!HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx)
544	{
545	rc = RTPowerNotificationDeregister(hwaccmR0PowerCallback, 0);
546	AssertRC(rc);
547	}
548	else
549	rc = VINF_SUCCESS;
550
551	/* Only disable VT-x/AMD-V on all CPUs if we enabled it before. */
552	if (HWACCMR0Globals.fGlobalInit)
553	{
554	int aRc[RTCPUSET_MAX_CPUS];
555
556	memset(aRc, 0, sizeof(aRc));
557	rc = RTMpOnAll(hwaccmR0DisableCpuCallback, aRc, NULL);
558	Assert(RT_SUCCESS(rc) \|\| rc == VERR_NOT_SUPPORTED);
559	#ifdef VBOX_STRICT
560	for (unsigned i=0;i<RT_ELEMENTS(HWACCMR0Globals.aCpuInfo);i++)
561	AssertMsgRC(aRc[i], ("hwaccmR0DisableCpuCallback failed for cpu %d with rc=%d\n", i, aRc[i]));
562	#endif
563	}
564
565	/* Free the per-cpu pages used for VT-x and AMD-V */
566	for (unsigned i=0;i<RT_ELEMENTS(HWACCMR0Globals.aCpuInfo);i++)
567	{
568	if (HWACCMR0Globals.aCpuInfo[i].pMemObj != NIL_RTR0MEMOBJ)
569	{
570	RTR0MemObjFree(HWACCMR0Globals.aCpuInfo[i].pMemObj, false);
571	HWACCMR0Globals.aCpuInfo[i].pMemObj = NIL_RTR0MEMOBJ;
572	}
573	}
574	}
575	return rc;
576	}
577
578
579	/**
580	* Worker function passed to RTMpOnAll, RTMpOnOthers and RTMpOnSpecific that
581	* is to be called on the target cpus.
582	*
583	* @param idCpu The identifier for the CPU the function is called on.
584	* @param pvUser1 The 1st user argument.
585	* @param pvUser2 The 2nd user argument.
586	*/
587	static DECLCALLBACK(void) HWACCMR0InitCPU(RTCPUID idCpu, void pvUser1, void pvUser2)
588	{
589	unsigned u32VendorEBX = (uintptr_t)pvUser1;
590	int paRc = (int )pvUser2;
591	uint64_t val;
592
593	#if defined(LOG_ENABLED) && !defined(DEBUG_bird)
594	SUPR0Printf("HWACCMR0InitCPU cpu %d\n", idCpu);
595	#endif
596	Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /// @todo fix idCpu == index assumption (rainy day)
597
598	if (u32VendorEBX == X86_CPUID_VENDOR_INTEL_EBX)
599	{
600	val = ASMRdMsr(MSR_IA32_FEATURE_CONTROL);
601
602	/*
603	* Both the LOCK and VMXON bit must be set; otherwise VMXON will generate a #GP.
604	* Once the lock bit is set, this MSR can no longer be modified.
605	*/
606	if ( !(val & (MSR_IA32_FEATURE_CONTROL_VMXON\|MSR_IA32_FEATURE_CONTROL_LOCK))
607	\|\| ((val & (MSR_IA32_FEATURE_CONTROL_VMXON\|MSR_IA32_FEATURE_CONTROL_LOCK)) == MSR_IA32_FEATURE_CONTROL_VMXON) /* Some BIOSes forget to set the locked bit. */
608	)
609	{
610	/* MSR is not yet locked; we can change it ourselves here */
611	ASMWrMsr(MSR_IA32_FEATURE_CONTROL, HWACCMR0Globals.vmx.msr.feature_ctrl \| MSR_IA32_FEATURE_CONTROL_VMXON \| MSR_IA32_FEATURE_CONTROL_LOCK);
612	val = ASMRdMsr(MSR_IA32_FEATURE_CONTROL);
613	}
614	if ( (val & (MSR_IA32_FEATURE_CONTROL_VMXON\|MSR_IA32_FEATURE_CONTROL_LOCK))
615	== (MSR_IA32_FEATURE_CONTROL_VMXON\|MSR_IA32_FEATURE_CONTROL_LOCK))
616	paRc[idCpu] = VINF_SUCCESS;
617	else
618	paRc[idCpu] = VERR_VMX_MSR_LOCKED_OR_DISABLED;
619	}
620	else
621	if (u32VendorEBX == X86_CPUID_VENDOR_AMD_EBX)
622	{
623	/* Check if SVM is disabled */
624	val = ASMRdMsr(MSR_K8_VM_CR);
625	if (!(val & MSR_K8_VM_CR_SVM_DISABLE))
626	{
627	/* Turn on SVM in the EFER MSR. */
628	val = ASMRdMsr(MSR_K6_EFER);
629	if (val & MSR_K6_EFER_SVME)
630	{
631	paRc[idCpu] = VERR_SVM_IN_USE;
632	}
633	else
634	{
635	ASMWrMsr(MSR_K6_EFER, val \| MSR_K6_EFER_SVME);
636
637	/* Paranoia. */
638	val = ASMRdMsr(MSR_K6_EFER);
639	if (val & MSR_K6_EFER_SVME)
640	{
641	/* Restore previous value. */
642	ASMWrMsr(MSR_K6_EFER, val & ~MSR_K6_EFER_SVME);
643	paRc[idCpu] = VINF_SUCCESS;
644	}
645	else
646	paRc[idCpu] = VERR_SVM_ILLEGAL_EFER_MSR;
647	}
648	}
649	else
650	paRc[idCpu] = VERR_SVM_DISABLED;
651	}
652	else
653	AssertFailed(); /* can't happen */
654	return;
655	}
656
657
658	/**
659	* Sets up HWACCM on all cpus.
660	*
661	* @returns VBox status code.
662	* @param pVM The VM to operate on.
663	*
664	*/
665	VMMR0DECL(int) HWACCMR0EnableAllCpus(PVM pVM)
666	{
667	AssertCompile(sizeof(HWACCMR0Globals.enmHwAccmState) == sizeof(uint32_t));
668
669	/* Make sure we don't touch hwaccm after we've disabled hwaccm in preparation of a suspend. */
670	if (ASMAtomicReadBool(&HWACCMR0Globals.fSuspended))
671	return VERR_HWACCM_SUSPEND_PENDING;
672
673	if (ASMAtomicCmpXchgU32((volatile uint32_t *)&HWACCMR0Globals.enmHwAccmState, HWACCMSTATE_ENABLED, HWACCMSTATE_UNINITIALIZED))
674	{
675	int rc;
676
677	HWACCMR0Globals.fGlobalInit = pVM->hwaccm.s.fGlobalInit;
678
679	if ( HWACCMR0Globals.vmx.fSupported
680	&& HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx)
681	{
682	rc = SUPR0EnableVTx(true /* fEnable */);
683	if (RT_SUCCESS(rc))
684	{
685	for (unsigned iCpu = 0; iCpu < RT_ELEMENTS(HWACCMR0Globals.aCpuInfo); iCpu++)
686	{
687	HWACCMR0Globals.aCpuInfo[iCpu].fConfigured = true;
688	Assert(HWACCMR0Globals.aCpuInfo[iCpu].pMemObj == NIL_RTR0MEMOBJ);
689	}
690	/* If the host provides a VT-x init API, then we'll rely on that for global init. */
691	HWACCMR0Globals.fGlobalInit = pVM->hwaccm.s.fGlobalInit = true;
692	}
693	else
694	AssertMsgFailed(("HWACCMR0EnableAllCpus/SUPR0EnableVTx: rc=%Rrc\n", rc));
695	}
696	else
697	{
698	int aRc[RTCPUSET_MAX_CPUS];
699	RTCPUID idCpu = 0;
700
701	memset(aRc, 0, sizeof(aRc));
702
703	/* Allocate one page per cpu for the global vt-x and amd-v pages */
704	for (unsigned i=0;i<RT_ELEMENTS(HWACCMR0Globals.aCpuInfo);i++)
705	{
706	Assert(!HWACCMR0Globals.aCpuInfo[i].pMemObj);
707
708	/** @todo this is rather dangerous if cpus can be taken offline; we don't care for now */
709	if (RTMpIsCpuOnline(i))
710	{
711	rc = RTR0MemObjAllocCont(&HWACCMR0Globals.aCpuInfo[i].pMemObj, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
712	AssertRC(rc);
713	if (RT_FAILURE(rc))
714	return rc;
715
716	void *pvR0 = RTR0MemObjAddress(HWACCMR0Globals.aCpuInfo[i].pMemObj);
717	Assert(pvR0);
718	ASMMemZeroPage(pvR0);
719
720	#if defined(LOG_ENABLED) && !defined(DEBUG_bird)
721	SUPR0Printf("address %x phys %x\n", pvR0, (uint32_t)RTR0MemObjGetPagePhysAddr(HWACCMR0Globals.aCpuInfo[i].pMemObj, 0));
722	#endif
723	}
724	}
725	if (HWACCMR0Globals.fGlobalInit)
726	{
727	/* First time, so initialize each cpu/core */
728	rc = RTMpOnAll(hwaccmR0EnableCpuCallback, (void *)pVM, aRc);
729
730	/* Check the return code of all invocations. */
731	if (RT_SUCCESS(rc))
732	rc = hwaccmR0CheckCpuRcArray(aRc, RT_ELEMENTS(aRc), &idCpu);
733	AssertMsgRC(rc, ("HWACCMR0EnableAllCpus failed for cpu %d with rc=%d\n", idCpu, rc));
734	}
735	else
736	rc = VINF_SUCCESS;
737	}
738
739	return rc;
740	}
741	return VINF_SUCCESS;
742	}
743
744	/**
745	* Disable VT-x or AMD-V on the current CPU
746	*
747	* @returns VBox status code.
748	* @param pVM VM handle (can be 0!)
749	* @param idCpu The identifier for the CPU the function is called on.
750	*/
751	static int hwaccmR0EnableCpu(PVM pVM, RTCPUID idCpu)
752	{
753	void *pvPageCpu;
754	RTHCPHYS pPageCpuPhys;
755	PHWACCM_CPUINFO pCpu = &HWACCMR0Globals.aCpuInfo[idCpu];
756
757	Assert(!HWACCMR0Globals.vmx.fSupported \|\| !HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx);
758	Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /// @todo fix idCpu == index assumption (rainy day)
759	Assert(idCpu < RT_ELEMENTS(HWACCMR0Globals.aCpuInfo));
760	Assert(!pCpu->fConfigured);
761	Assert(!HWACCMR0Globals.fGlobalInit \|\| ASMAtomicReadBool(&pCpu->fInUse) == false);
762
763	pCpu->idCpu = idCpu;
764
765	/* Make sure we start with a clean TLB. */
766	pCpu->fFlushTLB = true;
767
768	pCpu->uCurrentASID = 0; /* we'll aways increment this the first time (host uses ASID 0) */
769	pCpu->cTLBFlushes = 0;
770
771	/* Should never happen */
772	if (!pCpu->pMemObj)
773	{
774	AssertFailed();
775	return VERR_INTERNAL_ERROR;
776	}
777
778	pvPageCpu = RTR0MemObjAddress(pCpu->pMemObj);
779	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
780
781	int rc = HWACCMR0Globals.pfnEnableCpu(pCpu, pVM, pvPageCpu, pPageCpuPhys);
782	AssertRC(rc);
783	if (RT_SUCCESS(rc))
784	pCpu->fConfigured = true;
785
786	return rc;
787	}
788
789
790	/**
791	* Worker function passed to RTMpOnAll, RTMpOnOthers and RTMpOnSpecific that
792	* is to be called on the target cpus.
793	*
794	* @param idCpu The identifier for the CPU the function is called on.
795	* @param pvUser1 The 1st user argument.
796	* @param pvUser2 The 2nd user argument.
797	*/
798	static DECLCALLBACK(void) hwaccmR0EnableCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2)
799	{
800	PVM pVM = (PVM)pvUser1; /* can be NULL! */
801	int paRc = (int )pvUser2;
802
803	if (!HWACCMR0Globals.fGlobalInit)
804	{
805	paRc[idCpu] = VINF_SUCCESS;
806	AssertFailed();
807	return;
808	}
809
810	paRc[idCpu] = hwaccmR0EnableCpu(pVM, idCpu);
811	}
812
813
814	/**
815	* Disable VT-x or AMD-V on the current CPU
816	*
817	* @returns VBox status code.
818	* @param idCpu The identifier for the CPU the function is called on.
819	*/
820	static int hwaccmR0DisableCpu(RTCPUID idCpu)
821	{
822	void *pvPageCpu;
823	RTHCPHYS pPageCpuPhys;
824	int rc;
825	PHWACCM_CPUINFO pCpu = &HWACCMR0Globals.aCpuInfo[idCpu];
826
827	Assert(!HWACCMR0Globals.vmx.fSupported \|\| !HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx);
828	Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /// @todo fix idCpu == index assumption (rainy day)
829	Assert(idCpu < RT_ELEMENTS(HWACCMR0Globals.aCpuInfo));
830	Assert(!HWACCMR0Globals.fGlobalInit \|\| ASMAtomicReadBool(&pCpu->fInUse) == false);
831	Assert(!pCpu->fConfigured \|\| pCpu->pMemObj);
832
833	if (!pCpu->pMemObj)
834	return (pCpu->fConfigured) ? VERR_NO_MEMORY : VINF_SUCCESS /* not initialized. */;
835
836	pvPageCpu = RTR0MemObjAddress(pCpu->pMemObj);
837	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
838
839	if (pCpu->fConfigured)
840	{
841	rc = HWACCMR0Globals.pfnDisableCpu(pCpu, pvPageCpu, pPageCpuPhys);
842	AssertRC(rc);
843	pCpu->fConfigured = false;
844	}
845	else
846	rc = VINF_SUCCESS; /* nothing to do */
847
848	pCpu->uCurrentASID = 0;
849	return rc;
850	}
851
852	/**
853	* Worker function passed to RTMpOnAll, RTMpOnOthers and RTMpOnSpecific that
854	* is to be called on the target cpus.
855	*
856	* @param idCpu The identifier for the CPU the function is called on.
857	* @param pvUser1 The 1st user argument.
858	* @param pvUser2 The 2nd user argument.
859	*/
860	static DECLCALLBACK(void) hwaccmR0DisableCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2)
861	{
862	int paRc = (int )pvUser1;
863
864	if (!HWACCMR0Globals.fGlobalInit)
865	{
866	paRc[idCpu] = VINF_SUCCESS;
867	AssertFailed();
868	return;
869	}
870
871	paRc[idCpu] = hwaccmR0DisableCpu(idCpu);
872	}
873
874	/**
875	* Called whenever a system power state change occurs.
876	*
877	* @param enmEvent Power event
878	* @param pvUser User argument
879	*/
880	static DECLCALLBACK(void) hwaccmR0PowerCallback(RTPOWEREVENT enmEvent, void *pvUser)
881	{
882	NOREF(pvUser);
883	Assert(!HWACCMR0Globals.vmx.fSupported \|\| !HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx);
884
885	#ifdef LOG_ENABLED
886	if (enmEvent == RTPOWEREVENT_SUSPEND)
887	SUPR0Printf("hwaccmR0PowerCallback RTPOWEREVENT_SUSPEND\n");
888	else
889	SUPR0Printf("hwaccmR0PowerCallback RTPOWEREVENT_RESUME\n");
890	#endif
891
892	if (enmEvent == RTPOWEREVENT_SUSPEND)
893	ASMAtomicWriteBool(&HWACCMR0Globals.fSuspended, true);
894
895	if (HWACCMR0Globals.enmHwAccmState == HWACCMSTATE_ENABLED)
896	{
897	int aRc[RTCPUSET_MAX_CPUS];
898	int rc;
899	RTCPUID idCpu;
900
901	memset(aRc, 0, sizeof(aRc));
902	if (enmEvent == RTPOWEREVENT_SUSPEND)
903	{
904	if (HWACCMR0Globals.fGlobalInit)
905	{
906	/* Turn off VT-x or AMD-V on all CPUs. */
907	rc = RTMpOnAll(hwaccmR0DisableCpuCallback, aRc, NULL);
908	Assert(RT_SUCCESS(rc) \|\| rc == VERR_NOT_SUPPORTED);
909	}
910	/* else nothing to do here for the local init case */
911	}
912	else
913	{
914	/* Reinit the CPUs from scratch as the suspend state might have messed with the MSRs. (lousy BIOSes as usual) */
915	rc = RTMpOnAll(HWACCMR0InitCPU, (void *)((HWACCMR0Globals.vmx.fSupported) ? X86_CPUID_VENDOR_INTEL_EBX : X86_CPUID_VENDOR_AMD_EBX), aRc);
916	Assert(RT_SUCCESS(rc) \|\| rc == VERR_NOT_SUPPORTED);
917
918	if (RT_SUCCESS(rc))
919	rc = hwaccmR0CheckCpuRcArray(aRc, RT_ELEMENTS(aRc), &idCpu);
920	#ifdef LOG_ENABLED
921	if (RT_FAILURE(rc))
922	SUPR0Printf("hwaccmR0PowerCallback HWACCMR0InitCPU failed with %d\n", rc);
923	#endif
924
925	if (HWACCMR0Globals.fGlobalInit)
926	{
927	/* Turn VT-x or AMD-V back on on all CPUs. */
928	rc = RTMpOnAll(hwaccmR0EnableCpuCallback, NULL, aRc);
929	Assert(RT_SUCCESS(rc) \|\| rc == VERR_NOT_SUPPORTED);
930	}
931	/* else nothing to do here for the local init case */
932	}
933	}
934	if (enmEvent == RTPOWEREVENT_RESUME)
935	ASMAtomicWriteBool(&HWACCMR0Globals.fSuspended, false);
936	}
937
938
939	/**
940	* Does Ring-0 per VM HWACCM initialization.
941	*
942	* This is mainly to check that the Host CPU mode is compatible
943	* with VMX.
944	*
945	* @returns VBox status code.
946	* @param pVM The VM to operate on.
947	*/
948	VMMR0DECL(int) HWACCMR0InitVM(PVM pVM)
949	{
950	int rc;
951
952	AssertReturn(pVM, VERR_INVALID_PARAMETER);
953
954	#ifdef LOG_ENABLED
955	SUPR0Printf("HWACCMR0InitVM: %p\n", pVM);
956	#endif
957
958	/* Make sure we don't touch hwaccm after we've disabled hwaccm in preparation of a suspend. */
959	if (ASMAtomicReadBool(&HWACCMR0Globals.fSuspended))
960	return VERR_HWACCM_SUSPEND_PENDING;
961
962	pVM->hwaccm.s.vmx.fSupported = HWACCMR0Globals.vmx.fSupported;
963	pVM->hwaccm.s.svm.fSupported = HWACCMR0Globals.svm.fSupported;
964
965	pVM->hwaccm.s.vmx.fUsePreemptTimer = HWACCMR0Globals.vmx.fUsePreemptTimer;
966	pVM->hwaccm.s.vmx.cPreemptTimerShift = HWACCMR0Globals.vmx.cPreemptTimerShift;
967	pVM->hwaccm.s.vmx.msr.feature_ctrl = HWACCMR0Globals.vmx.msr.feature_ctrl;
968	pVM->hwaccm.s.vmx.hostCR4 = HWACCMR0Globals.vmx.hostCR4;
969	pVM->hwaccm.s.vmx.hostEFER = HWACCMR0Globals.vmx.hostEFER;
970	pVM->hwaccm.s.vmx.msr.vmx_basic_info = HWACCMR0Globals.vmx.msr.vmx_basic_info;
971	pVM->hwaccm.s.vmx.msr.vmx_pin_ctls = HWACCMR0Globals.vmx.msr.vmx_pin_ctls;
972	pVM->hwaccm.s.vmx.msr.vmx_proc_ctls = HWACCMR0Globals.vmx.msr.vmx_proc_ctls;
973	pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2 = HWACCMR0Globals.vmx.msr.vmx_proc_ctls2;
974	pVM->hwaccm.s.vmx.msr.vmx_exit = HWACCMR0Globals.vmx.msr.vmx_exit;
975	pVM->hwaccm.s.vmx.msr.vmx_entry = HWACCMR0Globals.vmx.msr.vmx_entry;
976	pVM->hwaccm.s.vmx.msr.vmx_misc = HWACCMR0Globals.vmx.msr.vmx_misc;
977	pVM->hwaccm.s.vmx.msr.vmx_cr0_fixed0 = HWACCMR0Globals.vmx.msr.vmx_cr0_fixed0;
978	pVM->hwaccm.s.vmx.msr.vmx_cr0_fixed1 = HWACCMR0Globals.vmx.msr.vmx_cr0_fixed1;
979	pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0 = HWACCMR0Globals.vmx.msr.vmx_cr4_fixed0;
980	pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed1 = HWACCMR0Globals.vmx.msr.vmx_cr4_fixed1;
981	pVM->hwaccm.s.vmx.msr.vmx_vmcs_enum = HWACCMR0Globals.vmx.msr.vmx_vmcs_enum;
982	pVM->hwaccm.s.vmx.msr.vmx_eptcaps = HWACCMR0Globals.vmx.msr.vmx_eptcaps;
983	pVM->hwaccm.s.svm.msrHWCR = HWACCMR0Globals.svm.msrHWCR;
984	pVM->hwaccm.s.svm.u32Rev = HWACCMR0Globals.svm.u32Rev;
985	pVM->hwaccm.s.svm.u32Features = HWACCMR0Globals.svm.u32Features;
986	pVM->hwaccm.s.cpuid.u32AMDFeatureECX = HWACCMR0Globals.cpuid.u32AMDFeatureECX;
987	pVM->hwaccm.s.cpuid.u32AMDFeatureEDX = HWACCMR0Globals.cpuid.u32AMDFeatureEDX;
988	pVM->hwaccm.s.lLastError = HWACCMR0Globals.lLastError;
989
990	pVM->hwaccm.s.uMaxASID = HWACCMR0Globals.uMaxASID;
991
992
993	if (!pVM->hwaccm.s.cMaxResumeLoops) /* allow ring-3 overrides */
994	{
995	pVM->hwaccm.s.cMaxResumeLoops = 1024;
996	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
997	if (RTThreadPreemptIsPendingTrusty())
998	pVM->hwaccm.s.cMaxResumeLoops = 8192;
999	#endif
1000	}
1001
1002	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1003	{
1004	PVMCPU pVCpu = &pVM->aCpus[i];
1005
1006	pVCpu->hwaccm.s.idEnteredCpu = NIL_RTCPUID;
1007
1008	/* Invalidate the last cpu we were running on. */
1009	pVCpu->hwaccm.s.idLastCpu = NIL_RTCPUID;
1010
1011	/* we'll aways increment this the first time (host uses ASID 0) */
1012	pVCpu->hwaccm.s.uCurrentASID = 0;
1013	}
1014
1015	RTCCUINTREG fFlags = ASMIntDisableFlags();
1016	PHWACCM_CPUINFO pCpu = HWACCMR0GetCurrentCpu();
1017
1018	/* Note: Not correct as we can be rescheduled to a different cpu, but the fInUse case is mostly for debugging. */
1019	ASMAtomicWriteBool(&pCpu->fInUse, true);
1020	ASMSetFlags(fFlags);
1021
1022	/* Init a VT-x or AMD-V VM. */
1023	rc = HWACCMR0Globals.pfnInitVM(pVM);
1024
1025	ASMAtomicWriteBool(&pCpu->fInUse, false);
1026	return rc;
1027	}
1028
1029
1030	/**
1031	* Does Ring-0 per VM HWACCM termination.
1032	*
1033	* @returns VBox status code.
1034	* @param pVM The VM to operate on.
1035	*/
1036	VMMR0DECL(int) HWACCMR0TermVM(PVM pVM)
1037	{
1038	int rc;
1039
1040	AssertReturn(pVM, VERR_INVALID_PARAMETER);
1041
1042	#ifdef LOG_ENABLED
1043	SUPR0Printf("HWACCMR0TermVM: %p\n", pVM);
1044	#endif
1045
1046	/* Make sure we don't touch hwaccm after we've disabled hwaccm in preparation of a suspend. */
1047	AssertReturn(!ASMAtomicReadBool(&HWACCMR0Globals.fSuspended), VERR_HWACCM_SUSPEND_PENDING);
1048
1049	/* @note Not correct as we can be rescheduled to a different cpu, but the fInUse case is mostly for debugging. */
1050	RTCCUINTREG fFlags = ASMIntDisableFlags();
1051	PHWACCM_CPUINFO pCpu = HWACCMR0GetCurrentCpu();
1052
1053	ASMAtomicWriteBool(&pCpu->fInUse, true);
1054	ASMSetFlags(fFlags);
1055
1056	/* Terminate a VT-x or AMD-V VM. */
1057	rc = HWACCMR0Globals.pfnTermVM(pVM);
1058
1059	ASMAtomicWriteBool(&pCpu->fInUse, false);
1060	return rc;
1061	}
1062
1063
1064	/**
1065	* Sets up a VT-x or AMD-V session
1066	*
1067	* @returns VBox status code.
1068	* @param pVM The VM to operate on.
1069	*/
1070	VMMR0DECL(int) HWACCMR0SetupVM(PVM pVM)
1071	{
1072	int rc;
1073	RTCPUID idCpu = RTMpCpuId();
1074	PHWACCM_CPUINFO pCpu = &HWACCMR0Globals.aCpuInfo[idCpu];
1075
1076	AssertReturn(pVM, VERR_INVALID_PARAMETER);
1077
1078	/* Make sure we don't touch hwaccm after we've disabled hwaccm in preparation of a suspend. */
1079	AssertReturn(!ASMAtomicReadBool(&HWACCMR0Globals.fSuspended), VERR_HWACCM_SUSPEND_PENDING);
1080
1081	#ifdef LOG_ENABLED
1082	SUPR0Printf("HWACCMR0SetupVM: %p\n", pVM);
1083	#endif
1084
1085	ASMAtomicWriteBool(&pCpu->fInUse, true);
1086
1087	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1088	{
1089	/* On first entry we'll sync everything. */
1090	pVM->aCpus[i].hwaccm.s.fContextUseFlags = HWACCM_CHANGED_ALL;
1091	}
1092
1093	/* Enable VT-x or AMD-V if local init is required. */
1094	if (!HWACCMR0Globals.fGlobalInit)
1095	{
1096	rc = hwaccmR0EnableCpu(pVM, idCpu);
1097	AssertRCReturn(rc, rc);
1098	}
1099
1100	/* Setup VT-x or AMD-V. */
1101	rc = HWACCMR0Globals.pfnSetupVM(pVM);
1102
1103	/* Disable VT-x or AMD-V if local init was done before. */
1104	if (!HWACCMR0Globals.fGlobalInit)
1105	{
1106	rc = hwaccmR0DisableCpu(idCpu);
1107	AssertRC(rc);
1108	}
1109
1110	ASMAtomicWriteBool(&pCpu->fInUse, false);
1111
1112	return rc;
1113	}
1114
1115
1116	/**
1117	* Enters the VT-x or AMD-V session
1118	*
1119	* @returns VBox status code.
1120	* @param pVM The VM to operate on.
1121	* @param pVCpu VMCPUD id.
1122	*/
1123	VMMR0DECL(int) HWACCMR0Enter(PVM pVM, PVMCPU pVCpu)
1124	{
1125	PCPUMCTX pCtx;
1126	int rc;
1127	RTCPUID idCpu = RTMpCpuId();
1128	PHWACCM_CPUINFO pCpu = &HWACCMR0Globals.aCpuInfo[idCpu];
1129
1130	/* Make sure we can't enter a session after we've disabled hwaccm in preparation of a suspend. */
1131	AssertReturn(!ASMAtomicReadBool(&HWACCMR0Globals.fSuspended), VERR_HWACCM_SUSPEND_PENDING);
1132	ASMAtomicWriteBool(&pCpu->fInUse, true);
1133
1134	AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == NIL_RTCPUID, ("%d", (int)pVCpu->hwaccm.s.idEnteredCpu));
1135	pVCpu->hwaccm.s.idEnteredCpu = idCpu;
1136
1137	pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1138
1139	/* Always load the guest's FPU/XMM state on-demand. */
1140	CPUMDeactivateGuestFPUState(pVCpu);
1141
1142	/* Always load the guest's debug state on-demand. */
1143	CPUMDeactivateGuestDebugState(pVCpu);
1144
1145	/* Always reload the host context and the guest's CR0 register. (!!!!) */
1146	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0 \| HWACCM_CHANGED_HOST_CONTEXT;
1147
1148	/* Setup the register and mask according to the current execution mode. */
1149	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1150	pVM->hwaccm.s.u64RegisterMask = UINT64_C(0xFFFFFFFFFFFFFFFF);
1151	else
1152	pVM->hwaccm.s.u64RegisterMask = UINT64_C(0xFFFFFFFF);
1153
1154	/* Enable VT-x or AMD-V if local init is required. */
1155	if (!HWACCMR0Globals.fGlobalInit)
1156	{
1157	rc = hwaccmR0EnableCpu(pVM, idCpu);
1158	AssertRCReturn(rc, rc);
1159	}
1160
1161	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1162	bool fStartedSet = PGMR0DynMapStartOrMigrateAutoSet(pVCpu);
1163	#endif
1164
1165	rc = HWACCMR0Globals.pfnEnterSession(pVM, pVCpu, pCpu);
1166	AssertRC(rc);
1167	/* We must save the host context here (VT-x) as we might be rescheduled on a different cpu after a long jump back to ring 3. */
1168	rc \|= HWACCMR0Globals.pfnSaveHostState(pVM, pVCpu);
1169	AssertRC(rc);
1170	rc \|= HWACCMR0Globals.pfnLoadGuestState(pVM, pVCpu, pCtx);
1171	AssertRC(rc);
1172
1173	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1174	if (fStartedSet)
1175	PGMRZDynMapReleaseAutoSet(pVCpu);
1176	#endif
1177
1178	/* keep track of the CPU owning the VMCS for debugging scheduling weirdness and ring-3 calls. */
1179	if (RT_FAILURE(rc))
1180	pVCpu->hwaccm.s.idEnteredCpu = NIL_RTCPUID;
1181	return rc;
1182	}
1183
1184
1185	/**
1186	* Leaves the VT-x or AMD-V session
1187	*
1188	* @returns VBox status code.
1189	* @param pVM The VM to operate on.
1190	* @param pVCpu VMCPUD id.
1191	*/
1192	VMMR0DECL(int) HWACCMR0Leave(PVM pVM, PVMCPU pVCpu)
1193	{
1194	PCPUMCTX pCtx;
1195	int rc;
1196	RTCPUID idCpu = RTMpCpuId();
1197	PHWACCM_CPUINFO pCpu = &HWACCMR0Globals.aCpuInfo[idCpu];
1198
1199	AssertReturn(!ASMAtomicReadBool(&HWACCMR0Globals.fSuspended), VERR_HWACCM_SUSPEND_PENDING);
1200
1201	pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1202
1203	/* Note: It's rather tricky with longjmps done by e.g. Log statements or the page fault handler.
1204	* We must restore the host FPU here to make absolutely sure we don't leave the guest FPU state active
1205	* or trash somebody else's FPU state.
1206	*/
1207	/* Save the guest FPU and XMM state if necessary. */
1208	if (CPUMIsGuestFPUStateActive(pVCpu))
1209	{
1210	Log2(("CPUMR0SaveGuestFPU\n"));
1211	CPUMR0SaveGuestFPU(pVM, pVCpu, pCtx);
1212
1213	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
1214	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
1215	}
1216
1217	rc = HWACCMR0Globals.pfnLeaveSession(pVM, pVCpu, pCtx);
1218
1219	/* We don't pass on invlpg information to the recompiler for nested paging guests, so we must make sure the recompiler flushes its TLB
1220	* the next time it executes code.
1221	*/
1222	if ( pVM->hwaccm.s.fNestedPaging
1223	&& CPUMIsGuestInPagedProtectedModeEx(pCtx))
1224	{
1225	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1226	}
1227
1228	/* keep track of the CPU owning the VMCS for debugging scheduling weirdness and ring-3 calls. */
1229	#ifdef RT_STRICT
1230	if (RT_UNLIKELY( pVCpu->hwaccm.s.idEnteredCpu != idCpu
1231	&& RT_FAILURE(rc)))
1232	{
1233	AssertMsgFailed(("Owner is %d, I'm %d", (int)pVCpu->hwaccm.s.idEnteredCpu, (int)idCpu));
1234	rc = VERR_INTERNAL_ERROR;
1235	}
1236	#endif
1237	pVCpu->hwaccm.s.idEnteredCpu = NIL_RTCPUID;
1238
1239	/* Disable VT-x or AMD-V if local init was done before. */
1240	if (!HWACCMR0Globals.fGlobalInit)
1241	{
1242	rc = hwaccmR0DisableCpu(idCpu);
1243	AssertRC(rc);
1244
1245	/* Reset these to force a TLB flush for the next entry. (-> EXPENSIVE) */
1246	pVCpu->hwaccm.s.idLastCpu = NIL_RTCPUID;
1247	pVCpu->hwaccm.s.uCurrentASID = 0;
1248	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1249	}
1250
1251	ASMAtomicWriteBool(&pCpu->fInUse, false);
1252	return rc;
1253	}
1254
1255	/**
1256	* Runs guest code in a hardware accelerated VM.
1257	*
1258	* @returns VBox status code.
1259	* @param pVM The VM to operate on.
1260	* @param pVCpu VMCPUD id.
1261	*/
1262	VMMR0DECL(int) HWACCMR0RunGuestCode(PVM pVM, PVMCPU pVCpu)
1263	{
1264	CPUMCTX *pCtx;
1265	int rc;
1266	#ifdef VBOX_STRICT
1267	RTCPUID idCpu = RTMpCpuId(); NOREF(idCpu);
1268	PHWACCM_CPUINFO pCpu = &HWACCMR0Globals.aCpuInfo[idCpu];
1269	#endif
1270
1271	Assert(!VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1272	Assert(HWACCMR0Globals.aCpuInfo[idCpu].fConfigured);
1273	AssertReturn(!ASMAtomicReadBool(&HWACCMR0Globals.fSuspended), VERR_HWACCM_SUSPEND_PENDING);
1274	Assert(ASMAtomicReadBool(&pCpu->fInUse) == true);
1275
1276	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1277	PGMRZDynMapStartAutoSet(pVCpu);
1278	#endif
1279
1280	pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1281
1282	rc = HWACCMR0Globals.pfnRunGuestCode(pVM, pVCpu, pCtx);
1283
1284	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1285	PGMRZDynMapReleaseAutoSet(pVCpu);
1286	#endif
1287	return rc;
1288	}
1289
1290
1291	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1292	/**
1293	* Save guest FPU/XMM state (64 bits guest mode & 32 bits host only)
1294	*
1295	* @returns VBox status code.
1296	* @param pVM VM handle.
1297	* @param pVCpu VMCPU handle.
1298	* @param pCtx CPU context
1299	*/
1300	VMMR0DECL(int) HWACCMR0SaveFPUState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1301	{
1302	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFpu64SwitchBack);
1303	if (pVM->hwaccm.s.vmx.fSupported)
1304	return VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnSaveGuestFPU64, 0, NULL);
1305
1306	return SVMR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnSaveGuestFPU64, 0, NULL);
1307	}
1308
1309	/**
1310	* Save guest debug state (64 bits guest mode & 32 bits host only)
1311	*
1312	* @returns VBox status code.
1313	* @param pVM VM handle.
1314	* @param pVCpu VMCPU handle.
1315	* @param pCtx CPU context
1316	*/
1317	VMMR0DECL(int) HWACCMR0SaveDebugState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1318	{
1319	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDebug64SwitchBack);
1320	if (pVM->hwaccm.s.vmx.fSupported)
1321	return VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnSaveGuestDebug64, 0, NULL);
1322
1323	return SVMR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnSaveGuestDebug64, 0, NULL);
1324	}
1325
1326	/**
1327	* Test the 32->64 bits switcher
1328	*
1329	* @returns VBox status code.
1330	* @param pVM VM handle.
1331	*/
1332	VMMR0DECL(int) HWACCMR0TestSwitcher3264(PVM pVM)
1333	{
1334	PVMCPU pVCpu = &pVM->aCpus[0];
1335	CPUMCTX *pCtx;
1336	uint32_t aParam[5] = {0, 1, 2, 3, 4};
1337	int rc;
1338
1339	pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1340
1341	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
1342	if (pVM->hwaccm.s.vmx.fSupported)
1343	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnTest64, 5, &aParam[0]);
1344	else
1345	rc = SVMR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnTest64, 5, &aParam[0]);
1346	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
1347	return rc;
1348	}
1349
1350	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
1351
1352	/**
1353	* Returns suspend status of the host
1354	*
1355	* @returns Suspend pending or not
1356	*/
1357	VMMR0DECL(bool) HWACCMR0SuspendPending()
1358	{
1359	return ASMAtomicReadBool(&HWACCMR0Globals.fSuspended);
1360	}
1361
1362	/**
1363	* Returns the cpu structure for the current cpu.
1364	* Keep in mind that there is no guarantee it will stay the same (long jumps to ring 3!!!).
1365	*
1366	* @returns cpu structure pointer
1367	*/
1368	VMMR0DECL(PHWACCM_CPUINFO) HWACCMR0GetCurrentCpu()
1369	{
1370	RTCPUID idCpu = RTMpCpuId();
1371
1372	return &HWACCMR0Globals.aCpuInfo[idCpu];
1373	}
1374
1375	/**
1376	* Returns the cpu structure for the current cpu.
1377	* Keep in mind that there is no guarantee it will stay the same (long jumps to ring 3!!!).
1378	*
1379	* @returns cpu structure pointer
1380	* @param idCpu id of the VCPU
1381	*/
1382	VMMR0DECL(PHWACCM_CPUINFO) HWACCMR0GetCurrentCpuEx(RTCPUID idCpu)
1383	{
1384	return &HWACCMR0Globals.aCpuInfo[idCpu];
1385	}
1386
1387	/**
1388	* Save a pending IO read.
1389	*
1390	* @param pVCpu The VMCPU to operate on.
1391	* @param GCPtrRip Address of IO instruction
1392	* @param GCPtrRipNext Address of the next instruction
1393	* @param uPort Port address
1394	* @param uAndVal And mask for saving the result in eax
1395	* @param cbSize Read size
1396	*/
1397	VMMR0DECL(void) HWACCMR0SavePendingIOPortRead(PVMCPU pVCpu, RTGCPTR GCPtrRip, RTGCPTR GCPtrRipNext, unsigned uPort, unsigned uAndVal, unsigned cbSize)
1398	{
1399	pVCpu->hwaccm.s.PendingIO.enmType = HWACCMPENDINGIO_PORT_READ;
1400	pVCpu->hwaccm.s.PendingIO.GCPtrRip = GCPtrRip;
1401	pVCpu->hwaccm.s.PendingIO.GCPtrRipNext = GCPtrRipNext;
1402	pVCpu->hwaccm.s.PendingIO.s.Port.uPort = uPort;
1403	pVCpu->hwaccm.s.PendingIO.s.Port.uAndVal = uAndVal;
1404	pVCpu->hwaccm.s.PendingIO.s.Port.cbSize = cbSize;
1405	return;
1406	}
1407
1408	/**
1409	* Save a pending IO write.
1410	*
1411	* @param pVCpu The VMCPU to operate on.
1412	* @param GCPtrRIP Address of IO instruction
1413	* @param uPort Port address
1414	* @param uAndVal And mask for fetching the result from eax
1415	* @param cbSize Read size
1416	*/
1417	VMMR0DECL(void) HWACCMR0SavePendingIOPortWrite(PVMCPU pVCpu, RTGCPTR GCPtrRip, RTGCPTR GCPtrRipNext, unsigned uPort, unsigned uAndVal, unsigned cbSize)
1418	{
1419	pVCpu->hwaccm.s.PendingIO.enmType = HWACCMPENDINGIO_PORT_WRITE;
1420	pVCpu->hwaccm.s.PendingIO.GCPtrRip = GCPtrRip;
1421	pVCpu->hwaccm.s.PendingIO.GCPtrRipNext = GCPtrRipNext;
1422	pVCpu->hwaccm.s.PendingIO.s.Port.uPort = uPort;
1423	pVCpu->hwaccm.s.PendingIO.s.Port.uAndVal = uAndVal;
1424	pVCpu->hwaccm.s.PendingIO.s.Port.cbSize = cbSize;
1425	return;
1426	}
1427
1428	/**
1429	* Disable VT-x if it's active and the current switcher turns off paging
1430	*
1431	* @returns VBox status code.
1432	* @param pVM VM handle.
1433	* @param pfVTxDisabled VT-x was disabled or not (out)
1434	*/
1435	VMMR0DECL(int) HWACCMR0EnterSwitcher(PVM pVM, bool *pfVTxDisabled)
1436	{
1437	Assert(!(ASMGetFlags() & X86_EFL_IF) \|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438
1439	*pfVTxDisabled = false;
1440
1441	if ( HWACCMR0Globals.enmHwAccmState != HWACCMSTATE_ENABLED
1442	\|\| !HWACCMR0Globals.vmx.fSupported /* no such issues with AMD-V */
1443	\|\| !HWACCMR0Globals.fGlobalInit /* Local init implies the CPU is currently not in VMX root mode. */)
1444	return VINF_SUCCESS; /* nothing to do */
1445
1446	switch(VMMGetSwitcher(pVM))
1447	{
1448	case VMMSWITCHER_32_TO_32:
1449	case VMMSWITCHER_PAE_TO_PAE:
1450	return VINF_SUCCESS; /* safe switchers as they don't turn off paging */
1451
1452	case VMMSWITCHER_32_TO_PAE:
1453	case VMMSWITCHER_PAE_TO_32: /* is this one actually used?? */
1454	case VMMSWITCHER_AMD64_TO_32:
1455	case VMMSWITCHER_AMD64_TO_PAE:
1456	break; /* unsafe switchers */
1457
1458	default:
1459	AssertFailed();
1460	return VERR_INTERNAL_ERROR;
1461	}
1462
1463	PHWACCM_CPUINFO pCpu = HWACCMR0GetCurrentCpu();
1464	void *pvPageCpu;
1465	RTHCPHYS pPageCpuPhys;
1466
1467	AssertReturn(pCpu && pCpu->pMemObj, VERR_INTERNAL_ERROR);
1468	pvPageCpu = RTR0MemObjAddress(pCpu->pMemObj);
1469	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
1470
1471	*pfVTxDisabled = true;
1472	return VMXR0DisableCpu(pCpu, pvPageCpu, pPageCpuPhys);
1473	}
1474
1475	/**
1476	* Reeable VT-x if was active and the current switcher turned off paging
1477	*
1478	* @returns VBox status code.
1479	* @param pVM VM handle.
1480	* @param fVTxDisabled VT-x was disabled or not
1481	*/
1482	VMMR0DECL(int) HWACCMR0LeaveSwitcher(PVM pVM, bool fVTxDisabled)
1483	{
1484	Assert(!(ASMGetFlags() & X86_EFL_IF));
1485
1486	if (!fVTxDisabled)
1487	return VINF_SUCCESS; /* nothing to do */
1488
1489	Assert( HWACCMR0Globals.enmHwAccmState == HWACCMSTATE_ENABLED
1490	&& HWACCMR0Globals.vmx.fSupported
1491	&& HWACCMR0Globals.fGlobalInit);
1492
1493	PHWACCM_CPUINFO pCpu = HWACCMR0GetCurrentCpu();
1494	void *pvPageCpu;
1495	RTHCPHYS pPageCpuPhys;
1496
1497	AssertReturn(pCpu && pCpu->pMemObj, VERR_INTERNAL_ERROR);
1498	pvPageCpu = RTR0MemObjAddress(pCpu->pMemObj);
1499	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
1500
1501	return VMXR0EnableCpu(pCpu, pVM, pvPageCpu, pPageCpuPhys);
1502	}
1503
1504	#ifdef VBOX_STRICT
1505	/**
1506	* Dumps a descriptor.
1507	*
1508	* @param pDesc Descriptor to dump.
1509	* @param Sel Selector number.
1510	* @param pszMsg Message to prepend the log entry with.
1511	*/
1512	VMMR0DECL(void) HWACCMR0DumpDescriptor(PCX86DESCHC pDesc, RTSEL Sel, const char *pszMsg)
1513	{
1514	/*
1515	* Make variable description string.
1516	*/
1517	static struct
1518	{
1519	unsigned cch;
1520	const char *psz;
1521	} const s_aTypes[32] =
1522	{
1523	# define STRENTRY(str) { sizeof(str) - 1, str }
1524
1525	/* system */
1526	# if HC_ARCH_BITS == 64
1527	STRENTRY("Reserved0 "), /* 0x00 */
1528	STRENTRY("Reserved1 "), /* 0x01 */
1529	STRENTRY("LDT "), /* 0x02 */
1530	STRENTRY("Reserved3 "), /* 0x03 */
1531	STRENTRY("Reserved4 "), /* 0x04 */
1532	STRENTRY("Reserved5 "), /* 0x05 */
1533	STRENTRY("Reserved6 "), /* 0x06 */
1534	STRENTRY("Reserved7 "), /* 0x07 */
1535	STRENTRY("Reserved8 "), /* 0x08 */
1536	STRENTRY("TSS64Avail "), /* 0x09 */
1537	STRENTRY("ReservedA "), /* 0x0a */
1538	STRENTRY("TSS64Busy "), /* 0x0b */
1539	STRENTRY("Call64 "), /* 0x0c */
1540	STRENTRY("ReservedD "), /* 0x0d */
1541	STRENTRY("Int64 "), /* 0x0e */
1542	STRENTRY("Trap64 "), /* 0x0f */
1543	# else
1544	STRENTRY("Reserved0 "), /* 0x00 */
1545	STRENTRY("TSS16Avail "), /* 0x01 */
1546	STRENTRY("LDT "), /* 0x02 */
1547	STRENTRY("TSS16Busy "), /* 0x03 */
1548	STRENTRY("Call16 "), /* 0x04 */
1549	STRENTRY("Task "), /* 0x05 */
1550	STRENTRY("Int16 "), /* 0x06 */
1551	STRENTRY("Trap16 "), /* 0x07 */
1552	STRENTRY("Reserved8 "), /* 0x08 */
1553	STRENTRY("TSS32Avail "), /* 0x09 */
1554	STRENTRY("ReservedA "), /* 0x0a */
1555	STRENTRY("TSS32Busy "), /* 0x0b */
1556	STRENTRY("Call32 "), /* 0x0c */
1557	STRENTRY("ReservedD "), /* 0x0d */
1558	STRENTRY("Int32 "), /* 0x0e */
1559	STRENTRY("Trap32 "), /* 0x0f */
1560	# endif
1561	/* non system */
1562	STRENTRY("DataRO "), /* 0x10 */
1563	STRENTRY("DataRO Accessed "), /* 0x11 */
1564	STRENTRY("DataRW "), /* 0x12 */
1565	STRENTRY("DataRW Accessed "), /* 0x13 */
1566	STRENTRY("DataDownRO "), /* 0x14 */
1567	STRENTRY("DataDownRO Accessed "), /* 0x15 */
1568	STRENTRY("DataDownRW "), /* 0x16 */
1569	STRENTRY("DataDownRW Accessed "), /* 0x17 */
1570	STRENTRY("CodeEO "), /* 0x18 */
1571	STRENTRY("CodeEO Accessed "), /* 0x19 */
1572	STRENTRY("CodeER "), /* 0x1a */
1573	STRENTRY("CodeER Accessed "), /* 0x1b */
1574	STRENTRY("CodeConfEO "), /* 0x1c */
1575	STRENTRY("CodeConfEO Accessed "), /* 0x1d */
1576	STRENTRY("CodeConfER "), /* 0x1e */
1577	STRENTRY("CodeConfER Accessed ") /* 0x1f */
1578	# undef SYSENTRY
1579	};
1580	# define ADD_STR(psz, pszAdd) do { strcpy(psz, pszAdd); psz += strlen(pszAdd); } while (0)
1581	char szMsg[128];
1582	char *psz = &szMsg[0];
1583	unsigned i = pDesc->Gen.u1DescType << 4 \| pDesc->Gen.u4Type;
1584	memcpy(psz, s_aTypes[i].psz, s_aTypes[i].cch);
1585	psz += s_aTypes[i].cch;
1586
1587	if (pDesc->Gen.u1Present)
1588	ADD_STR(psz, "Present ");
1589	else
1590	ADD_STR(psz, "Not-Present ");
1591	# if HC_ARCH_BITS == 64
1592	if (pDesc->Gen.u1Long)
1593	ADD_STR(psz, "64-bit ");
1594	else
1595	ADD_STR(psz, "Comp ");
1596	# else
1597	if (pDesc->Gen.u1Granularity)
1598	ADD_STR(psz, "Page ");
1599	if (pDesc->Gen.u1DefBig)
1600	ADD_STR(psz, "32-bit ");
1601	else
1602	ADD_STR(psz, "16-bit ");
1603	# endif
1604	# undef ADD_STR
1605	*psz = '\0';
1606
1607	/*
1608	* Limit and Base and format the output.
1609	*/
1610	uint32_t u32Limit = X86DESC_LIMIT(*pDesc);
1611	if (pDesc->Gen.u1Granularity)
1612	u32Limit = u32Limit << PAGE_SHIFT \| PAGE_OFFSET_MASK;
1613
1614	# if HC_ARCH_BITS == 64
1615	uint64_t u32Base = X86DESC64_BASE(*pDesc);
1616
1617	Log(("%s %04x - %RX64 %RX64 - base=%RX64 limit=%08x dpl=%d %s\n", pszMsg,
1618	Sel, pDesc->au64[0], pDesc->au64[1], u32Base, u32Limit, pDesc->Gen.u2Dpl, szMsg));
1619	# else
1620	uint32_t u32Base = X86DESC_BASE(*pDesc);
1621
1622	Log(("%s %04x - %08x %08x - base=%08x limit=%08x dpl=%d %s\n", pszMsg,
1623	Sel, pDesc->au32[0], pDesc->au32[1], u32Base, u32Limit, pDesc->Gen.u2Dpl, szMsg));
1624	# endif
1625	}
1626
1627	/**
1628	* Formats a full register dump.
1629	*
1630	* @param pVM The VM to operate on.
1631	* @param pVCpu The VMCPU to operate on.
1632	* @param pCtx The context to format.
1633	*/
1634	VMMR0DECL(void) HWACCMDumpRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1635	{
1636	/*
1637	* Format the flags.
1638	*/
1639	static struct
1640	{
1641	const char pszSet; const char pszClear; uint32_t fFlag;
1642	} aFlags[] =
1643	{
1644	{ "vip",NULL, X86_EFL_VIP },
1645	{ "vif",NULL, X86_EFL_VIF },
1646	{ "ac", NULL, X86_EFL_AC },
1647	{ "vm", NULL, X86_EFL_VM },
1648	{ "rf", NULL, X86_EFL_RF },
1649	{ "nt", NULL, X86_EFL_NT },
1650	{ "ov", "nv", X86_EFL_OF },
1651	{ "dn", "up", X86_EFL_DF },
1652	{ "ei", "di", X86_EFL_IF },
1653	{ "tf", NULL, X86_EFL_TF },
1654	{ "nt", "pl", X86_EFL_SF },
1655	{ "nz", "zr", X86_EFL_ZF },
1656	{ "ac", "na", X86_EFL_AF },
1657	{ "po", "pe", X86_EFL_PF },
1658	{ "cy", "nc", X86_EFL_CF },
1659	};
1660	char szEFlags[80];
1661	char *psz = szEFlags;
1662	uint32_t efl = pCtx->eflags.u32;
1663	for (unsigned i = 0; i < RT_ELEMENTS(aFlags); i++)
1664	{
1665	const char *pszAdd = aFlags[i].fFlag & efl ? aFlags[i].pszSet : aFlags[i].pszClear;
1666	if (pszAdd)
1667	{
1668	strcpy(psz, pszAdd);
1669	psz += strlen(pszAdd);
1670	*psz++ = ' ';
1671	}
1672	}
1673	psz[-1] = '\0';
1674
1675
1676	/*
1677	* Format the registers.
1678	*/
1679	if (CPUMIsGuestIn64BitCode(pVCpu, CPUMCTX2CORE(pCtx)))
1680	{
1681	Log(("rax=%016RX64 rbx=%016RX64 rcx=%016RX64 rdx=%016RX64\n"
1682	"rsi=%016RX64 rdi=%016RX64 r8 =%016RX64 r9 =%016RX64\n"
1683	"r10=%016RX64 r11=%016RX64 r12=%016RX64 r13=%016RX64\n"
1684	"r14=%016RX64 r15=%016RX64\n"
1685	"rip=%016RX64 rsp=%016RX64 rbp=%016RX64 iopl=%d %*s\n"
1686	"cs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1687	"ds={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1688	"es={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1689	"fs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1690	"gs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1691	"ss={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1692	"cr0=%016RX64 cr2=%016RX64 cr3=%016RX64 cr4=%016RX64\n"
1693	"dr0=%016RX64 dr1=%016RX64 dr2=%016RX64 dr3=%016RX64\n"
1694	"dr4=%016RX64 dr5=%016RX64 dr6=%016RX64 dr7=%016RX64\n"
1695	"gdtr=%016RX64:%04x idtr=%016RX64:%04x eflags=%08x\n"
1696	"ldtr={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1697	"tr ={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1698	"SysEnter={cs=%04llx eip=%08llx esp=%08llx}\n"
1699	,
1700	pCtx->rax, pCtx->rbx, pCtx->rcx, pCtx->rdx, pCtx->rsi, pCtx->rdi,
1701	pCtx->r8, pCtx->r9, pCtx->r10, pCtx->r11, pCtx->r12, pCtx->r13,
1702	pCtx->r14, pCtx->r15,
1703	pCtx->rip, pCtx->rsp, pCtx->rbp, X86_EFL_GET_IOPL(efl), 31, szEFlags,
1704	(RTSEL)pCtx->cs, pCtx->csHid.u64Base, pCtx->csHid.u32Limit, pCtx->csHid.Attr.u,
1705	(RTSEL)pCtx->ds, pCtx->dsHid.u64Base, pCtx->dsHid.u32Limit, pCtx->dsHid.Attr.u,
1706	(RTSEL)pCtx->es, pCtx->esHid.u64Base, pCtx->esHid.u32Limit, pCtx->esHid.Attr.u,
1707	(RTSEL)pCtx->fs, pCtx->fsHid.u64Base, pCtx->fsHid.u32Limit, pCtx->fsHid.Attr.u,
1708	(RTSEL)pCtx->gs, pCtx->gsHid.u64Base, pCtx->gsHid.u32Limit, pCtx->gsHid.Attr.u,
1709	(RTSEL)pCtx->ss, pCtx->ssHid.u64Base, pCtx->ssHid.u32Limit, pCtx->ssHid.Attr.u,
1710	pCtx->cr0, pCtx->cr2, pCtx->cr3, pCtx->cr4,
1711	pCtx->dr[0], pCtx->dr[1], pCtx->dr[2], pCtx->dr[3],
1712	pCtx->dr[4], pCtx->dr[5], pCtx->dr[6], pCtx->dr[7],
1713	pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pCtx->idtr.pIdt, pCtx->idtr.cbIdt, efl,
1714	(RTSEL)pCtx->ldtr, pCtx->ldtrHid.u64Base, pCtx->ldtrHid.u32Limit, pCtx->ldtrHid.Attr.u,
1715	(RTSEL)pCtx->tr, pCtx->trHid.u64Base, pCtx->trHid.u32Limit, pCtx->trHid.Attr.u,
1716	pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp));
1717	}
1718	else
1719	Log(("eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
1720	"eip=%08x esp=%08x ebp=%08x iopl=%d %*s\n"
1721	"cs={%04x base=%016RX64 limit=%08x flags=%08x} dr0=%08RX64 dr1=%08RX64\n"
1722	"ds={%04x base=%016RX64 limit=%08x flags=%08x} dr2=%08RX64 dr3=%08RX64\n"
1723	"es={%04x base=%016RX64 limit=%08x flags=%08x} dr4=%08RX64 dr5=%08RX64\n"
1724	"fs={%04x base=%016RX64 limit=%08x flags=%08x} dr6=%08RX64 dr7=%08RX64\n"
1725	"gs={%04x base=%016RX64 limit=%08x flags=%08x} cr0=%08RX64 cr2=%08RX64\n"
1726	"ss={%04x base=%016RX64 limit=%08x flags=%08x} cr3=%08RX64 cr4=%08RX64\n"
1727	"gdtr=%016RX64:%04x idtr=%016RX64:%04x eflags=%08x\n"
1728	"ldtr={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1729	"tr ={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1730	"SysEnter={cs=%04llx eip=%08llx esp=%08llx}\n"
1731	,
1732	pCtx->eax, pCtx->ebx, pCtx->ecx, pCtx->edx, pCtx->esi, pCtx->edi,
1733	pCtx->eip, pCtx->esp, pCtx->ebp, X86_EFL_GET_IOPL(efl), 31, szEFlags,
1734	(RTSEL)pCtx->cs, pCtx->csHid.u64Base, pCtx->csHid.u32Limit, pCtx->csHid.Attr.u, pCtx->dr[0], pCtx->dr[1],
1735	(RTSEL)pCtx->ds, pCtx->dsHid.u64Base, pCtx->dsHid.u32Limit, pCtx->dsHid.Attr.u, pCtx->dr[2], pCtx->dr[3],
1736	(RTSEL)pCtx->es, pCtx->esHid.u64Base, pCtx->esHid.u32Limit, pCtx->esHid.Attr.u, pCtx->dr[4], pCtx->dr[5],
1737	(RTSEL)pCtx->fs, pCtx->fsHid.u64Base, pCtx->fsHid.u32Limit, pCtx->fsHid.Attr.u, pCtx->dr[6], pCtx->dr[7],
1738	(RTSEL)pCtx->gs, pCtx->gsHid.u64Base, pCtx->gsHid.u32Limit, pCtx->gsHid.Attr.u, pCtx->cr0, pCtx->cr2,
1739	(RTSEL)pCtx->ss, pCtx->ssHid.u64Base, pCtx->ssHid.u32Limit, pCtx->ssHid.Attr.u, pCtx->cr3, pCtx->cr4,
1740	pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pCtx->idtr.pIdt, pCtx->idtr.cbIdt, efl,
1741	(RTSEL)pCtx->ldtr, pCtx->ldtrHid.u64Base, pCtx->ldtrHid.u32Limit, pCtx->ldtrHid.Attr.u,
1742	(RTSEL)pCtx->tr, pCtx->trHid.u64Base, pCtx->trHid.u32Limit, pCtx->trHid.Attr.u,
1743	pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp));
1744
1745	Log(("FPU:\n"
1746	"FCW=%04x FSW=%04x FTW=%02x\n"
1747	"FOP=%04x FPUIP=%08x CS=%04x Rsvrd1=%04x\n"
1748	"FPUDP=%04x DS=%04x Rsvrd2=%04x MXCSR=%08x MXCSR_MASK=%08x\n"
1749	,
1750	pCtx->fpu.FCW, pCtx->fpu.FSW, pCtx->fpu.FTW,
1751	pCtx->fpu.FOP, pCtx->fpu.FPUIP, pCtx->fpu.CS, pCtx->fpu.Rsvrd1,
1752	pCtx->fpu.FPUDP, pCtx->fpu.DS, pCtx->fpu.Rsrvd2,
1753	pCtx->fpu.MXCSR, pCtx->fpu.MXCSR_MASK));
1754
1755
1756	Log(("MSR:\n"
1757	"EFER =%016RX64\n"
1758	"PAT =%016RX64\n"
1759	"STAR =%016RX64\n"
1760	"CSTAR =%016RX64\n"
1761	"LSTAR =%016RX64\n"
1762	"SFMASK =%016RX64\n"
1763	"KERNELGSBASE =%016RX64\n",
1764	pCtx->msrEFER,
1765	pCtx->msrPAT,
1766	pCtx->msrSTAR,
1767	pCtx->msrCSTAR,
1768	pCtx->msrLSTAR,
1769	pCtx->msrSFMASK,
1770	pCtx->msrKERNELGSBASE));
1771
1772	}
1773	#endif /* VBOX_STRICT */
1774
1775	/* Dummy callback handlers. */
1776	VMMR0DECL(int) HWACCMR0DummyEnter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
1777	{
1778	return VINF_SUCCESS;
1779	}
1780
1781	VMMR0DECL(int) HWACCMR0DummyLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1782	{
1783	return VINF_SUCCESS;
1784	}
1785
1786	VMMR0DECL(int) HWACCMR0DummyEnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
1787	{
1788	return VINF_SUCCESS;
1789	}
1790
1791	VMMR0DECL(int) HWACCMR0DummyDisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
1792	{
1793	return VINF_SUCCESS;
1794	}
1795
1796	VMMR0DECL(int) HWACCMR0DummyInitVM(PVM pVM)
1797	{
1798	return VINF_SUCCESS;
1799	}
1800
1801	VMMR0DECL(int) HWACCMR0DummyTermVM(PVM pVM)
1802	{
1803	return VINF_SUCCESS;
1804	}
1805
1806	VMMR0DECL(int) HWACCMR0DummySetupVM(PVM pVM)
1807	{
1808	return VINF_SUCCESS;
1809	}
1810
1811	VMMR0DECL(int) HWACCMR0DummyRunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1812	{
1813	return VINF_SUCCESS;
1814	}
1815
1816	VMMR0DECL(int) HWACCMR0DummySaveHostState(PVM pVM, PVMCPU pVCpu)
1817	{
1818	return VINF_SUCCESS;
1819	}
1820
1821	VMMR0DECL(int) HWACCMR0DummyLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1822	{
1823	return VINF_SUCCESS;
1824	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWACCMR0.cpp@ 32963

Download in other formats: