HMR0.cpp@ 88726

Last change on this file since 88726 was 87606, checked in by vboxsync, 4 years ago
VMM/HMVMX: Translate fMdsClearOnSched and fL1dFlushOnSched to world switcher flags too and use them in VMXR0Enter and VMXR0ThreadCtxCallback. Added missing MDS flushing to the latter. Moved the flushing up to the start of the functions. bugref:9453 bugref:9087
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 68.4 KB

Line
1	/* $Id: HMR0.cpp 87606 2021-02-04 13:35:36Z vboxsync $ */
2	/** @file
3	* Hardware Assisted Virtualization Manager (HM) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <VBox/vmm/hm.h>
25	#include <VBox/vmm/pgm.h>
26	#include "HMInternal.h"
27	#include <VBox/vmm/vmcc.h>
28	#include <VBox/vmm/hm_svm.h>
29	#include <VBox/vmm/hmvmxinline.h>
30	#include <VBox/err.h>
31	#include <VBox/log.h>
32	#include <iprt/assert.h>
33	#include <iprt/asm.h>
34	#include <iprt/asm-amd64-x86.h>
35	#include <iprt/cpuset.h>
36	#include <iprt/mem.h>
37	#include <iprt/memobj.h>
38	#include <iprt/once.h>
39	#include <iprt/param.h>
40	#include <iprt/power.h>
41	#include <iprt/string.h>
42	#include <iprt/thread.h>
43	#include <iprt/x86.h>
44	#include "HMVMXR0.h"
45	#include "HMSVMR0.h"
46
47
48	/*********************************************************************************************************************************
49	* Internal Functions *
50	*********************************************************************************************************************************/
51	static DECLCALLBACK(void) hmR0EnableCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2);
52	static DECLCALLBACK(void) hmR0DisableCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2);
53	static DECLCALLBACK(void) hmR0PowerCallback(RTPOWEREVENT enmEvent, void *pvUser);
54	static DECLCALLBACK(void) hmR0MpEventCallback(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvData);
55
56
57	/*********************************************************************************************************************************
58	* Structures and Typedefs *
59	*********************************************************************************************************************************/
60	/**
61	* This is used to manage the status code of a RTMpOnAll in HM.
62	*/
63	typedef struct HMR0FIRSTRC
64	{
65	/** The status code. */
66	int32_t volatile rc;
67	/** The ID of the CPU reporting the first failure. */
68	RTCPUID volatile idCpu;
69	} HMR0FIRSTRC;
70	/** Pointer to a first return code structure. */
71	typedef HMR0FIRSTRC *PHMR0FIRSTRC;
72
73	/**
74	* Ring-0 method table for AMD-V and VT-x specific operations.
75	*/
76	typedef struct HMR0VTABLE
77	{
78	DECLR0CALLBACKMEMBER(int, pfnEnterSession, (PVMCPUCC pVCpu));
79	DECLR0CALLBACKMEMBER(void, pfnThreadCtxCallback, (RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit));
80	DECLR0CALLBACKMEMBER(int, pfnCallRing3Callback, (PVMCPUCC pVCpu, VMMCALLRING3 enmOperation));
81	DECLR0CALLBACKMEMBER(int, pfnExportHostState, (PVMCPUCC pVCpu));
82	DECLR0CALLBACKMEMBER(VBOXSTRICTRC, pfnRunGuestCode, (PVMCPUCC pVCpu));
83	DECLR0CALLBACKMEMBER(int, pfnEnableCpu, (PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage,
84	bool fEnabledByHost, PCSUPHWVIRTMSRS pHwvirtMsrs));
85	DECLR0CALLBACKMEMBER(int, pfnDisableCpu, (PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage));
86	DECLR0CALLBACKMEMBER(int, pfnInitVM, (PVMCC pVM));
87	DECLR0CALLBACKMEMBER(int, pfnTermVM, (PVMCC pVM));
88	DECLR0CALLBACKMEMBER(int, pfnSetupVM, (PVMCC pVM));
89	} HMR0VTABLE;
90
91
92	/*********************************************************************************************************************************
93	* Global Variables *
94	*********************************************************************************************************************************/
95	/** The active ring-0 HM operations (copied from one of the table at init). */
96	static HMR0VTABLE g_HmR0Ops;
97	/** Indicates whether the host is suspending or not. We'll refuse a few
98	* actions when the host is being suspended to speed up the suspending and
99	* avoid trouble. */
100	static bool volatile g_fHmSuspended;
101	/** If set, VT-x/AMD-V is enabled globally at init time, otherwise it's
102	* enabled and disabled each time it's used to execute guest code. */
103	static bool g_fHmGlobalInit;
104	/** Host kernel flags that HM might need to know (SUPKERNELFEATURES_XXX). */
105	uint32_t g_fHmHostKernelFeatures;
106	/** Maximum allowed ASID/VPID (inclusive).
107	* @todo r=bird: This is exclusive for VT-x according to source code comment.
108	* Couldn't immediately find any docs on AMD-V, but suspect it is
109	* exclusive there as well given how hmR0SvmFlushTaggedTlb() use it. */
110	uint32_t g_uHmMaxAsid;
111
112
113	/** Set if VT-x (VMX) is supported by the CPU. */
114	bool g_fHmVmxSupported = false;
115	/** VMX: Whether we're using the preemption timer or not. */
116	bool g_fHmVmxUsePreemptTimer;
117	/** VMX: The shift mask employed by the VMX-Preemption timer. */
118	uint8_t g_cHmVmxPreemptTimerShift;
119	/** VMX: Set if swapping EFER is supported. */
120	bool g_fHmVmxSupportsVmcsEfer = false;
121	/** VMX: Whether we're using SUPR0EnableVTx or not. */
122	static bool g_fHmVmxUsingSUPR0EnableVTx = false;
123	/** VMX: Set if we've called SUPR0EnableVTx(true) and should disable it during
124	* module termination. */
125	static bool g_fHmVmxCalledSUPR0EnableVTx = false;
126	/** VMX: Host CR4 value (set by ring-0 VMX init) */
127	uint64_t g_uHmVmxHostCr4;
128	/** VMX: Host EFER value (set by ring-0 VMX init) */
129	uint64_t g_uHmVmxHostMsrEfer;
130	/** VMX: Host SMM monitor control (used for logging/diagnostics) */
131	uint64_t g_uHmVmxHostSmmMonitorCtl;
132
133
134	/** Set if AMD-V is supported by the CPU. */
135	bool g_fHmSvmSupported = false;
136	/** SVM revision. */
137	uint32_t g_uHmSvmRev;
138	/** SVM feature bits from cpuid 0x8000000a */
139	uint32_t g_fHmSvmFeatures;
140
141
142	/** MSRs. */
143	SUPHWVIRTMSRS g_HmMsrs;
144
145	/** Last recorded error code during HM ring-0 init. */
146	static int32_t g_rcHmInit = VINF_SUCCESS;
147
148	/** Per CPU globals. */
149	static HMPHYSCPU g_aHmCpuInfo[RTCPUSET_MAX_CPUS];
150
151	/** Whether we've already initialized all CPUs.
152	* @remarks We could check the EnableAllCpusOnce state, but this is
153	* simpler and hopefully easier to understand. */
154	static bool g_fHmEnabled = false;
155	/** Serialize initialization in HMR0EnableAllCpus. */
156	static RTONCE g_HmEnableAllCpusOnce = RTONCE_INITIALIZER;
157
158
159	/** HM ring-0 operations for VT-x. */
160	static HMR0VTABLE const g_HmR0OpsVmx =
161	{
162	/* .pfnEnterSession = */ VMXR0Enter,
163	/* .pfnThreadCtxCallback = */ VMXR0ThreadCtxCallback,
164	/* .pfnCallRing3Callback = */ VMXR0CallRing3Callback,
165	/* .pfnExportHostState = */ VMXR0ExportHostState,
166	/* .pfnRunGuestCode = */ VMXR0RunGuestCode,
167	/* .pfnEnableCpu = */ VMXR0EnableCpu,
168	/* .pfnDisableCpu = */ VMXR0DisableCpu,
169	/* .pfnInitVM = */ VMXR0InitVM,
170	/* .pfnTermVM = */ VMXR0TermVM,
171	/* .pfnSetupVM = */ VMXR0SetupVM,
172	};
173
174	/** HM ring-0 operations for AMD-V. */
175	static HMR0VTABLE const g_HmR0OpsSvm =
176	{
177	/* .pfnEnterSession = */ SVMR0Enter,
178	/* .pfnThreadCtxCallback = */ SVMR0ThreadCtxCallback,
179	/* .pfnCallRing3Callback = */ SVMR0CallRing3Callback,
180	/* .pfnExportHostState = */ SVMR0ExportHostState,
181	/* .pfnRunGuestCode = */ SVMR0RunGuestCode,
182	/* .pfnEnableCpu = */ SVMR0EnableCpu,
183	/* .pfnDisableCpu = */ SVMR0DisableCpu,
184	/* .pfnInitVM = */ SVMR0InitVM,
185	/* .pfnTermVM = */ SVMR0TermVM,
186	/* .pfnSetupVM = */ SVMR0SetupVM,
187	};
188
189
190	/** @name Dummy callback handlers for when neither VT-x nor AMD-V is supported.
191	* @{ */
192
193	static DECLCALLBACK(int) hmR0DummyEnter(PVMCPUCC pVCpu)
194	{
195	RT_NOREF(pVCpu);
196	return VINF_SUCCESS;
197	}
198
199	static DECLCALLBACK(void) hmR0DummyThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
200	{
201	RT_NOREF(enmEvent, pVCpu, fGlobalInit);
202	}
203
204	static DECLCALLBACK(int) hmR0DummyEnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage,
205	bool fEnabledBySystem, PCSUPHWVIRTMSRS pHwvirtMsrs)
206	{
207	RT_NOREF(pHostCpu, pVM, pvCpuPage, HCPhysCpuPage, fEnabledBySystem, pHwvirtMsrs);
208	return VINF_SUCCESS;
209	}
210
211	static DECLCALLBACK(int) hmR0DummyDisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
212	{
213	RT_NOREF(pHostCpu, pvCpuPage, HCPhysCpuPage);
214	return VINF_SUCCESS;
215	}
216
217	static DECLCALLBACK(int) hmR0DummyInitVM(PVMCC pVM)
218	{
219	RT_NOREF(pVM);
220	return VINF_SUCCESS;
221	}
222
223	static DECLCALLBACK(int) hmR0DummyTermVM(PVMCC pVM)
224	{
225	RT_NOREF(pVM);
226	return VINF_SUCCESS;
227	}
228
229	static DECLCALLBACK(int) hmR0DummySetupVM(PVMCC pVM)
230	{
231	RT_NOREF(pVM);
232	return VINF_SUCCESS;
233	}
234
235	static DECLCALLBACK(int) hmR0DummyCallRing3Callback(PVMCPUCC pVCpu, VMMCALLRING3 enmOperation)
236	{
237	RT_NOREF(pVCpu, enmOperation);
238	return VINF_SUCCESS;
239	}
240
241	static DECLCALLBACK(VBOXSTRICTRC) hmR0DummyRunGuestCode(PVMCPUCC pVCpu)
242	{
243	RT_NOREF(pVCpu);
244	return VERR_NOT_SUPPORTED;
245	}
246
247	static DECLCALLBACK(int) hmR0DummyExportHostState(PVMCPUCC pVCpu)
248	{
249	RT_NOREF(pVCpu);
250	return VINF_SUCCESS;
251	}
252
253	/** Dummy ops. */
254	static HMR0VTABLE const g_HmR0OpsDummy =
255	{
256	/* .pfnEnterSession = */ hmR0DummyEnter,
257	/* .pfnThreadCtxCallback = */ hmR0DummyThreadCtxCallback,
258	/* .pfnCallRing3Callback = */ hmR0DummyCallRing3Callback,
259	/* .pfnExportHostState = */ hmR0DummyExportHostState,
260	/* .pfnRunGuestCode = */ hmR0DummyRunGuestCode,
261	/* .pfnEnableCpu = */ hmR0DummyEnableCpu,
262	/* .pfnDisableCpu = */ hmR0DummyDisableCpu,
263	/* .pfnInitVM = */ hmR0DummyInitVM,
264	/* .pfnTermVM = */ hmR0DummyTermVM,
265	/* .pfnSetupVM = */ hmR0DummySetupVM,
266	};
267
268	/** @} */
269
270
271	/**
272	* Initializes a first return code structure.
273	*
274	* @param pFirstRc The structure to init.
275	*/
276	static void hmR0FirstRcInit(PHMR0FIRSTRC pFirstRc)
277	{
278	pFirstRc->rc = VINF_SUCCESS;
279	pFirstRc->idCpu = NIL_RTCPUID;
280	}
281
282
283	/**
284	* Try set the status code (success ignored).
285	*
286	* @param pFirstRc The first return code structure.
287	* @param rc The status code.
288	*/
289	static void hmR0FirstRcSetStatus(PHMR0FIRSTRC pFirstRc, int rc)
290	{
291	if ( RT_FAILURE(rc)
292	&& ASMAtomicCmpXchgS32(&pFirstRc->rc, rc, VINF_SUCCESS))
293	pFirstRc->idCpu = RTMpCpuId();
294	}
295
296
297	/**
298	* Get the status code of a first return code structure.
299	*
300	* @returns The status code; VINF_SUCCESS or error status, no informational or
301	* warning errors.
302	* @param pFirstRc The first return code structure.
303	*/
304	static int hmR0FirstRcGetStatus(PHMR0FIRSTRC pFirstRc)
305	{
306	return pFirstRc->rc;
307	}
308
309
310	#ifdef VBOX_STRICT
311	# ifndef DEBUG_bird
312	/**
313	* Get the CPU ID on which the failure status code was reported.
314	*
315	* @returns The CPU ID, NIL_RTCPUID if no failure was reported.
316	* @param pFirstRc The first return code structure.
317	*/
318	static RTCPUID hmR0FirstRcGetCpuId(PHMR0FIRSTRC pFirstRc)
319	{
320	return pFirstRc->idCpu;
321	}
322	# endif
323	#endif /* VBOX_STRICT */
324
325
326
327	/**
328	* Worker function used by hmR0PowerCallback() and HMR0Init() to initalize VT-x
329	* on a CPU.
330	*
331	* @param idCpu The identifier for the CPU the function is called on.
332	* @param pvUser1 Pointer to the first RC structure.
333	* @param pvUser2 Ignored.
334	*/
335	static DECLCALLBACK(void) hmR0InitIntelCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
336	{
337	PHMR0FIRSTRC pFirstRc = (PHMR0FIRSTRC)pvUser1;
338	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
339	Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /** @todo fix idCpu == index assumption (rainy day) */
340	NOREF(idCpu); NOREF(pvUser2);
341
342	int rc = SUPR0GetVmxUsability(NULL /* pfIsSmxModeAmbiguous */);
343	hmR0FirstRcSetStatus(pFirstRc, rc);
344	}
345
346
347	/**
348	* Intel specific initialization code.
349	*
350	* @returns VBox status code (will only fail if out of memory).
351	*/
352	static int hmR0InitIntel(void)
353	{
354	/* Read this MSR now as it may be useful for error reporting when initializing VT-x fails. */
355	g_HmMsrs.u.vmx.u64FeatCtrl = ASMRdMsr(MSR_IA32_FEATURE_CONTROL);
356
357	/*
358	* First try use native kernel API for controlling VT-x.
359	* (This is only supported by some Mac OS X kernels atm.)
360	*/
361	int rc;
362	g_rcHmInit = rc = SUPR0EnableVTx(true /* fEnable */);
363	g_fHmVmxUsingSUPR0EnableVTx = rc != VERR_NOT_SUPPORTED;
364	if (g_fHmVmxUsingSUPR0EnableVTx)
365	{
366	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VERR_VMX_IN_VMX_ROOT_MODE \|\| rc == VERR_VMX_NO_VMX, ("%Rrc\n", rc));
367	if (RT_SUCCESS(rc))
368	{
369	g_fHmVmxSupported = true;
370	rc = SUPR0EnableVTx(false /* fEnable */);
371	AssertLogRelRC(rc);
372	rc = VINF_SUCCESS;
373	}
374	}
375	else
376	{
377	HMR0FIRSTRC FirstRc;
378	hmR0FirstRcInit(&FirstRc);
379	g_rcHmInit = rc = RTMpOnAll(hmR0InitIntelCpu, &FirstRc, NULL);
380	if (RT_SUCCESS(rc))
381	g_rcHmInit = rc = hmR0FirstRcGetStatus(&FirstRc);
382	}
383
384	if (RT_SUCCESS(rc))
385	{
386	/* Read CR4 and EFER for logging/diagnostic purposes. */
387	g_uHmVmxHostCr4 = ASMGetCR4();
388	g_uHmVmxHostMsrEfer = ASMRdMsr(MSR_K6_EFER);
389
390	/* Get VMX MSRs for determining VMX features we can ultimately use. */
391	SUPR0GetHwvirtMsrs(&g_HmMsrs, SUPVTCAPS_VT_X, false /* fForce */);
392
393	/*
394	* Nested KVM workaround: Intel SDM section 34.15.5 describes that
395	* MSR_IA32_SMM_MONITOR_CTL depends on bit 49 of MSR_IA32_VMX_BASIC while
396	* table 35-2 says that this MSR is available if either VMX or SMX is supported.
397	*/
398	uint64_t const uVmxBasicMsr = g_HmMsrs.u.vmx.u64Basic;
399	if (RT_BF_GET(uVmxBasicMsr, VMX_BF_BASIC_DUAL_MON))
400	g_uHmVmxHostSmmMonitorCtl = ASMRdMsr(MSR_IA32_SMM_MONITOR_CTL);
401
402	/* Initialize VPID - 16 bits ASID. */
403	g_uHmMaxAsid = 0x10000; /* exclusive */
404
405	/*
406	* If the host OS has not enabled VT-x for us, try enter VMX root mode
407	* to really verify if VT-x is usable.
408	*/
409	if (!g_fHmVmxUsingSUPR0EnableVTx)
410	{
411	/* Allocate a temporary VMXON region. */
412	RTR0MEMOBJ hScatchMemObj;
413	rc = RTR0MemObjAllocCont(&hScatchMemObj, PAGE_SIZE, false /* fExecutable */);
414	if (RT_FAILURE(rc))
415	{
416	LogRel(("hmR0InitIntel: RTR0MemObjAllocCont(,PAGE_SIZE,false) -> %Rrc\n", rc));
417	return rc;
418	}
419	void *pvScatchPage = RTR0MemObjAddress(hScatchMemObj);
420	RTHCPHYS const HCPhysScratchPage = RTR0MemObjGetPagePhysAddr(hScatchMemObj, 0);
421	ASMMemZeroPage(pvScatchPage);
422
423	/* Set revision dword at the beginning of the VMXON structure. */
424	(uint32_t )pvScatchPage = RT_BF_GET(uVmxBasicMsr, VMX_BF_BASIC_VMCS_ID);
425
426	/* Make sure we don't get rescheduled to another CPU during this probe. */
427	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
428
429	/* Enable CR4.VMXE if it isn't already set. */
430	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
431
432	/*
433	* The only way of checking if we're in VMX root mode or not is to try and enter it.
434	* There is no instruction or control bit that tells us if we're in VMX root mode.
435	* Therefore, try and enter VMX root mode here.
436	*/
437	rc = VMXEnable(HCPhysScratchPage);
438	if (RT_SUCCESS(rc))
439	{
440	g_fHmVmxSupported = true;
441	VMXDisable();
442	}
443	else
444	{
445	/*
446	* KVM leaves the CPU in VMX root mode. Not only is this not allowed,
447	* it will crash the host when we enter raw mode, because:
448	*
449	* (a) clearing X86_CR4_VMXE in CR4 causes a #GP (we no longer modify
450	* this bit), and
451	* (b) turning off paging causes a #GP (unavoidable when switching
452	* from long to 32 bits mode or 32 bits to PAE).
453	*
454	* They should fix their code, but until they do we simply refuse to run.
455	*/
456	g_rcHmInit = VERR_VMX_IN_VMX_ROOT_MODE;
457	Assert(g_fHmVmxSupported == false);
458	}
459
460	/* Restore CR4.VMXE if it wasn't set prior to us setting it above. */
461	if (!(uOldCr4 & X86_CR4_VMXE))
462	SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
463
464	/* Restore interrupts. */
465	ASMSetFlags(fEFlags);
466
467	RTR0MemObjFree(hScatchMemObj, false);
468	}
469
470	if (g_fHmVmxSupported)
471	{
472	rc = VMXR0GlobalInit();
473	if (RT_SUCCESS(rc))
474	{
475	/*
476	* Install the VT-x methods.
477	*/
478	g_HmR0Ops = g_HmR0OpsVmx;
479
480	/*
481	* Check for the VMX-Preemption Timer and adjust for the "VMX-Preemption
482	* Timer Does Not Count Down at the Rate Specified" CPU erratum.
483	*/
484	if (g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER)
485	{
486	g_fHmVmxUsePreemptTimer = true;
487	g_cHmVmxPreemptTimerShift = RT_BF_GET(g_HmMsrs.u.vmx.u64Misc, VMX_BF_MISC_PREEMPT_TIMER_TSC);
488	if (HMIsSubjectToVmxPreemptTimerErratum())
489	g_cHmVmxPreemptTimerShift = 0; /* This is about right most of the time here. */
490	}
491	else
492	g_fHmVmxUsePreemptTimer = false;
493
494	/*
495	* Check for EFER swapping support.
496	*/
497	g_fHmVmxSupportsVmcsEfer = (g_HmMsrs.u.vmx.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
498	&& (g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
499	&& (g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR);
500	}
501	else
502	{
503	g_rcHmInit = rc;
504	g_fHmVmxSupported = false;
505	}
506	}
507	}
508	#ifdef LOG_ENABLED
509	else
510	SUPR0Printf("hmR0InitIntelCpu failed with rc=%Rrc\n", g_rcHmInit);
511	#endif
512	return VINF_SUCCESS;
513	}
514
515
516	/**
517	* Worker function used by hmR0PowerCallback() and HMR0Init() to initalize AMD-V
518	* on a CPU.
519	*
520	* @param idCpu The identifier for the CPU the function is called on.
521	* @param pvUser1 Pointer to the first RC structure.
522	* @param pvUser2 Ignored.
523	*/
524	static DECLCALLBACK(void) hmR0InitAmdCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
525	{
526	PHMR0FIRSTRC pFirstRc = (PHMR0FIRSTRC)pvUser1;
527	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
528	Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /** @todo fix idCpu == index assumption (rainy day) */
529	NOREF(idCpu); NOREF(pvUser2);
530
531	int rc = SUPR0GetSvmUsability(true /* fInitSvm */);
532	hmR0FirstRcSetStatus(pFirstRc, rc);
533	}
534
535
536	/**
537	* AMD-specific initialization code.
538	*
539	* @returns VBox status code (will only fail if out of memory).
540	*/
541	static int hmR0InitAmd(void)
542	{
543	/* Call the global AMD-V initialization routine (should only fail in out-of-memory situations). */
544	int rc = SVMR0GlobalInit();
545	if (RT_SUCCESS(rc))
546	{
547	/*
548	* Install the AMD-V methods.
549	*/
550	g_HmR0Ops = g_HmR0OpsSvm;
551
552	/* Query AMD features. */
553	uint32_t u32Dummy;
554	ASMCpuId(0x8000000a, &g_uHmSvmRev, &g_uHmMaxAsid, &u32Dummy, &g_fHmSvmFeatures);
555
556	/*
557	* We need to check if AMD-V has been properly initialized on all CPUs.
558	* Some BIOSes might do a poor job.
559	*/
560	HMR0FIRSTRC FirstRc;
561	hmR0FirstRcInit(&FirstRc);
562	rc = RTMpOnAll(hmR0InitAmdCpu, &FirstRc, NULL);
563	AssertRC(rc);
564	if (RT_SUCCESS(rc))
565	rc = hmR0FirstRcGetStatus(&FirstRc);
566	#ifndef DEBUG_bird
567	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_SVM_IN_USE,
568	("hmR0InitAmdCpu failed for cpu %d with rc=%Rrc\n", hmR0FirstRcGetCpuId(&FirstRc), rc));
569	#endif
570	if (RT_SUCCESS(rc))
571	{
572	SUPR0GetHwvirtMsrs(&g_HmMsrs, SUPVTCAPS_AMD_V, false /* fForce */);
573	g_fHmSvmSupported = true;
574	}
575	else
576	{
577	g_rcHmInit = rc;
578	if (rc == VERR_SVM_DISABLED \|\| rc == VERR_SVM_IN_USE)
579	rc = VINF_SUCCESS; /* Don't fail if AMD-V is disabled or in use. */
580	}
581	}
582	else
583	g_rcHmInit = rc;
584	return rc;
585	}
586
587
588	/**
589	* Does global Ring-0 HM initialization (at module init).
590	*
591	* @returns VBox status code.
592	*/
593	VMMR0_INT_DECL(int) HMR0Init(void)
594	{
595	/*
596	* Initialize the globals.
597	*/
598	g_fHmEnabled = false;
599	for (unsigned i = 0; i < RT_ELEMENTS(g_aHmCpuInfo); i++)
600	{
601	g_aHmCpuInfo[i].idCpu = NIL_RTCPUID;
602	g_aHmCpuInfo[i].hMemObj = NIL_RTR0MEMOBJ;
603	g_aHmCpuInfo[i].HCPhysMemObj = NIL_RTHCPHYS;
604	g_aHmCpuInfo[i].pvMemObj = NULL;
605	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
606	g_aHmCpuInfo[i].n.svm.hNstGstMsrpm = NIL_RTR0MEMOBJ;
607	g_aHmCpuInfo[i].n.svm.HCPhysNstGstMsrpm = NIL_RTHCPHYS;
608	g_aHmCpuInfo[i].n.svm.pvNstGstMsrpm = NULL;
609	#endif
610	}
611
612	/* Fill in all callbacks with placeholders. */
613	g_HmR0Ops = g_HmR0OpsDummy;
614
615	/* Default is global VT-x/AMD-V init. */
616	g_fHmGlobalInit = true;
617
618	g_fHmVmxSupported = false;
619	g_fHmSvmSupported = false;
620	g_uHmMaxAsid = 0;
621
622	/*
623	* Get host kernel features that HM might need to know in order
624	* to co-operate and function properly with the host OS (e.g. SMAP).
625	*/
626	g_fHmHostKernelFeatures = SUPR0GetKernelFeatures();
627
628	/*
629	* Make sure aCpuInfo is big enough for all the CPUs on this system.
630	*/
631	if (RTMpGetArraySize() > RT_ELEMENTS(g_aHmCpuInfo))
632	{
633	LogRel(("HM: Too many real CPUs/cores/threads - %u, max %u\n", RTMpGetArraySize(), RT_ELEMENTS(g_aHmCpuInfo)));
634	return VERR_TOO_MANY_CPUS;
635	}
636
637	/*
638	* Check for VT-x or AMD-V support.
639	* Return failure only in out-of-memory situations.
640	*/
641	uint32_t fCaps = 0;
642	int rc = SUPR0GetVTSupport(&fCaps);
643	if (RT_SUCCESS(rc))
644	{
645	if (fCaps & SUPVTCAPS_VT_X)
646	rc = hmR0InitIntel();
647	else
648	{
649	Assert(fCaps & SUPVTCAPS_AMD_V);
650	rc = hmR0InitAmd();
651	}
652	if (RT_SUCCESS(rc))
653	{
654	/*
655	* Register notification callbacks that we can use to disable/enable CPUs
656	* when brought offline/online or suspending/resuming.
657	*/
658	if (!g_fHmVmxUsingSUPR0EnableVTx)
659	{
660	rc = RTMpNotificationRegister(hmR0MpEventCallback, NULL);
661	if (RT_SUCCESS(rc))
662	{
663	rc = RTPowerNotificationRegister(hmR0PowerCallback, NULL);
664	if (RT_FAILURE(rc))
665	RTMpNotificationDeregister(hmR0MpEventCallback, NULL);
666	}
667	if (RT_FAILURE(rc))
668	{
669	/* There shouldn't be any per-cpu allocations at this point,
670	so just have to call SVMR0GlobalTerm and VMXR0GlobalTerm. */
671	if (fCaps & SUPVTCAPS_VT_X)
672	VMXR0GlobalTerm();
673	else
674	SVMR0GlobalTerm();
675	g_HmR0Ops = g_HmR0OpsDummy;
676	g_rcHmInit = rc;
677	g_fHmSvmSupported = false;
678	g_fHmVmxSupported = false;
679	}
680	}
681	}
682	}
683	else
684	{
685	g_rcHmInit = rc;
686	rc = VINF_SUCCESS; /* We return success here because module init shall not fail if HM fails to initialize. */
687	}
688	return rc;
689	}
690
691
692	/**
693	* Does global Ring-0 HM termination (at module termination).
694	*
695	* @returns VBox status code (ignored).
696	*/
697	VMMR0_INT_DECL(int) HMR0Term(void)
698	{
699	int rc;
700	if ( g_fHmVmxSupported
701	&& g_fHmVmxUsingSUPR0EnableVTx)
702	{
703	/*
704	* Simple if the host OS manages VT-x.
705	*/
706	Assert(g_fHmGlobalInit);
707
708	if (g_fHmVmxCalledSUPR0EnableVTx)
709	{
710	rc = SUPR0EnableVTx(false /* fEnable */);
711	g_fHmVmxCalledSUPR0EnableVTx = false;
712	}
713	else
714	rc = VINF_SUCCESS;
715
716	for (unsigned iCpu = 0; iCpu < RT_ELEMENTS(g_aHmCpuInfo); iCpu++)
717	{
718	g_aHmCpuInfo[iCpu].fConfigured = false;
719	Assert(g_aHmCpuInfo[iCpu].hMemObj == NIL_RTR0MEMOBJ);
720	}
721	}
722	else
723	{
724	Assert(!g_fHmVmxSupported \|\| !g_fHmVmxUsingSUPR0EnableVTx);
725
726	/* Doesn't really matter if this fails. */
727	RTMpNotificationDeregister(hmR0MpEventCallback, NULL);
728	RTPowerNotificationDeregister(hmR0PowerCallback, NULL);
729	rc = VINF_SUCCESS;
730
731	/*
732	* Disable VT-x/AMD-V on all CPUs if we enabled it before.
733	*/
734	if (g_fHmGlobalInit)
735	{
736	HMR0FIRSTRC FirstRc;
737	hmR0FirstRcInit(&FirstRc);
738	rc = RTMpOnAll(hmR0DisableCpuCallback, NULL /* pvUser 1 */, &FirstRc);
739	Assert(RT_SUCCESS(rc) \|\| rc == VERR_NOT_SUPPORTED);
740	if (RT_SUCCESS(rc))
741	rc = hmR0FirstRcGetStatus(&FirstRc);
742	}
743
744	/*
745	* Free the per-cpu pages used for VT-x and AMD-V.
746	*/
747	for (unsigned i = 0; i < RT_ELEMENTS(g_aHmCpuInfo); i++)
748	{
749	if (g_aHmCpuInfo[i].hMemObj != NIL_RTR0MEMOBJ)
750	{
751	RTR0MemObjFree(g_aHmCpuInfo[i].hMemObj, false);
752	g_aHmCpuInfo[i].hMemObj = NIL_RTR0MEMOBJ;
753	g_aHmCpuInfo[i].HCPhysMemObj = NIL_RTHCPHYS;
754	g_aHmCpuInfo[i].pvMemObj = NULL;
755	}
756	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
757	if (g_aHmCpuInfo[i].n.svm.hNstGstMsrpm != NIL_RTR0MEMOBJ)
758	{
759	RTR0MemObjFree(g_aHmCpuInfo[i].n.svm.hNstGstMsrpm, false);
760	g_aHmCpuInfo[i].n.svm.hNstGstMsrpm = NIL_RTR0MEMOBJ;
761	g_aHmCpuInfo[i].n.svm.HCPhysNstGstMsrpm = NIL_RTHCPHYS;
762	g_aHmCpuInfo[i].n.svm.pvNstGstMsrpm = NULL;
763	}
764	#endif
765	}
766	}
767
768	/** @todo This needs cleaning up. There's no matching
769	* hmR0TermIntel()/hmR0TermAmd() and all the VT-x/AMD-V specific bits
770	* should move into their respective modules. */
771	/* Finally, call global VT-x/AMD-V termination. */
772	if (g_fHmVmxSupported)
773	VMXR0GlobalTerm();
774	else if (g_fHmSvmSupported)
775	SVMR0GlobalTerm();
776
777	return rc;
778	}
779
780
781	/**
782	* Enable VT-x or AMD-V on the current CPU
783	*
784	* @returns VBox status code.
785	* @param pVM The cross context VM structure. Can be NULL.
786	* @param idCpu The identifier for the CPU the function is called on.
787	*
788	* @remarks Maybe called with interrupts disabled!
789	*/
790	static int hmR0EnableCpu(PVMCC pVM, RTCPUID idCpu)
791	{
792	PHMPHYSCPU pHostCpu = &g_aHmCpuInfo[idCpu];
793
794	Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /** @todo fix idCpu == index assumption (rainy day) */
795	Assert(idCpu < RT_ELEMENTS(g_aHmCpuInfo));
796	Assert(!pHostCpu->fConfigured);
797	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
798
799	pHostCpu->idCpu = idCpu;
800	/* Do NOT reset cTlbFlushes here, see @bugref{6255}. */
801
802	int rc;
803	if ( g_fHmVmxSupported
804	&& g_fHmVmxUsingSUPR0EnableVTx)
805	rc = g_HmR0Ops.pfnEnableCpu(pHostCpu, pVM, NULL /* pvCpuPage */, NIL_RTHCPHYS, true, &g_HmMsrs);
806	else
807	{
808	AssertLogRelMsgReturn(pHostCpu->hMemObj != NIL_RTR0MEMOBJ, ("hmR0EnableCpu failed idCpu=%u.\n", idCpu), VERR_HM_IPE_1);
809	rc = g_HmR0Ops.pfnEnableCpu(pHostCpu, pVM, pHostCpu->pvMemObj, pHostCpu->HCPhysMemObj, false, &g_HmMsrs);
810	}
811	if (RT_SUCCESS(rc))
812	pHostCpu->fConfigured = true;
813	return rc;
814	}
815
816
817	/**
818	* Worker function passed to RTMpOnAll() that is to be called on all CPUs.
819	*
820	* @param idCpu The identifier for the CPU the function is called on.
821	* @param pvUser1 Opaque pointer to the VM (can be NULL!).
822	* @param pvUser2 The 2nd user argument.
823	*/
824	static DECLCALLBACK(void) hmR0EnableCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2)
825	{
826	PVMCC pVM = (PVMCC)pvUser1; /* can be NULL! */
827	PHMR0FIRSTRC pFirstRc = (PHMR0FIRSTRC)pvUser2;
828	AssertReturnVoid(g_fHmGlobalInit);
829	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
830	hmR0FirstRcSetStatus(pFirstRc, hmR0EnableCpu(pVM, idCpu));
831	}
832
833
834	/**
835	* RTOnce callback employed by HMR0EnableAllCpus.
836	*
837	* @returns VBox status code.
838	* @param pvUser Pointer to the VM.
839	*/
840	static DECLCALLBACK(int32_t) hmR0EnableAllCpuOnce(void *pvUser)
841	{
842	PVMCC pVM = (PVMCC)pvUser;
843
844	/*
845	* Indicate that we've initialized.
846	*
847	* Note! There is a potential race between this function and the suspend
848	* notification. Kind of unlikely though, so ignored for now.
849	*/
850	AssertReturn(!g_fHmEnabled, VERR_HM_ALREADY_ENABLED_IPE);
851	ASMAtomicWriteBool(&g_fHmEnabled, true);
852
853	/*
854	* The global init variable is set by the first VM.
855	*/
856	g_fHmGlobalInit = pVM->hm.s.fGlobalInit;
857
858	#ifdef VBOX_STRICT
859	for (unsigned i = 0; i < RT_ELEMENTS(g_aHmCpuInfo); i++)
860	{
861	Assert(g_aHmCpuInfo[i].hMemObj == NIL_RTR0MEMOBJ);
862	Assert(g_aHmCpuInfo[i].HCPhysMemObj == NIL_RTHCPHYS);
863	Assert(g_aHmCpuInfo[i].pvMemObj == NULL);
864	Assert(!g_aHmCpuInfo[i].fConfigured);
865	Assert(!g_aHmCpuInfo[i].cTlbFlushes);
866	Assert(!g_aHmCpuInfo[i].uCurrentAsid);
867	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
868	Assert(g_aHmCpuInfo[i].n.svm.hNstGstMsrpm == NIL_RTR0MEMOBJ);
869	Assert(g_aHmCpuInfo[i].n.svm.HCPhysNstGstMsrpm == NIL_RTHCPHYS);
870	Assert(g_aHmCpuInfo[i].n.svm.pvNstGstMsrpm == NULL);
871	# endif
872	}
873	#endif
874
875	int rc;
876	if ( g_fHmVmxSupported
877	&& g_fHmVmxUsingSUPR0EnableVTx)
878	{
879	/*
880	* Global VT-x initialization API (only darwin for now).
881	*/
882	rc = SUPR0EnableVTx(true /* fEnable */);
883	if (RT_SUCCESS(rc))
884	{
885	g_fHmVmxCalledSUPR0EnableVTx = true;
886	/* If the host provides a VT-x init API, then we'll rely on that for global init. */
887	g_fHmGlobalInit = pVM->hm.s.fGlobalInit = true;
888	}
889	else
890	AssertMsgFailed(("hmR0EnableAllCpuOnce/SUPR0EnableVTx: rc=%Rrc\n", rc));
891	}
892	else
893	{
894	/*
895	* We're doing the job ourselves.
896	*/
897	/* Allocate one page per cpu for the global VT-x and AMD-V pages */
898	for (unsigned i = 0; i < RT_ELEMENTS(g_aHmCpuInfo); i++)
899	{
900	Assert(g_aHmCpuInfo[i].hMemObj == NIL_RTR0MEMOBJ);
901	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
902	Assert(g_aHmCpuInfo[i].n.svm.hNstGstMsrpm == NIL_RTR0MEMOBJ);
903	#endif
904	if (RTMpIsCpuPossible(RTMpCpuIdFromSetIndex(i)))
905	{
906	/** @todo NUMA */
907	rc = RTR0MemObjAllocCont(&g_aHmCpuInfo[i].hMemObj, PAGE_SIZE, false /* executable R0 mapping */);
908	AssertLogRelRCReturn(rc, rc);
909
910	g_aHmCpuInfo[i].HCPhysMemObj = RTR0MemObjGetPagePhysAddr(g_aHmCpuInfo[i].hMemObj, 0);
911	Assert(g_aHmCpuInfo[i].HCPhysMemObj != NIL_RTHCPHYS);
912	Assert(!(g_aHmCpuInfo[i].HCPhysMemObj & PAGE_OFFSET_MASK));
913
914	g_aHmCpuInfo[i].pvMemObj = RTR0MemObjAddress(g_aHmCpuInfo[i].hMemObj);
915	AssertPtr(g_aHmCpuInfo[i].pvMemObj);
916	ASMMemZeroPage(g_aHmCpuInfo[i].pvMemObj);
917
918	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
919	rc = RTR0MemObjAllocCont(&g_aHmCpuInfo[i].n.svm.hNstGstMsrpm, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT,
920	false /* executable R0 mapping */);
921	AssertLogRelRCReturn(rc, rc);
922
923	g_aHmCpuInfo[i].n.svm.HCPhysNstGstMsrpm = RTR0MemObjGetPagePhysAddr(g_aHmCpuInfo[i].n.svm.hNstGstMsrpm, 0);
924	Assert(g_aHmCpuInfo[i].n.svm.HCPhysNstGstMsrpm != NIL_RTHCPHYS);
925	Assert(!(g_aHmCpuInfo[i].n.svm.HCPhysNstGstMsrpm & PAGE_OFFSET_MASK));
926
927	g_aHmCpuInfo[i].n.svm.pvNstGstMsrpm = RTR0MemObjAddress(g_aHmCpuInfo[i].n.svm.hNstGstMsrpm);
928	AssertPtr(g_aHmCpuInfo[i].n.svm.pvNstGstMsrpm);
929	ASMMemFill32(g_aHmCpuInfo[i].n.svm.pvNstGstMsrpm, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
930	#endif
931	}
932	}
933
934	rc = VINF_SUCCESS;
935	}
936
937	if ( RT_SUCCESS(rc)
938	&& g_fHmGlobalInit)
939	{
940	/* First time, so initialize each cpu/core. */
941	HMR0FIRSTRC FirstRc;
942	hmR0FirstRcInit(&FirstRc);
943	rc = RTMpOnAll(hmR0EnableCpuCallback, (void *)pVM, &FirstRc);
944	if (RT_SUCCESS(rc))
945	rc = hmR0FirstRcGetStatus(&FirstRc);
946	}
947
948	return rc;
949	}
950
951
952	/**
953	* Sets up HM on all cpus.
954	*
955	* @returns VBox status code.
956	* @param pVM The cross context VM structure.
957	*/
958	VMMR0_INT_DECL(int) HMR0EnableAllCpus(PVMCC pVM)
959	{
960	/* Make sure we don't touch HM after we've disabled HM in preparation of a suspend. */
961	if (ASMAtomicReadBool(&g_fHmSuspended))
962	return VERR_HM_SUSPEND_PENDING;
963
964	return RTOnce(&g_HmEnableAllCpusOnce, hmR0EnableAllCpuOnce, pVM);
965	}
966
967
968	/**
969	* Disable VT-x or AMD-V on the current CPU.
970	*
971	* @returns VBox status code.
972	* @param idCpu The identifier for the CPU this function is called on.
973	*
974	* @remarks Must be called with preemption disabled.
975	*/
976	static int hmR0DisableCpu(RTCPUID idCpu)
977	{
978	PHMPHYSCPU pHostCpu = &g_aHmCpuInfo[idCpu];
979
980	Assert(!g_fHmVmxSupported \|\| !g_fHmVmxUsingSUPR0EnableVTx);
981	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
982	Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /** @todo fix idCpu == index assumption (rainy day) */
983	Assert(idCpu < RT_ELEMENTS(g_aHmCpuInfo));
984	Assert(!pHostCpu->fConfigured \|\| pHostCpu->hMemObj != NIL_RTR0MEMOBJ);
985	AssertRelease(idCpu == RTMpCpuId());
986
987	if (pHostCpu->hMemObj == NIL_RTR0MEMOBJ)
988	return pHostCpu->fConfigured ? VERR_NO_MEMORY : VINF_SUCCESS /* not initialized. */;
989	AssertPtr(pHostCpu->pvMemObj);
990	Assert(pHostCpu->HCPhysMemObj != NIL_RTHCPHYS);
991
992	int rc;
993	if (pHostCpu->fConfigured)
994	{
995	rc = g_HmR0Ops.pfnDisableCpu(pHostCpu, pHostCpu->pvMemObj, pHostCpu->HCPhysMemObj);
996	AssertRCReturn(rc, rc);
997
998	pHostCpu->fConfigured = false;
999	pHostCpu->idCpu = NIL_RTCPUID;
1000	}
1001	else
1002	rc = VINF_SUCCESS; /* nothing to do */
1003	return rc;
1004	}
1005
1006
1007	/**
1008	* Worker function passed to RTMpOnAll() that is to be called on the target
1009	* CPUs.
1010	*
1011	* @param idCpu The identifier for the CPU the function is called on.
1012	* @param pvUser1 The 1st user argument.
1013	* @param pvUser2 Opaque pointer to the FirstRc.
1014	*/
1015	static DECLCALLBACK(void) hmR0DisableCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2)
1016	{
1017	PHMR0FIRSTRC pFirstRc = (PHMR0FIRSTRC)pvUser2; NOREF(pvUser1);
1018	AssertReturnVoid(g_fHmGlobalInit);
1019	hmR0FirstRcSetStatus(pFirstRc, hmR0DisableCpu(idCpu));
1020	}
1021
1022
1023	/**
1024	* Worker function passed to RTMpOnSpecific() that is to be called on the target
1025	* CPU.
1026	*
1027	* @param idCpu The identifier for the CPU the function is called on.
1028	* @param pvUser1 Null, not used.
1029	* @param pvUser2 Null, not used.
1030	*/
1031	static DECLCALLBACK(void) hmR0DisableCpuOnSpecificCallback(RTCPUID idCpu, void pvUser1, void pvUser2)
1032	{
1033	NOREF(pvUser1);
1034	NOREF(pvUser2);
1035	hmR0DisableCpu(idCpu);
1036	}
1037
1038
1039	/**
1040	* Callback function invoked when a cpu goes online or offline.
1041	*
1042	* @param enmEvent The Mp event.
1043	* @param idCpu The identifier for the CPU the function is called on.
1044	* @param pvData Opaque data (PVMCC pointer).
1045	*/
1046	static DECLCALLBACK(void) hmR0MpEventCallback(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvData)
1047	{
1048	NOREF(pvData);
1049	Assert(!g_fHmVmxSupported \|\| !g_fHmVmxUsingSUPR0EnableVTx);
1050
1051	/*
1052	* We only care about uninitializing a CPU that is going offline. When a
1053	* CPU comes online, the initialization is done lazily in HMR0Enter().
1054	*/
1055	switch (enmEvent)
1056	{
1057	case RTMPEVENT_OFFLINE:
1058	{
1059	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
1060	RTThreadPreemptDisable(&PreemptState);
1061	if (idCpu == RTMpCpuId())
1062	{
1063	int rc = hmR0DisableCpu(idCpu);
1064	AssertRC(rc);
1065	RTThreadPreemptRestore(&PreemptState);
1066	}
1067	else
1068	{
1069	RTThreadPreemptRestore(&PreemptState);
1070	RTMpOnSpecific(idCpu, hmR0DisableCpuOnSpecificCallback, NULL /* pvUser1 /, NULL / pvUser2 */);
1071	}
1072	break;
1073	}
1074
1075	default:
1076	break;
1077	}
1078	}
1079
1080
1081	/**
1082	* Called whenever a system power state change occurs.
1083	*
1084	* @param enmEvent The Power event.
1085	* @param pvUser User argument.
1086	*/
1087	static DECLCALLBACK(void) hmR0PowerCallback(RTPOWEREVENT enmEvent, void *pvUser)
1088	{
1089	NOREF(pvUser);
1090	Assert(!g_fHmVmxSupported \|\| !g_fHmVmxUsingSUPR0EnableVTx);
1091
1092	#ifdef LOG_ENABLED
1093	if (enmEvent == RTPOWEREVENT_SUSPEND)
1094	SUPR0Printf("hmR0PowerCallback RTPOWEREVENT_SUSPEND\n");
1095	else
1096	SUPR0Printf("hmR0PowerCallback RTPOWEREVENT_RESUME\n");
1097	#endif
1098
1099	if (enmEvent == RTPOWEREVENT_SUSPEND)
1100	ASMAtomicWriteBool(&g_fHmSuspended, true);
1101
1102	if (g_fHmEnabled)
1103	{
1104	int rc;
1105	HMR0FIRSTRC FirstRc;
1106	hmR0FirstRcInit(&FirstRc);
1107
1108	if (enmEvent == RTPOWEREVENT_SUSPEND)
1109	{
1110	if (g_fHmGlobalInit)
1111	{
1112	/* Turn off VT-x or AMD-V on all CPUs. */
1113	rc = RTMpOnAll(hmR0DisableCpuCallback, NULL /* pvUser 1 */, &FirstRc);
1114	Assert(RT_SUCCESS(rc) \|\| rc == VERR_NOT_SUPPORTED);
1115	}
1116	/* else nothing to do here for the local init case */
1117	}
1118	else
1119	{
1120	/* Reinit the CPUs from scratch as the suspend state might have
1121	messed with the MSRs. (lousy BIOSes as usual) */
1122	if (g_fHmVmxSupported)
1123	rc = RTMpOnAll(hmR0InitIntelCpu, &FirstRc, NULL);
1124	else
1125	rc = RTMpOnAll(hmR0InitAmdCpu, &FirstRc, NULL);
1126	Assert(RT_SUCCESS(rc) \|\| rc == VERR_NOT_SUPPORTED);
1127	if (RT_SUCCESS(rc))
1128	rc = hmR0FirstRcGetStatus(&FirstRc);
1129	#ifdef LOG_ENABLED
1130	if (RT_FAILURE(rc))
1131	SUPR0Printf("hmR0PowerCallback hmR0InitXxxCpu failed with %Rc\n", rc);
1132	#endif
1133	if (g_fHmGlobalInit)
1134	{
1135	/* Turn VT-x or AMD-V back on on all CPUs. */
1136	rc = RTMpOnAll(hmR0EnableCpuCallback, NULL /* pVM /, &FirstRc / output ignored */);
1137	Assert(RT_SUCCESS(rc) \|\| rc == VERR_NOT_SUPPORTED);
1138	}
1139	/* else nothing to do here for the local init case */
1140	}
1141	}
1142
1143	if (enmEvent == RTPOWEREVENT_RESUME)
1144	ASMAtomicWriteBool(&g_fHmSuspended, false);
1145	}
1146
1147
1148	/**
1149	* Does ring-0 per-VM HM initialization.
1150	*
1151	* This will call the CPU specific init. routine which may initialize and allocate
1152	* resources for virtual CPUs.
1153	*
1154	* @returns VBox status code.
1155	* @param pVM The cross context VM structure.
1156	*
1157	* @remarks This is called after HMR3Init(), see vmR3CreateU() and
1158	* vmR3InitRing3().
1159	*/
1160	VMMR0_INT_DECL(int) HMR0InitVM(PVMCC pVM)
1161	{
1162	AssertReturn(pVM, VERR_INVALID_PARAMETER);
1163
1164	/* Make sure we don't touch HM after we've disabled HM in preparation of a suspend. */
1165	if (ASMAtomicReadBool(&g_fHmSuspended))
1166	return VERR_HM_SUSPEND_PENDING;
1167
1168	/*
1169	* Copy globals to the VM structure.
1170	*/
1171	Assert(!(pVM->hm.s.vmx.fSupported && pVM->hm.s.svm.fSupported));
1172	if (pVM->hm.s.vmx.fSupported)
1173	{
1174	pVM->hmr0.s.vmx.fUsePreemptTimer = pVM->hm.s.vmx.fUsePreemptTimerCfg && g_fHmVmxUsePreemptTimer;
1175	pVM->hm.s.vmx.fUsePreemptTimerCfg = pVM->hmr0.s.vmx.fUsePreemptTimer;
1176	pVM->hm.s.vmx.cPreemptTimerShift = g_cHmVmxPreemptTimerShift;
1177	pVM->hm.s.ForR3.vmx.u64HostCr4 = g_uHmVmxHostCr4;
1178	pVM->hm.s.ForR3.vmx.u64HostMsrEfer = g_uHmVmxHostMsrEfer;
1179	pVM->hm.s.ForR3.vmx.u64HostSmmMonitorCtl = g_uHmVmxHostSmmMonitorCtl;
1180	HMGetVmxMsrsFromHwvirtMsrs(&g_HmMsrs, &pVM->hm.s.ForR3.vmx.Msrs);
1181	/* If you need to tweak host MSRs for testing VMX R0 code, do it here. */
1182
1183	/* Enable VPID if supported and configured. */
1184	if (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VPID)
1185	pVM->hm.s.ForR3.vmx.fVpid = pVM->hmr0.s.vmx.fVpid = pVM->hm.s.vmx.fAllowVpid; /* Can be overridden by CFGM in HMR3Init(). */
1186
1187	/* Use VMCS shadowing if supported. */
1188	pVM->hmr0.s.vmx.fUseVmcsShadowing = pVM->cpum.ro.GuestFeatures.fVmx
1189	&& (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VMCS_SHADOWING);
1190	pVM->hm.s.ForR3.vmx.fUseVmcsShadowing = pVM->hmr0.s.vmx.fUseVmcsShadowing;
1191
1192	/* Use the VMCS controls for swapping the EFER MSR if supported. */
1193	pVM->hm.s.ForR3.vmx.fSupportsVmcsEfer = g_fHmVmxSupportsVmcsEfer;
1194
1195	#if 0
1196	/* Enable APIC register virtualization and virtual-interrupt delivery if supported. */
1197	if ( (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT)
1198	&& (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY))
1199	pVM->hm.s.fVirtApicRegs = true;
1200
1201	/* Enable posted-interrupt processing if supported. */
1202	/** @todo Add and query IPRT API for host OS support for posted-interrupt IPI
1203	* here. */
1204	if ( (g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT)
1205	&& (g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT))
1206	pVM->hm.s.fPostedIntrs = true;
1207	#endif
1208	}
1209	else if (pVM->hm.s.svm.fSupported)
1210	{
1211	pVM->hm.s.ForR3.svm.u32Rev = g_uHmSvmRev;
1212	pVM->hm.s.ForR3.svm.fFeatures = g_fHmSvmFeatures;
1213	pVM->hm.s.ForR3.svm.u64MsrHwcr = g_HmMsrs.u.svm.u64MsrHwcr;
1214	/* If you need to tweak host MSRs for testing SVM R0 code, do it here. */
1215	}
1216	pVM->hm.s.ForR3.rcInit = g_rcHmInit;
1217	pVM->hm.s.ForR3.uMaxAsid = g_uHmMaxAsid;
1218
1219	/*
1220	* Set default maximum inner loops in ring-0 before returning to ring-3.
1221	* Can be overriden using CFGM.
1222	*/
1223	uint32_t cMaxResumeLoops = pVM->hm.s.cMaxResumeLoopsCfg;
1224	if (!cMaxResumeLoops)
1225	{
1226	cMaxResumeLoops = 1024;
1227	if (RTThreadPreemptIsPendingTrusty())
1228	cMaxResumeLoops = 8192;
1229	}
1230	else if (cMaxResumeLoops > 16384)
1231	cMaxResumeLoops = 16384;
1232	else if (cMaxResumeLoops < 32)
1233	cMaxResumeLoops = 32;
1234	pVM->hm.s.cMaxResumeLoopsCfg = pVM->hmr0.s.cMaxResumeLoops = cMaxResumeLoops;
1235
1236	/*
1237	* Initialize some per-VCPU fields.
1238	*/
1239	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1240	{
1241	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1242	pVCpu->hmr0.s.idEnteredCpu = NIL_RTCPUID;
1243	pVCpu->hmr0.s.idLastCpu = NIL_RTCPUID;
1244
1245	/* We'll aways increment this the first time (host uses ASID 0). */
1246	AssertReturn(!pVCpu->hmr0.s.uCurrentAsid, VERR_HM_IPE_3);
1247	}
1248
1249	/*
1250	* Configure defences against spectre and other CPU bugs.
1251	*/
1252	uint32_t fWorldSwitcher = 0;
1253	uint32_t cLastStdLeaf = ASMCpuId_EAX(0);
1254	if (cLastStdLeaf >= 0x00000007 && ASMIsValidStdRange(cLastStdLeaf))
1255	{
1256	uint32_t uEdx = 0;
1257	ASMCpuIdExSlow(0x00000007, 0, 0, 0, NULL, NULL, NULL, &uEdx);
1258
1259	if (uEdx & X86_CPUID_STEXT_FEATURE_EDX_IBRS_IBPB)
1260	{
1261	if (pVM->hm.s.fIbpbOnVmExit)
1262	fWorldSwitcher \|= HM_WSF_IBPB_EXIT;
1263	if (pVM->hm.s.fIbpbOnVmEntry)
1264	fWorldSwitcher \|= HM_WSF_IBPB_ENTRY;
1265	}
1266	if (uEdx & X86_CPUID_STEXT_FEATURE_EDX_FLUSH_CMD)
1267	{
1268	if (pVM->hm.s.fL1dFlushOnVmEntry)
1269	fWorldSwitcher \|= HM_WSF_L1D_ENTRY;
1270	else if (pVM->hm.s.fL1dFlushOnSched)
1271	fWorldSwitcher \|= HM_WSF_L1D_SCHED;
1272	}
1273	if (uEdx & X86_CPUID_STEXT_FEATURE_EDX_MD_CLEAR)
1274	{
1275	if (pVM->hm.s.fMdsClearOnVmEntry)
1276	fWorldSwitcher \|= HM_WSF_MDS_ENTRY;
1277	else if (pVM->hm.s.fMdsClearOnSched)
1278	fWorldSwitcher \|= HM_WSF_MDS_SCHED;
1279	}
1280	}
1281	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1282	{
1283	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1284	pVCpu->hmr0.s.fWorldSwitcher = fWorldSwitcher;
1285	}
1286	pVM->hm.s.ForR3.fWorldSwitcher = fWorldSwitcher;
1287
1288
1289	/*
1290	* Call the hardware specific initialization method.
1291	*/
1292	return g_HmR0Ops.pfnInitVM(pVM);
1293	}
1294
1295
1296	/**
1297	* Does ring-0 per VM HM termination.
1298	*
1299	* @returns VBox status code.
1300	* @param pVM The cross context VM structure.
1301	*/
1302	VMMR0_INT_DECL(int) HMR0TermVM(PVMCC pVM)
1303	{
1304	Log(("HMR0TermVM: %p\n", pVM));
1305	AssertReturn(pVM, VERR_INVALID_PARAMETER);
1306
1307	/*
1308	* Call the hardware specific method.
1309	*
1310	* Note! We might be preparing for a suspend, so the pfnTermVM() functions should probably not
1311	* mess with VT-x/AMD-V features on the CPU, currently all they do is free memory so this is safe.
1312	*/
1313	return g_HmR0Ops.pfnTermVM(pVM);
1314	}
1315
1316
1317	/**
1318	* Sets up a VT-x or AMD-V session.
1319	*
1320	* This is mostly about setting up the hardware VM state.
1321	*
1322	* @returns VBox status code.
1323	* @param pVM The cross context VM structure.
1324	*/
1325	VMMR0_INT_DECL(int) HMR0SetupVM(PVMCC pVM)
1326	{
1327	Log(("HMR0SetupVM: %p\n", pVM));
1328	AssertReturn(pVM, VERR_INVALID_PARAMETER);
1329
1330	/* Make sure we don't touch HM after we've disabled HM in preparation of a suspend. */
1331	AssertReturn(!ASMAtomicReadBool(&g_fHmSuspended), VERR_HM_SUSPEND_PENDING);
1332
1333	/* On first entry we'll sync everything. */
1334	VMCC_FOR_EACH_VMCPU_STMT(pVM, pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
1335
1336	/*
1337	* Call the hardware specific setup VM method. This requires the CPU to be
1338	* enabled for AMD-V/VT-x and preemption to be prevented.
1339	*/
1340	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
1341	RTThreadPreemptDisable(&PreemptState);
1342	RTCPUID const idCpu = RTMpCpuId();
1343
1344	/* Enable VT-x or AMD-V if local init is required. */
1345	int rc;
1346	if (!g_fHmGlobalInit)
1347	{
1348	Assert(!g_fHmVmxSupported \|\| !g_fHmVmxUsingSUPR0EnableVTx);
1349	rc = hmR0EnableCpu(pVM, idCpu);
1350	if (RT_FAILURE(rc))
1351	{
1352	RTThreadPreemptRestore(&PreemptState);
1353	return rc;
1354	}
1355	}
1356
1357	/* Setup VT-x or AMD-V. */
1358	rc = g_HmR0Ops.pfnSetupVM(pVM);
1359
1360	/* Disable VT-x or AMD-V if local init was done before. */
1361	if (!g_fHmGlobalInit)
1362	{
1363	Assert(!g_fHmVmxSupported \|\| !g_fHmVmxUsingSUPR0EnableVTx);
1364	int rc2 = hmR0DisableCpu(idCpu);
1365	AssertRC(rc2);
1366	}
1367
1368	RTThreadPreemptRestore(&PreemptState);
1369	return rc;
1370	}
1371
1372
1373	/**
1374	* Notification callback before performing a longjump to ring-3.
1375	*
1376	* @returns VBox status code.
1377	* @param pVCpu The cross context virtual CPU structure.
1378	* @param enmOperation The operation causing the ring-3 longjump.
1379	* @param pvUser User argument, currently unused, NULL.
1380	*/
1381	static DECLCALLBACK(int) hmR0CallRing3Callback(PVMCPUCC pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
1382	{
1383	RT_NOREF(pvUser);
1384	Assert(pVCpu);
1385	Assert(g_HmR0Ops.pfnCallRing3Callback);
1386	return g_HmR0Ops.pfnCallRing3Callback(pVCpu, enmOperation);
1387	}
1388
1389
1390	/**
1391	* Turns on HM on the CPU if necessary and initializes the bare minimum state
1392	* required for entering HM context.
1393	*
1394	* @returns VBox status code.
1395	* @param pVCpu The cross context virtual CPU structure.
1396	*
1397	* @remarks No-long-jump zone!!!
1398	*/
1399	VMMR0_INT_DECL(int) hmR0EnterCpu(PVMCPUCC pVCpu)
1400	{
1401	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1402
1403	int rc = VINF_SUCCESS;
1404	RTCPUID const idCpu = RTMpCpuId();
1405	PHMPHYSCPU pHostCpu = &g_aHmCpuInfo[idCpu];
1406	AssertPtr(pHostCpu);
1407
1408	/* Enable VT-x or AMD-V if local init is required, or enable if it's a freshly onlined CPU. */
1409	if (!pHostCpu->fConfigured)
1410	rc = hmR0EnableCpu(pVCpu->CTX_SUFF(pVM), idCpu);
1411
1412	/* Register a callback to fire prior to performing a longjmp to ring-3 so HM can disable VT-x/AMD-V if needed. */
1413	VMMRZCallRing3SetNotification(pVCpu, hmR0CallRing3Callback, NULL /* pvUser */);
1414
1415	/* Reload host-state (back from ring-3/migrated CPUs) and shared guest/host bits. */
1416	if (g_fHmVmxSupported)
1417	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE;
1418	else
1419	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE;
1420
1421	Assert(pHostCpu->idCpu == idCpu && pHostCpu->idCpu != NIL_RTCPUID);
1422	pVCpu->hmr0.s.idEnteredCpu = idCpu;
1423	return rc;
1424	}
1425
1426
1427	/**
1428	* Enters the VT-x or AMD-V session.
1429	*
1430	* @returns VBox status code.
1431	* @param pVCpu The cross context virtual CPU structure.
1432	*
1433	* @remarks This is called with preemption disabled.
1434	*/
1435	VMMR0_INT_DECL(int) HMR0Enter(PVMCPUCC pVCpu)
1436	{
1437	/* Make sure we can't enter a session after we've disabled HM in preparation of a suspend. */
1438	AssertReturn(!ASMAtomicReadBool(&g_fHmSuspended), VERR_HM_SUSPEND_PENDING);
1439	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1440
1441	/* Load the bare minimum state required for entering HM. */
1442	int rc = hmR0EnterCpu(pVCpu);
1443	if (RT_SUCCESS(rc))
1444	{
1445	if (g_fHmVmxSupported)
1446	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
1447	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
1448	else
1449	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
1450	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
1451
1452	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1453	AssertReturn(!VMMR0ThreadCtxHookIsEnabled(pVCpu), VERR_HM_IPE_5);
1454	bool const fStartedSet = PGMR0DynMapStartOrMigrateAutoSet(pVCpu);
1455	#endif
1456
1457	/* Keep track of the CPU owning the VMCS for debugging scheduling weirdness and ring-3 calls. */
1458	rc = g_HmR0Ops.pfnEnterSession(pVCpu);
1459	AssertMsgRCReturnStmt(rc, ("rc=%Rrc pVCpu=%p\n", rc, pVCpu), pVCpu->hmr0.s.idEnteredCpu = NIL_RTCPUID, rc);
1460
1461	/* Exports the host-state as we may be resuming code after a longjmp and quite
1462	possibly now be scheduled on a different CPU. */
1463	rc = g_HmR0Ops.pfnExportHostState(pVCpu);
1464	AssertMsgRCReturnStmt(rc, ("rc=%Rrc pVCpu=%p\n", rc, pVCpu), pVCpu->hmr0.s.idEnteredCpu = NIL_RTCPUID, rc);
1465
1466	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1467	if (fStartedSet)
1468	PGMRZDynMapReleaseAutoSet(pVCpu);
1469	#endif
1470	}
1471	return rc;
1472	}
1473
1474
1475	/**
1476	* Deinitializes the bare minimum state used for HM context and if necessary
1477	* disable HM on the CPU.
1478	*
1479	* @returns VBox status code.
1480	* @param pVCpu The cross context virtual CPU structure.
1481	*
1482	* @remarks No-long-jump zone!!!
1483	*/
1484	VMMR0_INT_DECL(int) HMR0LeaveCpu(PVMCPUCC pVCpu)
1485	{
1486	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1487	VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_HM_WRONG_CPU);
1488
1489	RTCPUID const idCpu = RTMpCpuId();
1490	PCHMPHYSCPU pHostCpu = &g_aHmCpuInfo[idCpu];
1491
1492	if ( !g_fHmGlobalInit
1493	&& pHostCpu->fConfigured)
1494	{
1495	int rc = hmR0DisableCpu(idCpu);
1496	AssertRCReturn(rc, rc);
1497	Assert(!pHostCpu->fConfigured);
1498	Assert(pHostCpu->idCpu == NIL_RTCPUID);
1499
1500	/* For obtaining a non-zero ASID/VPID on next re-entry. */
1501	pVCpu->hmr0.s.idLastCpu = NIL_RTCPUID;
1502	}
1503
1504	/* Clear it while leaving HM context, hmPokeCpuForTlbFlush() relies on this. */
1505	pVCpu->hmr0.s.idEnteredCpu = NIL_RTCPUID;
1506
1507	/* De-register the longjmp-to-ring 3 callback now that we have reliquished hardware resources. */
1508	VMMRZCallRing3RemoveNotification(pVCpu);
1509	return VINF_SUCCESS;
1510	}
1511
1512
1513	/**
1514	* Thread-context hook for HM.
1515	*
1516	* @param enmEvent The thread-context event.
1517	* @param pvUser Opaque pointer to the VMCPU.
1518	*/
1519	VMMR0_INT_DECL(void) HMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, void *pvUser)
1520	{
1521	PVMCPUCC pVCpu = (PVMCPUCC)pvUser;
1522	Assert(pVCpu);
1523	Assert(g_HmR0Ops.pfnThreadCtxCallback);
1524
1525	g_HmR0Ops.pfnThreadCtxCallback(enmEvent, pVCpu, g_fHmGlobalInit);
1526	}
1527
1528
1529	/**
1530	* Runs guest code in a hardware accelerated VM.
1531	*
1532	* @returns Strict VBox status code. (VBOXSTRICTRC isn't used because it's
1533	* called from setjmp assembly.)
1534	* @param pVM The cross context VM structure.
1535	* @param pVCpu The cross context virtual CPU structure.
1536	*
1537	* @remarks Can be called with preemption enabled if thread-context hooks are
1538	* used!!!
1539	*/
1540	VMMR0_INT_DECL(int) HMR0RunGuestCode(PVMCC pVM, PVMCPUCC pVCpu)
1541	{
1542	RT_NOREF(pVM);
1543
1544	#ifdef VBOX_STRICT
1545	/* With thread-context hooks we would be running this code with preemption enabled. */
1546	if (!RTThreadPreemptIsEnabled(NIL_RTTHREAD))
1547	{
1548	PCHMPHYSCPU pHostCpu = &g_aHmCpuInfo[RTMpCpuId()];
1549	Assert(!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1550	Assert(pHostCpu->fConfigured);
1551	AssertReturn(!ASMAtomicReadBool(&g_fHmSuspended), VERR_HM_SUSPEND_PENDING);
1552	}
1553	#endif
1554
1555	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1556	AssertReturn(!VMMR0ThreadCtxHookIsEnabled(pVCpu), VERR_HM_IPE_4);
1557	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1558	PGMRZDynMapStartAutoSet(pVCpu);
1559	#endif
1560
1561	VBOXSTRICTRC rcStrict = g_HmR0Ops.pfnRunGuestCode(pVCpu);
1562
1563	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1564	PGMRZDynMapReleaseAutoSet(pVCpu);
1565	#endif
1566	return VBOXSTRICTRC_VAL(rcStrict);
1567	}
1568
1569
1570	/**
1571	* Notification from CPUM that it has unloaded the guest FPU/SSE/AVX state from
1572	* the host CPU and that guest access to it must be intercepted.
1573	*
1574	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1575	*/
1576	VMMR0_INT_DECL(void) HMR0NotifyCpumUnloadedGuestFpuState(PVMCPUCC pVCpu)
1577	{
1578	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
1579	}
1580
1581
1582	/**
1583	* Notification from CPUM that it has modified the host CR0 (because of FPU).
1584	*
1585	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1586	*/
1587	VMMR0_INT_DECL(void) HMR0NotifyCpumModifiedHostCr0(PVMCPUCC pVCpu)
1588	{
1589	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT);
1590	}
1591
1592
1593	/**
1594	* Returns suspend status of the host.
1595	*
1596	* @returns Suspend pending or not.
1597	*/
1598	VMMR0_INT_DECL(bool) HMR0SuspendPending(void)
1599	{
1600	return ASMAtomicReadBool(&g_fHmSuspended);
1601	}
1602
1603
1604	/**
1605	* Invalidates a guest page from the host TLB.
1606	*
1607	* @param pVCpu The cross context virtual CPU structure.
1608	* @param GCVirt Page to invalidate.
1609	*/
1610	VMMR0_INT_DECL(int) HMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
1611	{
1612	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1613	if (pVM->hm.s.vmx.fSupported)
1614	return VMXR0InvalidatePage(pVCpu, GCVirt);
1615	return SVMR0InvalidatePage(pVCpu, GCVirt);
1616	}
1617
1618
1619	/**
1620	* Returns the cpu structure for the current cpu.
1621	* Keep in mind that there is no guarantee it will stay the same (long jumps to ring 3!!!).
1622	*
1623	* @returns The cpu structure pointer.
1624	*/
1625	VMMR0_INT_DECL(PHMPHYSCPU) hmR0GetCurrentCpu(void)
1626	{
1627	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1628	RTCPUID const idCpu = RTMpCpuId();
1629	Assert(idCpu < RT_ELEMENTS(g_aHmCpuInfo));
1630	return &g_aHmCpuInfo[idCpu];
1631	}
1632
1633
1634	/**
1635	* Interface for importing state on demand (used by IEM).
1636	*
1637	* @returns VBox status code.
1638	* @param pVCpu The cross context CPU structure.
1639	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
1640	*/
1641	VMMR0_INT_DECL(int) HMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
1642	{
1643	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
1644	return VMXR0ImportStateOnDemand(pVCpu, fWhat);
1645	return SVMR0ImportStateOnDemand(pVCpu, fWhat);
1646	}
1647
1648	#ifdef VBOX_STRICT
1649
1650	/**
1651	* Dumps a descriptor.
1652	*
1653	* @param pDesc Descriptor to dump.
1654	* @param Sel The selector.
1655	* @param pszSel The name of the selector.
1656	*/
1657	VMMR0_INT_DECL(void) hmR0DumpDescriptor(PCX86DESCHC pDesc, RTSEL Sel, const char *pszSel)
1658	{
1659	/*
1660	* Make variable description string.
1661	*/
1662	static struct
1663	{
1664	unsigned cch;
1665	const char *psz;
1666	} const s_aTypes[32] =
1667	{
1668	# define STRENTRY(str) { sizeof(str) - 1, str }
1669
1670	/* system */
1671	# if HC_ARCH_BITS == 64
1672	STRENTRY("Reserved0 "), /* 0x00 */
1673	STRENTRY("Reserved1 "), /* 0x01 */
1674	STRENTRY("LDT "), /* 0x02 */
1675	STRENTRY("Reserved3 "), /* 0x03 */
1676	STRENTRY("Reserved4 "), /* 0x04 */
1677	STRENTRY("Reserved5 "), /* 0x05 */
1678	STRENTRY("Reserved6 "), /* 0x06 */
1679	STRENTRY("Reserved7 "), /* 0x07 */
1680	STRENTRY("Reserved8 "), /* 0x08 */
1681	STRENTRY("TSS64Avail "), /* 0x09 */
1682	STRENTRY("ReservedA "), /* 0x0a */
1683	STRENTRY("TSS64Busy "), /* 0x0b */
1684	STRENTRY("Call64 "), /* 0x0c */
1685	STRENTRY("ReservedD "), /* 0x0d */
1686	STRENTRY("Int64 "), /* 0x0e */
1687	STRENTRY("Trap64 "), /* 0x0f */
1688	# else
1689	STRENTRY("Reserved0 "), /* 0x00 */
1690	STRENTRY("TSS16Avail "), /* 0x01 */
1691	STRENTRY("LDT "), /* 0x02 */
1692	STRENTRY("TSS16Busy "), /* 0x03 */
1693	STRENTRY("Call16 "), /* 0x04 */
1694	STRENTRY("Task "), /* 0x05 */
1695	STRENTRY("Int16 "), /* 0x06 */
1696	STRENTRY("Trap16 "), /* 0x07 */
1697	STRENTRY("Reserved8 "), /* 0x08 */
1698	STRENTRY("TSS32Avail "), /* 0x09 */
1699	STRENTRY("ReservedA "), /* 0x0a */
1700	STRENTRY("TSS32Busy "), /* 0x0b */
1701	STRENTRY("Call32 "), /* 0x0c */
1702	STRENTRY("ReservedD "), /* 0x0d */
1703	STRENTRY("Int32 "), /* 0x0e */
1704	STRENTRY("Trap32 "), /* 0x0f */
1705	# endif
1706	/* non system */
1707	STRENTRY("DataRO "), /* 0x10 */
1708	STRENTRY("DataRO Accessed "), /* 0x11 */
1709	STRENTRY("DataRW "), /* 0x12 */
1710	STRENTRY("DataRW Accessed "), /* 0x13 */
1711	STRENTRY("DataDownRO "), /* 0x14 */
1712	STRENTRY("DataDownRO Accessed "), /* 0x15 */
1713	STRENTRY("DataDownRW "), /* 0x16 */
1714	STRENTRY("DataDownRW Accessed "), /* 0x17 */
1715	STRENTRY("CodeEO "), /* 0x18 */
1716	STRENTRY("CodeEO Accessed "), /* 0x19 */
1717	STRENTRY("CodeER "), /* 0x1a */
1718	STRENTRY("CodeER Accessed "), /* 0x1b */
1719	STRENTRY("CodeConfEO "), /* 0x1c */
1720	STRENTRY("CodeConfEO Accessed "), /* 0x1d */
1721	STRENTRY("CodeConfER "), /* 0x1e */
1722	STRENTRY("CodeConfER Accessed ") /* 0x1f */
1723	# undef SYSENTRY
1724	};
1725	# define ADD_STR(psz, pszAdd) do { strcpy(psz, pszAdd); psz += strlen(pszAdd); } while (0)
1726	char szMsg[128];
1727	char *psz = &szMsg[0];
1728	unsigned i = pDesc->Gen.u1DescType << 4 \| pDesc->Gen.u4Type;
1729	memcpy(psz, s_aTypes[i].psz, s_aTypes[i].cch);
1730	psz += s_aTypes[i].cch;
1731
1732	if (pDesc->Gen.u1Present)
1733	ADD_STR(psz, "Present ");
1734	else
1735	ADD_STR(psz, "Not-Present ");
1736	# if HC_ARCH_BITS == 64
1737	if (pDesc->Gen.u1Long)
1738	ADD_STR(psz, "64-bit ");
1739	else
1740	ADD_STR(psz, "Comp ");
1741	# else
1742	if (pDesc->Gen.u1Granularity)
1743	ADD_STR(psz, "Page ");
1744	if (pDesc->Gen.u1DefBig)
1745	ADD_STR(psz, "32-bit ");
1746	else
1747	ADD_STR(psz, "16-bit ");
1748	# endif
1749	# undef ADD_STR
1750	*psz = '\0';
1751
1752	/*
1753	* Limit and Base and format the output.
1754	*/
1755	#ifdef LOG_ENABLED
1756	uint32_t u32Limit = X86DESC_LIMIT_G(pDesc);
1757
1758	# if HC_ARCH_BITS == 64
1759	uint64_t const u64Base = X86DESC64_BASE(pDesc);
1760	Log((" %s { %#04x - %#RX64 %#RX64 - base=%#RX64 limit=%#08x dpl=%d } %s\n", pszSel,
1761	Sel, pDesc->au64[0], pDesc->au64[1], u64Base, u32Limit, pDesc->Gen.u2Dpl, szMsg));
1762	# else
1763	uint32_t const u32Base = X86DESC_BASE(pDesc);
1764	Log((" %s { %#04x - %#08x %#08x - base=%#08x limit=%#08x dpl=%d } %s\n", pszSel,
1765	Sel, pDesc->au32[0], pDesc->au32[1], u32Base, u32Limit, pDesc->Gen.u2Dpl, szMsg));
1766	# endif
1767	#else
1768	NOREF(Sel); NOREF(pszSel);
1769	#endif
1770	}
1771
1772
1773	/**
1774	* Formats a full register dump.
1775	*
1776	* @param pVCpu The cross context virtual CPU structure.
1777	* @param fFlags The dumping flags (HM_DUMP_REG_FLAGS_XXX).
1778	*/
1779	VMMR0_INT_DECL(void) hmR0DumpRegs(PVMCPUCC pVCpu, uint32_t fFlags)
1780	{
1781	/*
1782	* Format the flags.
1783	*/
1784	static struct
1785	{
1786	const char *pszSet;
1787	const char *pszClear;
1788	uint32_t fFlag;
1789	} const s_aFlags[] =
1790	{
1791	{ "vip", NULL, X86_EFL_VIP },
1792	{ "vif", NULL, X86_EFL_VIF },
1793	{ "ac", NULL, X86_EFL_AC },
1794	{ "vm", NULL, X86_EFL_VM },
1795	{ "rf", NULL, X86_EFL_RF },
1796	{ "nt", NULL, X86_EFL_NT },
1797	{ "ov", "nv", X86_EFL_OF },
1798	{ "dn", "up", X86_EFL_DF },
1799	{ "ei", "di", X86_EFL_IF },
1800	{ "tf", NULL, X86_EFL_TF },
1801	{ "nt", "pl", X86_EFL_SF },
1802	{ "nz", "zr", X86_EFL_ZF },
1803	{ "ac", "na", X86_EFL_AF },
1804	{ "po", "pe", X86_EFL_PF },
1805	{ "cy", "nc", X86_EFL_CF },
1806	};
1807	char szEFlags[80];
1808	char *psz = szEFlags;
1809	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1810	uint32_t uEFlags = pCtx->eflags.u32;
1811	for (unsigned i = 0; i < RT_ELEMENTS(s_aFlags); i++)
1812	{
1813	const char *pszAdd = s_aFlags[i].fFlag & uEFlags ? s_aFlags[i].pszSet : s_aFlags[i].pszClear;
1814	if (pszAdd)
1815	{
1816	strcpy(psz, pszAdd);
1817	psz += strlen(pszAdd);
1818	*psz++ = ' ';
1819	}
1820	}
1821	psz[-1] = '\0';
1822
1823	if (fFlags & HM_DUMP_REG_FLAGS_GPRS)
1824	{
1825	/*
1826	* Format the registers.
1827	*/
1828	if (CPUMIsGuestIn64BitCode(pVCpu))
1829	{
1830	Log(("rax=%016RX64 rbx=%016RX64 rcx=%016RX64 rdx=%016RX64\n"
1831	"rsi=%016RX64 rdi=%016RX64 r8 =%016RX64 r9 =%016RX64\n"
1832	"r10=%016RX64 r11=%016RX64 r12=%016RX64 r13=%016RX64\n"
1833	"r14=%016RX64 r15=%016RX64\n"
1834	"rip=%016RX64 rsp=%016RX64 rbp=%016RX64 iopl=%d %*s\n"
1835	"cs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1836	"ds={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1837	"es={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1838	"fs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1839	"gs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1840	"ss={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1841	"cr0=%016RX64 cr2=%016RX64 cr3=%016RX64 cr4=%016RX64\n"
1842	"dr0=%016RX64 dr1=%016RX64 dr2=%016RX64 dr3=%016RX64\n"
1843	"dr4=%016RX64 dr5=%016RX64 dr6=%016RX64 dr7=%016RX64\n"
1844	"gdtr=%016RX64:%04x idtr=%016RX64:%04x eflags=%08x\n"
1845	"ldtr={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1846	"tr ={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1847	"SysEnter={cs=%04llx eip=%08llx esp=%08llx}\n"
1848	,
1849	pCtx->rax, pCtx->rbx, pCtx->rcx, pCtx->rdx, pCtx->rsi, pCtx->rdi,
1850	pCtx->r8, pCtx->r9, pCtx->r10, pCtx->r11, pCtx->r12, pCtx->r13,
1851	pCtx->r14, pCtx->r15,
1852	pCtx->rip, pCtx->rsp, pCtx->rbp, X86_EFL_GET_IOPL(uEFlags), 31, szEFlags,
1853	pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit, pCtx->cs.Attr.u,
1854	pCtx->ds.Sel, pCtx->ds.u64Base, pCtx->ds.u32Limit, pCtx->ds.Attr.u,
1855	pCtx->es.Sel, pCtx->es.u64Base, pCtx->es.u32Limit, pCtx->es.Attr.u,
1856	pCtx->fs.Sel, pCtx->fs.u64Base, pCtx->fs.u32Limit, pCtx->fs.Attr.u,
1857	pCtx->gs.Sel, pCtx->gs.u64Base, pCtx->gs.u32Limit, pCtx->gs.Attr.u,
1858	pCtx->ss.Sel, pCtx->ss.u64Base, pCtx->ss.u32Limit, pCtx->ss.Attr.u,
1859	pCtx->cr0, pCtx->cr2, pCtx->cr3, pCtx->cr4,
1860	pCtx->dr[0], pCtx->dr[1], pCtx->dr[2], pCtx->dr[3],
1861	pCtx->dr[4], pCtx->dr[5], pCtx->dr[6], pCtx->dr[7],
1862	pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pCtx->idtr.pIdt, pCtx->idtr.cbIdt, uEFlags,
1863	pCtx->ldtr.Sel, pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit, pCtx->ldtr.Attr.u,
1864	pCtx->tr.Sel, pCtx->tr.u64Base, pCtx->tr.u32Limit, pCtx->tr.Attr.u,
1865	pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp));
1866	}
1867	else
1868	Log(("eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
1869	"eip=%08x esp=%08x ebp=%08x iopl=%d %*s\n"
1870	"cs={%04x base=%016RX64 limit=%08x flags=%08x} dr0=%08RX64 dr1=%08RX64\n"
1871	"ds={%04x base=%016RX64 limit=%08x flags=%08x} dr2=%08RX64 dr3=%08RX64\n"
1872	"es={%04x base=%016RX64 limit=%08x flags=%08x} dr4=%08RX64 dr5=%08RX64\n"
1873	"fs={%04x base=%016RX64 limit=%08x flags=%08x} dr6=%08RX64 dr7=%08RX64\n"
1874	"gs={%04x base=%016RX64 limit=%08x flags=%08x} cr0=%08RX64 cr2=%08RX64\n"
1875	"ss={%04x base=%016RX64 limit=%08x flags=%08x} cr3=%08RX64 cr4=%08RX64\n"
1876	"gdtr=%016RX64:%04x idtr=%016RX64:%04x eflags=%08x\n"
1877	"ldtr={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1878	"tr ={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1879	"SysEnter={cs=%04llx eip=%08llx esp=%08llx}\n"
1880	,
1881	pCtx->eax, pCtx->ebx, pCtx->ecx, pCtx->edx, pCtx->esi, pCtx->edi,
1882	pCtx->eip, pCtx->esp, pCtx->ebp, X86_EFL_GET_IOPL(uEFlags), 31, szEFlags,
1883	pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit, pCtx->cs.Attr.u, pCtx->dr[0], pCtx->dr[1],
1884	pCtx->ds.Sel, pCtx->ds.u64Base, pCtx->ds.u32Limit, pCtx->ds.Attr.u, pCtx->dr[2], pCtx->dr[3],
1885	pCtx->es.Sel, pCtx->es.u64Base, pCtx->es.u32Limit, pCtx->es.Attr.u, pCtx->dr[4], pCtx->dr[5],
1886	pCtx->fs.Sel, pCtx->fs.u64Base, pCtx->fs.u32Limit, pCtx->fs.Attr.u, pCtx->dr[6], pCtx->dr[7],
1887	pCtx->gs.Sel, pCtx->gs.u64Base, pCtx->gs.u32Limit, pCtx->gs.Attr.u, pCtx->cr0, pCtx->cr2,
1888	pCtx->ss.Sel, pCtx->ss.u64Base, pCtx->ss.u32Limit, pCtx->ss.Attr.u, pCtx->cr3, pCtx->cr4,
1889	pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pCtx->idtr.pIdt, pCtx->idtr.cbIdt, uEFlags,
1890	pCtx->ldtr.Sel, pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit, pCtx->ldtr.Attr.u,
1891	pCtx->tr.Sel, pCtx->tr.u64Base, pCtx->tr.u32Limit, pCtx->tr.Attr.u,
1892	pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp));
1893	}
1894
1895	if (fFlags & HM_DUMP_REG_FLAGS_FPU)
1896	{
1897	PCX86FXSTATE pFpuCtx = &pCtx->CTX_SUFF(pXState)->x87;
1898	Log(("FPU:\n"
1899	"FCW=%04x FSW=%04x FTW=%02x\n"
1900	"FOP=%04x FPUIP=%08x CS=%04x Rsrvd1=%04x\n"
1901	"FPUDP=%04x DS=%04x Rsvrd2=%04x MXCSR=%08x MXCSR_MASK=%08x\n"
1902	,
1903	pFpuCtx->FCW, pFpuCtx->FSW, pFpuCtx->FTW,
1904	pFpuCtx->FOP, pFpuCtx->FPUIP, pFpuCtx->CS, pFpuCtx->Rsrvd1,
1905	pFpuCtx->FPUDP, pFpuCtx->DS, pFpuCtx->Rsrvd2,
1906	pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK));
1907	NOREF(pFpuCtx);
1908	}
1909
1910	if (fFlags & HM_DUMP_REG_FLAGS_MSRS)
1911	{
1912	Log(("MSR:\n"
1913	"EFER =%016RX64\n"
1914	"PAT =%016RX64\n"
1915	"STAR =%016RX64\n"
1916	"CSTAR =%016RX64\n"
1917	"LSTAR =%016RX64\n"
1918	"SFMASK =%016RX64\n"
1919	"KERNELGSBASE =%016RX64\n",
1920	pCtx->msrEFER,
1921	pCtx->msrPAT,
1922	pCtx->msrSTAR,
1923	pCtx->msrCSTAR,
1924	pCtx->msrLSTAR,
1925	pCtx->msrSFMASK,
1926	pCtx->msrKERNELGSBASE));
1927	}
1928	}
1929
1930	#endif /* VBOX_STRICT */
1931

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp@ 88726

Download in other formats: