GVMMR0.cpp@ 71075

Last change on this file since 71075 was 71075, checked in by vboxsync, 7 years ago
VMM,SUPDrv: More NEM/win experimentation. bugref:9044
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 104.1 KB

Line
1	/* $Id: GVMMR0.cpp 71075 2018-02-20 21:10:45Z vboxsync $ */
2	/** @file
3	* GVMM - Global VM Manager.
4	*/
5
6	/*
7	* Copyright (C) 2007-2017 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/** @page pg_gvmm GVMM - The Global VM Manager
20	*
21	* The Global VM Manager lives in ring-0. Its main function at the moment is
22	* to manage a list of all running VMs, keep a ring-0 only structure (GVM) for
23	* each of them, and assign them unique identifiers (so GMM can track page
24	* owners). The GVMM also manage some of the host CPU resources, like the
25	* periodic preemption timer.
26	*
27	* The GVMM will create a ring-0 object for each VM when it is registered, this
28	* is both for session cleanup purposes and for having a point where it is
29	* possible to implement usage polices later (in SUPR0ObjRegister).
30	*
31	*
32	* @section sec_gvmm_ppt Periodic Preemption Timer (PPT)
33	*
34	* On system that sports a high resolution kernel timer API, we use per-cpu
35	* timers to generate interrupts that preempts VT-x, AMD-V and raw-mode guest
36	* execution. The timer frequency is calculating by taking the max
37	* TMCalcHostTimerFrequency for all VMs running on a CPU for the last ~160 ms
38	* (RT_ELEMENTS((PGVMMHOSTCPU)0, Ppt.aHzHistory) *
39	* GVMMHOSTCPU_PPT_HIST_INTERVAL_NS).
40	*
41	* The TMCalcHostTimerFrequency() part of the things gets its takes the max
42	* TMTimerSetFrequencyHint() value and adjusts by the current catch-up percent,
43	* warp drive percent and some fudge factors. VMMR0.cpp reports the result via
44	* GVMMR0SchedUpdatePeriodicPreemptionTimer() before switching to the VT-x,
45	* AMD-V and raw-mode execution environments.
46	*/
47
48
49	/*********************************************************************************************************************************
50	* Header Files *
51	*********************************************************************************************************************************/
52	#define LOG_GROUP LOG_GROUP_GVMM
53	#include <VBox/vmm/gvmm.h>
54	#include <VBox/vmm/gmm.h>
55	#include "GVMMR0Internal.h"
56	#include <VBox/vmm/gvm.h>
57	#include <VBox/vmm/vm.h>
58	#include <VBox/vmm/vmcpuset.h>
59	#include <VBox/vmm/vmm.h>
60	#ifdef VBOX_WITH_NEM_R0
61	# include <VBox/vmm/nem.h>
62	#endif
63	#include <VBox/param.h>
64	#include <VBox/err.h>
65
66	#include <iprt/asm.h>
67	#include <iprt/asm-amd64-x86.h>
68	#include <iprt/critsect.h>
69	#include <iprt/mem.h>
70	#include <iprt/semaphore.h>
71	#include <iprt/time.h>
72	#include <VBox/log.h>
73	#include <iprt/thread.h>
74	#include <iprt/process.h>
75	#include <iprt/param.h>
76	#include <iprt/string.h>
77	#include <iprt/assert.h>
78	#include <iprt/mem.h>
79	#include <iprt/memobj.h>
80	#include <iprt/mp.h>
81	#include <iprt/cpuset.h>
82	#include <iprt/spinlock.h>
83	#include <iprt/timer.h>
84
85	#include "dtrace/VBoxVMM.h"
86
87
88	/*********************************************************************************************************************************
89	* Defined Constants And Macros *
90	*********************************************************************************************************************************/
91	#if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS) \|\| defined(DOXYGEN_RUNNING)
92	/** Define this to enable the periodic preemption timer. */
93	# define GVMM_SCHED_WITH_PPT
94	#endif
95
96
97	/** @def GVMM_CHECK_SMAP_SETUP
98	* SMAP check setup. */
99	/** @def GVMM_CHECK_SMAP_CHECK
100	* Checks that the AC flag is set if SMAP is enabled. If AC is not set,
101	* it will be logged and @a a_BadExpr is executed. */
102	/** @def GVMM_CHECK_SMAP_CHECK2
103	* Checks that the AC flag is set if SMAP is enabled. If AC is not set, it will
104	* be logged, written to the VMs assertion text buffer, and @a a_BadExpr is
105	* executed. */
106	#if defined(VBOX_STRICT) \|\| 1
107	# define GVMM_CHECK_SMAP_SETUP() uint32_t const fKernelFeatures = SUPR0GetKernelFeatures()
108	# define GVMM_CHECK_SMAP_CHECK(a_BadExpr) \
109	do { \
110	if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
111	{ \
112	RTCCUINTREG fEflCheck = ASMGetFlags(); \
113	if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
114	{ /* likely */ } \
115	else \
116	{ \
117	SUPR0Printf("%s, line %d: EFLAGS.AC is clear! (%#x)\n", __FUNCTION__, __LINE__, (uint32_t)fEflCheck); \
118	a_BadExpr; \
119	} \
120	} \
121	} while (0)
122	# define GVMM_CHECK_SMAP_CHECK2(a_pVM, a_BadExpr) \
123	do { \
124	if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
125	{ \
126	RTCCUINTREG fEflCheck = ASMGetFlags(); \
127	if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
128	{ /* likely */ } \
129	else \
130	{ \
131	SUPR0BadContext((a_pVM) ? (a_pVM)->pSession : NULL, __FILE__, __LINE__, "EFLAGS.AC is zero!"); \
132	a_BadExpr; \
133	} \
134	} \
135	} while (0)
136	#else
137	# define GVMM_CHECK_SMAP_SETUP() uint32_t const fKernelFeatures = 0
138	# define GVMM_CHECK_SMAP_CHECK(a_BadExpr) NOREF(fKernelFeatures)
139	# define GVMM_CHECK_SMAP_CHECK2(a_pVM, a_BadExpr) NOREF(fKernelFeatures)
140	#endif
141
142
143
144	/*********************************************************************************************************************************
145	* Structures and Typedefs *
146	*********************************************************************************************************************************/
147
148	/**
149	* Global VM handle.
150	*/
151	typedef struct GVMHANDLE
152	{
153	/** The index of the next handle in the list (free or used). (0 is nil.) */
154	uint16_t volatile iNext;
155	/** Our own index / handle value. */
156	uint16_t iSelf;
157	/** The process ID of the handle owner.
158	* This is used for access checks. */
159	RTPROCESS ProcId;
160	/** The pointer to the ring-0 only (aka global) VM structure. */
161	PGVM pGVM;
162	/** The ring-0 mapping of the shared VM instance data. */
163	PVM pVM;
164	/** The virtual machine object. */
165	void *pvObj;
166	/** The session this VM is associated with. */
167	PSUPDRVSESSION pSession;
168	/** The ring-0 handle of the EMT0 thread.
169	* This is used for ownership checks as well as looking up a VM handle by thread
170	* at times like assertions. */
171	RTNATIVETHREAD hEMT0;
172	} GVMHANDLE;
173	/** Pointer to a global VM handle. */
174	typedef GVMHANDLE *PGVMHANDLE;
175
176	/** Number of GVM handles (including the NIL handle). */
177	#if HC_ARCH_BITS == 64
178	# define GVMM_MAX_HANDLES 8192
179	#else
180	# define GVMM_MAX_HANDLES 128
181	#endif
182
183	/**
184	* Per host CPU GVMM data.
185	*/
186	typedef struct GVMMHOSTCPU
187	{
188	/** Magic number (GVMMHOSTCPU_MAGIC). */
189	uint32_t volatile u32Magic;
190	/** The CPU ID. */
191	RTCPUID idCpu;
192	/** The CPU set index. */
193	uint32_t idxCpuSet;
194
195	#ifdef GVMM_SCHED_WITH_PPT
196	/** Periodic preemption timer data. */
197	struct
198	{
199	/** The handle to the periodic preemption timer. */
200	PRTTIMER pTimer;
201	/** Spinlock protecting the data below. */
202	RTSPINLOCK hSpinlock;
203	/** The smalles Hz that we need to care about. (static) */
204	uint32_t uMinHz;
205	/** The number of ticks between each historization. */
206	uint32_t cTicksHistoriziationInterval;
207	/** The current historization tick (counting up to
208	* cTicksHistoriziationInterval and then resetting). */
209	uint32_t iTickHistorization;
210	/** The current timer interval. This is set to 0 when inactive. */
211	uint32_t cNsInterval;
212	/** The current timer frequency. This is set to 0 when inactive. */
213	uint32_t uTimerHz;
214	/** The current max frequency reported by the EMTs.
215	* This gets historicize and reset by the timer callback. This is
216	* read without holding the spinlock, so needs atomic updating. */
217	uint32_t volatile uDesiredHz;
218	/** Whether the timer was started or not. */
219	bool volatile fStarted;
220	/** Set if we're starting timer. */
221	bool volatile fStarting;
222	/** The index of the next history entry (mod it). */
223	uint32_t iHzHistory;
224	/** Historicized uDesiredHz values. The array wraps around, new entries
225	* are added at iHzHistory. This is updated approximately every
226	* GVMMHOSTCPU_PPT_HIST_INTERVAL_NS by the timer callback. */
227	uint32_t aHzHistory[8];
228	/** Statistics counter for recording the number of interval changes. */
229	uint32_t cChanges;
230	/** Statistics counter for recording the number of timer starts. */
231	uint32_t cStarts;
232	} Ppt;
233	#endif /* GVMM_SCHED_WITH_PPT */
234
235	} GVMMHOSTCPU;
236	/** Pointer to the per host CPU GVMM data. */
237	typedef GVMMHOSTCPU *PGVMMHOSTCPU;
238	/** The GVMMHOSTCPU::u32Magic value (Petra, Tanya & Rachel Haden). */
239	#define GVMMHOSTCPU_MAGIC UINT32_C(0x19711011)
240	/** The interval on history entry should cover (approximately) give in
241	* nanoseconds. */
242	#define GVMMHOSTCPU_PPT_HIST_INTERVAL_NS UINT32_C(20000000)
243
244
245	/**
246	* The GVMM instance data.
247	*/
248	typedef struct GVMM
249	{
250	/** Eyecatcher / magic. */
251	uint32_t u32Magic;
252	/** The index of the head of the free handle chain. (0 is nil.) */
253	uint16_t volatile iFreeHead;
254	/** The index of the head of the active handle chain. (0 is nil.) */
255	uint16_t volatile iUsedHead;
256	/** The number of VMs. */
257	uint16_t volatile cVMs;
258	/** Alignment padding. */
259	uint16_t u16Reserved;
260	/** The number of EMTs. */
261	uint32_t volatile cEMTs;
262	/** The number of EMTs that have halted in GVMMR0SchedHalt. */
263	uint32_t volatile cHaltedEMTs;
264	/** Mini lock for restricting early wake-ups to one thread. */
265	bool volatile fDoingEarlyWakeUps;
266	bool afPadding[3]; /*< explicit alignment padding. /
267	/** When the next halted or sleeping EMT will wake up.
268	* This is set to 0 when it needs recalculating and to UINT64_MAX when
269	* there are no halted or sleeping EMTs in the GVMM. */
270	uint64_t uNsNextEmtWakeup;
271	/** The lock used to serialize VM creation, destruction and associated events that
272	* isn't performance critical. Owners may acquire the list lock. */
273	RTCRITSECT CreateDestroyLock;
274	/** The lock used to serialize used list updates and accesses.
275	* This indirectly includes scheduling since the scheduler will have to walk the
276	* used list to examin running VMs. Owners may not acquire any other locks. */
277	RTCRITSECTRW UsedLock;
278	/** The handle array.
279	* The size of this array defines the maximum number of currently running VMs.
280	* The first entry is unused as it represents the NIL handle. */
281	GVMHANDLE aHandles[GVMM_MAX_HANDLES];
282
283	/** @gcfgm{/GVMM/cEMTsMeansCompany, 32-bit, 0, UINT32_MAX, 1}
284	* The number of EMTs that means we no longer consider ourselves alone on a
285	* CPU/Core.
286	*/
287	uint32_t cEMTsMeansCompany;
288	/** @gcfgm{/GVMM/MinSleepAlone,32-bit, 0, 100000000, 750000, ns}
289	* The minimum sleep time for when we're alone, in nano seconds.
290	*/
291	uint32_t nsMinSleepAlone;
292	/** @gcfgm{/GVMM/MinSleepCompany,32-bit,0, 100000000, 15000, ns}
293	* The minimum sleep time for when we've got company, in nano seconds.
294	*/
295	uint32_t nsMinSleepCompany;
296	/** @gcfgm{/GVMM/EarlyWakeUp1, 32-bit, 0, 100000000, 25000, ns}
297	* The limit for the first round of early wake-ups, given in nano seconds.
298	*/
299	uint32_t nsEarlyWakeUp1;
300	/** @gcfgm{/GVMM/EarlyWakeUp2, 32-bit, 0, 100000000, 50000, ns}
301	* The limit for the second round of early wake-ups, given in nano seconds.
302	*/
303	uint32_t nsEarlyWakeUp2;
304
305	/** Set if we're doing early wake-ups.
306	* This reflects nsEarlyWakeUp1 and nsEarlyWakeUp2. */
307	bool volatile fDoEarlyWakeUps;
308
309	/** The number of entries in the host CPU array (aHostCpus). */
310	uint32_t cHostCpus;
311	/** Per host CPU data (variable length). */
312	GVMMHOSTCPU aHostCpus[1];
313	} GVMM;
314	AssertCompileMemberAlignment(GVMM, CreateDestroyLock, 8);
315	AssertCompileMemberAlignment(GVMM, UsedLock, 8);
316	AssertCompileMemberAlignment(GVMM, uNsNextEmtWakeup, 8);
317	/** Pointer to the GVMM instance data. */
318	typedef GVMM *PGVMM;
319
320	/** The GVMM::u32Magic value (Charlie Haden). */
321	#define GVMM_MAGIC UINT32_C(0x19370806)
322
323
324
325	/*********************************************************************************************************************************
326	* Global Variables *
327	*********************************************************************************************************************************/
328	/** Pointer to the GVMM instance data.
329	* (Just my general dislike for global variables.) */
330	static PGVMM g_pGVMM = NULL;
331
332	/** Macro for obtaining and validating the g_pGVMM pointer.
333	* On failure it will return from the invoking function with the specified return value.
334	*
335	* @param pGVMM The name of the pGVMM variable.
336	* @param rc The return value on failure. Use VERR_GVMM_INSTANCE for VBox
337	* status codes.
338	*/
339	#define GVMM_GET_VALID_INSTANCE(pGVMM, rc) \
340	do { \
341	(pGVMM) = g_pGVMM;\
342	AssertPtrReturn((pGVMM), (rc)); \
343	AssertMsgReturn((pGVMM)->u32Magic == GVMM_MAGIC, ("%p - %#x\n", (pGVMM), (pGVMM)->u32Magic), (rc)); \
344	} while (0)
345
346	/** Macro for obtaining and validating the g_pGVMM pointer, void function variant.
347	* On failure it will return from the invoking function.
348	*
349	* @param pGVMM The name of the pGVMM variable.
350	*/
351	#define GVMM_GET_VALID_INSTANCE_VOID(pGVMM) \
352	do { \
353	(pGVMM) = g_pGVMM;\
354	AssertPtrReturnVoid((pGVMM)); \
355	AssertMsgReturnVoid((pGVMM)->u32Magic == GVMM_MAGIC, ("%p - %#x\n", (pGVMM), (pGVMM)->u32Magic)); \
356	} while (0)
357
358
359	/*********************************************************************************************************************************
360	* Internal Functions *
361	*********************************************************************************************************************************/
362	static void gvmmR0InitPerVMData(PGVM pGVM);
363	static DECLCALLBACK(void) gvmmR0HandleObjDestructor(void pvObj, void pvGVMM, void *pvHandle);
364	static int gvmmR0ByGVMandVM(PGVM pGVM, PVM pVM, PGVMM *ppGVMM, bool fTakeUsedLock);
365	static int gvmmR0ByGVMandVMandEMT(PGVM pGVM, PVM pVM, VMCPUID idCpu, PGVMM *ppGVMM);
366
367	#ifdef GVMM_SCHED_WITH_PPT
368	static DECLCALLBACK(void) gvmmR0SchedPeriodicPreemptionTimerCallback(PRTTIMER pTimer, void *pvUser, uint64_t iTick);
369	#endif
370
371
372	/**
373	* Initializes the GVMM.
374	*
375	* This is called while owning the loader semaphore (see supdrvIOCtl_LdrLoad()).
376	*
377	* @returns VBox status code.
378	*/
379	GVMMR0DECL(int) GVMMR0Init(void)
380	{
381	LogFlow(("GVMMR0Init:\n"));
382
383	/*
384	* Allocate and initialize the instance data.
385	*/
386	uint32_t cHostCpus = RTMpGetArraySize();
387	AssertMsgReturn(cHostCpus > 0 && cHostCpus < _64K, ("%d", (int)cHostCpus), VERR_GVMM_HOST_CPU_RANGE);
388
389	PGVMM pGVMM = (PGVMM)RTMemAllocZ(RT_UOFFSETOF(GVMM, aHostCpus[cHostCpus]));
390	if (!pGVMM)
391	return VERR_NO_MEMORY;
392	int rc = RTCritSectInitEx(&pGVMM->CreateDestroyLock, 0, NIL_RTLOCKVALCLASS, RTLOCKVAL_SUB_CLASS_NONE,
393	"GVMM-CreateDestroyLock");
394	if (RT_SUCCESS(rc))
395	{
396	rc = RTCritSectRwInitEx(&pGVMM->UsedLock, 0, NIL_RTLOCKVALCLASS, RTLOCKVAL_SUB_CLASS_NONE, "GVMM-UsedLock");
397	if (RT_SUCCESS(rc))
398	{
399	pGVMM->u32Magic = GVMM_MAGIC;
400	pGVMM->iUsedHead = 0;
401	pGVMM->iFreeHead = 1;
402
403	/* the nil handle */
404	pGVMM->aHandles[0].iSelf = 0;
405	pGVMM->aHandles[0].iNext = 0;
406
407	/* the tail */
408	unsigned i = RT_ELEMENTS(pGVMM->aHandles) - 1;
409	pGVMM->aHandles[i].iSelf = i;
410	pGVMM->aHandles[i].iNext = 0; /* nil */
411
412	/* the rest */
413	while (i-- > 1)
414	{
415	pGVMM->aHandles[i].iSelf = i;
416	pGVMM->aHandles[i].iNext = i + 1;
417	}
418
419	/* The default configuration values. */
420	uint32_t cNsResolution = RTSemEventMultiGetResolution();
421	pGVMM->cEMTsMeansCompany = 1; /** @todo should be adjusted to relative to the cpu count or something... */
422	if (cNsResolution >= 5*RT_NS_100US)
423	{
424	pGVMM->nsMinSleepAlone = 750000 /* ns (0.750 ms) /; /* @todo this should be adjusted to be 75% (or something) of the scheduler granularity... */
425	pGVMM->nsMinSleepCompany = 15000 /* ns (0.015 ms) */;
426	pGVMM->nsEarlyWakeUp1 = 25000 /* ns (0.025 ms) */;
427	pGVMM->nsEarlyWakeUp2 = 50000 /* ns (0.050 ms) */;
428	}
429	else if (cNsResolution > RT_NS_100US)
430	{
431	pGVMM->nsMinSleepAlone = cNsResolution / 2;
432	pGVMM->nsMinSleepCompany = cNsResolution / 4;
433	pGVMM->nsEarlyWakeUp1 = 0;
434	pGVMM->nsEarlyWakeUp2 = 0;
435	}
436	else
437	{
438	pGVMM->nsMinSleepAlone = 2000;
439	pGVMM->nsMinSleepCompany = 2000;
440	pGVMM->nsEarlyWakeUp1 = 0;
441	pGVMM->nsEarlyWakeUp2 = 0;
442	}
443	pGVMM->fDoEarlyWakeUps = pGVMM->nsEarlyWakeUp1 > 0 && pGVMM->nsEarlyWakeUp2 > 0;
444
445	/* The host CPU data. */
446	pGVMM->cHostCpus = cHostCpus;
447	uint32_t iCpu = cHostCpus;
448	RTCPUSET PossibleSet;
449	RTMpGetSet(&PossibleSet);
450	while (iCpu-- > 0)
451	{
452	pGVMM->aHostCpus[iCpu].idxCpuSet = iCpu;
453	#ifdef GVMM_SCHED_WITH_PPT
454	pGVMM->aHostCpus[iCpu].Ppt.pTimer = NULL;
455	pGVMM->aHostCpus[iCpu].Ppt.hSpinlock = NIL_RTSPINLOCK;
456	pGVMM->aHostCpus[iCpu].Ppt.uMinHz = 5; /** @todo Add some API which figures this one out. (not that important) */
457	pGVMM->aHostCpus[iCpu].Ppt.cTicksHistoriziationInterval = 1;
458	//pGVMM->aHostCpus[iCpu].Ppt.iTickHistorization = 0;
459	//pGVMM->aHostCpus[iCpu].Ppt.cNsInterval = 0;
460	//pGVMM->aHostCpus[iCpu].Ppt.uTimerHz = 0;
461	//pGVMM->aHostCpus[iCpu].Ppt.uDesiredHz = 0;
462	//pGVMM->aHostCpus[iCpu].Ppt.fStarted = false;
463	//pGVMM->aHostCpus[iCpu].Ppt.fStarting = false;
464	//pGVMM->aHostCpus[iCpu].Ppt.iHzHistory = 0;
465	//pGVMM->aHostCpus[iCpu].Ppt.aHzHistory = {0};
466	#endif
467
468	if (RTCpuSetIsMember(&PossibleSet, iCpu))
469	{
470	pGVMM->aHostCpus[iCpu].idCpu = RTMpCpuIdFromSetIndex(iCpu);
471	pGVMM->aHostCpus[iCpu].u32Magic = GVMMHOSTCPU_MAGIC;
472
473	#ifdef GVMM_SCHED_WITH_PPT
474	rc = RTTimerCreateEx(&pGVMM->aHostCpus[iCpu].Ppt.pTimer,
475	5010001000 /* whatever */,
476	RTTIMER_FLAGS_CPU(iCpu) \| RTTIMER_FLAGS_HIGH_RES,
477	gvmmR0SchedPeriodicPreemptionTimerCallback,
478	&pGVMM->aHostCpus[iCpu]);
479	if (RT_SUCCESS(rc))
480	rc = RTSpinlockCreate(&pGVMM->aHostCpus[iCpu].Ppt.hSpinlock, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "GVMM/CPU");
481	if (RT_FAILURE(rc))
482	{
483	while (iCpu < cHostCpus)
484	{
485	RTTimerDestroy(pGVMM->aHostCpus[iCpu].Ppt.pTimer);
486	RTSpinlockDestroy(pGVMM->aHostCpus[iCpu].Ppt.hSpinlock);
487	pGVMM->aHostCpus[iCpu].Ppt.hSpinlock = NIL_RTSPINLOCK;
488	iCpu++;
489	}
490	break;
491	}
492	#endif
493	}
494	else
495	{
496	pGVMM->aHostCpus[iCpu].idCpu = NIL_RTCPUID;
497	pGVMM->aHostCpus[iCpu].u32Magic = 0;
498	}
499	}
500	if (RT_SUCCESS(rc))
501	{
502	g_pGVMM = pGVMM;
503	LogFlow(("GVMMR0Init: pGVMM=%p cHostCpus=%u\n", pGVMM, cHostCpus));
504	return VINF_SUCCESS;
505	}
506
507	/* bail out. */
508	RTCritSectRwDelete(&pGVMM->UsedLock);
509	}
510	RTCritSectDelete(&pGVMM->CreateDestroyLock);
511	}
512
513	RTMemFree(pGVMM);
514	return rc;
515	}
516
517
518	/**
519	* Terminates the GVM.
520	*
521	* This is called while owning the loader semaphore (see supdrvLdrFree()).
522	* And unless something is wrong, there should be absolutely no VMs
523	* registered at this point.
524	*/
525	GVMMR0DECL(void) GVMMR0Term(void)
526	{
527	LogFlow(("GVMMR0Term:\n"));
528
529	PGVMM pGVMM = g_pGVMM;
530	g_pGVMM = NULL;
531	if (RT_UNLIKELY(!VALID_PTR(pGVMM)))
532	{
533	SUPR0Printf("GVMMR0Term: pGVMM=%RKv\n", pGVMM);
534	return;
535	}
536
537	/*
538	* First of all, stop all active timers.
539	*/
540	uint32_t cActiveTimers = 0;
541	uint32_t iCpu = pGVMM->cHostCpus;
542	while (iCpu-- > 0)
543	{
544	ASMAtomicWriteU32(&pGVMM->aHostCpus[iCpu].u32Magic, ~GVMMHOSTCPU_MAGIC);
545	#ifdef GVMM_SCHED_WITH_PPT
546	if ( pGVMM->aHostCpus[iCpu].Ppt.pTimer != NULL
547	&& RT_SUCCESS(RTTimerStop(pGVMM->aHostCpus[iCpu].Ppt.pTimer)))
548	cActiveTimers++;
549	#endif
550	}
551	if (cActiveTimers)
552	RTThreadSleep(1); /* fudge */
553
554	/*
555	* Invalidate the and free resources.
556	*/
557	pGVMM->u32Magic = ~GVMM_MAGIC;
558	RTCritSectRwDelete(&pGVMM->UsedLock);
559	RTCritSectDelete(&pGVMM->CreateDestroyLock);
560
561	pGVMM->iFreeHead = 0;
562	if (pGVMM->iUsedHead)
563	{
564	SUPR0Printf("GVMMR0Term: iUsedHead=%#x! (cVMs=%#x cEMTs=%#x)\n", pGVMM->iUsedHead, pGVMM->cVMs, pGVMM->cEMTs);
565	pGVMM->iUsedHead = 0;
566	}
567
568	#ifdef GVMM_SCHED_WITH_PPT
569	iCpu = pGVMM->cHostCpus;
570	while (iCpu-- > 0)
571	{
572	RTTimerDestroy(pGVMM->aHostCpus[iCpu].Ppt.pTimer);
573	pGVMM->aHostCpus[iCpu].Ppt.pTimer = NULL;
574	RTSpinlockDestroy(pGVMM->aHostCpus[iCpu].Ppt.hSpinlock);
575	pGVMM->aHostCpus[iCpu].Ppt.hSpinlock = NIL_RTSPINLOCK;
576	}
577	#endif
578
579	RTMemFree(pGVMM);
580	}
581
582
583	/**
584	* A quick hack for setting global config values.
585	*
586	* @returns VBox status code.
587	*
588	* @param pSession The session handle. Used for authentication.
589	* @param pszName The variable name.
590	* @param u64Value The new value.
591	*/
592	GVMMR0DECL(int) GVMMR0SetConfig(PSUPDRVSESSION pSession, const char *pszName, uint64_t u64Value)
593	{
594	/*
595	* Validate input.
596	*/
597	PGVMM pGVMM;
598	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
599	AssertPtrReturn(pSession, VERR_INVALID_HANDLE);
600	AssertPtrReturn(pszName, VERR_INVALID_POINTER);
601
602	/*
603	* String switch time!
604	*/
605	if (strncmp(pszName, RT_STR_TUPLE("/GVMM/")))
606	return VERR_CFGM_VALUE_NOT_FOUND; /* borrow status codes from CFGM... */
607	int rc = VINF_SUCCESS;
608	pszName += sizeof("/GVMM/") - 1;
609	if (!strcmp(pszName, "cEMTsMeansCompany"))
610	{
611	if (u64Value <= UINT32_MAX)
612	pGVMM->cEMTsMeansCompany = u64Value;
613	else
614	rc = VERR_OUT_OF_RANGE;
615	}
616	else if (!strcmp(pszName, "MinSleepAlone"))
617	{
618	if (u64Value <= RT_NS_100MS)
619	pGVMM->nsMinSleepAlone = u64Value;
620	else
621	rc = VERR_OUT_OF_RANGE;
622	}
623	else if (!strcmp(pszName, "MinSleepCompany"))
624	{
625	if (u64Value <= RT_NS_100MS)
626	pGVMM->nsMinSleepCompany = u64Value;
627	else
628	rc = VERR_OUT_OF_RANGE;
629	}
630	else if (!strcmp(pszName, "EarlyWakeUp1"))
631	{
632	if (u64Value <= RT_NS_100MS)
633	{
634	pGVMM->nsEarlyWakeUp1 = u64Value;
635	pGVMM->fDoEarlyWakeUps = pGVMM->nsEarlyWakeUp1 > 0 && pGVMM->nsEarlyWakeUp2 > 0;
636	}
637	else
638	rc = VERR_OUT_OF_RANGE;
639	}
640	else if (!strcmp(pszName, "EarlyWakeUp2"))
641	{
642	if (u64Value <= RT_NS_100MS)
643	{
644	pGVMM->nsEarlyWakeUp2 = u64Value;
645	pGVMM->fDoEarlyWakeUps = pGVMM->nsEarlyWakeUp1 > 0 && pGVMM->nsEarlyWakeUp2 > 0;
646	}
647	else
648	rc = VERR_OUT_OF_RANGE;
649	}
650	else
651	rc = VERR_CFGM_VALUE_NOT_FOUND;
652	return rc;
653	}
654
655
656	/**
657	* A quick hack for getting global config values.
658	*
659	* @returns VBox status code.
660	*
661	* @param pSession The session handle. Used for authentication.
662	* @param pszName The variable name.
663	* @param pu64Value Where to return the value.
664	*/
665	GVMMR0DECL(int) GVMMR0QueryConfig(PSUPDRVSESSION pSession, const char pszName, uint64_t pu64Value)
666	{
667	/*
668	* Validate input.
669	*/
670	PGVMM pGVMM;
671	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
672	AssertPtrReturn(pSession, VERR_INVALID_HANDLE);
673	AssertPtrReturn(pszName, VERR_INVALID_POINTER);
674	AssertPtrReturn(pu64Value, VERR_INVALID_POINTER);
675
676	/*
677	* String switch time!
678	*/
679	if (strncmp(pszName, RT_STR_TUPLE("/GVMM/")))
680	return VERR_CFGM_VALUE_NOT_FOUND; /* borrow status codes from CFGM... */
681	int rc = VINF_SUCCESS;
682	pszName += sizeof("/GVMM/") - 1;
683	if (!strcmp(pszName, "cEMTsMeansCompany"))
684	*pu64Value = pGVMM->cEMTsMeansCompany;
685	else if (!strcmp(pszName, "MinSleepAlone"))
686	*pu64Value = pGVMM->nsMinSleepAlone;
687	else if (!strcmp(pszName, "MinSleepCompany"))
688	*pu64Value = pGVMM->nsMinSleepCompany;
689	else if (!strcmp(pszName, "EarlyWakeUp1"))
690	*pu64Value = pGVMM->nsEarlyWakeUp1;
691	else if (!strcmp(pszName, "EarlyWakeUp2"))
692	*pu64Value = pGVMM->nsEarlyWakeUp2;
693	else
694	rc = VERR_CFGM_VALUE_NOT_FOUND;
695	return rc;
696	}
697
698
699	/**
700	* Acquire the 'used' lock in shared mode.
701	*
702	* This prevents destruction of the VM while we're in ring-0.
703	*
704	* @returns IPRT status code, see RTSemFastMutexRequest.
705	* @param a_pGVMM The GVMM instance data.
706	* @sa GVMMR0_USED_SHARED_UNLOCK, GVMMR0_USED_EXCLUSIVE_LOCK
707	*/
708	#define GVMMR0_USED_SHARED_LOCK(a_pGVMM) RTCritSectRwEnterShared(&(a_pGVMM)->UsedLock)
709
710	/**
711	* Release the 'used' lock in when owning it in shared mode.
712	*
713	* @returns IPRT status code, see RTSemFastMutexRequest.
714	* @param a_pGVMM The GVMM instance data.
715	* @sa GVMMR0_USED_SHARED_LOCK
716	*/
717	#define GVMMR0_USED_SHARED_UNLOCK(a_pGVMM) RTCritSectRwLeaveShared(&(a_pGVMM)->UsedLock)
718
719	/**
720	* Acquire the 'used' lock in exclusive mode.
721	*
722	* Only use this function when making changes to the used list.
723	*
724	* @returns IPRT status code, see RTSemFastMutexRequest.
725	* @param a_pGVMM The GVMM instance data.
726	* @sa GVMMR0_USED_EXCLUSIVE_UNLOCK
727	*/
728	#define GVMMR0_USED_EXCLUSIVE_LOCK(a_pGVMM) RTCritSectRwEnterExcl(&(a_pGVMM)->UsedLock)
729
730	/**
731	* Release the 'used' lock when owning it in exclusive mode.
732	*
733	* @returns IPRT status code, see RTSemFastMutexRelease.
734	* @param a_pGVMM The GVMM instance data.
735	* @sa GVMMR0_USED_EXCLUSIVE_LOCK, GVMMR0_USED_SHARED_UNLOCK
736	*/
737	#define GVMMR0_USED_EXCLUSIVE_UNLOCK(a_pGVMM) RTCritSectRwLeaveExcl(&(a_pGVMM)->UsedLock)
738
739
740	/**
741	* Try acquire the 'create & destroy' lock.
742	*
743	* @returns IPRT status code, see RTSemFastMutexRequest.
744	* @param pGVMM The GVMM instance data.
745	*/
746	DECLINLINE(int) gvmmR0CreateDestroyLock(PGVMM pGVMM)
747	{
748	LogFlow(("++gvmmR0CreateDestroyLock(%p)\n", pGVMM));
749	int rc = RTCritSectEnter(&pGVMM->CreateDestroyLock);
750	LogFlow(("gvmmR0CreateDestroyLock(%p)->%Rrc\n", pGVMM, rc));
751	return rc;
752	}
753
754
755	/**
756	* Release the 'create & destroy' lock.
757	*
758	* @returns IPRT status code, see RTSemFastMutexRequest.
759	* @param pGVMM The GVMM instance data.
760	*/
761	DECLINLINE(int) gvmmR0CreateDestroyUnlock(PGVMM pGVMM)
762	{
763	LogFlow(("--gvmmR0CreateDestroyUnlock(%p)\n", pGVMM));
764	int rc = RTCritSectLeave(&pGVMM->CreateDestroyLock);
765	AssertRC(rc);
766	return rc;
767	}
768
769
770	/**
771	* Request wrapper for the GVMMR0CreateVM API.
772	*
773	* @returns VBox status code.
774	* @param pReq The request buffer.
775	* @param pSession The session handle. The VM will be associated with this.
776	*/
777	GVMMR0DECL(int) GVMMR0CreateVMReq(PGVMMCREATEVMREQ pReq, PSUPDRVSESSION pSession)
778	{
779	/*
780	* Validate the request.
781	*/
782	if (!VALID_PTR(pReq))
783	return VERR_INVALID_POINTER;
784	if (pReq->Hdr.cbReq != sizeof(*pReq))
785	return VERR_INVALID_PARAMETER;
786	if (pReq->pSession != pSession)
787	return VERR_INVALID_POINTER;
788
789	/*
790	* Execute it.
791	*/
792	PVM pVM;
793	pReq->pVMR0 = NULL;
794	pReq->pVMR3 = NIL_RTR3PTR;
795	int rc = GVMMR0CreateVM(pSession, pReq->cCpus, &pVM);
796	if (RT_SUCCESS(rc))
797	{
798	pReq->pVMR0 = pVM;
799	pReq->pVMR3 = pVM->pVMR3;
800	}
801	return rc;
802	}
803
804
805	/**
806	* Allocates the VM structure and registers it with GVM.
807	*
808	* The caller will become the VM owner and there by the EMT.
809	*
810	* @returns VBox status code.
811	* @param pSession The support driver session.
812	* @param cCpus Number of virtual CPUs for the new VM.
813	* @param ppVM Where to store the pointer to the VM structure.
814	*
815	* @thread EMT.
816	*/
817	GVMMR0DECL(int) GVMMR0CreateVM(PSUPDRVSESSION pSession, uint32_t cCpus, PVM *ppVM)
818	{
819	LogFlow(("GVMMR0CreateVM: pSession=%p\n", pSession));
820	PGVMM pGVMM;
821	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
822
823	AssertPtrReturn(ppVM, VERR_INVALID_POINTER);
824	*ppVM = NULL;
825
826	if ( cCpus == 0
827	\|\| cCpus > VMM_MAX_CPU_COUNT)
828	return VERR_INVALID_PARAMETER;
829
830	RTNATIVETHREAD hEMT0 = RTThreadNativeSelf();
831	AssertReturn(hEMT0 != NIL_RTNATIVETHREAD, VERR_GVMM_BROKEN_IPRT);
832	RTPROCESS ProcId = RTProcSelf();
833	AssertReturn(ProcId != NIL_RTPROCESS, VERR_GVMM_BROKEN_IPRT);
834
835	/*
836	* The whole allocation process is protected by the lock.
837	*/
838	int rc = gvmmR0CreateDestroyLock(pGVMM);
839	AssertRCReturn(rc, rc);
840
841	/*
842	* Only one VM per session.
843	*/
844	if (SUPR0GetSessionVM(pSession) != NULL)
845	{
846	gvmmR0CreateDestroyUnlock(pGVMM);
847	SUPR0Printf("GVMMR0CreateVM: The session %p already got a VM: %p\n", pSession, SUPR0GetSessionVM(pSession));
848	return VERR_ALREADY_EXISTS;
849	}
850
851	/*
852	* Allocate a handle first so we don't waste resources unnecessarily.
853	*/
854	uint16_t iHandle = pGVMM->iFreeHead;
855	if (iHandle)
856	{
857	PGVMHANDLE pHandle = &pGVMM->aHandles[iHandle];
858
859	/* consistency checks, a bit paranoid as always. */
860	if ( !pHandle->pVM
861	&& !pHandle->pGVM
862	&& !pHandle->pvObj
863	&& pHandle->iSelf == iHandle)
864	{
865	pHandle->pvObj = SUPR0ObjRegister(pSession, SUPDRVOBJTYPE_VM, gvmmR0HandleObjDestructor, pGVMM, pHandle);
866	if (pHandle->pvObj)
867	{
868	/*
869	* Move the handle from the free to used list and perform permission checks.
870	*/
871	rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
872	AssertRC(rc);
873
874	pGVMM->iFreeHead = pHandle->iNext;
875	pHandle->iNext = pGVMM->iUsedHead;
876	pGVMM->iUsedHead = iHandle;
877	pGVMM->cVMs++;
878
879	pHandle->pVM = NULL;
880	pHandle->pGVM = NULL;
881	pHandle->pSession = pSession;
882	pHandle->hEMT0 = NIL_RTNATIVETHREAD;
883	pHandle->ProcId = NIL_RTPROCESS;
884
885	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
886
887	rc = SUPR0ObjVerifyAccess(pHandle->pvObj, pSession, NULL);
888	if (RT_SUCCESS(rc))
889	{
890	/*
891	* Allocate the global VM structure (GVM) and initialize it.
892	*/
893	PGVM pGVM = (PGVM)RTMemAllocZ(RT_UOFFSETOF(GVM, aCpus[cCpus]));
894	if (pGVM)
895	{
896	pGVM->u32Magic = GVM_MAGIC;
897	pGVM->hSelf = iHandle;
898	pGVM->pVM = NULL;
899	pGVM->cCpus = cCpus;
900	pGVM->pSession = pSession;
901
902	gvmmR0InitPerVMData(pGVM);
903	GMMR0InitPerVMData(pGVM);
904
905	/*
906	* Allocate the shared VM structure and associated page array.
907	*/
908	const uint32_t cbVM = RT_UOFFSETOF(VM, aCpus[cCpus]);
909	const uint32_t cPages = RT_ALIGN_32(cbVM, PAGE_SIZE) >> PAGE_SHIFT;
910	rc = RTR0MemObjAllocLow(&pGVM->gvmm.s.VMMemObj, cPages << PAGE_SHIFT, false /* fExecutable */);
911	if (RT_SUCCESS(rc))
912	{
913	PVM pVM = (PVM)RTR0MemObjAddress(pGVM->gvmm.s.VMMemObj); AssertPtr(pVM);
914	memset(pVM, 0, cPages << PAGE_SHIFT);
915	pVM->enmVMState = VMSTATE_CREATING;
916	pVM->pVMR0 = pVM;
917	pVM->pSession = pSession;
918	pVM->hSelf = iHandle;
919	pVM->cbSelf = cbVM;
920	pVM->cCpus = cCpus;
921	pVM->uCpuExecutionCap = 100; /* default is no cap. */
922	pVM->offVMCPU = RT_UOFFSETOF(VM, aCpus);
923	AssertCompileMemberAlignment(VM, cpum, 64);
924	AssertCompileMemberAlignment(VM, tm, 64);
925	AssertCompileMemberAlignment(VM, aCpus, PAGE_SIZE);
926
927	rc = RTR0MemObjAllocPage(&pGVM->gvmm.s.VMPagesMemObj, cPages * sizeof(SUPPAGE), false /* fExecutable */);
928	if (RT_SUCCESS(rc))
929	{
930	PSUPPAGE paPages = (PSUPPAGE)RTR0MemObjAddress(pGVM->gvmm.s.VMPagesMemObj); AssertPtr(paPages);
931	for (uint32_t iPage = 0; iPage < cPages; iPage++)
932	{
933	paPages[iPage].uReserved = 0;
934	paPages[iPage].Phys = RTR0MemObjGetPagePhysAddr(pGVM->gvmm.s.VMMemObj, iPage);
935	Assert(paPages[iPage].Phys != NIL_RTHCPHYS);
936	}
937
938	/*
939	* Map them into ring-3.
940	*/
941	rc = RTR0MemObjMapUser(&pGVM->gvmm.s.VMMapObj, pGVM->gvmm.s.VMMemObj, (RTR3PTR)-1, 0,
942	RTMEM_PROT_READ \| RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
943	if (RT_SUCCESS(rc))
944	{
945	pVM->pVMR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMMapObj);
946	AssertPtr((void *)pVM->pVMR3);
947
948	/* Initialize all the VM pointers. */
949	for (uint32_t i = 0; i < cCpus; i++)
950	{
951	pVM->aCpus[i].pVMR0 = pVM;
952	pVM->aCpus[i].pVMR3 = pVM->pVMR3;
953	pVM->aCpus[i].idHostCpu = NIL_RTCPUID;
954	pVM->aCpus[i].hNativeThreadR0 = NIL_RTNATIVETHREAD;
955	}
956
957	rc = RTR0MemObjMapUser(&pGVM->gvmm.s.VMPagesMapObj, pGVM->gvmm.s.VMPagesMemObj, (RTR3PTR)-1,
958	0 /* uAlignment */, RTMEM_PROT_READ \| RTMEM_PROT_WRITE,
959	NIL_RTR0PROCESS);
960	if (RT_SUCCESS(rc))
961	{
962	pVM->paVMPagesR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMPagesMapObj);
963	AssertPtr((void *)pVM->paVMPagesR3);
964
965	/* complete the handle - take the UsedLock sem just to be careful. */
966	rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
967	AssertRC(rc);
968
969	pHandle->pVM = pVM;
970	pHandle->pGVM = pGVM;
971	pHandle->hEMT0 = hEMT0;
972	pHandle->ProcId = ProcId;
973	pGVM->pVM = pVM;
974	pGVM->aCpus[0].hEMT = hEMT0;
975	pVM->aCpus[0].hNativeThreadR0 = hEMT0;
976	pGVMM->cEMTs += cCpus;
977
978	/* Associate it with the session and create the context hook for EMT0. */
979	rc = SUPR0SetSessionVM(pSession, pGVM, pVM);
980	if (RT_SUCCESS(rc))
981	{
982	rc = VMMR0ThreadCtxHookCreateForEmt(&pVM->aCpus[0]);
983	if (RT_SUCCESS(rc))
984	{
985	/*
986	* Done!
987	*/
988	VBOXVMM_R0_GVMM_VM_CREATED(pGVM, pVM, ProcId, (void *)hEMT0, cCpus);
989
990	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
991	gvmmR0CreateDestroyUnlock(pGVMM);
992
993	CPUMR0RegisterVCpuThread(&pVM->aCpus[0]);
994
995	*ppVM = pVM;
996	Log(("GVMMR0CreateVM: pVM=%p pVMR3=%p pGVM=%p hGVM=%d\n", pVM, pVM->pVMR3, pGVM, iHandle));
997	return VINF_SUCCESS;
998	}
999
1000	SUPR0SetSessionVM(pSession, NULL, NULL);
1001	}
1002	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1003	}
1004
1005	RTR0MemObjFree(pGVM->gvmm.s.VMMapObj, false /* fFreeMappings */);
1006	pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
1007	}
1008	RTR0MemObjFree(pGVM->gvmm.s.VMPagesMemObj, false /* fFreeMappings */);
1009	pGVM->gvmm.s.VMPagesMemObj = NIL_RTR0MEMOBJ;
1010	}
1011	RTR0MemObjFree(pGVM->gvmm.s.VMMemObj, false /* fFreeMappings */);
1012	pGVM->gvmm.s.VMMemObj = NIL_RTR0MEMOBJ;
1013	}
1014	}
1015	}
1016	/* else: The user wasn't permitted to create this VM. */
1017
1018	/*
1019	* The handle will be freed by gvmmR0HandleObjDestructor as we release the
1020	* object reference here. A little extra mess because of non-recursive lock.
1021	*/
1022	void *pvObj = pHandle->pvObj;
1023	pHandle->pvObj = NULL;
1024	gvmmR0CreateDestroyUnlock(pGVMM);
1025
1026	SUPR0ObjRelease(pvObj, pSession);
1027
1028	SUPR0Printf("GVMMR0CreateVM: failed, rc=%d\n", rc);
1029	return rc;
1030	}
1031
1032	rc = VERR_NO_MEMORY;
1033	}
1034	else
1035	rc = VERR_GVMM_IPE_1;
1036	}
1037	else
1038	rc = VERR_GVM_TOO_MANY_VMS;
1039
1040	gvmmR0CreateDestroyUnlock(pGVMM);
1041	return rc;
1042	}
1043
1044
1045	/**
1046	* Initializes the per VM data belonging to GVMM.
1047	*
1048	* @param pGVM Pointer to the global VM structure.
1049	*/
1050	static void gvmmR0InitPerVMData(PGVM pGVM)
1051	{
1052	AssertCompile(RT_SIZEOFMEMB(GVM,gvmm.s) <= RT_SIZEOFMEMB(GVM,gvmm.padding));
1053	AssertCompile(RT_SIZEOFMEMB(GVMCPU,gvmm.s) <= RT_SIZEOFMEMB(GVMCPU,gvmm.padding));
1054	pGVM->gvmm.s.VMMemObj = NIL_RTR0MEMOBJ;
1055	pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
1056	pGVM->gvmm.s.VMPagesMemObj = NIL_RTR0MEMOBJ;
1057	pGVM->gvmm.s.VMPagesMapObj = NIL_RTR0MEMOBJ;
1058	pGVM->gvmm.s.fDoneVMMR0Init = false;
1059	pGVM->gvmm.s.fDoneVMMR0Term = false;
1060
1061	for (VMCPUID i = 0; i < pGVM->cCpus; i++)
1062	{
1063	pGVM->aCpus[i].gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
1064	pGVM->aCpus[i].hEMT = NIL_RTNATIVETHREAD;
1065	}
1066	}
1067
1068
1069	/**
1070	* Does the VM initialization.
1071	*
1072	* @returns VBox status code.
1073	* @param pGVM The global (ring-0) VM structure.
1074	*/
1075	GVMMR0DECL(int) GVMMR0InitVM(PGVM pGVM)
1076	{
1077	LogFlow(("GVMMR0InitVM: pGVM=%p\n", pGVM));
1078
1079	int rc = VERR_INTERNAL_ERROR_3;
1080	if ( !pGVM->gvmm.s.fDoneVMMR0Init
1081	&& pGVM->aCpus[0].gvmm.s.HaltEventMulti == NIL_RTSEMEVENTMULTI)
1082	{
1083	for (VMCPUID i = 0; i < pGVM->cCpus; i++)
1084	{
1085	rc = RTSemEventMultiCreate(&pGVM->aCpus[i].gvmm.s.HaltEventMulti);
1086	if (RT_FAILURE(rc))
1087	{
1088	pGVM->aCpus[i].gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
1089	break;
1090	}
1091	}
1092	}
1093	else
1094	rc = VERR_WRONG_ORDER;
1095
1096	LogFlow(("GVMMR0InitVM: returns %Rrc\n", rc));
1097	return rc;
1098	}
1099
1100
1101	/**
1102	* Indicates that we're done with the ring-0 initialization
1103	* of the VM.
1104	*
1105	* @param pGVM The global (ring-0) VM structure.
1106	* @thread EMT(0)
1107	*/
1108	GVMMR0DECL(void) GVMMR0DoneInitVM(PGVM pGVM)
1109	{
1110	/* Set the indicator. */
1111	pGVM->gvmm.s.fDoneVMMR0Init = true;
1112	}
1113
1114
1115	/**
1116	* Indicates that we're doing the ring-0 termination of the VM.
1117	*
1118	* @returns true if termination hasn't been done already, false if it has.
1119	* @param pGVM Pointer to the global VM structure. Optional.
1120	* @thread EMT(0) or session cleanup thread.
1121	*/
1122	GVMMR0DECL(bool) GVMMR0DoingTermVM(PGVM pGVM)
1123	{
1124	/* Validate the VM structure, state and handle. */
1125	AssertPtrReturn(pGVM, false);
1126
1127	/* Set the indicator. */
1128	if (pGVM->gvmm.s.fDoneVMMR0Term)
1129	return false;
1130	pGVM->gvmm.s.fDoneVMMR0Term = true;
1131	return true;
1132	}
1133
1134
1135	/**
1136	* Destroys the VM, freeing all associated resources (the ring-0 ones anyway).
1137	*
1138	* This is call from the vmR3DestroyFinalBit and from a error path in VMR3Create,
1139	* and the caller is not the EMT thread, unfortunately. For security reasons, it
1140	* would've been nice if the caller was actually the EMT thread or that we somehow
1141	* could've associated the calling thread with the VM up front.
1142	*
1143	* @returns VBox status code.
1144	* @param pGVM The global (ring-0) VM structure.
1145	* @param pVM The cross context VM structure.
1146	*
1147	* @thread EMT(0) if it's associated with the VM, otherwise any thread.
1148	*/
1149	GVMMR0DECL(int) GVMMR0DestroyVM(PGVM pGVM, PVM pVM)
1150	{
1151	LogFlow(("GVMMR0DestroyVM: pGVM=%p pVM=%p\n", pGVM, pVM));
1152	PGVMM pGVMM;
1153	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
1154
1155	/*
1156	* Validate the VM structure, state and caller.
1157	*/
1158	AssertPtrReturn(pGVM, VERR_INVALID_POINTER);
1159	AssertPtrReturn(pVM, VERR_INVALID_POINTER);
1160	AssertReturn(!((uintptr_t)pVM & PAGE_OFFSET_MASK), VERR_INVALID_POINTER);
1161	AssertReturn(pGVM->pVM == pVM, VERR_INVALID_POINTER);
1162	AssertMsgReturn(pVM->enmVMState >= VMSTATE_CREATING && pVM->enmVMState <= VMSTATE_TERMINATED, ("%d\n", pVM->enmVMState),
1163	VERR_WRONG_ORDER);
1164
1165	uint32_t hGVM = pGVM->hSelf;
1166	AssertReturn(hGVM != NIL_GVM_HANDLE, VERR_INVALID_VM_HANDLE);
1167	AssertReturn(hGVM < RT_ELEMENTS(pGVMM->aHandles), VERR_INVALID_VM_HANDLE);
1168
1169	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1170	AssertReturn(pHandle->pVM == pVM, VERR_NOT_OWNER);
1171
1172	RTPROCESS ProcId = RTProcSelf();
1173	RTNATIVETHREAD hSelf = RTThreadNativeSelf();
1174	AssertReturn( ( pHandle->hEMT0 == hSelf
1175	&& pHandle->ProcId == ProcId)
1176	\|\| pHandle->hEMT0 == NIL_RTNATIVETHREAD, VERR_NOT_OWNER);
1177
1178	/*
1179	* Lookup the handle and destroy the object.
1180	* Since the lock isn't recursive and we'll have to leave it before dereferencing the
1181	* object, we take some precautions against racing callers just in case...
1182	*/
1183	int rc = gvmmR0CreateDestroyLock(pGVMM);
1184	AssertRC(rc);
1185
1186	/* Be careful here because we might theoretically be racing someone else cleaning up. */
1187	if ( pHandle->pVM == pVM
1188	&& ( ( pHandle->hEMT0 == hSelf
1189	&& pHandle->ProcId == ProcId)
1190	\|\| pHandle->hEMT0 == NIL_RTNATIVETHREAD)
1191	&& VALID_PTR(pHandle->pvObj)
1192	&& VALID_PTR(pHandle->pSession)
1193	&& VALID_PTR(pHandle->pGVM)
1194	&& pHandle->pGVM->u32Magic == GVM_MAGIC)
1195	{
1196	/* Check that other EMTs have deregistered. */
1197	uint32_t cNotDeregistered = 0;
1198	for (VMCPUID idCpu = 1; idCpu < pGVM->cCpus; idCpu++)
1199	cNotDeregistered += pGVM->aCpus[idCpu].hEMT != ~(RTNATIVETHREAD)1; /* see GVMMR0DeregisterVCpu for the value */
1200	if (cNotDeregistered == 0)
1201	{
1202	/* Grab the object pointer. */
1203	void *pvObj = pHandle->pvObj;
1204	pHandle->pvObj = NULL;
1205	gvmmR0CreateDestroyUnlock(pGVMM);
1206
1207	SUPR0ObjRelease(pvObj, pHandle->pSession);
1208	}
1209	else
1210	{
1211	gvmmR0CreateDestroyUnlock(pGVMM);
1212	rc = VERR_GVMM_NOT_ALL_EMTS_DEREGISTERED;
1213	}
1214	}
1215	else
1216	{
1217	SUPR0Printf("GVMMR0DestroyVM: pHandle=%RKv:{.pVM=%p, .hEMT0=%p, .ProcId=%u, .pvObj=%p} pVM=%p hSelf=%p\n",
1218	pHandle, pHandle->pVM, pHandle->hEMT0, pHandle->ProcId, pHandle->pvObj, pVM, hSelf);
1219	gvmmR0CreateDestroyUnlock(pGVMM);
1220	rc = VERR_GVMM_IPE_2;
1221	}
1222
1223	return rc;
1224	}
1225
1226
1227	/**
1228	* Performs VM cleanup task as part of object destruction.
1229	*
1230	* @param pGVM The GVM pointer.
1231	*/
1232	static void gvmmR0CleanupVM(PGVM pGVM)
1233	{
1234	if ( pGVM->gvmm.s.fDoneVMMR0Init
1235	&& !pGVM->gvmm.s.fDoneVMMR0Term)
1236	{
1237	if ( pGVM->gvmm.s.VMMemObj != NIL_RTR0MEMOBJ
1238	&& RTR0MemObjAddress(pGVM->gvmm.s.VMMemObj) == pGVM->pVM)
1239	{
1240	LogFlow(("gvmmR0CleanupVM: Calling VMMR0TermVM\n"));
1241	VMMR0TermVM(pGVM, pGVM->pVM, NIL_VMCPUID);
1242	}
1243	else
1244	AssertMsgFailed(("gvmmR0CleanupVM: VMMemObj=%p pVM=%p\n", pGVM->gvmm.s.VMMemObj, pGVM->pVM));
1245	}
1246
1247	GMMR0CleanupVM(pGVM);
1248	#ifdef VBOX_WITH_NEM_R0
1249	NEMR0CleanupVM(pGVM);
1250	#endif
1251
1252	AssertCompile((uintptr_t)NIL_RTTHREADCTXHOOK == 0); /* Depends on zero initialized memory working for NIL at the moment. */
1253	for (VMCPUID idCpu = 0; idCpu < pGVM->cCpus; idCpu++)
1254	{
1255	/** @todo Can we busy wait here for all thread-context hooks to be
1256	* deregistered before releasing (destroying) it? Only until we find a
1257	* solution for not deregistering hooks everytime we're leaving HMR0
1258	* context. */
1259	VMMR0ThreadCtxHookDestroyForEmt(&pGVM->pVM->aCpus[idCpu]);
1260	}
1261	}
1262
1263
1264	/**
1265	* @callback_method_impl{FNSUPDRVDESTRUCTOR,VM handle destructor}
1266	*
1267	* pvUser1 is the GVM instance pointer.
1268	* pvUser2 is the handle pointer.
1269	*/
1270	static DECLCALLBACK(void) gvmmR0HandleObjDestructor(void pvObj, void pvUser1, void *pvUser2)
1271	{
1272	LogFlow(("gvmmR0HandleObjDestructor: %p %p %p\n", pvObj, pvUser1, pvUser2));
1273
1274	NOREF(pvObj);
1275
1276	/*
1277	* Some quick, paranoid, input validation.
1278	*/
1279	PGVMHANDLE pHandle = (PGVMHANDLE)pvUser2;
1280	AssertPtr(pHandle);
1281	PGVMM pGVMM = (PGVMM)pvUser1;
1282	Assert(pGVMM == g_pGVMM);
1283	const uint16_t iHandle = pHandle - &pGVMM->aHandles[0];
1284	if ( !iHandle
1285	\|\| iHandle >= RT_ELEMENTS(pGVMM->aHandles)
1286	\|\| iHandle != pHandle->iSelf)
1287	{
1288	SUPR0Printf("GVM: handle %d is out of range or corrupt (iSelf=%d)!\n", iHandle, pHandle->iSelf);
1289	return;
1290	}
1291
1292	int rc = gvmmR0CreateDestroyLock(pGVMM);
1293	AssertRC(rc);
1294	rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
1295	AssertRC(rc);
1296
1297	/*
1298	* This is a tad slow but a doubly linked list is too much hassle.
1299	*/
1300	if (RT_UNLIKELY(pHandle->iNext >= RT_ELEMENTS(pGVMM->aHandles)))
1301	{
1302	SUPR0Printf("GVM: used list index %d is out of range!\n", pHandle->iNext);
1303	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1304	gvmmR0CreateDestroyUnlock(pGVMM);
1305	return;
1306	}
1307
1308	if (pGVMM->iUsedHead == iHandle)
1309	pGVMM->iUsedHead = pHandle->iNext;
1310	else
1311	{
1312	uint16_t iPrev = pGVMM->iUsedHead;
1313	int c = RT_ELEMENTS(pGVMM->aHandles) + 2;
1314	while (iPrev)
1315	{
1316	if (RT_UNLIKELY(iPrev >= RT_ELEMENTS(pGVMM->aHandles)))
1317	{
1318	SUPR0Printf("GVM: used list index %d is out of range!\n", iPrev);
1319	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1320	gvmmR0CreateDestroyUnlock(pGVMM);
1321	return;
1322	}
1323	if (RT_UNLIKELY(c-- <= 0))
1324	{
1325	iPrev = 0;
1326	break;
1327	}
1328
1329	if (pGVMM->aHandles[iPrev].iNext == iHandle)
1330	break;
1331	iPrev = pGVMM->aHandles[iPrev].iNext;
1332	}
1333	if (!iPrev)
1334	{
1335	SUPR0Printf("GVM: can't find the handle previous previous of %d!\n", pHandle->iSelf);
1336	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1337	gvmmR0CreateDestroyUnlock(pGVMM);
1338	return;
1339	}
1340
1341	Assert(pGVMM->aHandles[iPrev].iNext == iHandle);
1342	pGVMM->aHandles[iPrev].iNext = pHandle->iNext;
1343	}
1344	pHandle->iNext = 0;
1345	pGVMM->cVMs--;
1346
1347	/*
1348	* Do the global cleanup round.
1349	*/
1350	PGVM pGVM = pHandle->pGVM;
1351	if ( VALID_PTR(pGVM)
1352	&& pGVM->u32Magic == GVM_MAGIC)
1353	{
1354	pGVMM->cEMTs -= pGVM->cCpus;
1355
1356	if (pGVM->pSession)
1357	SUPR0SetSessionVM(pGVM->pSession, NULL, NULL);
1358
1359	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1360
1361	gvmmR0CleanupVM(pGVM);
1362
1363	/*
1364	* Do the GVMM cleanup - must be done last.
1365	*/
1366	/* The VM and VM pages mappings/allocations. */
1367	if (pGVM->gvmm.s.VMPagesMapObj != NIL_RTR0MEMOBJ)
1368	{
1369	rc = RTR0MemObjFree(pGVM->gvmm.s.VMPagesMapObj, false /* fFreeMappings */); AssertRC(rc);
1370	pGVM->gvmm.s.VMPagesMapObj = NIL_RTR0MEMOBJ;
1371	}
1372
1373	if (pGVM->gvmm.s.VMMapObj != NIL_RTR0MEMOBJ)
1374	{
1375	rc = RTR0MemObjFree(pGVM->gvmm.s.VMMapObj, false /* fFreeMappings */); AssertRC(rc);
1376	pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
1377	}
1378
1379	if (pGVM->gvmm.s.VMPagesMemObj != NIL_RTR0MEMOBJ)
1380	{
1381	rc = RTR0MemObjFree(pGVM->gvmm.s.VMPagesMemObj, false /* fFreeMappings */); AssertRC(rc);
1382	pGVM->gvmm.s.VMPagesMemObj = NIL_RTR0MEMOBJ;
1383	}
1384
1385	if (pGVM->gvmm.s.VMMemObj != NIL_RTR0MEMOBJ)
1386	{
1387	rc = RTR0MemObjFree(pGVM->gvmm.s.VMMemObj, false /* fFreeMappings */); AssertRC(rc);
1388	pGVM->gvmm.s.VMMemObj = NIL_RTR0MEMOBJ;
1389	}
1390
1391	for (VMCPUID i = 0; i < pGVM->cCpus; i++)
1392	{
1393	if (pGVM->aCpus[i].gvmm.s.HaltEventMulti != NIL_RTSEMEVENTMULTI)
1394	{
1395	rc = RTSemEventMultiDestroy(pGVM->aCpus[i].gvmm.s.HaltEventMulti); AssertRC(rc);
1396	pGVM->aCpus[i].gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
1397	}
1398	}
1399
1400	/* the GVM structure itself. */
1401	pGVM->u32Magic \|= UINT32_C(0x80000000);
1402	RTMemFree(pGVM);
1403
1404	/* Re-acquire the UsedLock before freeing the handle since we're updating handle fields. */
1405	rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
1406	AssertRC(rc);
1407	}
1408	/* else: GVMMR0CreateVM cleanup. */
1409
1410	/*
1411	* Free the handle.
1412	*/
1413	pHandle->iNext = pGVMM->iFreeHead;
1414	pGVMM->iFreeHead = iHandle;
1415	ASMAtomicWriteNullPtr(&pHandle->pGVM);
1416	ASMAtomicWriteNullPtr(&pHandle->pVM);
1417	ASMAtomicWriteNullPtr(&pHandle->pvObj);
1418	ASMAtomicWriteNullPtr(&pHandle->pSession);
1419	ASMAtomicWriteHandle(&pHandle->hEMT0, NIL_RTNATIVETHREAD);
1420	ASMAtomicWriteU32(&pHandle->ProcId, NIL_RTPROCESS);
1421
1422	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1423	gvmmR0CreateDestroyUnlock(pGVMM);
1424	LogFlow(("gvmmR0HandleObjDestructor: returns\n"));
1425	}
1426
1427
1428	/**
1429	* Registers the calling thread as the EMT of a Virtual CPU.
1430	*
1431	* Note that VCPU 0 is automatically registered during VM creation.
1432	*
1433	* @returns VBox status code
1434	* @param pGVM The global (ring-0) VM structure.
1435	* @param pVM The cross context VM structure.
1436	* @param idCpu VCPU id to register the current thread as.
1437	*/
1438	GVMMR0DECL(int) GVMMR0RegisterVCpu(PGVM pGVM, PVM pVM, VMCPUID idCpu)
1439	{
1440	AssertReturn(idCpu != 0, VERR_INVALID_FUNCTION);
1441
1442	/*
1443	* Validate the VM structure, state and handle.
1444	*/
1445	PGVMM pGVMM;
1446	int rc = gvmmR0ByGVMandVM(pGVM, pVM, &pGVMM, false /* fTakeUsedLock /); /* @todo take lock here. */
1447	if (RT_SUCCESS(rc))
1448	{
1449	if (idCpu < pGVM->cCpus)
1450	{
1451	/* Check that the EMT isn't already assigned to a thread. */
1452	if (pGVM->aCpus[idCpu].hEMT == NIL_RTNATIVETHREAD)
1453	{
1454	Assert(pVM->aCpus[idCpu].hNativeThreadR0 == NIL_RTNATIVETHREAD);
1455
1456	/* A thread may only be one EMT. */
1457	RTNATIVETHREAD const hNativeSelf = RTThreadNativeSelf();
1458	for (VMCPUID iCpu = 0; iCpu < pGVM->cCpus; iCpu++)
1459	AssertBreakStmt(pGVM->aCpus[iCpu].hEMT != hNativeSelf, rc = VERR_INVALID_PARAMETER);
1460	if (RT_SUCCESS(rc))
1461	{
1462	/*
1463	* Do the assignment, then try setup the hook. Undo if that fails.
1464	*/
1465	pVM->aCpus[idCpu].hNativeThreadR0 = pGVM->aCpus[idCpu].hEMT = RTThreadNativeSelf();
1466
1467	rc = VMMR0ThreadCtxHookCreateForEmt(&pVM->aCpus[idCpu]);
1468	if (RT_SUCCESS(rc))
1469	CPUMR0RegisterVCpuThread(&pVM->aCpus[idCpu]);
1470	else
1471	pVM->aCpus[idCpu].hNativeThreadR0 = pGVM->aCpus[idCpu].hEMT = NIL_RTNATIVETHREAD;
1472	}
1473	}
1474	else
1475	rc = VERR_ACCESS_DENIED;
1476	}
1477	else
1478	rc = VERR_INVALID_CPU_ID;
1479	}
1480	return rc;
1481	}
1482
1483
1484	/**
1485	* Deregisters the calling thread as the EMT of a Virtual CPU.
1486	*
1487	* Note that VCPU 0 shall call GVMMR0DestroyVM intead of this API.
1488	*
1489	* @returns VBox status code
1490	* @param pGVM The global (ring-0) VM structure.
1491	* @param pVM The cross context VM structure.
1492	* @param idCpu VCPU id to register the current thread as.
1493	*/
1494	GVMMR0DECL(int) GVMMR0DeregisterVCpu(PGVM pGVM, PVM pVM, VMCPUID idCpu)
1495	{
1496	AssertReturn(idCpu != 0, VERR_INVALID_FUNCTION);
1497
1498	/*
1499	* Validate the VM structure, state and handle.
1500	*/
1501	PGVMM pGVMM;
1502	int rc = gvmmR0ByGVMandVMandEMT(pGVM, pVM, idCpu, &pGVMM);
1503	if (RT_SUCCESS(rc))
1504	{
1505	/*
1506	* Take the destruction lock and recheck the handle state to
1507	* prevent racing GVMMR0DestroyVM.
1508	*/
1509	gvmmR0CreateDestroyLock(pGVMM);
1510	uint32_t hSelf = pGVM->hSelf;
1511	if ( hSelf < RT_ELEMENTS(pGVMM->aHandles)
1512	&& pGVMM->aHandles[hSelf].pvObj != NULL
1513	&& pGVMM->aHandles[hSelf].pGVM == pGVM)
1514	{
1515	/*
1516	* Do per-EMT cleanups.
1517	*/
1518	VMMR0ThreadCtxHookDestroyForEmt(&pVM->aCpus[idCpu]);
1519
1520	/*
1521	* Invalidate hEMT. We don't use NIL here as that would allow
1522	* GVMMR0RegisterVCpu to be called again, and we don't want that.
1523	*/
1524	AssertCompile(~(RTNATIVETHREAD)1 != NIL_RTNATIVETHREAD);
1525	pGVM->aCpus[idCpu].hEMT = ~(RTNATIVETHREAD)1;
1526	pVM->aCpus[idCpu].hNativeThreadR0 = NIL_RTNATIVETHREAD;
1527	}
1528
1529	gvmmR0CreateDestroyUnlock(pGVMM);
1530	}
1531	return rc;
1532	}
1533
1534
1535	/**
1536	* Lookup a GVM structure by its handle.
1537	*
1538	* @returns The GVM pointer on success, NULL on failure.
1539	* @param hGVM The global VM handle. Asserts on bad handle.
1540	*/
1541	GVMMR0DECL(PGVM) GVMMR0ByHandle(uint32_t hGVM)
1542	{
1543	PGVMM pGVMM;
1544	GVMM_GET_VALID_INSTANCE(pGVMM, NULL);
1545
1546	/*
1547	* Validate.
1548	*/
1549	AssertReturn(hGVM != NIL_GVM_HANDLE, NULL);
1550	AssertReturn(hGVM < RT_ELEMENTS(pGVMM->aHandles), NULL);
1551
1552	/*
1553	* Look it up.
1554	*/
1555	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1556	AssertPtrReturn(pHandle->pVM, NULL);
1557	AssertPtrReturn(pHandle->pvObj, NULL);
1558	PGVM pGVM = pHandle->pGVM;
1559	AssertPtrReturn(pGVM, NULL);
1560	AssertReturn(pGVM->pVM == pHandle->pVM, NULL);
1561
1562	return pHandle->pGVM;
1563	}
1564
1565
1566	/**
1567	* Lookup a GVM structure by the shared VM structure.
1568	*
1569	* The calling thread must be in the same process as the VM. All current lookups
1570	* are by threads inside the same process, so this will not be an issue.
1571	*
1572	* @returns VBox status code.
1573	* @param pVM The cross context VM structure.
1574	* @param ppGVM Where to store the GVM pointer.
1575	* @param ppGVMM Where to store the pointer to the GVMM instance data.
1576	* @param fTakeUsedLock Whether to take the used lock or not. We take it in
1577	* shared mode when requested.
1578	*
1579	* Be very careful if not taking the lock as it's
1580	* possible that the VM will disappear then!
1581	*
1582	* @remark This will not assert on an invalid pVM but try return silently.
1583	*/
1584	static int gvmmR0ByVM(PVM pVM, PGVM ppGVM, PGVMM ppGVMM, bool fTakeUsedLock)
1585	{
1586	RTPROCESS ProcId = RTProcSelf();
1587	PGVMM pGVMM;
1588	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
1589
1590	/*
1591	* Validate.
1592	*/
1593	if (RT_UNLIKELY( !VALID_PTR(pVM)
1594	\|\| ((uintptr_t)pVM & PAGE_OFFSET_MASK)))
1595	return VERR_INVALID_POINTER;
1596	if (RT_UNLIKELY( pVM->enmVMState < VMSTATE_CREATING
1597	\|\| pVM->enmVMState >= VMSTATE_TERMINATED))
1598	return VERR_INVALID_POINTER;
1599
1600	uint16_t hGVM = pVM->hSelf;
1601	if (RT_UNLIKELY( hGVM == NIL_GVM_HANDLE
1602	\|\| hGVM >= RT_ELEMENTS(pGVMM->aHandles)))
1603	return VERR_INVALID_HANDLE;
1604
1605	/*
1606	* Look it up.
1607	*/
1608	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1609	PGVM pGVM;
1610	if (fTakeUsedLock)
1611	{
1612	int rc = GVMMR0_USED_SHARED_LOCK(pGVMM);
1613	AssertRCReturn(rc, rc);
1614
1615	pGVM = pHandle->pGVM;
1616	if (RT_UNLIKELY( pHandle->pVM != pVM
1617	\|\| pHandle->ProcId != ProcId
1618	\|\| !VALID_PTR(pHandle->pvObj)
1619	\|\| !VALID_PTR(pGVM)
1620	\|\| pGVM->pVM != pVM))
1621	{
1622	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
1623	return VERR_INVALID_HANDLE;
1624	}
1625	}
1626	else
1627	{
1628	if (RT_UNLIKELY(pHandle->pVM != pVM))
1629	return VERR_INVALID_HANDLE;
1630	if (RT_UNLIKELY(pHandle->ProcId != ProcId))
1631	return VERR_INVALID_HANDLE;
1632	if (RT_UNLIKELY(!VALID_PTR(pHandle->pvObj)))
1633	return VERR_INVALID_HANDLE;
1634
1635	pGVM = pHandle->pGVM;
1636	if (RT_UNLIKELY(!VALID_PTR(pGVM)))
1637	return VERR_INVALID_HANDLE;
1638	if (RT_UNLIKELY(pGVM->pVM != pVM))
1639	return VERR_INVALID_HANDLE;
1640	}
1641
1642	*ppGVM = pGVM;
1643	*ppGVMM = pGVMM;
1644	return VINF_SUCCESS;
1645	}
1646
1647
1648	/**
1649	* Check that the given GVM and VM structures match up.
1650	*
1651	* The calling thread must be in the same process as the VM. All current lookups
1652	* are by threads inside the same process, so this will not be an issue.
1653	*
1654	* @returns VBox status code.
1655	* @param pGVM The global (ring-0) VM structure.
1656	* @param pVM The cross context VM structure.
1657	* @param ppGVMM Where to store the pointer to the GVMM instance data.
1658	* @param fTakeUsedLock Whether to take the used lock or not. We take it in
1659	* shared mode when requested.
1660	*
1661	* Be very careful if not taking the lock as it's
1662	* possible that the VM will disappear then!
1663	*
1664	* @remark This will not assert on an invalid pVM but try return silently.
1665	*/
1666	static int gvmmR0ByGVMandVM(PGVM pGVM, PVM pVM, PGVMM *ppGVMM, bool fTakeUsedLock)
1667	{
1668	/*
1669	* Check the pointers.
1670	*/
1671	int rc;
1672	if (RT_LIKELY(RT_VALID_PTR(pGVM)))
1673	{
1674	if (RT_LIKELY( RT_VALID_PTR(pVM)
1675	&& ((uintptr_t)pVM & PAGE_OFFSET_MASK) == 0))
1676	{
1677	if (RT_LIKELY(pGVM->pVM == pVM))
1678	{
1679	/*
1680	* Get the pGVMM instance and check the VM handle.
1681	*/
1682	PGVMM pGVMM;
1683	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
1684
1685	uint16_t hGVM = pGVM->hSelf;
1686	if (RT_LIKELY( hGVM != NIL_GVM_HANDLE
1687	&& hGVM < RT_ELEMENTS(pGVMM->aHandles)))
1688	{
1689	RTPROCESS const pidSelf = RTProcSelf();
1690	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1691	if (fTakeUsedLock)
1692	{
1693	rc = GVMMR0_USED_SHARED_LOCK(pGVMM);
1694	AssertRCReturn(rc, rc);
1695	}
1696
1697	if (RT_LIKELY( pHandle->pGVM == pGVM
1698	&& pHandle->pVM == pVM
1699	&& pHandle->ProcId == pidSelf
1700	&& RT_VALID_PTR(pHandle->pvObj)))
1701	{
1702	/*
1703	* Some more VM data consistency checks.
1704	*/
1705	if (RT_LIKELY( pVM->cCpus == pGVM->cCpus
1706	&& pVM->hSelf == hGVM
1707	&& pVM->enmVMState >= VMSTATE_CREATING
1708	&& pVM->enmVMState <= VMSTATE_TERMINATED
1709	&& pVM->pVMR0 == pVM))
1710	{
1711	*ppGVMM = pGVMM;
1712	return VINF_SUCCESS;
1713	}
1714	}
1715
1716	if (fTakeUsedLock)
1717	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
1718	}
1719	}
1720	rc = VERR_INVALID_VM_HANDLE;
1721	}
1722	else
1723	rc = VERR_INVALID_POINTER;
1724	}
1725	else
1726	rc = VERR_INVALID_POINTER;
1727	return rc;
1728	}
1729
1730
1731	/**
1732	* Check that the given GVM and VM structures match up.
1733	*
1734	* The calling thread must be in the same process as the VM. All current lookups
1735	* are by threads inside the same process, so this will not be an issue.
1736	*
1737	* @returns VBox status code.
1738	* @param pGVM The global (ring-0) VM structure.
1739	* @param pVM The cross context VM structure.
1740	* @param idCpu The (alleged) Virtual CPU ID of the calling EMT.
1741	* @param ppGVMM Where to store the pointer to the GVMM instance data.
1742	* @thread EMT
1743	*
1744	* @remarks This will assert in all failure paths.
1745	*/
1746	static int gvmmR0ByGVMandVMandEMT(PGVM pGVM, PVM pVM, VMCPUID idCpu, PGVMM *ppGVMM)
1747	{
1748	/*
1749	* Check the pointers.
1750	*/
1751	AssertPtrReturn(pGVM, VERR_INVALID_POINTER);
1752
1753	AssertPtrReturn(pVM, VERR_INVALID_POINTER);
1754	AssertReturn(((uintptr_t)pVM & PAGE_OFFSET_MASK) == 0, VERR_INVALID_POINTER);
1755	AssertReturn(pGVM->pVM == pVM, VERR_INVALID_VM_HANDLE);
1756
1757
1758	/*
1759	* Get the pGVMM instance and check the VM handle.
1760	*/
1761	PGVMM pGVMM;
1762	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
1763
1764	uint16_t hGVM = pGVM->hSelf;
1765	AssertReturn( hGVM != NIL_GVM_HANDLE
1766	&& hGVM < RT_ELEMENTS(pGVMM->aHandles), VERR_INVALID_VM_HANDLE);
1767
1768	RTPROCESS const pidSelf = RTProcSelf();
1769	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1770	AssertReturn( pHandle->pGVM == pGVM
1771	&& pHandle->pVM == pVM
1772	&& pHandle->ProcId == pidSelf
1773	&& RT_VALID_PTR(pHandle->pvObj),
1774	VERR_INVALID_HANDLE);
1775
1776	/*
1777	* Check the EMT claim.
1778	*/
1779	RTNATIVETHREAD const hAllegedEMT = RTThreadNativeSelf();
1780	AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID);
1781	AssertReturn(pGVM->aCpus[idCpu].hEMT == hAllegedEMT, VERR_NOT_OWNER);
1782
1783	/*
1784	* Some more VM data consistency checks.
1785	*/
1786	AssertReturn(pVM->cCpus == pGVM->cCpus, VERR_INCONSISTENT_VM_HANDLE);
1787	AssertReturn(pVM->hSelf == hGVM, VERR_INCONSISTENT_VM_HANDLE);
1788	AssertReturn(pVM->pVMR0 == pVM, VERR_INCONSISTENT_VM_HANDLE);
1789	AssertReturn( pVM->enmVMState >= VMSTATE_CREATING
1790	&& pVM->enmVMState <= VMSTATE_TERMINATED, VERR_INCONSISTENT_VM_HANDLE);
1791
1792	*ppGVMM = pGVMM;
1793	return VINF_SUCCESS;
1794	}
1795
1796
1797	/**
1798	* Validates a GVM/VM pair.
1799	*
1800	* @returns VBox status code.
1801	* @param pGVM The global (ring-0) VM structure.
1802	* @param pVM The cross context VM structure.
1803	*/
1804	GVMMR0DECL(int) GVMMR0ValidateGVMandVM(PGVM pGVM, PVM pVM)
1805	{
1806	PGVMM pGVMM;
1807	return gvmmR0ByGVMandVM(pGVM, pVM, &pGVMM, false /fTakeUsedLock/);
1808	}
1809
1810
1811
1812	/**
1813	* Validates a GVM/VM/EMT combo.
1814	*
1815	* @returns VBox status code.
1816	* @param pGVM The global (ring-0) VM structure.
1817	* @param pVM The cross context VM structure.
1818	* @param idCpu The Virtual CPU ID of the calling EMT.
1819	* @thread EMT(idCpu)
1820	*/
1821	GVMMR0DECL(int) GVMMR0ValidateGVMandVMandEMT(PGVM pGVM, PVM pVM, VMCPUID idCpu)
1822	{
1823	PGVMM pGVMM;
1824	return gvmmR0ByGVMandVMandEMT(pGVM, pVM, idCpu, &pGVMM);
1825	}
1826
1827
1828	/**
1829	* Looks up the VM belonging to the specified EMT thread.
1830	*
1831	* This is used by the assertion machinery in VMMR0.cpp to avoid causing
1832	* unnecessary kernel panics when the EMT thread hits an assertion. The
1833	* call may or not be an EMT thread.
1834	*
1835	* @returns Pointer to the VM on success, NULL on failure.
1836	* @param hEMT The native thread handle of the EMT.
1837	* NIL_RTNATIVETHREAD means the current thread
1838	*/
1839	GVMMR0DECL(PVM) GVMMR0GetVMByEMT(RTNATIVETHREAD hEMT)
1840	{
1841	/*
1842	* No Assertions here as we're usually called in a AssertMsgN or
1843	* RTAssert* context.
1844	*/
1845	PGVMM pGVMM = g_pGVMM;
1846	if ( !VALID_PTR(pGVMM)
1847	\|\| pGVMM->u32Magic != GVMM_MAGIC)
1848	return NULL;
1849
1850	if (hEMT == NIL_RTNATIVETHREAD)
1851	hEMT = RTThreadNativeSelf();
1852	RTPROCESS ProcId = RTProcSelf();
1853
1854	/*
1855	* Search the handles in a linear fashion as we don't dare to take the lock (assert).
1856	*/
1857	for (unsigned i = 1; i < RT_ELEMENTS(pGVMM->aHandles); i++)
1858	{
1859	if ( pGVMM->aHandles[i].iSelf == i
1860	&& pGVMM->aHandles[i].ProcId == ProcId
1861	&& VALID_PTR(pGVMM->aHandles[i].pvObj)
1862	&& VALID_PTR(pGVMM->aHandles[i].pVM)
1863	&& VALID_PTR(pGVMM->aHandles[i].pGVM))
1864	{
1865	if (pGVMM->aHandles[i].hEMT0 == hEMT)
1866	return pGVMM->aHandles[i].pVM;
1867
1868	/* This is fearly safe with the current process per VM approach. */
1869	PGVM pGVM = pGVMM->aHandles[i].pGVM;
1870	VMCPUID const cCpus = pGVM->cCpus;
1871	if ( cCpus < 1
1872	\|\| cCpus > VMM_MAX_CPU_COUNT)
1873	continue;
1874	for (VMCPUID idCpu = 1; idCpu < cCpus; idCpu++)
1875	if (pGVM->aCpus[idCpu].hEMT == hEMT)
1876	return pGVMM->aHandles[i].pVM;
1877	}
1878	}
1879	return NULL;
1880	}
1881
1882
1883	/**
1884	* This is will wake up expired and soon-to-be expired VMs.
1885	*
1886	* @returns Number of VMs that has been woken up.
1887	* @param pGVMM Pointer to the GVMM instance data.
1888	* @param u64Now The current time.
1889	*/
1890	static unsigned gvmmR0SchedDoWakeUps(PGVMM pGVMM, uint64_t u64Now)
1891	{
1892	/*
1893	* Skip this if we've got disabled because of high resolution wakeups or by
1894	* the user.
1895	*/
1896	if (!pGVMM->fDoEarlyWakeUps)
1897	return 0;
1898
1899	/** @todo Rewrite this algorithm. See performance defect XYZ. */
1900
1901	/*
1902	* A cheap optimization to stop wasting so much time here on big setups.
1903	*/
1904	const uint64_t uNsEarlyWakeUp2 = u64Now + pGVMM->nsEarlyWakeUp2;
1905	if ( pGVMM->cHaltedEMTs == 0
1906	\|\| uNsEarlyWakeUp2 > pGVMM->uNsNextEmtWakeup)
1907	return 0;
1908
1909	/*
1910	* Only one thread doing this at a time.
1911	*/
1912	if (!ASMAtomicCmpXchgBool(&pGVMM->fDoingEarlyWakeUps, true, false))
1913	return 0;
1914
1915	/*
1916	* The first pass will wake up VMs which have actually expired
1917	* and look for VMs that should be woken up in the 2nd and 3rd passes.
1918	*/
1919	const uint64_t uNsEarlyWakeUp1 = u64Now + pGVMM->nsEarlyWakeUp1;
1920	uint64_t u64Min = UINT64_MAX;
1921	unsigned cWoken = 0;
1922	unsigned cHalted = 0;
1923	unsigned cTodo2nd = 0;
1924	unsigned cTodo3rd = 0;
1925	for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
1926	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
1927	i = pGVMM->aHandles[i].iNext)
1928	{
1929	PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
1930	if ( VALID_PTR(pCurGVM)
1931	&& pCurGVM->u32Magic == GVM_MAGIC)
1932	{
1933	for (VMCPUID idCpu = 0; idCpu < pCurGVM->cCpus; idCpu++)
1934	{
1935	PGVMCPU pCurGVCpu = &pCurGVM->aCpus[idCpu];
1936	uint64_t u64 = ASMAtomicUoReadU64(&pCurGVCpu->gvmm.s.u64HaltExpire);
1937	if (u64)
1938	{
1939	if (u64 <= u64Now)
1940	{
1941	if (ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0))
1942	{
1943	int rc = RTSemEventMultiSignal(pCurGVCpu->gvmm.s.HaltEventMulti);
1944	AssertRC(rc);
1945	cWoken++;
1946	}
1947	}
1948	else
1949	{
1950	cHalted++;
1951	if (u64 <= uNsEarlyWakeUp1)
1952	cTodo2nd++;
1953	else if (u64 <= uNsEarlyWakeUp2)
1954	cTodo3rd++;
1955	else if (u64 < u64Min)
1956	u64 = u64Min;
1957	}
1958	}
1959	}
1960	}
1961	AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
1962	}
1963
1964	if (cTodo2nd)
1965	{
1966	for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
1967	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
1968	i = pGVMM->aHandles[i].iNext)
1969	{
1970	PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
1971	if ( VALID_PTR(pCurGVM)
1972	&& pCurGVM->u32Magic == GVM_MAGIC)
1973	{
1974	for (VMCPUID idCpu = 0; idCpu < pCurGVM->cCpus; idCpu++)
1975	{
1976	PGVMCPU pCurGVCpu = &pCurGVM->aCpus[idCpu];
1977	uint64_t u64 = ASMAtomicUoReadU64(&pCurGVCpu->gvmm.s.u64HaltExpire);
1978	if ( u64
1979	&& u64 <= uNsEarlyWakeUp1)
1980	{
1981	if (ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0))
1982	{
1983	int rc = RTSemEventMultiSignal(pCurGVCpu->gvmm.s.HaltEventMulti);
1984	AssertRC(rc);
1985	cWoken++;
1986	}
1987	}
1988	}
1989	}
1990	AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
1991	}
1992	}
1993
1994	if (cTodo3rd)
1995	{
1996	for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
1997	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
1998	i = pGVMM->aHandles[i].iNext)
1999	{
2000	PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
2001	if ( VALID_PTR(pCurGVM)
2002	&& pCurGVM->u32Magic == GVM_MAGIC)
2003	{
2004	for (VMCPUID idCpu = 0; idCpu < pCurGVM->cCpus; idCpu++)
2005	{
2006	PGVMCPU pCurGVCpu = &pCurGVM->aCpus[idCpu];
2007	uint64_t u64 = ASMAtomicUoReadU64(&pCurGVCpu->gvmm.s.u64HaltExpire);
2008	if ( u64
2009	&& u64 <= uNsEarlyWakeUp2)
2010	{
2011	if (ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0))
2012	{
2013	int rc = RTSemEventMultiSignal(pCurGVCpu->gvmm.s.HaltEventMulti);
2014	AssertRC(rc);
2015	cWoken++;
2016	}
2017	}
2018	}
2019	}
2020	AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
2021	}
2022	}
2023
2024	/*
2025	* Set the minimum value.
2026	*/
2027	pGVMM->uNsNextEmtWakeup = u64Min;
2028
2029	ASMAtomicWriteBool(&pGVMM->fDoingEarlyWakeUps, false);
2030	return cWoken;
2031	}
2032
2033
2034	/**
2035	* Halt the EMT thread.
2036	*
2037	* @returns VINF_SUCCESS normal wakeup (timeout or kicked by other thread).
2038	* VERR_INTERRUPTED if a signal was scheduled for the thread.
2039	* @param pGVM The global (ring-0) VM structure.
2040	* @param pVM The cross context VM structure.
2041	* @param idCpu The Virtual CPU ID of the calling EMT.
2042	* @param u64ExpireGipTime The time for the sleep to expire expressed as GIP time.
2043	* @thread EMT(idCpu).
2044	*/
2045	GVMMR0DECL(int) GVMMR0SchedHalt(PGVM pGVM, PVM pVM, VMCPUID idCpu, uint64_t u64ExpireGipTime)
2046	{
2047	LogFlow(("GVMMR0SchedHalt: pGVM=%p pVM=%p idCpu=%#x u64ExpireGipTime=%#RX64\n", pGVM, pVM, idCpu, u64ExpireGipTime));
2048	GVMM_CHECK_SMAP_SETUP();
2049	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2050
2051	/*
2052	* Validate the VM structure, state and handle.
2053	*/
2054	PGVMM pGVMM;
2055	int rc = gvmmR0ByGVMandVMandEMT(pGVM, pVM, idCpu, &pGVMM);
2056	if (RT_FAILURE(rc))
2057	return rc;
2058	pGVM->gvmm.s.StatsSched.cHaltCalls++;
2059	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2060
2061	PGVMCPU pCurGVCpu = &pGVM->aCpus[idCpu];
2062	Assert(!pCurGVCpu->gvmm.s.u64HaltExpire);
2063
2064	/*
2065	* If we're doing early wake-ups, we must take the UsedList lock before we
2066	* start querying the current time.
2067	* Note! Interrupts must NOT be disabled at this point because we ask for GIP time!
2068	*/
2069	bool const fDoEarlyWakeUps = pGVMM->fDoEarlyWakeUps;
2070	if (fDoEarlyWakeUps)
2071	{
2072	rc = GVMMR0_USED_SHARED_LOCK(pGVMM); AssertRC(rc);
2073	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2074	}
2075
2076	pCurGVCpu->gvmm.s.iCpuEmt = ASMGetApicId();
2077
2078	/* GIP hack: We might are frequently sleeping for short intervals where the
2079	difference between GIP and system time matters on systems with high resolution
2080	system time. So, convert the input from GIP to System time in that case. */
2081	Assert(ASMGetFlags() & X86_EFL_IF);
2082	const uint64_t u64NowSys = RTTimeSystemNanoTS();
2083	const uint64_t u64NowGip = RTTimeNanoTS();
2084	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2085
2086	if (fDoEarlyWakeUps)
2087	{
2088	pGVM->gvmm.s.StatsSched.cHaltWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64NowGip);
2089	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2090	}
2091
2092	/*
2093	* Go to sleep if we must...
2094	* Cap the sleep time to 1 second to be on the safe side.
2095	*/
2096	uint64_t cNsInterval = u64ExpireGipTime - u64NowGip;
2097	if ( u64NowGip < u64ExpireGipTime
2098	&& cNsInterval >= (pGVMM->cEMTs > pGVMM->cEMTsMeansCompany
2099	? pGVMM->nsMinSleepCompany
2100	: pGVMM->nsMinSleepAlone))
2101	{
2102	pGVM->gvmm.s.StatsSched.cHaltBlocking++;
2103	if (cNsInterval > RT_NS_1SEC)
2104	u64ExpireGipTime = u64NowGip + RT_NS_1SEC;
2105	ASMAtomicWriteU64(&pCurGVCpu->gvmm.s.u64HaltExpire, u64ExpireGipTime);
2106	ASMAtomicIncU32(&pGVMM->cHaltedEMTs);
2107	if (fDoEarlyWakeUps)
2108	{
2109	if (u64ExpireGipTime < pGVMM->uNsNextEmtWakeup)
2110	pGVMM->uNsNextEmtWakeup = u64ExpireGipTime;
2111	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2112	}
2113	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2114
2115	rc = RTSemEventMultiWaitEx(pCurGVCpu->gvmm.s.HaltEventMulti,
2116	RTSEMWAIT_FLAGS_ABSOLUTE \| RTSEMWAIT_FLAGS_NANOSECS \| RTSEMWAIT_FLAGS_INTERRUPTIBLE,
2117	u64NowGip > u64NowSys ? u64ExpireGipTime : u64NowSys + cNsInterval);
2118	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2119
2120	ASMAtomicWriteU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0);
2121	ASMAtomicDecU32(&pGVMM->cHaltedEMTs);
2122
2123	/* Reset the semaphore to try prevent a few false wake-ups. */
2124	if (rc == VINF_SUCCESS)
2125	{
2126	RTSemEventMultiReset(pCurGVCpu->gvmm.s.HaltEventMulti);
2127	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2128	}
2129	else if (rc == VERR_TIMEOUT)
2130	{
2131	pGVM->gvmm.s.StatsSched.cHaltTimeouts++;
2132	rc = VINF_SUCCESS;
2133	}
2134	}
2135	else
2136	{
2137	pGVM->gvmm.s.StatsSched.cHaltNotBlocking++;
2138	if (fDoEarlyWakeUps)
2139	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2140	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2141	RTSemEventMultiReset(pCurGVCpu->gvmm.s.HaltEventMulti);
2142	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2143	}
2144
2145	return rc;
2146	}
2147
2148
2149	/**
2150	* Worker for GVMMR0SchedWakeUp and GVMMR0SchedWakeUpAndPokeCpus that wakes up
2151	* the a sleeping EMT.
2152	*
2153	* @retval VINF_SUCCESS if successfully woken up.
2154	* @retval VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
2155	*
2156	* @param pGVM The global (ring-0) VM structure.
2157	* @param pGVCpu The global (ring-0) VCPU structure.
2158	*/
2159	DECLINLINE(int) gvmmR0SchedWakeUpOne(PGVM pGVM, PGVMCPU pGVCpu)
2160	{
2161	pGVM->gvmm.s.StatsSched.cWakeUpCalls++;
2162
2163	/*
2164	* Signal the semaphore regardless of whether it's current blocked on it.
2165	*
2166	* The reason for this is that there is absolutely no way we can be 100%
2167	* certain that it isn't about go to go to sleep on it and just got
2168	* delayed a bit en route. So, we will always signal the semaphore when
2169	* the it is flagged as halted in the VMM.
2170	*/
2171	/** @todo we can optimize some of that by means of the pVCpu->enmState now. */
2172	int rc;
2173	if (pGVCpu->gvmm.s.u64HaltExpire)
2174	{
2175	rc = VINF_SUCCESS;
2176	ASMAtomicWriteU64(&pGVCpu->gvmm.s.u64HaltExpire, 0);
2177	}
2178	else
2179	{
2180	rc = VINF_GVM_NOT_BLOCKED;
2181	pGVM->gvmm.s.StatsSched.cWakeUpNotHalted++;
2182	}
2183
2184	int rc2 = RTSemEventMultiSignal(pGVCpu->gvmm.s.HaltEventMulti);
2185	AssertRC(rc2);
2186
2187	return rc;
2188	}
2189
2190
2191	/**
2192	* Wakes up the halted EMT thread so it can service a pending request.
2193	*
2194	* @returns VBox status code.
2195	* @retval VINF_SUCCESS if successfully woken up.
2196	* @retval VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
2197	*
2198	* @param pGVM The global (ring-0) VM structure.
2199	* @param pVM The cross context VM structure.
2200	* @param idCpu The Virtual CPU ID of the EMT to wake up.
2201	* @param fTakeUsedLock Take the used lock or not
2202	* @thread Any but EMT(idCpu).
2203	*/
2204	GVMMR0DECL(int) GVMMR0SchedWakeUpEx(PGVM pGVM, PVM pVM, VMCPUID idCpu, bool fTakeUsedLock)
2205	{
2206	GVMM_CHECK_SMAP_SETUP();
2207	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2208
2209	/*
2210	* Validate input and take the UsedLock.
2211	*/
2212	PGVMM pGVMM;
2213	int rc = gvmmR0ByGVMandVM(pGVM, pVM, &pGVMM, fTakeUsedLock);
2214	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2215	if (RT_SUCCESS(rc))
2216	{
2217	if (idCpu < pGVM->cCpus)
2218	{
2219	/*
2220	* Do the actual job.
2221	*/
2222	rc = gvmmR0SchedWakeUpOne(pGVM, &pGVM->aCpus[idCpu]);
2223	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2224
2225	if (fTakeUsedLock && pGVMM->fDoEarlyWakeUps)
2226	{
2227	/*
2228	* While we're here, do a round of scheduling.
2229	*/
2230	Assert(ASMGetFlags() & X86_EFL_IF);
2231	const uint64_t u64Now = RTTimeNanoTS(); /* (GIP time) */
2232	pGVM->gvmm.s.StatsSched.cWakeUpWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64Now);
2233	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2234	}
2235	}
2236	else
2237	rc = VERR_INVALID_CPU_ID;
2238
2239	if (fTakeUsedLock)
2240	{
2241	int rc2 = GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2242	AssertRC(rc2);
2243	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2244	}
2245	}
2246
2247	LogFlow(("GVMMR0SchedWakeUpEx: returns %Rrc\n", rc));
2248	return rc;
2249	}
2250
2251
2252	/**
2253	* Wakes up the halted EMT thread so it can service a pending request.
2254	*
2255	* @returns VBox status code.
2256	* @retval VINF_SUCCESS if successfully woken up.
2257	* @retval VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
2258	*
2259	* @param pGVM The global (ring-0) VM structure.
2260	* @param pVM The cross context VM structure.
2261	* @param idCpu The Virtual CPU ID of the EMT to wake up.
2262	* @thread Any but EMT(idCpu).
2263	*/
2264	GVMMR0DECL(int) GVMMR0SchedWakeUp(PGVM pGVM, PVM pVM, VMCPUID idCpu)
2265	{
2266	return GVMMR0SchedWakeUpEx(pGVM, pVM, idCpu, true /* fTakeUsedLock */);
2267	}
2268
2269
2270	/**
2271	* Wakes up the halted EMT thread so it can service a pending request, no GVM
2272	* parameter and no used locking.
2273	*
2274	* @returns VBox status code.
2275	* @retval VINF_SUCCESS if successfully woken up.
2276	* @retval VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
2277	*
2278	* @param pVM The cross context VM structure.
2279	* @param idCpu The Virtual CPU ID of the EMT to wake up.
2280	* @thread Any but EMT(idCpu).
2281	* @deprecated Don't use in new code if possible! Use the GVM variant.
2282	*/
2283	GVMMR0DECL(int) GVMMR0SchedWakeUpNoGVMNoLock(PVM pVM, VMCPUID idCpu)
2284	{
2285	GVMM_CHECK_SMAP_SETUP();
2286	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2287	PGVM pGVM;
2288	PGVMM pGVMM;
2289	int rc = gvmmR0ByVM(pVM, &pGVM, &pGVMM, false /fTakeUsedLock/);
2290	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2291	if (RT_SUCCESS(rc))
2292	rc = GVMMR0SchedWakeUpEx(pGVM, pVM, idCpu, false /fTakeUsedLock/);
2293	return rc;
2294	}
2295
2296
2297	/**
2298	* Worker common to GVMMR0SchedPoke and GVMMR0SchedWakeUpAndPokeCpus that pokes
2299	* the Virtual CPU if it's still busy executing guest code.
2300	*
2301	* @returns VBox status code.
2302	* @retval VINF_SUCCESS if poked successfully.
2303	* @retval VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
2304	*
2305	* @param pGVM The global (ring-0) VM structure.
2306	* @param pVCpu The cross context virtual CPU structure.
2307	*/
2308	DECLINLINE(int) gvmmR0SchedPokeOne(PGVM pGVM, PVMCPU pVCpu)
2309	{
2310	pGVM->gvmm.s.StatsSched.cPokeCalls++;
2311
2312	RTCPUID idHostCpu = pVCpu->idHostCpu;
2313	if ( idHostCpu == NIL_RTCPUID
2314	\|\| VMCPU_GET_STATE(pVCpu) != VMCPUSTATE_STARTED_EXEC)
2315	{
2316	pGVM->gvmm.s.StatsSched.cPokeNotBusy++;
2317	return VINF_GVM_NOT_BUSY_IN_GC;
2318	}
2319
2320	/* Note: this function is not implemented on Darwin and Linux (kernel < 2.6.19) */
2321	RTMpPokeCpu(idHostCpu);
2322	return VINF_SUCCESS;
2323	}
2324
2325
2326	/**
2327	* Pokes an EMT if it's still busy running guest code.
2328	*
2329	* @returns VBox status code.
2330	* @retval VINF_SUCCESS if poked successfully.
2331	* @retval VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
2332	*
2333	* @param pGVM The global (ring-0) VM structure.
2334	* @param pVM The cross context VM structure.
2335	* @param idCpu The ID of the virtual CPU to poke.
2336	* @param fTakeUsedLock Take the used lock or not
2337	*/
2338	GVMMR0DECL(int) GVMMR0SchedPokeEx(PGVM pGVM, PVM pVM, VMCPUID idCpu, bool fTakeUsedLock)
2339	{
2340	/*
2341	* Validate input and take the UsedLock.
2342	*/
2343	PGVMM pGVMM;
2344	int rc = gvmmR0ByGVMandVM(pGVM, pVM, &pGVMM, fTakeUsedLock);
2345	if (RT_SUCCESS(rc))
2346	{
2347	if (idCpu < pGVM->cCpus)
2348	rc = gvmmR0SchedPokeOne(pGVM, &pVM->aCpus[idCpu]);
2349	else
2350	rc = VERR_INVALID_CPU_ID;
2351
2352	if (fTakeUsedLock)
2353	{
2354	int rc2 = GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2355	AssertRC(rc2);
2356	}
2357	}
2358
2359	LogFlow(("GVMMR0SchedWakeUpAndPokeCpus: returns %Rrc\n", rc));
2360	return rc;
2361	}
2362
2363
2364	/**
2365	* Pokes an EMT if it's still busy running guest code.
2366	*
2367	* @returns VBox status code.
2368	* @retval VINF_SUCCESS if poked successfully.
2369	* @retval VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
2370	*
2371	* @param pGVM The global (ring-0) VM structure.
2372	* @param pVM The cross context VM structure.
2373	* @param idCpu The ID of the virtual CPU to poke.
2374	*/
2375	GVMMR0DECL(int) GVMMR0SchedPoke(PGVM pGVM, PVM pVM, VMCPUID idCpu)
2376	{
2377	return GVMMR0SchedPokeEx(pGVM, pVM, idCpu, true /* fTakeUsedLock */);
2378	}
2379
2380
2381	/**
2382	* Pokes an EMT if it's still busy running guest code, no GVM parameter and no
2383	* used locking.
2384	*
2385	* @returns VBox status code.
2386	* @retval VINF_SUCCESS if poked successfully.
2387	* @retval VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
2388	*
2389	* @param pVM The cross context VM structure.
2390	* @param idCpu The ID of the virtual CPU to poke.
2391	*
2392	* @deprecated Don't use in new code if possible! Use the GVM variant.
2393	*/
2394	GVMMR0DECL(int) GVMMR0SchedPokeNoGVMNoLock(PVM pVM, VMCPUID idCpu)
2395	{
2396	PGVM pGVM;
2397	PGVMM pGVMM;
2398	int rc = gvmmR0ByVM(pVM, &pGVM, &pGVMM, false /fTakeUsedLock/);
2399	if (RT_SUCCESS(rc))
2400	{
2401	if (idCpu < pGVM->cCpus)
2402	rc = gvmmR0SchedPokeOne(pGVM, &pVM->aCpus[idCpu]);
2403	else
2404	rc = VERR_INVALID_CPU_ID;
2405	}
2406	return rc;
2407	}
2408
2409
2410	/**
2411	* Wakes up a set of halted EMT threads so they can service pending request.
2412	*
2413	* @returns VBox status code, no informational stuff.
2414	*
2415	* @param pGVM The global (ring-0) VM structure.
2416	* @param pVM The cross context VM structure.
2417	* @param pSleepSet The set of sleepers to wake up.
2418	* @param pPokeSet The set of CPUs to poke.
2419	*/
2420	GVMMR0DECL(int) GVMMR0SchedWakeUpAndPokeCpus(PGVM pGVM, PVM pVM, PCVMCPUSET pSleepSet, PCVMCPUSET pPokeSet)
2421	{
2422	AssertPtrReturn(pSleepSet, VERR_INVALID_POINTER);
2423	AssertPtrReturn(pPokeSet, VERR_INVALID_POINTER);
2424	GVMM_CHECK_SMAP_SETUP();
2425	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2426	RTNATIVETHREAD hSelf = RTThreadNativeSelf();
2427
2428	/*
2429	* Validate input and take the UsedLock.
2430	*/
2431	PGVMM pGVMM;
2432	int rc = gvmmR0ByGVMandVM(pGVM, pVM, &pGVMM, true /* fTakeUsedLock */);
2433	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2434	if (RT_SUCCESS(rc))
2435	{
2436	rc = VINF_SUCCESS;
2437	VMCPUID idCpu = pGVM->cCpus;
2438	while (idCpu-- > 0)
2439	{
2440	/* Don't try poke or wake up ourselves. */
2441	if (pGVM->aCpus[idCpu].hEMT == hSelf)
2442	continue;
2443
2444	/* just ignore errors for now. */
2445	if (VMCPUSET_IS_PRESENT(pSleepSet, idCpu))
2446	{
2447	gvmmR0SchedWakeUpOne(pGVM, &pGVM->aCpus[idCpu]);
2448	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2449	}
2450	else if (VMCPUSET_IS_PRESENT(pPokeSet, idCpu))
2451	{
2452	gvmmR0SchedPokeOne(pGVM, &pVM->aCpus[idCpu]);
2453	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2454	}
2455	}
2456
2457	int rc2 = GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2458	AssertRC(rc2);
2459	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2460	}
2461
2462	LogFlow(("GVMMR0SchedWakeUpAndPokeCpus: returns %Rrc\n", rc));
2463	return rc;
2464	}
2465
2466
2467	/**
2468	* VMMR0 request wrapper for GVMMR0SchedWakeUpAndPokeCpus.
2469	*
2470	* @returns see GVMMR0SchedWakeUpAndPokeCpus.
2471	* @param pGVM The global (ring-0) VM structure.
2472	* @param pVM The cross context VM structure.
2473	* @param pReq Pointer to the request packet.
2474	*/
2475	GVMMR0DECL(int) GVMMR0SchedWakeUpAndPokeCpusReq(PGVM pGVM, PVM pVM, PGVMMSCHEDWAKEUPANDPOKECPUSREQ pReq)
2476	{
2477	/*
2478	* Validate input and pass it on.
2479	*/
2480	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
2481	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
2482
2483	return GVMMR0SchedWakeUpAndPokeCpus(pGVM, pVM, &pReq->SleepSet, &pReq->PokeSet);
2484	}
2485
2486
2487
2488	/**
2489	* Poll the schedule to see if someone else should get a chance to run.
2490	*
2491	* This is a bit hackish and will not work too well if the machine is
2492	* under heavy load from non-VM processes.
2493	*
2494	* @returns VINF_SUCCESS if not yielded.
2495	* VINF_GVM_YIELDED if an attempt to switch to a different VM task was made.
2496	* @param pGVM The global (ring-0) VM structure.
2497	* @param pVM The cross context VM structure.
2498	* @param idCpu The Virtual CPU ID of the calling EMT.
2499	* @param fYield Whether to yield or not.
2500	* This is for when we're spinning in the halt loop.
2501	* @thread EMT(idCpu).
2502	*/
2503	GVMMR0DECL(int) GVMMR0SchedPoll(PGVM pGVM, PVM pVM, VMCPUID idCpu, bool fYield)
2504	{
2505	/*
2506	* Validate input.
2507	*/
2508	PGVMM pGVMM;
2509	int rc = gvmmR0ByGVMandVMandEMT(pGVM, pVM, idCpu, &pGVMM);
2510	if (RT_SUCCESS(rc))
2511	{
2512	/*
2513	* We currently only implement helping doing wakeups (fYield = false), so don't
2514	* bother taking the lock if gvmmR0SchedDoWakeUps is not going to do anything.
2515	*/
2516	if (!fYield && pGVMM->fDoEarlyWakeUps)
2517	{
2518	rc = GVMMR0_USED_SHARED_LOCK(pGVMM); AssertRC(rc);
2519	pGVM->gvmm.s.StatsSched.cPollCalls++;
2520
2521	Assert(ASMGetFlags() & X86_EFL_IF);
2522	const uint64_t u64Now = RTTimeNanoTS(); /* (GIP time) */
2523
2524	pGVM->gvmm.s.StatsSched.cPollWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64Now);
2525
2526	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2527	}
2528	/*
2529	* Not quite sure what we could do here...
2530	*/
2531	else if (fYield)
2532	rc = VERR_NOT_IMPLEMENTED; /** @todo implement this... */
2533	else
2534	rc = VINF_SUCCESS;
2535	}
2536
2537	LogFlow(("GVMMR0SchedWakeUp: returns %Rrc\n", rc));
2538	return rc;
2539	}
2540
2541
2542	#ifdef GVMM_SCHED_WITH_PPT
2543	/**
2544	* Timer callback for the periodic preemption timer.
2545	*
2546	* @param pTimer The timer handle.
2547	* @param pvUser Pointer to the per cpu structure.
2548	* @param iTick The current tick.
2549	*/
2550	static DECLCALLBACK(void) gvmmR0SchedPeriodicPreemptionTimerCallback(PRTTIMER pTimer, void *pvUser, uint64_t iTick)
2551	{
2552	PGVMMHOSTCPU pCpu = (PGVMMHOSTCPU)pvUser;
2553	NOREF(pTimer); NOREF(iTick);
2554
2555	/*
2556	* Termination check
2557	*/
2558	if (pCpu->u32Magic != GVMMHOSTCPU_MAGIC)
2559	return;
2560
2561	/*
2562	* Do the house keeping.
2563	*/
2564	RTSpinlockAcquire(pCpu->Ppt.hSpinlock);
2565
2566	if (++pCpu->Ppt.iTickHistorization >= pCpu->Ppt.cTicksHistoriziationInterval)
2567	{
2568	/*
2569	* Historicize the max frequency.
2570	*/
2571	uint32_t iHzHistory = ++pCpu->Ppt.iHzHistory % RT_ELEMENTS(pCpu->Ppt.aHzHistory);
2572	pCpu->Ppt.aHzHistory[iHzHistory] = pCpu->Ppt.uDesiredHz;
2573	pCpu->Ppt.iTickHistorization = 0;
2574	pCpu->Ppt.uDesiredHz = 0;
2575
2576	/*
2577	* Check if the current timer frequency.
2578	*/
2579	uint32_t uHistMaxHz = 0;
2580	for (uint32_t i = 0; i < RT_ELEMENTS(pCpu->Ppt.aHzHistory); i++)
2581	if (pCpu->Ppt.aHzHistory[i] > uHistMaxHz)
2582	uHistMaxHz = pCpu->Ppt.aHzHistory[i];
2583	if (uHistMaxHz == pCpu->Ppt.uTimerHz)
2584	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
2585	else if (uHistMaxHz)
2586	{
2587	/*
2588	* Reprogram it.
2589	*/
2590	pCpu->Ppt.cChanges++;
2591	pCpu->Ppt.iTickHistorization = 0;
2592	pCpu->Ppt.uTimerHz = uHistMaxHz;
2593	uint32_t const cNsInterval = RT_NS_1SEC / uHistMaxHz;
2594	pCpu->Ppt.cNsInterval = cNsInterval;
2595	if (cNsInterval < GVMMHOSTCPU_PPT_HIST_INTERVAL_NS)
2596	pCpu->Ppt.cTicksHistoriziationInterval = ( GVMMHOSTCPU_PPT_HIST_INTERVAL_NS
2597	+ GVMMHOSTCPU_PPT_HIST_INTERVAL_NS / 2 - 1)
2598	/ cNsInterval;
2599	else
2600	pCpu->Ppt.cTicksHistoriziationInterval = 1;
2601	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
2602
2603	/SUPR0Printf("Cpu%u: change to %u Hz / %u ns\n", pCpu->idxCpuSet, uHistMaxHz, cNsInterval);/
2604	RTTimerChangeInterval(pTimer, cNsInterval);
2605	}
2606	else
2607	{
2608	/*
2609	* Stop it.
2610	*/
2611	pCpu->Ppt.fStarted = false;
2612	pCpu->Ppt.uTimerHz = 0;
2613	pCpu->Ppt.cNsInterval = 0;
2614	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
2615
2616	/SUPR0Printf("Cpu%u: stopping (%u Hz)\n", pCpu->idxCpuSet, uHistMaxHz);/
2617	RTTimerStop(pTimer);
2618	}
2619	}
2620	else
2621	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
2622	}
2623	#endif /* GVMM_SCHED_WITH_PPT */
2624
2625
2626	/**
2627	* Updates the periodic preemption timer for the calling CPU.
2628	*
2629	* The caller must have disabled preemption!
2630	* The caller must check that the host can do high resolution timers.
2631	*
2632	* @param pVM The cross context VM structure.
2633	* @param idHostCpu The current host CPU id.
2634	* @param uHz The desired frequency.
2635	*/
2636	GVMMR0DECL(void) GVMMR0SchedUpdatePeriodicPreemptionTimer(PVM pVM, RTCPUID idHostCpu, uint32_t uHz)
2637	{
2638	NOREF(pVM);
2639	#ifdef GVMM_SCHED_WITH_PPT
2640	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2641	Assert(RTTimerCanDoHighResolution());
2642
2643	/*
2644	* Resolve the per CPU data.
2645	*/
2646	uint32_t iCpu = RTMpCpuIdToSetIndex(idHostCpu);
2647	PGVMM pGVMM = g_pGVMM;
2648	if ( !VALID_PTR(pGVMM)
2649	\|\| pGVMM->u32Magic != GVMM_MAGIC)
2650	return;
2651	AssertMsgReturnVoid(iCpu < pGVMM->cHostCpus, ("iCpu=%d cHostCpus=%d\n", iCpu, pGVMM->cHostCpus));
2652	PGVMMHOSTCPU pCpu = &pGVMM->aHostCpus[iCpu];
2653	AssertMsgReturnVoid( pCpu->u32Magic == GVMMHOSTCPU_MAGIC
2654	&& pCpu->idCpu == idHostCpu,
2655	("u32Magic=%#x idCpu=% idHostCpu=%d\n", pCpu->u32Magic, pCpu->idCpu, idHostCpu));
2656
2657	/*
2658	* Check whether we need to do anything about the timer.
2659	* We have to be a little bit careful since we might be race the timer
2660	* callback here.
2661	*/
2662	if (uHz > 16384)
2663	uHz = 16384; /** @todo add a query method for this! */
2664	if (RT_UNLIKELY( uHz > ASMAtomicReadU32(&pCpu->Ppt.uDesiredHz)
2665	&& uHz >= pCpu->Ppt.uMinHz
2666	&& !pCpu->Ppt.fStarting /* solaris paranoia */))
2667	{
2668	RTSpinlockAcquire(pCpu->Ppt.hSpinlock);
2669
2670	pCpu->Ppt.uDesiredHz = uHz;
2671	uint32_t cNsInterval = 0;
2672	if (!pCpu->Ppt.fStarted)
2673	{
2674	pCpu->Ppt.cStarts++;
2675	pCpu->Ppt.fStarted = true;
2676	pCpu->Ppt.fStarting = true;
2677	pCpu->Ppt.iTickHistorization = 0;
2678	pCpu->Ppt.uTimerHz = uHz;
2679	pCpu->Ppt.cNsInterval = cNsInterval = RT_NS_1SEC / uHz;
2680	if (cNsInterval < GVMMHOSTCPU_PPT_HIST_INTERVAL_NS)
2681	pCpu->Ppt.cTicksHistoriziationInterval = ( GVMMHOSTCPU_PPT_HIST_INTERVAL_NS
2682	+ GVMMHOSTCPU_PPT_HIST_INTERVAL_NS / 2 - 1)
2683	/ cNsInterval;
2684	else
2685	pCpu->Ppt.cTicksHistoriziationInterval = 1;
2686	}
2687
2688	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
2689
2690	if (cNsInterval)
2691	{
2692	RTTimerChangeInterval(pCpu->Ppt.pTimer, cNsInterval);
2693	int rc = RTTimerStart(pCpu->Ppt.pTimer, cNsInterval);
2694	AssertRC(rc);
2695
2696	RTSpinlockAcquire(pCpu->Ppt.hSpinlock);
2697	if (RT_FAILURE(rc))
2698	pCpu->Ppt.fStarted = false;
2699	pCpu->Ppt.fStarting = false;
2700	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
2701	}
2702	}
2703	#else /* !GVMM_SCHED_WITH_PPT */
2704	NOREF(idHostCpu); NOREF(uHz);
2705	#endif /* !GVMM_SCHED_WITH_PPT */
2706	}
2707
2708
2709	/**
2710	* Retrieves the GVMM statistics visible to the caller.
2711	*
2712	* @returns VBox status code.
2713	*
2714	* @param pStats Where to put the statistics.
2715	* @param pSession The current session.
2716	* @param pGVM The GVM to obtain statistics for. Optional.
2717	* @param pVM The VM structure corresponding to @a pGVM.
2718	*/
2719	GVMMR0DECL(int) GVMMR0QueryStatistics(PGVMMSTATS pStats, PSUPDRVSESSION pSession, PGVM pGVM, PVM pVM)
2720	{
2721	LogFlow(("GVMMR0QueryStatistics: pStats=%p pSession=%p pGVM=%p pVM=%p\n", pStats, pSession, pGVM, pVM));
2722
2723	/*
2724	* Validate input.
2725	*/
2726	AssertPtrReturn(pSession, VERR_INVALID_POINTER);
2727	AssertPtrReturn(pStats, VERR_INVALID_POINTER);
2728	pStats->cVMs = 0; /* (crash before taking the sem...) */
2729
2730	/*
2731	* Take the lock and get the VM statistics.
2732	*/
2733	PGVMM pGVMM;
2734	if (pGVM)
2735	{
2736	int rc = gvmmR0ByGVMandVM(pGVM, pVM, &pGVMM, true /fTakeUsedLock/);
2737	if (RT_FAILURE(rc))
2738	return rc;
2739	pStats->SchedVM = pGVM->gvmm.s.StatsSched;
2740	}
2741	else
2742	{
2743	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
2744	memset(&pStats->SchedVM, 0, sizeof(pStats->SchedVM));
2745
2746	int rc = GVMMR0_USED_SHARED_LOCK(pGVMM);
2747	AssertRCReturn(rc, rc);
2748	}
2749
2750	/*
2751	* Enumerate the VMs and add the ones visible to the statistics.
2752	*/
2753	pStats->cVMs = 0;
2754	pStats->cEMTs = 0;
2755	memset(&pStats->SchedSum, 0, sizeof(pStats->SchedSum));
2756
2757	for (unsigned i = pGVMM->iUsedHead;
2758	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
2759	i = pGVMM->aHandles[i].iNext)
2760	{
2761	PGVM pOtherGVM = pGVMM->aHandles[i].pGVM;
2762	void *pvObj = pGVMM->aHandles[i].pvObj;
2763	if ( VALID_PTR(pvObj)
2764	&& VALID_PTR(pOtherGVM)
2765	&& pOtherGVM->u32Magic == GVM_MAGIC
2766	&& RT_SUCCESS(SUPR0ObjVerifyAccess(pvObj, pSession, NULL)))
2767	{
2768	pStats->cVMs++;
2769	pStats->cEMTs += pOtherGVM->cCpus;
2770
2771	pStats->SchedSum.cHaltCalls += pOtherGVM->gvmm.s.StatsSched.cHaltCalls;
2772	pStats->SchedSum.cHaltBlocking += pOtherGVM->gvmm.s.StatsSched.cHaltBlocking;
2773	pStats->SchedSum.cHaltTimeouts += pOtherGVM->gvmm.s.StatsSched.cHaltTimeouts;
2774	pStats->SchedSum.cHaltNotBlocking += pOtherGVM->gvmm.s.StatsSched.cHaltNotBlocking;
2775	pStats->SchedSum.cHaltWakeUps += pOtherGVM->gvmm.s.StatsSched.cHaltWakeUps;
2776
2777	pStats->SchedSum.cWakeUpCalls += pOtherGVM->gvmm.s.StatsSched.cWakeUpCalls;
2778	pStats->SchedSum.cWakeUpNotHalted += pOtherGVM->gvmm.s.StatsSched.cWakeUpNotHalted;
2779	pStats->SchedSum.cWakeUpWakeUps += pOtherGVM->gvmm.s.StatsSched.cWakeUpWakeUps;
2780
2781	pStats->SchedSum.cPokeCalls += pOtherGVM->gvmm.s.StatsSched.cPokeCalls;
2782	pStats->SchedSum.cPokeNotBusy += pOtherGVM->gvmm.s.StatsSched.cPokeNotBusy;
2783
2784	pStats->SchedSum.cPollCalls += pOtherGVM->gvmm.s.StatsSched.cPollCalls;
2785	pStats->SchedSum.cPollHalts += pOtherGVM->gvmm.s.StatsSched.cPollHalts;
2786	pStats->SchedSum.cPollWakeUps += pOtherGVM->gvmm.s.StatsSched.cPollWakeUps;
2787	}
2788	}
2789
2790	/*
2791	* Copy out the per host CPU statistics.
2792	*/
2793	uint32_t iDstCpu = 0;
2794	uint32_t cSrcCpus = pGVMM->cHostCpus;
2795	for (uint32_t iSrcCpu = 0; iSrcCpu < cSrcCpus; iSrcCpu++)
2796	{
2797	if (pGVMM->aHostCpus[iSrcCpu].idCpu != NIL_RTCPUID)
2798	{
2799	pStats->aHostCpus[iDstCpu].idCpu = pGVMM->aHostCpus[iSrcCpu].idCpu;
2800	pStats->aHostCpus[iDstCpu].idxCpuSet = pGVMM->aHostCpus[iSrcCpu].idxCpuSet;
2801	#ifdef GVMM_SCHED_WITH_PPT
2802	pStats->aHostCpus[iDstCpu].uDesiredHz = pGVMM->aHostCpus[iSrcCpu].Ppt.uDesiredHz;
2803	pStats->aHostCpus[iDstCpu].uTimerHz = pGVMM->aHostCpus[iSrcCpu].Ppt.uTimerHz;
2804	pStats->aHostCpus[iDstCpu].cChanges = pGVMM->aHostCpus[iSrcCpu].Ppt.cChanges;
2805	pStats->aHostCpus[iDstCpu].cStarts = pGVMM->aHostCpus[iSrcCpu].Ppt.cStarts;
2806	#else
2807	pStats->aHostCpus[iDstCpu].uDesiredHz = 0;
2808	pStats->aHostCpus[iDstCpu].uTimerHz = 0;
2809	pStats->aHostCpus[iDstCpu].cChanges = 0;
2810	pStats->aHostCpus[iDstCpu].cStarts = 0;
2811	#endif
2812	iDstCpu++;
2813	if (iDstCpu >= RT_ELEMENTS(pStats->aHostCpus))
2814	break;
2815	}
2816	}
2817	pStats->cHostCpus = iDstCpu;
2818
2819	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2820
2821	return VINF_SUCCESS;
2822	}
2823
2824
2825	/**
2826	* VMMR0 request wrapper for GVMMR0QueryStatistics.
2827	*
2828	* @returns see GVMMR0QueryStatistics.
2829	* @param pGVM The global (ring-0) VM structure. Optional.
2830	* @param pVM The cross context VM structure. Optional.
2831	* @param pReq Pointer to the request packet.
2832	* @param pSession The current session.
2833	*/
2834	GVMMR0DECL(int) GVMMR0QueryStatisticsReq(PGVM pGVM, PVM pVM, PGVMMQUERYSTATISTICSSREQ pReq, PSUPDRVSESSION pSession)
2835	{
2836	/*
2837	* Validate input and pass it on.
2838	*/
2839	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
2840	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
2841	AssertReturn(pReq->pSession == pSession, VERR_INVALID_PARAMETER);
2842
2843	return GVMMR0QueryStatistics(&pReq->Stats, pSession, pGVM, pVM);
2844	}
2845
2846
2847	/**
2848	* Resets the specified GVMM statistics.
2849	*
2850	* @returns VBox status code.
2851	*
2852	* @param pStats Which statistics to reset, that is, non-zero fields indicates which to reset.
2853	* @param pSession The current session.
2854	* @param pGVM The GVM to reset statistics for. Optional.
2855	* @param pVM The VM structure corresponding to @a pGVM.
2856	*/
2857	GVMMR0DECL(int) GVMMR0ResetStatistics(PCGVMMSTATS pStats, PSUPDRVSESSION pSession, PGVM pGVM, PVM pVM)
2858	{
2859	LogFlow(("GVMMR0ResetStatistics: pStats=%p pSession=%p pGVM=%p pVM=%p\n", pStats, pSession, pGVM, pVM));
2860
2861	/*
2862	* Validate input.
2863	*/
2864	AssertPtrReturn(pSession, VERR_INVALID_POINTER);
2865	AssertPtrReturn(pStats, VERR_INVALID_POINTER);
2866
2867	/*
2868	* Take the lock and get the VM statistics.
2869	*/
2870	PGVMM pGVMM;
2871	if (pGVM)
2872	{
2873	int rc = gvmmR0ByGVMandVM(pGVM, pVM, &pGVMM, true /fTakeUsedLock/);
2874	if (RT_FAILURE(rc))
2875	return rc;
2876	# define MAYBE_RESET_FIELD(field) \
2877	do { if (pStats->SchedVM. field ) { pGVM->gvmm.s.StatsSched. field = 0; } } while (0)
2878	MAYBE_RESET_FIELD(cHaltCalls);
2879	MAYBE_RESET_FIELD(cHaltBlocking);
2880	MAYBE_RESET_FIELD(cHaltTimeouts);
2881	MAYBE_RESET_FIELD(cHaltNotBlocking);
2882	MAYBE_RESET_FIELD(cHaltWakeUps);
2883	MAYBE_RESET_FIELD(cWakeUpCalls);
2884	MAYBE_RESET_FIELD(cWakeUpNotHalted);
2885	MAYBE_RESET_FIELD(cWakeUpWakeUps);
2886	MAYBE_RESET_FIELD(cPokeCalls);
2887	MAYBE_RESET_FIELD(cPokeNotBusy);
2888	MAYBE_RESET_FIELD(cPollCalls);
2889	MAYBE_RESET_FIELD(cPollHalts);
2890	MAYBE_RESET_FIELD(cPollWakeUps);
2891	# undef MAYBE_RESET_FIELD
2892	}
2893	else
2894	{
2895	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
2896
2897	int rc = GVMMR0_USED_SHARED_LOCK(pGVMM);
2898	AssertRCReturn(rc, rc);
2899	}
2900
2901	/*
2902	* Enumerate the VMs and add the ones visible to the statistics.
2903	*/
2904	if (!ASMMemIsZero(&pStats->SchedSum, sizeof(pStats->SchedSum)))
2905	{
2906	for (unsigned i = pGVMM->iUsedHead;
2907	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
2908	i = pGVMM->aHandles[i].iNext)
2909	{
2910	PGVM pOtherGVM = pGVMM->aHandles[i].pGVM;
2911	void *pvObj = pGVMM->aHandles[i].pvObj;
2912	if ( VALID_PTR(pvObj)
2913	&& VALID_PTR(pOtherGVM)
2914	&& pOtherGVM->u32Magic == GVM_MAGIC
2915	&& RT_SUCCESS(SUPR0ObjVerifyAccess(pvObj, pSession, NULL)))
2916	{
2917	# define MAYBE_RESET_FIELD(field) \
2918	do { if (pStats->SchedSum. field ) { pOtherGVM->gvmm.s.StatsSched. field = 0; } } while (0)
2919	MAYBE_RESET_FIELD(cHaltCalls);
2920	MAYBE_RESET_FIELD(cHaltBlocking);
2921	MAYBE_RESET_FIELD(cHaltTimeouts);
2922	MAYBE_RESET_FIELD(cHaltNotBlocking);
2923	MAYBE_RESET_FIELD(cHaltWakeUps);
2924	MAYBE_RESET_FIELD(cWakeUpCalls);
2925	MAYBE_RESET_FIELD(cWakeUpNotHalted);
2926	MAYBE_RESET_FIELD(cWakeUpWakeUps);
2927	MAYBE_RESET_FIELD(cPokeCalls);
2928	MAYBE_RESET_FIELD(cPokeNotBusy);
2929	MAYBE_RESET_FIELD(cPollCalls);
2930	MAYBE_RESET_FIELD(cPollHalts);
2931	MAYBE_RESET_FIELD(cPollWakeUps);
2932	# undef MAYBE_RESET_FIELD
2933	}
2934	}
2935	}
2936
2937	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2938
2939	return VINF_SUCCESS;
2940	}
2941
2942
2943	/**
2944	* VMMR0 request wrapper for GVMMR0ResetStatistics.
2945	*
2946	* @returns see GVMMR0ResetStatistics.
2947	* @param pGVM The global (ring-0) VM structure. Optional.
2948	* @param pVM The cross context VM structure. Optional.
2949	* @param pReq Pointer to the request packet.
2950	* @param pSession The current session.
2951	*/
2952	GVMMR0DECL(int) GVMMR0ResetStatisticsReq(PGVM pGVM, PVM pVM, PGVMMRESETSTATISTICSSREQ pReq, PSUPDRVSESSION pSession)
2953	{
2954	/*
2955	* Validate input and pass it on.
2956	*/
2957	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
2958	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
2959	AssertReturn(pReq->pSession == pSession, VERR_INVALID_PARAMETER);
2960
2961	return GVMMR0ResetStatistics(&pReq->Stats, pSession, pGVM, pVM);
2962	}
2963

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/GVMMR0.cpp@ 71075

Download in other formats: