GVMMR0.cpp@ 90597

Last change on this file since 90597 was 90597, checked in by vboxsync, 3 years ago
VMM: Speed up VMMGetCpu in ring-0 by using hash table (via new GVMMR0GetGVCpuByGVMandEMT call). bugref:6695
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 111.0 KB

Line
1	/* $Id: GVMMR0.cpp 90597 2021-08-10 13:08:35Z vboxsync $ */
2	/** @file
3	* GVMM - Global VM Manager.
4	*/
5
6	/*
7	* Copyright (C) 2007-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/** @page pg_gvmm GVMM - The Global VM Manager
20	*
21	* The Global VM Manager lives in ring-0. Its main function at the moment is
22	* to manage a list of all running VMs, keep a ring-0 only structure (GVM) for
23	* each of them, and assign them unique identifiers (so GMM can track page
24	* owners). The GVMM also manage some of the host CPU resources, like the
25	* periodic preemption timer.
26	*
27	* The GVMM will create a ring-0 object for each VM when it is registered, this
28	* is both for session cleanup purposes and for having a point where it is
29	* possible to implement usage polices later (in SUPR0ObjRegister).
30	*
31	*
32	* @section sec_gvmm_ppt Periodic Preemption Timer (PPT)
33	*
34	* On system that sports a high resolution kernel timer API, we use per-cpu
35	* timers to generate interrupts that preempts VT-x, AMD-V and raw-mode guest
36	* execution. The timer frequency is calculating by taking the max
37	* TMCalcHostTimerFrequency for all VMs running on a CPU for the last ~160 ms
38	* (RT_ELEMENTS((PGVMMHOSTCPU)0, Ppt.aHzHistory) *
39	* GVMMHOSTCPU_PPT_HIST_INTERVAL_NS).
40	*
41	* The TMCalcHostTimerFrequency() part of the things gets its takes the max
42	* TMTimerSetFrequencyHint() value and adjusts by the current catch-up percent,
43	* warp drive percent and some fudge factors. VMMR0.cpp reports the result via
44	* GVMMR0SchedUpdatePeriodicPreemptionTimer() before switching to the VT-x,
45	* AMD-V and raw-mode execution environments.
46	*/
47
48
49	/*********************************************************************************************************************************
50	* Header Files *
51	*********************************************************************************************************************************/
52	#define LOG_GROUP LOG_GROUP_GVMM
53	#include <VBox/vmm/gvmm.h>
54	#include <VBox/vmm/gmm.h>
55	#include "GVMMR0Internal.h"
56	#include <VBox/vmm/dbgf.h>
57	#include <VBox/vmm/iom.h>
58	#include <VBox/vmm/pdm.h>
59	#include <VBox/vmm/pgm.h>
60	#include <VBox/vmm/vmm.h>
61	#ifdef VBOX_WITH_NEM_R0
62	# include <VBox/vmm/nem.h>
63	#endif
64	#include <VBox/vmm/vmcpuset.h>
65	#include <VBox/vmm/vmcc.h>
66	#include <VBox/param.h>
67	#include <VBox/err.h>
68
69	#include <iprt/asm.h>
70	#include <iprt/asm-amd64-x86.h>
71	#include <iprt/critsect.h>
72	#include <iprt/mem.h>
73	#include <iprt/semaphore.h>
74	#include <iprt/time.h>
75	#include <VBox/log.h>
76	#include <iprt/thread.h>
77	#include <iprt/process.h>
78	#include <iprt/param.h>
79	#include <iprt/string.h>
80	#include <iprt/assert.h>
81	#include <iprt/mem.h>
82	#include <iprt/memobj.h>
83	#include <iprt/mp.h>
84	#include <iprt/cpuset.h>
85	#include <iprt/spinlock.h>
86	#include <iprt/timer.h>
87
88	#include "dtrace/VBoxVMM.h"
89
90
91	/*********************************************************************************************************************************
92	* Defined Constants And Macros *
93	*********************************************************************************************************************************/
94	#if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS) \|\| defined(DOXYGEN_RUNNING)
95	/** Define this to enable the periodic preemption timer. */
96	# define GVMM_SCHED_WITH_PPT
97	#endif
98
99
100	/** @def GVMM_CHECK_SMAP_SETUP
101	* SMAP check setup. */
102	/** @def GVMM_CHECK_SMAP_CHECK
103	* Checks that the AC flag is set if SMAP is enabled. If AC is not set,
104	* it will be logged and @a a_BadExpr is executed. */
105	/** @def GVMM_CHECK_SMAP_CHECK2
106	* Checks that the AC flag is set if SMAP is enabled. If AC is not set, it will
107	* be logged, written to the VMs assertion text buffer, and @a a_BadExpr is
108	* executed. */
109	#if (defined(VBOX_STRICT) \|\| 1) && !defined(VBOX_WITH_RAM_IN_KERNEL)
110	# define GVMM_CHECK_SMAP_SETUP() uint32_t const fKernelFeatures = SUPR0GetKernelFeatures()
111	# define GVMM_CHECK_SMAP_CHECK(a_BadExpr) \
112	do { \
113	if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
114	{ \
115	RTCCUINTREG fEflCheck = ASMGetFlags(); \
116	if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
117	{ /* likely */ } \
118	else \
119	{ \
120	SUPR0Printf("%s, line %d: EFLAGS.AC is clear! (%#x)\n", __FUNCTION__, __LINE__, (uint32_t)fEflCheck); \
121	a_BadExpr; \
122	} \
123	} \
124	} while (0)
125	# define GVMM_CHECK_SMAP_CHECK2(a_pGVM, a_BadExpr) \
126	do { \
127	if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
128	{ \
129	RTCCUINTREG fEflCheck = ASMGetFlags(); \
130	if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
131	{ /* likely */ } \
132	else \
133	{ \
134	SUPR0BadContext((a_pGVM) ? (a_pGVM)->pSession : NULL, __FILE__, __LINE__, "EFLAGS.AC is zero!"); \
135	a_BadExpr; \
136	} \
137	} \
138	} while (0)
139	#else
140	# define GVMM_CHECK_SMAP_SETUP() uint32_t const fKernelFeatures = 0
141	# define GVMM_CHECK_SMAP_CHECK(a_BadExpr) NOREF(fKernelFeatures)
142	# define GVMM_CHECK_SMAP_CHECK2(a_pGVM, a_BadExpr) NOREF(fKernelFeatures)
143	#endif
144
145	/** Special value that GVMMR0DeregisterVCpu sets. */
146	#define GVMM_RTNATIVETHREAD_DESTROYED (~(RTNATIVETHREAD)1)
147	AssertCompile(GVMM_RTNATIVETHREAD_DESTROYED != NIL_RTNATIVETHREAD);
148
149
150	/*********************************************************************************************************************************
151	* Structures and Typedefs *
152	*********************************************************************************************************************************/
153
154	/**
155	* Global VM handle.
156	*/
157	typedef struct GVMHANDLE
158	{
159	/** The index of the next handle in the list (free or used). (0 is nil.) */
160	uint16_t volatile iNext;
161	/** Our own index / handle value. */
162	uint16_t iSelf;
163	/** The process ID of the handle owner.
164	* This is used for access checks. */
165	RTPROCESS ProcId;
166	/** The pointer to the ring-0 only (aka global) VM structure. */
167	PGVM pGVM;
168	/** The virtual machine object. */
169	void *pvObj;
170	/** The session this VM is associated with. */
171	PSUPDRVSESSION pSession;
172	/** The ring-0 handle of the EMT0 thread.
173	* This is used for ownership checks as well as looking up a VM handle by thread
174	* at times like assertions. */
175	RTNATIVETHREAD hEMT0;
176	} GVMHANDLE;
177	/** Pointer to a global VM handle. */
178	typedef GVMHANDLE *PGVMHANDLE;
179
180	/** Number of GVM handles (including the NIL handle). */
181	#if HC_ARCH_BITS == 64
182	# define GVMM_MAX_HANDLES 8192
183	#else
184	# define GVMM_MAX_HANDLES 128
185	#endif
186
187	/**
188	* Per host CPU GVMM data.
189	*/
190	typedef struct GVMMHOSTCPU
191	{
192	/** Magic number (GVMMHOSTCPU_MAGIC). */
193	uint32_t volatile u32Magic;
194	/** The CPU ID. */
195	RTCPUID idCpu;
196	/** The CPU set index. */
197	uint32_t idxCpuSet;
198
199	#ifdef GVMM_SCHED_WITH_PPT
200	/** Periodic preemption timer data. */
201	struct
202	{
203	/** The handle to the periodic preemption timer. */
204	PRTTIMER pTimer;
205	/** Spinlock protecting the data below. */
206	RTSPINLOCK hSpinlock;
207	/** The smalles Hz that we need to care about. (static) */
208	uint32_t uMinHz;
209	/** The number of ticks between each historization. */
210	uint32_t cTicksHistoriziationInterval;
211	/** The current historization tick (counting up to
212	* cTicksHistoriziationInterval and then resetting). */
213	uint32_t iTickHistorization;
214	/** The current timer interval. This is set to 0 when inactive. */
215	uint32_t cNsInterval;
216	/** The current timer frequency. This is set to 0 when inactive. */
217	uint32_t uTimerHz;
218	/** The current max frequency reported by the EMTs.
219	* This gets historicize and reset by the timer callback. This is
220	* read without holding the spinlock, so needs atomic updating. */
221	uint32_t volatile uDesiredHz;
222	/** Whether the timer was started or not. */
223	bool volatile fStarted;
224	/** Set if we're starting timer. */
225	bool volatile fStarting;
226	/** The index of the next history entry (mod it). */
227	uint32_t iHzHistory;
228	/** Historicized uDesiredHz values. The array wraps around, new entries
229	* are added at iHzHistory. This is updated approximately every
230	* GVMMHOSTCPU_PPT_HIST_INTERVAL_NS by the timer callback. */
231	uint32_t aHzHistory[8];
232	/** Statistics counter for recording the number of interval changes. */
233	uint32_t cChanges;
234	/** Statistics counter for recording the number of timer starts. */
235	uint32_t cStarts;
236	} Ppt;
237	#endif /* GVMM_SCHED_WITH_PPT */
238
239	} GVMMHOSTCPU;
240	/** Pointer to the per host CPU GVMM data. */
241	typedef GVMMHOSTCPU *PGVMMHOSTCPU;
242	/** The GVMMHOSTCPU::u32Magic value (Petra, Tanya & Rachel Haden). */
243	#define GVMMHOSTCPU_MAGIC UINT32_C(0x19711011)
244	/** The interval on history entry should cover (approximately) give in
245	* nanoseconds. */
246	#define GVMMHOSTCPU_PPT_HIST_INTERVAL_NS UINT32_C(20000000)
247
248
249	/**
250	* The GVMM instance data.
251	*/
252	typedef struct GVMM
253	{
254	/** Eyecatcher / magic. */
255	uint32_t u32Magic;
256	/** The index of the head of the free handle chain. (0 is nil.) */
257	uint16_t volatile iFreeHead;
258	/** The index of the head of the active handle chain. (0 is nil.) */
259	uint16_t volatile iUsedHead;
260	/** The number of VMs. */
261	uint16_t volatile cVMs;
262	/** Alignment padding. */
263	uint16_t u16Reserved;
264	/** The number of EMTs. */
265	uint32_t volatile cEMTs;
266	/** The number of EMTs that have halted in GVMMR0SchedHalt. */
267	uint32_t volatile cHaltedEMTs;
268	/** Mini lock for restricting early wake-ups to one thread. */
269	bool volatile fDoingEarlyWakeUps;
270	bool afPadding[3]; /*< explicit alignment padding. /
271	/** When the next halted or sleeping EMT will wake up.
272	* This is set to 0 when it needs recalculating and to UINT64_MAX when
273	* there are no halted or sleeping EMTs in the GVMM. */
274	uint64_t uNsNextEmtWakeup;
275	/** The lock used to serialize VM creation, destruction and associated events that
276	* isn't performance critical. Owners may acquire the list lock. */
277	RTCRITSECT CreateDestroyLock;
278	/** The lock used to serialize used list updates and accesses.
279	* This indirectly includes scheduling since the scheduler will have to walk the
280	* used list to examin running VMs. Owners may not acquire any other locks. */
281	RTCRITSECTRW UsedLock;
282	/** The handle array.
283	* The size of this array defines the maximum number of currently running VMs.
284	* The first entry is unused as it represents the NIL handle. */
285	GVMHANDLE aHandles[GVMM_MAX_HANDLES];
286
287	/** @gcfgm{/GVMM/cEMTsMeansCompany, 32-bit, 0, UINT32_MAX, 1}
288	* The number of EMTs that means we no longer consider ourselves alone on a
289	* CPU/Core.
290	*/
291	uint32_t cEMTsMeansCompany;
292	/** @gcfgm{/GVMM/MinSleepAlone,32-bit, 0, 100000000, 750000, ns}
293	* The minimum sleep time for when we're alone, in nano seconds.
294	*/
295	uint32_t nsMinSleepAlone;
296	/** @gcfgm{/GVMM/MinSleepCompany,32-bit,0, 100000000, 15000, ns}
297	* The minimum sleep time for when we've got company, in nano seconds.
298	*/
299	uint32_t nsMinSleepCompany;
300	/** @gcfgm{/GVMM/EarlyWakeUp1, 32-bit, 0, 100000000, 25000, ns}
301	* The limit for the first round of early wake-ups, given in nano seconds.
302	*/
303	uint32_t nsEarlyWakeUp1;
304	/** @gcfgm{/GVMM/EarlyWakeUp2, 32-bit, 0, 100000000, 50000, ns}
305	* The limit for the second round of early wake-ups, given in nano seconds.
306	*/
307	uint32_t nsEarlyWakeUp2;
308
309	/** Set if we're doing early wake-ups.
310	* This reflects nsEarlyWakeUp1 and nsEarlyWakeUp2. */
311	bool volatile fDoEarlyWakeUps;
312
313	/** The number of entries in the host CPU array (aHostCpus). */
314	uint32_t cHostCpus;
315	/** Per host CPU data (variable length). */
316	GVMMHOSTCPU aHostCpus[1];
317	} GVMM;
318	AssertCompileMemberAlignment(GVMM, CreateDestroyLock, 8);
319	AssertCompileMemberAlignment(GVMM, UsedLock, 8);
320	AssertCompileMemberAlignment(GVMM, uNsNextEmtWakeup, 8);
321	/** Pointer to the GVMM instance data. */
322	typedef GVMM *PGVMM;
323
324	/** The GVMM::u32Magic value (Charlie Haden). */
325	#define GVMM_MAGIC UINT32_C(0x19370806)
326
327
328
329	/*********************************************************************************************************************************
330	* Global Variables *
331	*********************************************************************************************************************************/
332	/** Pointer to the GVMM instance data.
333	* (Just my general dislike for global variables.) */
334	static PGVMM g_pGVMM = NULL;
335
336	/** Macro for obtaining and validating the g_pGVMM pointer.
337	* On failure it will return from the invoking function with the specified return value.
338	*
339	* @param pGVMM The name of the pGVMM variable.
340	* @param rc The return value on failure. Use VERR_GVMM_INSTANCE for VBox
341	* status codes.
342	*/
343	#define GVMM_GET_VALID_INSTANCE(pGVMM, rc) \
344	do { \
345	(pGVMM) = g_pGVMM;\
346	AssertPtrReturn((pGVMM), (rc)); \
347	AssertMsgReturn((pGVMM)->u32Magic == GVMM_MAGIC, ("%p - %#x\n", (pGVMM), (pGVMM)->u32Magic), (rc)); \
348	} while (0)
349
350	/** Macro for obtaining and validating the g_pGVMM pointer, void function variant.
351	* On failure it will return from the invoking function.
352	*
353	* @param pGVMM The name of the pGVMM variable.
354	*/
355	#define GVMM_GET_VALID_INSTANCE_VOID(pGVMM) \
356	do { \
357	(pGVMM) = g_pGVMM;\
358	AssertPtrReturnVoid((pGVMM)); \
359	AssertMsgReturnVoid((pGVMM)->u32Magic == GVMM_MAGIC, ("%p - %#x\n", (pGVMM), (pGVMM)->u32Magic)); \
360	} while (0)
361
362
363	/*********************************************************************************************************************************
364	* Internal Functions *
365	*********************************************************************************************************************************/
366	static void gvmmR0InitPerVMData(PGVM pGVM, int16_t hSelf, VMCPUID cCpus, PSUPDRVSESSION pSession);
367	static DECLCALLBACK(void) gvmmR0HandleObjDestructor(void pvObj, void pvGVMM, void *pvHandle);
368	static int gvmmR0ByGVM(PGVM pGVM, PGVMM *ppGVMM, bool fTakeUsedLock);
369	static int gvmmR0ByGVMandEMT(PGVM pGVM, VMCPUID idCpu, PGVMM *ppGVMM);
370
371	#ifdef GVMM_SCHED_WITH_PPT
372	static DECLCALLBACK(void) gvmmR0SchedPeriodicPreemptionTimerCallback(PRTTIMER pTimer, void *pvUser, uint64_t iTick);
373	#endif
374
375
376	/**
377	* Initializes the GVMM.
378	*
379	* This is called while owning the loader semaphore (see supdrvIOCtl_LdrLoad()).
380	*
381	* @returns VBox status code.
382	*/
383	GVMMR0DECL(int) GVMMR0Init(void)
384	{
385	LogFlow(("GVMMR0Init:\n"));
386
387	/*
388	* Allocate and initialize the instance data.
389	*/
390	uint32_t cHostCpus = RTMpGetArraySize();
391	AssertMsgReturn(cHostCpus > 0 && cHostCpus < _64K, ("%d", (int)cHostCpus), VERR_GVMM_HOST_CPU_RANGE);
392
393	PGVMM pGVMM = (PGVMM)RTMemAllocZ(RT_UOFFSETOF_DYN(GVMM, aHostCpus[cHostCpus]));
394	if (!pGVMM)
395	return VERR_NO_MEMORY;
396	int rc = RTCritSectInitEx(&pGVMM->CreateDestroyLock, 0, NIL_RTLOCKVALCLASS, RTLOCKVAL_SUB_CLASS_NONE,
397	"GVMM-CreateDestroyLock");
398	if (RT_SUCCESS(rc))
399	{
400	rc = RTCritSectRwInitEx(&pGVMM->UsedLock, 0, NIL_RTLOCKVALCLASS, RTLOCKVAL_SUB_CLASS_NONE, "GVMM-UsedLock");
401	if (RT_SUCCESS(rc))
402	{
403	pGVMM->u32Magic = GVMM_MAGIC;
404	pGVMM->iUsedHead = 0;
405	pGVMM->iFreeHead = 1;
406
407	/* the nil handle */
408	pGVMM->aHandles[0].iSelf = 0;
409	pGVMM->aHandles[0].iNext = 0;
410
411	/* the tail */
412	unsigned i = RT_ELEMENTS(pGVMM->aHandles) - 1;
413	pGVMM->aHandles[i].iSelf = i;
414	pGVMM->aHandles[i].iNext = 0; /* nil */
415
416	/* the rest */
417	while (i-- > 1)
418	{
419	pGVMM->aHandles[i].iSelf = i;
420	pGVMM->aHandles[i].iNext = i + 1;
421	}
422
423	/* The default configuration values. */
424	uint32_t cNsResolution = RTSemEventMultiGetResolution();
425	pGVMM->cEMTsMeansCompany = 1; /** @todo should be adjusted to relative to the cpu count or something... */
426	if (cNsResolution >= 5*RT_NS_100US)
427	{
428	pGVMM->nsMinSleepAlone = 750000 /* ns (0.750 ms) /; /* @todo this should be adjusted to be 75% (or something) of the scheduler granularity... */
429	pGVMM->nsMinSleepCompany = 15000 /* ns (0.015 ms) */;
430	pGVMM->nsEarlyWakeUp1 = 25000 /* ns (0.025 ms) */;
431	pGVMM->nsEarlyWakeUp2 = 50000 /* ns (0.050 ms) */;
432	}
433	else if (cNsResolution > RT_NS_100US)
434	{
435	pGVMM->nsMinSleepAlone = cNsResolution / 2;
436	pGVMM->nsMinSleepCompany = cNsResolution / 4;
437	pGVMM->nsEarlyWakeUp1 = 0;
438	pGVMM->nsEarlyWakeUp2 = 0;
439	}
440	else
441	{
442	pGVMM->nsMinSleepAlone = 2000;
443	pGVMM->nsMinSleepCompany = 2000;
444	pGVMM->nsEarlyWakeUp1 = 0;
445	pGVMM->nsEarlyWakeUp2 = 0;
446	}
447	pGVMM->fDoEarlyWakeUps = pGVMM->nsEarlyWakeUp1 > 0 && pGVMM->nsEarlyWakeUp2 > 0;
448
449	/* The host CPU data. */
450	pGVMM->cHostCpus = cHostCpus;
451	uint32_t iCpu = cHostCpus;
452	RTCPUSET PossibleSet;
453	RTMpGetSet(&PossibleSet);
454	while (iCpu-- > 0)
455	{
456	pGVMM->aHostCpus[iCpu].idxCpuSet = iCpu;
457	#ifdef GVMM_SCHED_WITH_PPT
458	pGVMM->aHostCpus[iCpu].Ppt.pTimer = NULL;
459	pGVMM->aHostCpus[iCpu].Ppt.hSpinlock = NIL_RTSPINLOCK;
460	pGVMM->aHostCpus[iCpu].Ppt.uMinHz = 5; /** @todo Add some API which figures this one out. (not that important) */
461	pGVMM->aHostCpus[iCpu].Ppt.cTicksHistoriziationInterval = 1;
462	//pGVMM->aHostCpus[iCpu].Ppt.iTickHistorization = 0;
463	//pGVMM->aHostCpus[iCpu].Ppt.cNsInterval = 0;
464	//pGVMM->aHostCpus[iCpu].Ppt.uTimerHz = 0;
465	//pGVMM->aHostCpus[iCpu].Ppt.uDesiredHz = 0;
466	//pGVMM->aHostCpus[iCpu].Ppt.fStarted = false;
467	//pGVMM->aHostCpus[iCpu].Ppt.fStarting = false;
468	//pGVMM->aHostCpus[iCpu].Ppt.iHzHistory = 0;
469	//pGVMM->aHostCpus[iCpu].Ppt.aHzHistory = {0};
470	#endif
471
472	if (RTCpuSetIsMember(&PossibleSet, iCpu))
473	{
474	pGVMM->aHostCpus[iCpu].idCpu = RTMpCpuIdFromSetIndex(iCpu);
475	pGVMM->aHostCpus[iCpu].u32Magic = GVMMHOSTCPU_MAGIC;
476
477	#ifdef GVMM_SCHED_WITH_PPT
478	rc = RTTimerCreateEx(&pGVMM->aHostCpus[iCpu].Ppt.pTimer,
479	5010001000 /* whatever */,
480	RTTIMER_FLAGS_CPU(iCpu) \| RTTIMER_FLAGS_HIGH_RES,
481	gvmmR0SchedPeriodicPreemptionTimerCallback,
482	&pGVMM->aHostCpus[iCpu]);
483	if (RT_SUCCESS(rc))
484	rc = RTSpinlockCreate(&pGVMM->aHostCpus[iCpu].Ppt.hSpinlock, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "GVMM/CPU");
485	if (RT_FAILURE(rc))
486	{
487	while (iCpu < cHostCpus)
488	{
489	RTTimerDestroy(pGVMM->aHostCpus[iCpu].Ppt.pTimer);
490	RTSpinlockDestroy(pGVMM->aHostCpus[iCpu].Ppt.hSpinlock);
491	pGVMM->aHostCpus[iCpu].Ppt.hSpinlock = NIL_RTSPINLOCK;
492	iCpu++;
493	}
494	break;
495	}
496	#endif
497	}
498	else
499	{
500	pGVMM->aHostCpus[iCpu].idCpu = NIL_RTCPUID;
501	pGVMM->aHostCpus[iCpu].u32Magic = 0;
502	}
503	}
504	if (RT_SUCCESS(rc))
505	{
506	g_pGVMM = pGVMM;
507	LogFlow(("GVMMR0Init: pGVMM=%p cHostCpus=%u\n", pGVMM, cHostCpus));
508	return VINF_SUCCESS;
509	}
510
511	/* bail out. */
512	RTCritSectRwDelete(&pGVMM->UsedLock);
513	}
514	RTCritSectDelete(&pGVMM->CreateDestroyLock);
515	}
516
517	RTMemFree(pGVMM);
518	return rc;
519	}
520
521
522	/**
523	* Terminates the GVM.
524	*
525	* This is called while owning the loader semaphore (see supdrvLdrFree()).
526	* And unless something is wrong, there should be absolutely no VMs
527	* registered at this point.
528	*/
529	GVMMR0DECL(void) GVMMR0Term(void)
530	{
531	LogFlow(("GVMMR0Term:\n"));
532
533	PGVMM pGVMM = g_pGVMM;
534	g_pGVMM = NULL;
535	if (RT_UNLIKELY(!RT_VALID_PTR(pGVMM)))
536	{
537	SUPR0Printf("GVMMR0Term: pGVMM=%RKv\n", pGVMM);
538	return;
539	}
540
541	/*
542	* First of all, stop all active timers.
543	*/
544	uint32_t cActiveTimers = 0;
545	uint32_t iCpu = pGVMM->cHostCpus;
546	while (iCpu-- > 0)
547	{
548	ASMAtomicWriteU32(&pGVMM->aHostCpus[iCpu].u32Magic, ~GVMMHOSTCPU_MAGIC);
549	#ifdef GVMM_SCHED_WITH_PPT
550	if ( pGVMM->aHostCpus[iCpu].Ppt.pTimer != NULL
551	&& RT_SUCCESS(RTTimerStop(pGVMM->aHostCpus[iCpu].Ppt.pTimer)))
552	cActiveTimers++;
553	#endif
554	}
555	if (cActiveTimers)
556	RTThreadSleep(1); /* fudge */
557
558	/*
559	* Invalidate the and free resources.
560	*/
561	pGVMM->u32Magic = ~GVMM_MAGIC;
562	RTCritSectRwDelete(&pGVMM->UsedLock);
563	RTCritSectDelete(&pGVMM->CreateDestroyLock);
564
565	pGVMM->iFreeHead = 0;
566	if (pGVMM->iUsedHead)
567	{
568	SUPR0Printf("GVMMR0Term: iUsedHead=%#x! (cVMs=%#x cEMTs=%#x)\n", pGVMM->iUsedHead, pGVMM->cVMs, pGVMM->cEMTs);
569	pGVMM->iUsedHead = 0;
570	}
571
572	#ifdef GVMM_SCHED_WITH_PPT
573	iCpu = pGVMM->cHostCpus;
574	while (iCpu-- > 0)
575	{
576	RTTimerDestroy(pGVMM->aHostCpus[iCpu].Ppt.pTimer);
577	pGVMM->aHostCpus[iCpu].Ppt.pTimer = NULL;
578	RTSpinlockDestroy(pGVMM->aHostCpus[iCpu].Ppt.hSpinlock);
579	pGVMM->aHostCpus[iCpu].Ppt.hSpinlock = NIL_RTSPINLOCK;
580	}
581	#endif
582
583	RTMemFree(pGVMM);
584	}
585
586
587	/**
588	* A quick hack for setting global config values.
589	*
590	* @returns VBox status code.
591	*
592	* @param pSession The session handle. Used for authentication.
593	* @param pszName The variable name.
594	* @param u64Value The new value.
595	*/
596	GVMMR0DECL(int) GVMMR0SetConfig(PSUPDRVSESSION pSession, const char *pszName, uint64_t u64Value)
597	{
598	/*
599	* Validate input.
600	*/
601	PGVMM pGVMM;
602	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
603	AssertPtrReturn(pSession, VERR_INVALID_HANDLE);
604	AssertPtrReturn(pszName, VERR_INVALID_POINTER);
605
606	/*
607	* String switch time!
608	*/
609	if (strncmp(pszName, RT_STR_TUPLE("/GVMM/")))
610	return VERR_CFGM_VALUE_NOT_FOUND; /* borrow status codes from CFGM... */
611	int rc = VINF_SUCCESS;
612	pszName += sizeof("/GVMM/") - 1;
613	if (!strcmp(pszName, "cEMTsMeansCompany"))
614	{
615	if (u64Value <= UINT32_MAX)
616	pGVMM->cEMTsMeansCompany = u64Value;
617	else
618	rc = VERR_OUT_OF_RANGE;
619	}
620	else if (!strcmp(pszName, "MinSleepAlone"))
621	{
622	if (u64Value <= RT_NS_100MS)
623	pGVMM->nsMinSleepAlone = u64Value;
624	else
625	rc = VERR_OUT_OF_RANGE;
626	}
627	else if (!strcmp(pszName, "MinSleepCompany"))
628	{
629	if (u64Value <= RT_NS_100MS)
630	pGVMM->nsMinSleepCompany = u64Value;
631	else
632	rc = VERR_OUT_OF_RANGE;
633	}
634	else if (!strcmp(pszName, "EarlyWakeUp1"))
635	{
636	if (u64Value <= RT_NS_100MS)
637	{
638	pGVMM->nsEarlyWakeUp1 = u64Value;
639	pGVMM->fDoEarlyWakeUps = pGVMM->nsEarlyWakeUp1 > 0 && pGVMM->nsEarlyWakeUp2 > 0;
640	}
641	else
642	rc = VERR_OUT_OF_RANGE;
643	}
644	else if (!strcmp(pszName, "EarlyWakeUp2"))
645	{
646	if (u64Value <= RT_NS_100MS)
647	{
648	pGVMM->nsEarlyWakeUp2 = u64Value;
649	pGVMM->fDoEarlyWakeUps = pGVMM->nsEarlyWakeUp1 > 0 && pGVMM->nsEarlyWakeUp2 > 0;
650	}
651	else
652	rc = VERR_OUT_OF_RANGE;
653	}
654	else
655	rc = VERR_CFGM_VALUE_NOT_FOUND;
656	return rc;
657	}
658
659
660	/**
661	* A quick hack for getting global config values.
662	*
663	* @returns VBox status code.
664	*
665	* @param pSession The session handle. Used for authentication.
666	* @param pszName The variable name.
667	* @param pu64Value Where to return the value.
668	*/
669	GVMMR0DECL(int) GVMMR0QueryConfig(PSUPDRVSESSION pSession, const char pszName, uint64_t pu64Value)
670	{
671	/*
672	* Validate input.
673	*/
674	PGVMM pGVMM;
675	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
676	AssertPtrReturn(pSession, VERR_INVALID_HANDLE);
677	AssertPtrReturn(pszName, VERR_INVALID_POINTER);
678	AssertPtrReturn(pu64Value, VERR_INVALID_POINTER);
679
680	/*
681	* String switch time!
682	*/
683	if (strncmp(pszName, RT_STR_TUPLE("/GVMM/")))
684	return VERR_CFGM_VALUE_NOT_FOUND; /* borrow status codes from CFGM... */
685	int rc = VINF_SUCCESS;
686	pszName += sizeof("/GVMM/") - 1;
687	if (!strcmp(pszName, "cEMTsMeansCompany"))
688	*pu64Value = pGVMM->cEMTsMeansCompany;
689	else if (!strcmp(pszName, "MinSleepAlone"))
690	*pu64Value = pGVMM->nsMinSleepAlone;
691	else if (!strcmp(pszName, "MinSleepCompany"))
692	*pu64Value = pGVMM->nsMinSleepCompany;
693	else if (!strcmp(pszName, "EarlyWakeUp1"))
694	*pu64Value = pGVMM->nsEarlyWakeUp1;
695	else if (!strcmp(pszName, "EarlyWakeUp2"))
696	*pu64Value = pGVMM->nsEarlyWakeUp2;
697	else
698	rc = VERR_CFGM_VALUE_NOT_FOUND;
699	return rc;
700	}
701
702
703	/**
704	* Acquire the 'used' lock in shared mode.
705	*
706	* This prevents destruction of the VM while we're in ring-0.
707	*
708	* @returns IPRT status code, see RTSemFastMutexRequest.
709	* @param a_pGVMM The GVMM instance data.
710	* @sa GVMMR0_USED_SHARED_UNLOCK, GVMMR0_USED_EXCLUSIVE_LOCK
711	*/
712	#define GVMMR0_USED_SHARED_LOCK(a_pGVMM) RTCritSectRwEnterShared(&(a_pGVMM)->UsedLock)
713
714	/**
715	* Release the 'used' lock in when owning it in shared mode.
716	*
717	* @returns IPRT status code, see RTSemFastMutexRequest.
718	* @param a_pGVMM The GVMM instance data.
719	* @sa GVMMR0_USED_SHARED_LOCK
720	*/
721	#define GVMMR0_USED_SHARED_UNLOCK(a_pGVMM) RTCritSectRwLeaveShared(&(a_pGVMM)->UsedLock)
722
723	/**
724	* Acquire the 'used' lock in exclusive mode.
725	*
726	* Only use this function when making changes to the used list.
727	*
728	* @returns IPRT status code, see RTSemFastMutexRequest.
729	* @param a_pGVMM The GVMM instance data.
730	* @sa GVMMR0_USED_EXCLUSIVE_UNLOCK
731	*/
732	#define GVMMR0_USED_EXCLUSIVE_LOCK(a_pGVMM) RTCritSectRwEnterExcl(&(a_pGVMM)->UsedLock)
733
734	/**
735	* Release the 'used' lock when owning it in exclusive mode.
736	*
737	* @returns IPRT status code, see RTSemFastMutexRelease.
738	* @param a_pGVMM The GVMM instance data.
739	* @sa GVMMR0_USED_EXCLUSIVE_LOCK, GVMMR0_USED_SHARED_UNLOCK
740	*/
741	#define GVMMR0_USED_EXCLUSIVE_UNLOCK(a_pGVMM) RTCritSectRwLeaveExcl(&(a_pGVMM)->UsedLock)
742
743
744	/**
745	* Try acquire the 'create & destroy' lock.
746	*
747	* @returns IPRT status code, see RTSemFastMutexRequest.
748	* @param pGVMM The GVMM instance data.
749	*/
750	DECLINLINE(int) gvmmR0CreateDestroyLock(PGVMM pGVMM)
751	{
752	LogFlow(("++gvmmR0CreateDestroyLock(%p)\n", pGVMM));
753	int rc = RTCritSectEnter(&pGVMM->CreateDestroyLock);
754	LogFlow(("gvmmR0CreateDestroyLock(%p)->%Rrc\n", pGVMM, rc));
755	return rc;
756	}
757
758
759	/**
760	* Release the 'create & destroy' lock.
761	*
762	* @returns IPRT status code, see RTSemFastMutexRequest.
763	* @param pGVMM The GVMM instance data.
764	*/
765	DECLINLINE(int) gvmmR0CreateDestroyUnlock(PGVMM pGVMM)
766	{
767	LogFlow(("--gvmmR0CreateDestroyUnlock(%p)\n", pGVMM));
768	int rc = RTCritSectLeave(&pGVMM->CreateDestroyLock);
769	AssertRC(rc);
770	return rc;
771	}
772
773
774	/**
775	* Request wrapper for the GVMMR0CreateVM API.
776	*
777	* @returns VBox status code.
778	* @param pReq The request buffer.
779	* @param pSession The session handle. The VM will be associated with this.
780	*/
781	GVMMR0DECL(int) GVMMR0CreateVMReq(PGVMMCREATEVMREQ pReq, PSUPDRVSESSION pSession)
782	{
783	/*
784	* Validate the request.
785	*/
786	if (!RT_VALID_PTR(pReq))
787	return VERR_INVALID_POINTER;
788	if (pReq->Hdr.cbReq != sizeof(*pReq))
789	return VERR_INVALID_PARAMETER;
790	if (pReq->pSession != pSession)
791	return VERR_INVALID_POINTER;
792
793	/*
794	* Execute it.
795	*/
796	PGVM pGVM;
797	pReq->pVMR0 = NULL;
798	pReq->pVMR3 = NIL_RTR3PTR;
799	int rc = GVMMR0CreateVM(pSession, pReq->cCpus, &pGVM);
800	if (RT_SUCCESS(rc))
801	{
802	pReq->pVMR0 = pGVM; /** @todo don't expose this to ring-3, use a unique random number instead. */
803	pReq->pVMR3 = pGVM->pVMR3;
804	}
805	return rc;
806	}
807
808
809	/**
810	* Allocates the VM structure and registers it with GVM.
811	*
812	* The caller will become the VM owner and there by the EMT.
813	*
814	* @returns VBox status code.
815	* @param pSession The support driver session.
816	* @param cCpus Number of virtual CPUs for the new VM.
817	* @param ppGVM Where to store the pointer to the VM structure.
818	*
819	* @thread EMT.
820	*/
821	GVMMR0DECL(int) GVMMR0CreateVM(PSUPDRVSESSION pSession, uint32_t cCpus, PGVM *ppGVM)
822	{
823	LogFlow(("GVMMR0CreateVM: pSession=%p\n", pSession));
824	PGVMM pGVMM;
825	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
826
827	AssertPtrReturn(ppGVM, VERR_INVALID_POINTER);
828	*ppGVM = NULL;
829
830	if ( cCpus == 0
831	\|\| cCpus > VMM_MAX_CPU_COUNT)
832	return VERR_INVALID_PARAMETER;
833
834	RTNATIVETHREAD hEMT0 = RTThreadNativeSelf();
835	AssertReturn(hEMT0 != NIL_RTNATIVETHREAD, VERR_GVMM_BROKEN_IPRT);
836	RTPROCESS ProcId = RTProcSelf();
837	AssertReturn(ProcId != NIL_RTPROCESS, VERR_GVMM_BROKEN_IPRT);
838
839	/*
840	* The whole allocation process is protected by the lock.
841	*/
842	int rc = gvmmR0CreateDestroyLock(pGVMM);
843	AssertRCReturn(rc, rc);
844
845	/*
846	* Only one VM per session.
847	*/
848	if (SUPR0GetSessionVM(pSession) != NULL)
849	{
850	gvmmR0CreateDestroyUnlock(pGVMM);
851	SUPR0Printf("GVMMR0CreateVM: The session %p already got a VM: %p\n", pSession, SUPR0GetSessionVM(pSession));
852	return VERR_ALREADY_EXISTS;
853	}
854
855	/*
856	* Allocate a handle first so we don't waste resources unnecessarily.
857	*/
858	uint16_t iHandle = pGVMM->iFreeHead;
859	if (iHandle)
860	{
861	PGVMHANDLE pHandle = &pGVMM->aHandles[iHandle];
862
863	/* consistency checks, a bit paranoid as always. */
864	if ( !pHandle->pGVM
865	&& !pHandle->pvObj
866	&& pHandle->iSelf == iHandle)
867	{
868	pHandle->pvObj = SUPR0ObjRegister(pSession, SUPDRVOBJTYPE_VM, gvmmR0HandleObjDestructor, pGVMM, pHandle);
869	if (pHandle->pvObj)
870	{
871	/*
872	* Move the handle from the free to used list and perform permission checks.
873	*/
874	rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
875	AssertRC(rc);
876
877	pGVMM->iFreeHead = pHandle->iNext;
878	pHandle->iNext = pGVMM->iUsedHead;
879	pGVMM->iUsedHead = iHandle;
880	pGVMM->cVMs++;
881
882	pHandle->pGVM = NULL;
883	pHandle->pSession = pSession;
884	pHandle->hEMT0 = NIL_RTNATIVETHREAD;
885	pHandle->ProcId = NIL_RTPROCESS;
886
887	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
888
889	rc = SUPR0ObjVerifyAccess(pHandle->pvObj, pSession, NULL);
890	if (RT_SUCCESS(rc))
891	{
892	/*
893	* Allocate memory for the VM structure (combined VM + GVM).
894	*/
895	const uint32_t cbVM = RT_UOFFSETOF_DYN(GVM, aCpus[cCpus]);
896	const uint32_t cPages = RT_ALIGN_32(cbVM, PAGE_SIZE) >> PAGE_SHIFT;
897	RTR0MEMOBJ hVMMemObj = NIL_RTR0MEMOBJ;
898	rc = RTR0MemObjAllocPage(&hVMMemObj, cPages << PAGE_SHIFT, false /* fExecutable */);
899	if (RT_SUCCESS(rc))
900	{
901	PGVM pGVM = (PGVM)RTR0MemObjAddress(hVMMemObj);
902	AssertPtr(pGVM);
903
904	/*
905	* Initialise the structure.
906	*/
907	RT_BZERO(pGVM, cPages << PAGE_SHIFT);
908	gvmmR0InitPerVMData(pGVM, iHandle, cCpus, pSession);
909	pGVM->gvmm.s.VMMemObj = hVMMemObj;
910	rc = GMMR0InitPerVMData(pGVM);
911	int rc2 = PGMR0InitPerVMData(pGVM);
912	VMMR0InitPerVMData(pGVM);
913	DBGFR0InitPerVMData(pGVM);
914	PDMR0InitPerVMData(pGVM);
915	IOMR0InitPerVMData(pGVM);
916	TMR0InitPerVMData(pGVM);
917	if (RT_SUCCESS(rc) && RT_SUCCESS(rc2))
918	{
919	/*
920	* Allocate page array.
921	* This currently have to be made available to ring-3, but this is should change eventually.
922	*/
923	rc = RTR0MemObjAllocPage(&pGVM->gvmm.s.VMPagesMemObj, cPages * sizeof(SUPPAGE), false /* fExecutable */);
924	if (RT_SUCCESS(rc))
925	{
926	PSUPPAGE paPages = (PSUPPAGE)RTR0MemObjAddress(pGVM->gvmm.s.VMPagesMemObj); AssertPtr(paPages);
927	for (uint32_t iPage = 0; iPage < cPages; iPage++)
928	{
929	paPages[iPage].uReserved = 0;
930	paPages[iPage].Phys = RTR0MemObjGetPagePhysAddr(pGVM->gvmm.s.VMMemObj, iPage);
931	Assert(paPages[iPage].Phys != NIL_RTHCPHYS);
932	}
933
934	/*
935	* Map the page array, VM and VMCPU structures into ring-3.
936	*/
937	AssertCompileSizeAlignment(VM, PAGE_SIZE);
938	rc = RTR0MemObjMapUserEx(&pGVM->gvmm.s.VMMapObj, pGVM->gvmm.s.VMMemObj, (RTR3PTR)-1, 0,
939	RTMEM_PROT_READ \| RTMEM_PROT_WRITE, NIL_RTR0PROCESS,
940	0 /offSub/, sizeof(VM));
941	for (VMCPUID i = 0; i < cCpus && RT_SUCCESS(rc); i++)
942	{
943	AssertCompileSizeAlignment(VMCPU, PAGE_SIZE);
944	rc = RTR0MemObjMapUserEx(&pGVM->aCpus[i].gvmm.s.VMCpuMapObj, pGVM->gvmm.s.VMMemObj,
945	(RTR3PTR)-1, 0, RTMEM_PROT_READ \| RTMEM_PROT_WRITE, NIL_RTR0PROCESS,
946	RT_UOFFSETOF_DYN(GVM, aCpus[i]), sizeof(VMCPU));
947	}
948	if (RT_SUCCESS(rc))
949	rc = RTR0MemObjMapUser(&pGVM->gvmm.s.VMPagesMapObj, pGVM->gvmm.s.VMPagesMemObj, (RTR3PTR)-1,
950	0 /* uAlignment */, RTMEM_PROT_READ \| RTMEM_PROT_WRITE,
951	NIL_RTR0PROCESS);
952	if (RT_SUCCESS(rc))
953	{
954	/*
955	* Initialize all the VM pointers.
956	*/
957	PVMR3 pVMR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMMapObj);
958	AssertPtr((void *)pVMR3);
959
960	for (VMCPUID i = 0; i < cCpus; i++)
961	{
962	pGVM->aCpus[i].pVMR0 = pGVM;
963	pGVM->aCpus[i].pVMR3 = pVMR3;
964	pGVM->apCpusR3[i] = RTR0MemObjAddressR3(pGVM->aCpus[i].gvmm.s.VMCpuMapObj);
965	pGVM->aCpus[i].pVCpuR3 = pGVM->apCpusR3[i];
966	pGVM->apCpusR0[i] = &pGVM->aCpus[i];
967	AssertPtr((void *)pGVM->apCpusR3[i]);
968	}
969
970	pGVM->paVMPagesR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMPagesMapObj);
971	AssertPtr((void *)pGVM->paVMPagesR3);
972
973	/*
974	* Complete the handle - take the UsedLock sem just to be careful.
975	*/
976	rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
977	AssertRC(rc);
978
979	pHandle->pGVM = pGVM;
980	pHandle->hEMT0 = hEMT0;
981	pHandle->ProcId = ProcId;
982	pGVM->pVMR3 = pVMR3;
983	pGVM->pVMR3Unsafe = pVMR3;
984	pGVM->aCpus[0].hEMT = hEMT0;
985	pGVM->aCpus[0].hNativeThreadR0 = hEMT0;
986	pGVM->aCpus[0].cEmtHashCollisions = 0;
987	uint32_t const idxHash = GVMM_EMT_HASH_1(hEMT0);
988	pGVM->aCpus[0].gvmm.s.idxEmtHash = (uint16_t)idxHash;
989	pGVM->gvmm.s.aEmtHash[idxHash].hNativeEmt = hEMT0;
990	pGVM->gvmm.s.aEmtHash[idxHash].idVCpu = 0;
991	pGVMM->cEMTs += cCpus;
992
993	/* Associate it with the session and create the context hook for EMT0. */
994	rc = SUPR0SetSessionVM(pSession, pGVM, pGVM);
995	if (RT_SUCCESS(rc))
996	{
997	rc = VMMR0ThreadCtxHookCreateForEmt(&pGVM->aCpus[0]);
998	if (RT_SUCCESS(rc))
999	{
1000	/*
1001	* Done!
1002	*/
1003	VBOXVMM_R0_GVMM_VM_CREATED(pGVM, pGVM, ProcId, (void *)hEMT0, cCpus);
1004
1005	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1006	gvmmR0CreateDestroyUnlock(pGVMM);
1007
1008	CPUMR0RegisterVCpuThread(&pGVM->aCpus[0]);
1009
1010	*ppGVM = pGVM;
1011	Log(("GVMMR0CreateVM: pVMR3=%p pGVM=%p hGVM=%d\n", pVMR3, pGVM, iHandle));
1012	return VINF_SUCCESS;
1013	}
1014
1015	SUPR0SetSessionVM(pSession, NULL, NULL);
1016	}
1017	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1018	}
1019
1020	/* Cleanup mappings. */
1021	if (pGVM->gvmm.s.VMMapObj != NIL_RTR0MEMOBJ)
1022	{
1023	RTR0MemObjFree(pGVM->gvmm.s.VMMapObj, false /* fFreeMappings */);
1024	pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
1025	}
1026	for (VMCPUID i = 0; i < cCpus; i++)
1027	if (pGVM->aCpus[i].gvmm.s.VMCpuMapObj != NIL_RTR0MEMOBJ)
1028	{
1029	RTR0MemObjFree(pGVM->aCpus[i].gvmm.s.VMCpuMapObj, false /* fFreeMappings */);
1030	pGVM->aCpus[i].gvmm.s.VMCpuMapObj = NIL_RTR0MEMOBJ;
1031	}
1032	if (pGVM->gvmm.s.VMPagesMapObj != NIL_RTR0MEMOBJ)
1033	{
1034	RTR0MemObjFree(pGVM->gvmm.s.VMPagesMapObj, false /* fFreeMappings */);
1035	pGVM->gvmm.s.VMPagesMapObj = NIL_RTR0MEMOBJ;
1036	}
1037	}
1038	}
1039	else if (RT_SUCCESS(rc))
1040	rc = rc2;
1041	}
1042	}
1043	/* else: The user wasn't permitted to create this VM. */
1044
1045	/*
1046	* The handle will be freed by gvmmR0HandleObjDestructor as we release the
1047	* object reference here. A little extra mess because of non-recursive lock.
1048	*/
1049	void *pvObj = pHandle->pvObj;
1050	pHandle->pvObj = NULL;
1051	gvmmR0CreateDestroyUnlock(pGVMM);
1052
1053	SUPR0ObjRelease(pvObj, pSession);
1054
1055	SUPR0Printf("GVMMR0CreateVM: failed, rc=%Rrc\n", rc);
1056	return rc;
1057	}
1058
1059	rc = VERR_NO_MEMORY;
1060	}
1061	else
1062	rc = VERR_GVMM_IPE_1;
1063	}
1064	else
1065	rc = VERR_GVM_TOO_MANY_VMS;
1066
1067	gvmmR0CreateDestroyUnlock(pGVMM);
1068	return rc;
1069	}
1070
1071
1072	/**
1073	* Initializes the per VM data belonging to GVMM.
1074	*
1075	* @param pGVM Pointer to the global VM structure.
1076	* @param hSelf The handle.
1077	* @param cCpus The CPU count.
1078	* @param pSession The session this VM is associated with.
1079	*/
1080	static void gvmmR0InitPerVMData(PGVM pGVM, int16_t hSelf, VMCPUID cCpus, PSUPDRVSESSION pSession)
1081	{
1082	AssertCompile(RT_SIZEOFMEMB(GVM,gvmm.s) <= RT_SIZEOFMEMB(GVM,gvmm.padding));
1083	AssertCompile(RT_SIZEOFMEMB(GVMCPU,gvmm.s) <= RT_SIZEOFMEMB(GVMCPU,gvmm.padding));
1084	AssertCompileMemberAlignment(VM, cpum, 64);
1085	AssertCompileMemberAlignment(VM, tm, 64);
1086
1087	/* GVM: */
1088	pGVM->u32Magic = GVM_MAGIC;
1089	pGVM->hSelf = hSelf;
1090	pGVM->cCpus = cCpus;
1091	pGVM->pSession = pSession;
1092	pGVM->pSelf = pGVM;
1093
1094	/* VM: */
1095	pGVM->enmVMState = VMSTATE_CREATING;
1096	pGVM->hSelfUnsafe = hSelf;
1097	pGVM->pSessionUnsafe = pSession;
1098	pGVM->pVMR0ForCall = pGVM;
1099	pGVM->cCpusUnsafe = cCpus;
1100	pGVM->uCpuExecutionCap = 100; /* default is no cap. */
1101	pGVM->uStructVersion = 1;
1102	pGVM->cbSelf = sizeof(VM);
1103	pGVM->cbVCpu = sizeof(VMCPU);
1104
1105	/* GVMM: */
1106	pGVM->gvmm.s.VMMemObj = NIL_RTR0MEMOBJ;
1107	pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
1108	pGVM->gvmm.s.VMPagesMemObj = NIL_RTR0MEMOBJ;
1109	pGVM->gvmm.s.VMPagesMapObj = NIL_RTR0MEMOBJ;
1110	pGVM->gvmm.s.fDoneVMMR0Init = false;
1111	pGVM->gvmm.s.fDoneVMMR0Term = false;
1112	for (size_t i = 0; i < RT_ELEMENTS(pGVM->gvmm.s.aEmtHash); i++)
1113	{
1114	pGVM->gvmm.s.aEmtHash[i].hNativeEmt = NIL_RTNATIVETHREAD;
1115	pGVM->gvmm.s.aEmtHash[i].idVCpu = NIL_VMCPUID;
1116	}
1117
1118	/*
1119	* Per virtual CPU.
1120	*/
1121	for (VMCPUID i = 0; i < pGVM->cCpus; i++)
1122	{
1123	pGVM->aCpus[i].idCpu = i;
1124	pGVM->aCpus[i].idCpuUnsafe = i;
1125	pGVM->aCpus[i].gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
1126	pGVM->aCpus[i].gvmm.s.VMCpuMapObj = NIL_RTR0MEMOBJ;
1127	pGVM->aCpus[i].gvmm.s.idxEmtHash = UINT16_MAX;
1128	pGVM->aCpus[i].hEMT = NIL_RTNATIVETHREAD;
1129	pGVM->aCpus[i].pGVM = pGVM;
1130	pGVM->aCpus[i].idHostCpu = NIL_RTCPUID;
1131	pGVM->aCpus[i].iHostCpuSet = UINT32_MAX;
1132	pGVM->aCpus[i].hNativeThread = NIL_RTNATIVETHREAD;
1133	pGVM->aCpus[i].hNativeThreadR0 = NIL_RTNATIVETHREAD;
1134	pGVM->aCpus[i].enmState = VMCPUSTATE_STOPPED;
1135	pGVM->aCpus[i].pVCpuR0ForVtg = &pGVM->aCpus[i];
1136	}
1137	}
1138
1139
1140	/**
1141	* Does the VM initialization.
1142	*
1143	* @returns VBox status code.
1144	* @param pGVM The global (ring-0) VM structure.
1145	*/
1146	GVMMR0DECL(int) GVMMR0InitVM(PGVM pGVM)
1147	{
1148	LogFlow(("GVMMR0InitVM: pGVM=%p\n", pGVM));
1149
1150	int rc = VERR_INTERNAL_ERROR_3;
1151	if ( !pGVM->gvmm.s.fDoneVMMR0Init
1152	&& pGVM->aCpus[0].gvmm.s.HaltEventMulti == NIL_RTSEMEVENTMULTI)
1153	{
1154	for (VMCPUID i = 0; i < pGVM->cCpus; i++)
1155	{
1156	rc = RTSemEventMultiCreate(&pGVM->aCpus[i].gvmm.s.HaltEventMulti);
1157	if (RT_FAILURE(rc))
1158	{
1159	pGVM->aCpus[i].gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
1160	break;
1161	}
1162	}
1163	}
1164	else
1165	rc = VERR_WRONG_ORDER;
1166
1167	LogFlow(("GVMMR0InitVM: returns %Rrc\n", rc));
1168	return rc;
1169	}
1170
1171
1172	/**
1173	* Indicates that we're done with the ring-0 initialization
1174	* of the VM.
1175	*
1176	* @param pGVM The global (ring-0) VM structure.
1177	* @thread EMT(0)
1178	*/
1179	GVMMR0DECL(void) GVMMR0DoneInitVM(PGVM pGVM)
1180	{
1181	/* Set the indicator. */
1182	pGVM->gvmm.s.fDoneVMMR0Init = true;
1183	}
1184
1185
1186	/**
1187	* Indicates that we're doing the ring-0 termination of the VM.
1188	*
1189	* @returns true if termination hasn't been done already, false if it has.
1190	* @param pGVM Pointer to the global VM structure. Optional.
1191	* @thread EMT(0) or session cleanup thread.
1192	*/
1193	GVMMR0DECL(bool) GVMMR0DoingTermVM(PGVM pGVM)
1194	{
1195	/* Validate the VM structure, state and handle. */
1196	AssertPtrReturn(pGVM, false);
1197
1198	/* Set the indicator. */
1199	if (pGVM->gvmm.s.fDoneVMMR0Term)
1200	return false;
1201	pGVM->gvmm.s.fDoneVMMR0Term = true;
1202	return true;
1203	}
1204
1205
1206	/**
1207	* Destroys the VM, freeing all associated resources (the ring-0 ones anyway).
1208	*
1209	* This is call from the vmR3DestroyFinalBit and from a error path in VMR3Create,
1210	* and the caller is not the EMT thread, unfortunately. For security reasons, it
1211	* would've been nice if the caller was actually the EMT thread or that we somehow
1212	* could've associated the calling thread with the VM up front.
1213	*
1214	* @returns VBox status code.
1215	* @param pGVM The global (ring-0) VM structure.
1216	*
1217	* @thread EMT(0) if it's associated with the VM, otherwise any thread.
1218	*/
1219	GVMMR0DECL(int) GVMMR0DestroyVM(PGVM pGVM)
1220	{
1221	LogFlow(("GVMMR0DestroyVM: pGVM=%p\n", pGVM));
1222	PGVMM pGVMM;
1223	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
1224
1225	/*
1226	* Validate the VM structure, state and caller.
1227	*/
1228	AssertPtrReturn(pGVM, VERR_INVALID_POINTER);
1229	AssertReturn(!((uintptr_t)pGVM & PAGE_OFFSET_MASK), VERR_INVALID_POINTER);
1230	AssertMsgReturn(pGVM->enmVMState >= VMSTATE_CREATING && pGVM->enmVMState <= VMSTATE_TERMINATED, ("%d\n", pGVM->enmVMState),
1231	VERR_WRONG_ORDER);
1232
1233	uint32_t hGVM = pGVM->hSelf;
1234	ASMCompilerBarrier();
1235	AssertReturn(hGVM != NIL_GVM_HANDLE, VERR_INVALID_VM_HANDLE);
1236	AssertReturn(hGVM < RT_ELEMENTS(pGVMM->aHandles), VERR_INVALID_VM_HANDLE);
1237
1238	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1239	AssertReturn(pHandle->pGVM == pGVM, VERR_NOT_OWNER);
1240
1241	RTPROCESS ProcId = RTProcSelf();
1242	RTNATIVETHREAD hSelf = RTThreadNativeSelf();
1243	AssertReturn( ( pHandle->hEMT0 == hSelf
1244	&& pHandle->ProcId == ProcId)
1245	\|\| pHandle->hEMT0 == NIL_RTNATIVETHREAD, VERR_NOT_OWNER);
1246
1247	/*
1248	* Lookup the handle and destroy the object.
1249	* Since the lock isn't recursive and we'll have to leave it before dereferencing the
1250	* object, we take some precautions against racing callers just in case...
1251	*/
1252	int rc = gvmmR0CreateDestroyLock(pGVMM);
1253	AssertRC(rc);
1254
1255	/* Be careful here because we might theoretically be racing someone else cleaning up. */
1256	if ( pHandle->pGVM == pGVM
1257	&& ( ( pHandle->hEMT0 == hSelf
1258	&& pHandle->ProcId == ProcId)
1259	\|\| pHandle->hEMT0 == NIL_RTNATIVETHREAD)
1260	&& RT_VALID_PTR(pHandle->pvObj)
1261	&& RT_VALID_PTR(pHandle->pSession)
1262	&& RT_VALID_PTR(pHandle->pGVM)
1263	&& pHandle->pGVM->u32Magic == GVM_MAGIC)
1264	{
1265	/* Check that other EMTs have deregistered. */
1266	uint32_t cNotDeregistered = 0;
1267	for (VMCPUID idCpu = 1; idCpu < pGVM->cCpus; idCpu++)
1268	cNotDeregistered += pGVM->aCpus[idCpu].hEMT != GVMM_RTNATIVETHREAD_DESTROYED;
1269	if (cNotDeregistered == 0)
1270	{
1271	/* Grab the object pointer. */
1272	void *pvObj = pHandle->pvObj;
1273	pHandle->pvObj = NULL;
1274	gvmmR0CreateDestroyUnlock(pGVMM);
1275
1276	SUPR0ObjRelease(pvObj, pHandle->pSession);
1277	}
1278	else
1279	{
1280	gvmmR0CreateDestroyUnlock(pGVMM);
1281	rc = VERR_GVMM_NOT_ALL_EMTS_DEREGISTERED;
1282	}
1283	}
1284	else
1285	{
1286	SUPR0Printf("GVMMR0DestroyVM: pHandle=%RKv:{.pGVM=%p, .hEMT0=%p, .ProcId=%u, .pvObj=%p} pGVM=%p hSelf=%p\n",
1287	pHandle, pHandle->pGVM, pHandle->hEMT0, pHandle->ProcId, pHandle->pvObj, pGVM, hSelf);
1288	gvmmR0CreateDestroyUnlock(pGVMM);
1289	rc = VERR_GVMM_IPE_2;
1290	}
1291
1292	return rc;
1293	}
1294
1295
1296	/**
1297	* Performs VM cleanup task as part of object destruction.
1298	*
1299	* @param pGVM The GVM pointer.
1300	*/
1301	static void gvmmR0CleanupVM(PGVM pGVM)
1302	{
1303	if ( pGVM->gvmm.s.fDoneVMMR0Init
1304	&& !pGVM->gvmm.s.fDoneVMMR0Term)
1305	{
1306	if ( pGVM->gvmm.s.VMMemObj != NIL_RTR0MEMOBJ
1307	&& RTR0MemObjAddress(pGVM->gvmm.s.VMMemObj) == pGVM)
1308	{
1309	LogFlow(("gvmmR0CleanupVM: Calling VMMR0TermVM\n"));
1310	VMMR0TermVM(pGVM, NIL_VMCPUID);
1311	}
1312	else
1313	AssertMsgFailed(("gvmmR0CleanupVM: VMMemObj=%p pGVM=%p\n", pGVM->gvmm.s.VMMemObj, pGVM));
1314	}
1315
1316	GMMR0CleanupVM(pGVM);
1317	#ifdef VBOX_WITH_NEM_R0
1318	NEMR0CleanupVM(pGVM);
1319	#endif
1320	PDMR0CleanupVM(pGVM);
1321	IOMR0CleanupVM(pGVM);
1322	DBGFR0CleanupVM(pGVM);
1323	PGMR0CleanupVM(pGVM);
1324	TMR0CleanupVM(pGVM);
1325
1326	AssertCompile(NIL_RTTHREADCTXHOOK == (RTTHREADCTXHOOK)0); /* Depends on zero initialized memory working for NIL at the moment. */
1327	for (VMCPUID idCpu = 0; idCpu < pGVM->cCpus; idCpu++)
1328	{
1329	/** @todo Can we busy wait here for all thread-context hooks to be
1330	* deregistered before releasing (destroying) it? Only until we find a
1331	* solution for not deregistering hooks everytime we're leaving HMR0
1332	* context. */
1333	VMMR0ThreadCtxHookDestroyForEmt(&pGVM->aCpus[idCpu]);
1334	}
1335	}
1336
1337
1338	/**
1339	* @callback_method_impl{FNSUPDRVDESTRUCTOR,VM handle destructor}
1340	*
1341	* pvUser1 is the GVM instance pointer.
1342	* pvUser2 is the handle pointer.
1343	*/
1344	static DECLCALLBACK(void) gvmmR0HandleObjDestructor(void pvObj, void pvUser1, void *pvUser2)
1345	{
1346	LogFlow(("gvmmR0HandleObjDestructor: %p %p %p\n", pvObj, pvUser1, pvUser2));
1347
1348	NOREF(pvObj);
1349
1350	/*
1351	* Some quick, paranoid, input validation.
1352	*/
1353	PGVMHANDLE pHandle = (PGVMHANDLE)pvUser2;
1354	AssertPtr(pHandle);
1355	PGVMM pGVMM = (PGVMM)pvUser1;
1356	Assert(pGVMM == g_pGVMM);
1357	const uint16_t iHandle = pHandle - &pGVMM->aHandles[0];
1358	if ( !iHandle
1359	\|\| iHandle >= RT_ELEMENTS(pGVMM->aHandles)
1360	\|\| iHandle != pHandle->iSelf)
1361	{
1362	SUPR0Printf("GVM: handle %d is out of range or corrupt (iSelf=%d)!\n", iHandle, pHandle->iSelf);
1363	return;
1364	}
1365
1366	int rc = gvmmR0CreateDestroyLock(pGVMM);
1367	AssertRC(rc);
1368	rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
1369	AssertRC(rc);
1370
1371	/*
1372	* This is a tad slow but a doubly linked list is too much hassle.
1373	*/
1374	if (RT_UNLIKELY(pHandle->iNext >= RT_ELEMENTS(pGVMM->aHandles)))
1375	{
1376	SUPR0Printf("GVM: used list index %d is out of range!\n", pHandle->iNext);
1377	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1378	gvmmR0CreateDestroyUnlock(pGVMM);
1379	return;
1380	}
1381
1382	if (pGVMM->iUsedHead == iHandle)
1383	pGVMM->iUsedHead = pHandle->iNext;
1384	else
1385	{
1386	uint16_t iPrev = pGVMM->iUsedHead;
1387	int c = RT_ELEMENTS(pGVMM->aHandles) + 2;
1388	while (iPrev)
1389	{
1390	if (RT_UNLIKELY(iPrev >= RT_ELEMENTS(pGVMM->aHandles)))
1391	{
1392	SUPR0Printf("GVM: used list index %d is out of range!\n", iPrev);
1393	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1394	gvmmR0CreateDestroyUnlock(pGVMM);
1395	return;
1396	}
1397	if (RT_UNLIKELY(c-- <= 0))
1398	{
1399	iPrev = 0;
1400	break;
1401	}
1402
1403	if (pGVMM->aHandles[iPrev].iNext == iHandle)
1404	break;
1405	iPrev = pGVMM->aHandles[iPrev].iNext;
1406	}
1407	if (!iPrev)
1408	{
1409	SUPR0Printf("GVM: can't find the handle previous previous of %d!\n", pHandle->iSelf);
1410	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1411	gvmmR0CreateDestroyUnlock(pGVMM);
1412	return;
1413	}
1414
1415	Assert(pGVMM->aHandles[iPrev].iNext == iHandle);
1416	pGVMM->aHandles[iPrev].iNext = pHandle->iNext;
1417	}
1418	pHandle->iNext = 0;
1419	pGVMM->cVMs--;
1420
1421	/*
1422	* Do the global cleanup round.
1423	*/
1424	PGVM pGVM = pHandle->pGVM;
1425	if ( RT_VALID_PTR(pGVM)
1426	&& pGVM->u32Magic == GVM_MAGIC)
1427	{
1428	pGVMM->cEMTs -= pGVM->cCpus;
1429
1430	if (pGVM->pSession)
1431	SUPR0SetSessionVM(pGVM->pSession, NULL, NULL);
1432
1433	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1434
1435	gvmmR0CleanupVM(pGVM);
1436
1437	/*
1438	* Do the GVMM cleanup - must be done last.
1439	*/
1440	/* The VM and VM pages mappings/allocations. */
1441	if (pGVM->gvmm.s.VMPagesMapObj != NIL_RTR0MEMOBJ)
1442	{
1443	rc = RTR0MemObjFree(pGVM->gvmm.s.VMPagesMapObj, false /* fFreeMappings */); AssertRC(rc);
1444	pGVM->gvmm.s.VMPagesMapObj = NIL_RTR0MEMOBJ;
1445	}
1446
1447	if (pGVM->gvmm.s.VMMapObj != NIL_RTR0MEMOBJ)
1448	{
1449	rc = RTR0MemObjFree(pGVM->gvmm.s.VMMapObj, false /* fFreeMappings */); AssertRC(rc);
1450	pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
1451	}
1452
1453	if (pGVM->gvmm.s.VMPagesMemObj != NIL_RTR0MEMOBJ)
1454	{
1455	rc = RTR0MemObjFree(pGVM->gvmm.s.VMPagesMemObj, false /* fFreeMappings */); AssertRC(rc);
1456	pGVM->gvmm.s.VMPagesMemObj = NIL_RTR0MEMOBJ;
1457	}
1458
1459	for (VMCPUID i = 0; i < pGVM->cCpus; i++)
1460	{
1461	if (pGVM->aCpus[i].gvmm.s.HaltEventMulti != NIL_RTSEMEVENTMULTI)
1462	{
1463	rc = RTSemEventMultiDestroy(pGVM->aCpus[i].gvmm.s.HaltEventMulti); AssertRC(rc);
1464	pGVM->aCpus[i].gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
1465	}
1466	if (pGVM->aCpus[i].gvmm.s.VMCpuMapObj != NIL_RTR0MEMOBJ)
1467	{
1468	rc = RTR0MemObjFree(pGVM->aCpus[i].gvmm.s.VMCpuMapObj, false /* fFreeMappings */); AssertRC(rc);
1469	pGVM->aCpus[i].gvmm.s.VMCpuMapObj = NIL_RTR0MEMOBJ;
1470	}
1471	}
1472
1473	/* the GVM structure itself. */
1474	pGVM->u32Magic \|= UINT32_C(0x80000000);
1475	Assert(pGVM->gvmm.s.VMMemObj != NIL_RTR0MEMOBJ);
1476	rc = RTR0MemObjFree(pGVM->gvmm.s.VMMemObj, true /fFreeMappings/); AssertRC(rc);
1477	pGVM = NULL;
1478
1479	/* Re-acquire the UsedLock before freeing the handle since we're updating handle fields. */
1480	rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
1481	AssertRC(rc);
1482	}
1483	/* else: GVMMR0CreateVM cleanup. */
1484
1485	/*
1486	* Free the handle.
1487	*/
1488	pHandle->iNext = pGVMM->iFreeHead;
1489	pGVMM->iFreeHead = iHandle;
1490	ASMAtomicWriteNullPtr(&pHandle->pGVM);
1491	ASMAtomicWriteNullPtr(&pHandle->pvObj);
1492	ASMAtomicWriteNullPtr(&pHandle->pSession);
1493	ASMAtomicWriteHandle(&pHandle->hEMT0, NIL_RTNATIVETHREAD);
1494	ASMAtomicWriteU32(&pHandle->ProcId, NIL_RTPROCESS);
1495
1496	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1497	gvmmR0CreateDestroyUnlock(pGVMM);
1498	LogFlow(("gvmmR0HandleObjDestructor: returns\n"));
1499	}
1500
1501
1502	/**
1503	* Registers the calling thread as the EMT of a Virtual CPU.
1504	*
1505	* Note that VCPU 0 is automatically registered during VM creation.
1506	*
1507	* @returns VBox status code
1508	* @param pGVM The global (ring-0) VM structure.
1509	* @param idCpu VCPU id to register the current thread as.
1510	*/
1511	GVMMR0DECL(int) GVMMR0RegisterVCpu(PGVM pGVM, VMCPUID idCpu)
1512	{
1513	AssertReturn(idCpu != 0, VERR_INVALID_FUNCTION);
1514
1515	/*
1516	* Validate the VM structure, state and handle.
1517	*/
1518	PGVMM pGVMM;
1519	int rc = gvmmR0ByGVM(pGVM, &pGVMM, false /* fTakeUsedLock */);
1520	if (RT_SUCCESS(rc))
1521	{
1522	if (idCpu < pGVM->cCpus)
1523	{
1524	RTNATIVETHREAD const hNativeSelf = RTThreadNativeSelf();
1525
1526	gvmmR0CreateDestroyLock(pGVMM); /** @todo per-VM lock? */
1527
1528	/* Check that the EMT isn't already assigned to a thread. */
1529	if (pGVM->aCpus[idCpu].hEMT == NIL_RTNATIVETHREAD)
1530	{
1531	Assert(pGVM->aCpus[idCpu].hNativeThreadR0 == NIL_RTNATIVETHREAD);
1532
1533	/* A thread may only be one EMT (this makes sure hNativeSelf isn't NIL). */
1534	for (VMCPUID iCpu = 0; iCpu < pGVM->cCpus; iCpu++)
1535	AssertBreakStmt(pGVM->aCpus[iCpu].hEMT != hNativeSelf, rc = VERR_INVALID_PARAMETER);
1536	if (RT_SUCCESS(rc))
1537	{
1538	/*
1539	* Do the assignment, then try setup the hook. Undo if that fails.
1540	*/
1541	unsigned cCollisions = 0;
1542	uint32_t idxHash = GVMM_EMT_HASH_1(hNativeSelf);
1543	if (pGVM->gvmm.s.aEmtHash[idxHash].hNativeEmt != NIL_RTNATIVETHREAD)
1544	{
1545	uint32_t const idxHash2 = GVMM_EMT_HASH_2(hNativeSelf);
1546	do
1547	{
1548	cCollisions++;
1549	Assert(cCollisions < GVMM_EMT_HASH_SIZE);
1550	idxHash = (idxHash + idxHash2) % GVMM_EMT_HASH_SIZE;
1551	} while (pGVM->gvmm.s.aEmtHash[idxHash].hNativeEmt != NIL_RTNATIVETHREAD);
1552	}
1553	pGVM->gvmm.s.aEmtHash[idxHash].hNativeEmt = hNativeSelf;
1554	pGVM->gvmm.s.aEmtHash[idxHash].idVCpu = idCpu;
1555	pGVM->aCpus[idCpu].hNativeThreadR0 = hNativeSelf;
1556	pGVM->aCpus[idCpu].hEMT = hNativeSelf;
1557	pGVM->aCpus[idCpu].cEmtHashCollisions = (uint8_t)cCollisions;
1558	pGVM->aCpus[idCpu].gvmm.s.idxEmtHash = (uint16_t)idxHash;
1559
1560	rc = VMMR0ThreadCtxHookCreateForEmt(&pGVM->aCpus[idCpu]);
1561	if (RT_SUCCESS(rc))
1562	CPUMR0RegisterVCpuThread(&pGVM->aCpus[idCpu]);
1563	else
1564	{
1565	pGVM->aCpus[idCpu].hNativeThreadR0 = NIL_RTNATIVETHREAD;
1566	pGVM->aCpus[idCpu].hEMT = NIL_RTNATIVETHREAD;
1567	pGVM->gvmm.s.aEmtHash[idxHash].hNativeEmt = NIL_RTNATIVETHREAD;
1568	pGVM->gvmm.s.aEmtHash[idxHash].idVCpu = NIL_VMCPUID;
1569	pGVM->aCpus[idCpu].gvmm.s.idxEmtHash = UINT16_MAX;
1570	}
1571	}
1572	}
1573	else
1574	rc = VERR_ACCESS_DENIED;
1575
1576	gvmmR0CreateDestroyUnlock(pGVMM);
1577	}
1578	else
1579	rc = VERR_INVALID_CPU_ID;
1580	}
1581	return rc;
1582	}
1583
1584
1585	/**
1586	* Deregisters the calling thread as the EMT of a Virtual CPU.
1587	*
1588	* Note that VCPU 0 shall call GVMMR0DestroyVM intead of this API.
1589	*
1590	* @returns VBox status code
1591	* @param pGVM The global (ring-0) VM structure.
1592	* @param idCpu VCPU id to register the current thread as.
1593	*/
1594	GVMMR0DECL(int) GVMMR0DeregisterVCpu(PGVM pGVM, VMCPUID idCpu)
1595	{
1596	AssertReturn(idCpu != 0, VERR_INVALID_FUNCTION);
1597
1598	/*
1599	* Validate the VM structure, state and handle.
1600	*/
1601	PGVMM pGVMM;
1602	int rc = gvmmR0ByGVMandEMT(pGVM, idCpu, &pGVMM);
1603	if (RT_SUCCESS(rc))
1604	{
1605	/*
1606	* Take the destruction lock and recheck the handle state to
1607	* prevent racing GVMMR0DestroyVM.
1608	*/
1609	gvmmR0CreateDestroyLock(pGVMM);
1610
1611	uint32_t hSelf = pGVM->hSelf;
1612	ASMCompilerBarrier();
1613	if ( hSelf < RT_ELEMENTS(pGVMM->aHandles)
1614	&& pGVMM->aHandles[hSelf].pvObj != NULL
1615	&& pGVMM->aHandles[hSelf].pGVM == pGVM)
1616	{
1617	/*
1618	* Do per-EMT cleanups.
1619	*/
1620	VMMR0ThreadCtxHookDestroyForEmt(&pGVM->aCpus[idCpu]);
1621
1622	/*
1623	* Invalidate hEMT. We don't use NIL here as that would allow
1624	* GVMMR0RegisterVCpu to be called again, and we don't want that.
1625	*/
1626	pGVM->aCpus[idCpu].hEMT = GVMM_RTNATIVETHREAD_DESTROYED;
1627	pGVM->aCpus[idCpu].hNativeThreadR0 = NIL_RTNATIVETHREAD;
1628
1629	uint32_t const idxHash = pGVM->aCpus[idCpu].gvmm.s.idxEmtHash;
1630	if (idxHash < RT_ELEMENTS(pGVM->gvmm.s.aEmtHash))
1631	pGVM->gvmm.s.aEmtHash[idxHash].hNativeEmt = GVMM_RTNATIVETHREAD_DESTROYED;
1632	}
1633
1634	gvmmR0CreateDestroyUnlock(pGVMM);
1635	}
1636	return rc;
1637	}
1638
1639
1640	/**
1641	* Lookup a GVM structure by its handle.
1642	*
1643	* @returns The GVM pointer on success, NULL on failure.
1644	* @param hGVM The global VM handle. Asserts on bad handle.
1645	*/
1646	GVMMR0DECL(PGVM) GVMMR0ByHandle(uint32_t hGVM)
1647	{
1648	PGVMM pGVMM;
1649	GVMM_GET_VALID_INSTANCE(pGVMM, NULL);
1650
1651	/*
1652	* Validate.
1653	*/
1654	AssertReturn(hGVM != NIL_GVM_HANDLE, NULL);
1655	AssertReturn(hGVM < RT_ELEMENTS(pGVMM->aHandles), NULL);
1656
1657	/*
1658	* Look it up.
1659	*/
1660	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1661	AssertPtrReturn(pHandle->pvObj, NULL);
1662	PGVM pGVM = pHandle->pGVM;
1663	AssertPtrReturn(pGVM, NULL);
1664
1665	return pGVM;
1666	}
1667
1668
1669	/**
1670	* Check that the given GVM and VM structures match up.
1671	*
1672	* The calling thread must be in the same process as the VM. All current lookups
1673	* are by threads inside the same process, so this will not be an issue.
1674	*
1675	* @returns VBox status code.
1676	* @param pGVM The global (ring-0) VM structure.
1677	* @param ppGVMM Where to store the pointer to the GVMM instance data.
1678	* @param fTakeUsedLock Whether to take the used lock or not. We take it in
1679	* shared mode when requested.
1680	*
1681	* Be very careful if not taking the lock as it's
1682	* possible that the VM will disappear then!
1683	*
1684	* @remark This will not assert on an invalid pGVM but try return silently.
1685	*/
1686	static int gvmmR0ByGVM(PGVM pGVM, PGVMM *ppGVMM, bool fTakeUsedLock)
1687	{
1688	/*
1689	* Check the pointers.
1690	*/
1691	int rc;
1692	if (RT_LIKELY( RT_VALID_PTR(pGVM)
1693	&& ((uintptr_t)pGVM & PAGE_OFFSET_MASK) == 0 ))
1694	{
1695	/*
1696	* Get the pGVMM instance and check the VM handle.
1697	*/
1698	PGVMM pGVMM;
1699	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
1700
1701	uint16_t hGVM = pGVM->hSelf;
1702	if (RT_LIKELY( hGVM != NIL_GVM_HANDLE
1703	&& hGVM < RT_ELEMENTS(pGVMM->aHandles)))
1704	{
1705	RTPROCESS const pidSelf = RTProcSelf();
1706	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1707	if (fTakeUsedLock)
1708	{
1709	rc = GVMMR0_USED_SHARED_LOCK(pGVMM);
1710	AssertRCReturn(rc, rc);
1711	}
1712
1713	if (RT_LIKELY( pHandle->pGVM == pGVM
1714	&& pHandle->ProcId == pidSelf
1715	&& RT_VALID_PTR(pHandle->pvObj)))
1716	{
1717	/*
1718	* Some more VM data consistency checks.
1719	*/
1720	if (RT_LIKELY( pGVM->cCpusUnsafe == pGVM->cCpus
1721	&& pGVM->hSelfUnsafe == hGVM
1722	&& pGVM->pSelf == pGVM))
1723	{
1724	if (RT_LIKELY( pGVM->enmVMState >= VMSTATE_CREATING
1725	&& pGVM->enmVMState <= VMSTATE_TERMINATED))
1726	{
1727	*ppGVMM = pGVMM;
1728	return VINF_SUCCESS;
1729	}
1730	rc = VERR_INCONSISTENT_VM_HANDLE;
1731	}
1732	else
1733	rc = VERR_INCONSISTENT_VM_HANDLE;
1734	}
1735	else
1736	rc = VERR_INVALID_VM_HANDLE;
1737
1738	if (fTakeUsedLock)
1739	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
1740	}
1741	else
1742	rc = VERR_INVALID_VM_HANDLE;
1743	}
1744	else
1745	rc = VERR_INVALID_POINTER;
1746	return rc;
1747	}
1748
1749
1750	/**
1751	* Validates a GVM/VM pair.
1752	*
1753	* @returns VBox status code.
1754	* @param pGVM The global (ring-0) VM structure.
1755	*/
1756	GVMMR0DECL(int) GVMMR0ValidateGVM(PGVM pGVM)
1757	{
1758	PGVMM pGVMM;
1759	return gvmmR0ByGVM(pGVM, &pGVMM, false /fTakeUsedLock/);
1760	}
1761
1762
1763	/**
1764	* Check that the given GVM and VM structures match up.
1765	*
1766	* The calling thread must be in the same process as the VM. All current lookups
1767	* are by threads inside the same process, so this will not be an issue.
1768	*
1769	* @returns VBox status code.
1770	* @param pGVM The global (ring-0) VM structure.
1771	* @param idCpu The (alleged) Virtual CPU ID of the calling EMT.
1772	* @param ppGVMM Where to store the pointer to the GVMM instance data.
1773	* @thread EMT
1774	*
1775	* @remarks This will assert in all failure paths.
1776	*/
1777	static int gvmmR0ByGVMandEMT(PGVM pGVM, VMCPUID idCpu, PGVMM *ppGVMM)
1778	{
1779	/*
1780	* Check the pointers.
1781	*/
1782	AssertPtrReturn(pGVM, VERR_INVALID_POINTER);
1783	AssertReturn(((uintptr_t)pGVM & PAGE_OFFSET_MASK) == 0, VERR_INVALID_POINTER);
1784
1785	/*
1786	* Get the pGVMM instance and check the VM handle.
1787	*/
1788	PGVMM pGVMM;
1789	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
1790
1791	uint16_t hGVM = pGVM->hSelf;
1792	ASMCompilerBarrier();
1793	AssertReturn( hGVM != NIL_GVM_HANDLE
1794	&& hGVM < RT_ELEMENTS(pGVMM->aHandles), VERR_INVALID_VM_HANDLE);
1795
1796	RTPROCESS const pidSelf = RTProcSelf();
1797	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1798	AssertReturn( pHandle->pGVM == pGVM
1799	&& pHandle->ProcId == pidSelf
1800	&& RT_VALID_PTR(pHandle->pvObj),
1801	VERR_INVALID_HANDLE);
1802
1803	/*
1804	* Check the EMT claim.
1805	*/
1806	RTNATIVETHREAD const hAllegedEMT = RTThreadNativeSelf();
1807	AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID);
1808	AssertReturn(pGVM->aCpus[idCpu].hEMT == hAllegedEMT, VERR_NOT_OWNER);
1809
1810	/*
1811	* Some more VM data consistency checks.
1812	*/
1813	AssertReturn(pGVM->cCpusUnsafe == pGVM->cCpus, VERR_INCONSISTENT_VM_HANDLE);
1814	AssertReturn(pGVM->hSelfUnsafe == hGVM, VERR_INCONSISTENT_VM_HANDLE);
1815	AssertReturn( pGVM->enmVMState >= VMSTATE_CREATING
1816	&& pGVM->enmVMState <= VMSTATE_TERMINATED, VERR_INCONSISTENT_VM_HANDLE);
1817
1818	*ppGVMM = pGVMM;
1819	return VINF_SUCCESS;
1820	}
1821
1822
1823	/**
1824	* Validates a GVM/EMT pair.
1825	*
1826	* @returns VBox status code.
1827	* @param pGVM The global (ring-0) VM structure.
1828	* @param idCpu The Virtual CPU ID of the calling EMT.
1829	* @thread EMT(idCpu)
1830	*/
1831	GVMMR0DECL(int) GVMMR0ValidateGVMandEMT(PGVM pGVM, VMCPUID idCpu)
1832	{
1833	PGVMM pGVMM;
1834	return gvmmR0ByGVMandEMT(pGVM, idCpu, &pGVMM);
1835	}
1836
1837
1838	/**
1839	* Looks up the VM belonging to the specified EMT thread.
1840	*
1841	* This is used by the assertion machinery in VMMR0.cpp to avoid causing
1842	* unnecessary kernel panics when the EMT thread hits an assertion. The
1843	* call may or not be an EMT thread.
1844	*
1845	* @returns Pointer to the VM on success, NULL on failure.
1846	* @param hEMT The native thread handle of the EMT.
1847	* NIL_RTNATIVETHREAD means the current thread
1848	*/
1849	GVMMR0DECL(PVMCC) GVMMR0GetVMByEMT(RTNATIVETHREAD hEMT)
1850	{
1851	/*
1852	* No Assertions here as we're usually called in a AssertMsgN or
1853	* RTAssert* context.
1854	*/
1855	PGVMM pGVMM = g_pGVMM;
1856	if ( !RT_VALID_PTR(pGVMM)
1857	\|\| pGVMM->u32Magic != GVMM_MAGIC)
1858	return NULL;
1859
1860	if (hEMT == NIL_RTNATIVETHREAD)
1861	hEMT = RTThreadNativeSelf();
1862	RTPROCESS ProcId = RTProcSelf();
1863
1864	/*
1865	* Search the handles in a linear fashion as we don't dare to take the lock (assert).
1866	*/
1867	/** @todo introduce some pid hash table here, please. */
1868	for (unsigned i = 1; i < RT_ELEMENTS(pGVMM->aHandles); i++)
1869	{
1870	if ( pGVMM->aHandles[i].iSelf == i
1871	&& pGVMM->aHandles[i].ProcId == ProcId
1872	&& RT_VALID_PTR(pGVMM->aHandles[i].pvObj)
1873	&& RT_VALID_PTR(pGVMM->aHandles[i].pGVM))
1874	{
1875	if (pGVMM->aHandles[i].hEMT0 == hEMT)
1876	return pGVMM->aHandles[i].pGVM;
1877
1878	/* This is fearly safe with the current process per VM approach. */
1879	PGVM pGVM = pGVMM->aHandles[i].pGVM;
1880	VMCPUID const cCpus = pGVM->cCpus;
1881	ASMCompilerBarrier();
1882	if ( cCpus < 1
1883	\|\| cCpus > VMM_MAX_CPU_COUNT)
1884	continue;
1885	for (VMCPUID idCpu = 1; idCpu < cCpus; idCpu++)
1886	if (pGVM->aCpus[idCpu].hEMT == hEMT)
1887	return pGVMM->aHandles[i].pGVM;
1888	}
1889	}
1890	return NULL;
1891	}
1892
1893
1894	/**
1895	* Looks up the GVMCPU belonging to the specified EMT thread.
1896	*
1897	* This is used by the assertion machinery in VMMR0.cpp to avoid causing
1898	* unnecessary kernel panics when the EMT thread hits an assertion. The
1899	* call may or not be an EMT thread.
1900	*
1901	* @returns Pointer to the VM on success, NULL on failure.
1902	* @param hEMT The native thread handle of the EMT.
1903	* NIL_RTNATIVETHREAD means the current thread
1904	*/
1905	GVMMR0DECL(PGVMCPU) GVMMR0GetGVCpuByEMT(RTNATIVETHREAD hEMT)
1906	{
1907	/*
1908	* No Assertions here as we're usually called in a AssertMsgN,
1909	* RTAssert*, Log and LogRel contexts.
1910	*/
1911	PGVMM pGVMM = g_pGVMM;
1912	if ( !RT_VALID_PTR(pGVMM)
1913	\|\| pGVMM->u32Magic != GVMM_MAGIC)
1914	return NULL;
1915
1916	if (hEMT == NIL_RTNATIVETHREAD)
1917	hEMT = RTThreadNativeSelf();
1918	RTPROCESS ProcId = RTProcSelf();
1919
1920	/*
1921	* Search the handles in a linear fashion as we don't dare to take the lock (assert).
1922	*/
1923	/** @todo introduce some pid hash table here, please. */
1924	for (unsigned i = 1; i < RT_ELEMENTS(pGVMM->aHandles); i++)
1925	{
1926	if ( pGVMM->aHandles[i].iSelf == i
1927	&& pGVMM->aHandles[i].ProcId == ProcId
1928	&& RT_VALID_PTR(pGVMM->aHandles[i].pvObj)
1929	&& RT_VALID_PTR(pGVMM->aHandles[i].pGVM))
1930	{
1931	PGVM pGVM = pGVMM->aHandles[i].pGVM;
1932	if (pGVMM->aHandles[i].hEMT0 == hEMT)
1933	return &pGVM->aCpus[0];
1934
1935	/* This is fearly safe with the current process per VM approach. */
1936	VMCPUID const cCpus = pGVM->cCpus;
1937	ASMCompilerBarrier();
1938	ASMCompilerBarrier();
1939	if ( cCpus < 1
1940	\|\| cCpus > VMM_MAX_CPU_COUNT)
1941	continue;
1942	for (VMCPUID idCpu = 1; idCpu < cCpus; idCpu++)
1943	if (pGVM->aCpus[idCpu].hEMT == hEMT)
1944	return &pGVM->aCpus[idCpu];
1945	}
1946	}
1947	return NULL;
1948	}
1949
1950
1951	/**
1952	* Get the GVMCPU structure for the given EMT.
1953	*
1954	* @returns The VCpu structure for @a hEMT, NULL if not an EMT.
1955	* @param pGVM The global (ring-0) VM structure.
1956	* @param hEMT The native thread handle of the EMT.
1957	* NIL_RTNATIVETHREAD means the current thread
1958	*/
1959	GVMMR0DECL(PGVMCPU) GVMMR0GetGVCpuByGVMandEMT(PGVM pGVM, RTNATIVETHREAD hEMT)
1960	{
1961	/*
1962	* Validate & adjust input.
1963	*/
1964	AssertPtr(pGVM);
1965	Assert(pGVM->u32Magic == GVM_MAGIC);
1966	if (hEMT == NIL_RTNATIVETHREAD /* likely */)
1967	{
1968	hEMT = RTThreadNativeSelf();
1969	AssertReturn(hEMT != NIL_RTNATIVETHREAD, NULL);
1970	}
1971
1972	/*
1973	* Find the matching hash table entry.
1974	*/
1975	uint32_t idxHash = GVMM_EMT_HASH_1(hEMT);
1976	if (pGVM->gvmm.s.aEmtHash[idxHash].hNativeEmt == hEMT)
1977	{ /* likely */ }
1978	else
1979	{
1980	#ifdef VBOX_STRICT
1981	unsigned cCollisions = 0;
1982	#endif
1983	uint32_t const idxHash2 = GVMM_EMT_HASH_2(hEMT);
1984	for (;;)
1985	{
1986	Assert(cCollisions++ < GVMM_EMT_HASH_SIZE);
1987	idxHash = (idxHash + idxHash2) % GVMM_EMT_HASH_SIZE;
1988	if (pGVM->gvmm.s.aEmtHash[idxHash].hNativeEmt == hEMT)
1989	break;
1990	if (pGVM->gvmm.s.aEmtHash[idxHash].hNativeEmt == NIL_RTNATIVETHREAD)
1991	{
1992	#ifdef VBOX_STRICT
1993	uint32_t idxCpu = pGVM->cCpus;
1994	AssertStmt(idxCpu < VMM_MAX_CPU_COUNT, idxCpu = VMM_MAX_CPU_COUNT);
1995	while (idxCpu-- > 0)
1996	Assert(pGVM->aCpus[idxCpu].hNativeThreadR0 != hEMT);
1997	#endif
1998	return NULL;
1999	}
2000	}
2001	}
2002
2003	/*
2004	* Validate the VCpu number and translate it into a pointer.
2005	*/
2006	VMCPUID const idCpu = pGVM->gvmm.s.aEmtHash[idxHash].idVCpu;
2007	AssertReturn(idCpu < pGVM->cCpus, NULL);
2008	PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
2009	Assert(pGVCpu->hNativeThreadR0 == hEMT);
2010	Assert(pGVCpu->gvmm.s.idxEmtHash == idxHash);
2011	return pGVCpu;
2012	}
2013
2014
2015	/**
2016	* This is will wake up expired and soon-to-be expired VMs.
2017	*
2018	* @returns Number of VMs that has been woken up.
2019	* @param pGVMM Pointer to the GVMM instance data.
2020	* @param u64Now The current time.
2021	*/
2022	static unsigned gvmmR0SchedDoWakeUps(PGVMM pGVMM, uint64_t u64Now)
2023	{
2024	/*
2025	* Skip this if we've got disabled because of high resolution wakeups or by
2026	* the user.
2027	*/
2028	if (!pGVMM->fDoEarlyWakeUps)
2029	return 0;
2030
2031	/** @todo Rewrite this algorithm. See performance defect XYZ. */
2032
2033	/*
2034	* A cheap optimization to stop wasting so much time here on big setups.
2035	*/
2036	const uint64_t uNsEarlyWakeUp2 = u64Now + pGVMM->nsEarlyWakeUp2;
2037	if ( pGVMM->cHaltedEMTs == 0
2038	\|\| uNsEarlyWakeUp2 > pGVMM->uNsNextEmtWakeup)
2039	return 0;
2040
2041	/*
2042	* Only one thread doing this at a time.
2043	*/
2044	if (!ASMAtomicCmpXchgBool(&pGVMM->fDoingEarlyWakeUps, true, false))
2045	return 0;
2046
2047	/*
2048	* The first pass will wake up VMs which have actually expired
2049	* and look for VMs that should be woken up in the 2nd and 3rd passes.
2050	*/
2051	const uint64_t uNsEarlyWakeUp1 = u64Now + pGVMM->nsEarlyWakeUp1;
2052	uint64_t u64Min = UINT64_MAX;
2053	unsigned cWoken = 0;
2054	unsigned cHalted = 0;
2055	unsigned cTodo2nd = 0;
2056	unsigned cTodo3rd = 0;
2057	for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
2058	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
2059	i = pGVMM->aHandles[i].iNext)
2060	{
2061	PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
2062	if ( RT_VALID_PTR(pCurGVM)
2063	&& pCurGVM->u32Magic == GVM_MAGIC)
2064	{
2065	for (VMCPUID idCpu = 0; idCpu < pCurGVM->cCpus; idCpu++)
2066	{
2067	PGVMCPU pCurGVCpu = &pCurGVM->aCpus[idCpu];
2068	uint64_t u64 = ASMAtomicUoReadU64(&pCurGVCpu->gvmm.s.u64HaltExpire);
2069	if (u64)
2070	{
2071	if (u64 <= u64Now)
2072	{
2073	if (ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0))
2074	{
2075	int rc = RTSemEventMultiSignal(pCurGVCpu->gvmm.s.HaltEventMulti);
2076	AssertRC(rc);
2077	cWoken++;
2078	}
2079	}
2080	else
2081	{
2082	cHalted++;
2083	if (u64 <= uNsEarlyWakeUp1)
2084	cTodo2nd++;
2085	else if (u64 <= uNsEarlyWakeUp2)
2086	cTodo3rd++;
2087	else if (u64 < u64Min)
2088	u64 = u64Min;
2089	}
2090	}
2091	}
2092	}
2093	AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
2094	}
2095
2096	if (cTodo2nd)
2097	{
2098	for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
2099	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
2100	i = pGVMM->aHandles[i].iNext)
2101	{
2102	PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
2103	if ( RT_VALID_PTR(pCurGVM)
2104	&& pCurGVM->u32Magic == GVM_MAGIC)
2105	{
2106	for (VMCPUID idCpu = 0; idCpu < pCurGVM->cCpus; idCpu++)
2107	{
2108	PGVMCPU pCurGVCpu = &pCurGVM->aCpus[idCpu];
2109	uint64_t u64 = ASMAtomicUoReadU64(&pCurGVCpu->gvmm.s.u64HaltExpire);
2110	if ( u64
2111	&& u64 <= uNsEarlyWakeUp1)
2112	{
2113	if (ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0))
2114	{
2115	int rc = RTSemEventMultiSignal(pCurGVCpu->gvmm.s.HaltEventMulti);
2116	AssertRC(rc);
2117	cWoken++;
2118	}
2119	}
2120	}
2121	}
2122	AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
2123	}
2124	}
2125
2126	if (cTodo3rd)
2127	{
2128	for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
2129	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
2130	i = pGVMM->aHandles[i].iNext)
2131	{
2132	PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
2133	if ( RT_VALID_PTR(pCurGVM)
2134	&& pCurGVM->u32Magic == GVM_MAGIC)
2135	{
2136	for (VMCPUID idCpu = 0; idCpu < pCurGVM->cCpus; idCpu++)
2137	{
2138	PGVMCPU pCurGVCpu = &pCurGVM->aCpus[idCpu];
2139	uint64_t u64 = ASMAtomicUoReadU64(&pCurGVCpu->gvmm.s.u64HaltExpire);
2140	if ( u64
2141	&& u64 <= uNsEarlyWakeUp2)
2142	{
2143	if (ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0))
2144	{
2145	int rc = RTSemEventMultiSignal(pCurGVCpu->gvmm.s.HaltEventMulti);
2146	AssertRC(rc);
2147	cWoken++;
2148	}
2149	}
2150	}
2151	}
2152	AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
2153	}
2154	}
2155
2156	/*
2157	* Set the minimum value.
2158	*/
2159	pGVMM->uNsNextEmtWakeup = u64Min;
2160
2161	ASMAtomicWriteBool(&pGVMM->fDoingEarlyWakeUps, false);
2162	return cWoken;
2163	}
2164
2165
2166	/**
2167	* Halt the EMT thread.
2168	*
2169	* @returns VINF_SUCCESS normal wakeup (timeout or kicked by other thread).
2170	* VERR_INTERRUPTED if a signal was scheduled for the thread.
2171	* @param pGVM The global (ring-0) VM structure.
2172	* @param pGVCpu The global (ring-0) CPU structure of the calling
2173	* EMT.
2174	* @param u64ExpireGipTime The time for the sleep to expire expressed as GIP time.
2175	* @thread EMT(pGVCpu).
2176	*/
2177	GVMMR0DECL(int) GVMMR0SchedHalt(PGVM pGVM, PGVMCPU pGVCpu, uint64_t u64ExpireGipTime)
2178	{
2179	LogFlow(("GVMMR0SchedHalt: pGVM=%p pGVCpu=%p(%d) u64ExpireGipTime=%#RX64\n",
2180	pGVM, pGVCpu, pGVCpu->idCpu, u64ExpireGipTime));
2181	GVMM_CHECK_SMAP_SETUP();
2182	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2183
2184	PGVMM pGVMM;
2185	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
2186
2187	pGVM->gvmm.s.StatsSched.cHaltCalls++;
2188	Assert(!pGVCpu->gvmm.s.u64HaltExpire);
2189
2190	/*
2191	* If we're doing early wake-ups, we must take the UsedList lock before we
2192	* start querying the current time.
2193	* Note! Interrupts must NOT be disabled at this point because we ask for GIP time!
2194	*/
2195	bool const fDoEarlyWakeUps = pGVMM->fDoEarlyWakeUps;
2196	if (fDoEarlyWakeUps)
2197	{
2198	int rc2 = GVMMR0_USED_SHARED_LOCK(pGVMM); AssertRC(rc2);
2199	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2200	}
2201
2202	pGVCpu->gvmm.s.iCpuEmt = ASMGetApicId();
2203
2204	/* GIP hack: We might are frequently sleeping for short intervals where the
2205	difference between GIP and system time matters on systems with high resolution
2206	system time. So, convert the input from GIP to System time in that case. */
2207	Assert(ASMGetFlags() & X86_EFL_IF);
2208	const uint64_t u64NowSys = RTTimeSystemNanoTS();
2209	const uint64_t u64NowGip = RTTimeNanoTS();
2210	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2211
2212	if (fDoEarlyWakeUps)
2213	{
2214	pGVM->gvmm.s.StatsSched.cHaltWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64NowGip);
2215	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2216	}
2217
2218	/*
2219	* Go to sleep if we must...
2220	* Cap the sleep time to 1 second to be on the safe side.
2221	*/
2222	int rc;
2223	uint64_t cNsInterval = u64ExpireGipTime - u64NowGip;
2224	if ( u64NowGip < u64ExpireGipTime
2225	&& cNsInterval >= (pGVMM->cEMTs > pGVMM->cEMTsMeansCompany
2226	? pGVMM->nsMinSleepCompany
2227	: pGVMM->nsMinSleepAlone))
2228	{
2229	pGVM->gvmm.s.StatsSched.cHaltBlocking++;
2230	if (cNsInterval > RT_NS_1SEC)
2231	u64ExpireGipTime = u64NowGip + RT_NS_1SEC;
2232	ASMAtomicWriteU64(&pGVCpu->gvmm.s.u64HaltExpire, u64ExpireGipTime);
2233	ASMAtomicIncU32(&pGVMM->cHaltedEMTs);
2234	if (fDoEarlyWakeUps)
2235	{
2236	if (u64ExpireGipTime < pGVMM->uNsNextEmtWakeup)
2237	pGVMM->uNsNextEmtWakeup = u64ExpireGipTime;
2238	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2239	}
2240	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2241
2242	rc = RTSemEventMultiWaitEx(pGVCpu->gvmm.s.HaltEventMulti,
2243	RTSEMWAIT_FLAGS_ABSOLUTE \| RTSEMWAIT_FLAGS_NANOSECS \| RTSEMWAIT_FLAGS_INTERRUPTIBLE,
2244	u64NowGip > u64NowSys ? u64ExpireGipTime : u64NowSys + cNsInterval);
2245	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2246
2247	ASMAtomicWriteU64(&pGVCpu->gvmm.s.u64HaltExpire, 0);
2248	ASMAtomicDecU32(&pGVMM->cHaltedEMTs);
2249
2250	/* Reset the semaphore to try prevent a few false wake-ups. */
2251	if (rc == VINF_SUCCESS)
2252	{
2253	RTSemEventMultiReset(pGVCpu->gvmm.s.HaltEventMulti);
2254	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2255	}
2256	else if (rc == VERR_TIMEOUT)
2257	{
2258	pGVM->gvmm.s.StatsSched.cHaltTimeouts++;
2259	rc = VINF_SUCCESS;
2260	}
2261	}
2262	else
2263	{
2264	pGVM->gvmm.s.StatsSched.cHaltNotBlocking++;
2265	if (fDoEarlyWakeUps)
2266	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2267	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2268	RTSemEventMultiReset(pGVCpu->gvmm.s.HaltEventMulti);
2269	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2270	rc = VINF_SUCCESS;
2271	}
2272
2273	return rc;
2274	}
2275
2276
2277	/**
2278	* Halt the EMT thread.
2279	*
2280	* @returns VINF_SUCCESS normal wakeup (timeout or kicked by other thread).
2281	* VERR_INTERRUPTED if a signal was scheduled for the thread.
2282	* @param pGVM The global (ring-0) VM structure.
2283	* @param idCpu The Virtual CPU ID of the calling EMT.
2284	* @param u64ExpireGipTime The time for the sleep to expire expressed as GIP time.
2285	* @thread EMT(idCpu).
2286	*/
2287	GVMMR0DECL(int) GVMMR0SchedHaltReq(PGVM pGVM, VMCPUID idCpu, uint64_t u64ExpireGipTime)
2288	{
2289	GVMM_CHECK_SMAP_SETUP();
2290	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2291	PGVMM pGVMM;
2292	int rc = gvmmR0ByGVMandEMT(pGVM, idCpu, &pGVMM);
2293	if (RT_SUCCESS(rc))
2294	{
2295	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2296	rc = GVMMR0SchedHalt(pGVM, &pGVM->aCpus[idCpu], u64ExpireGipTime);
2297	}
2298	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2299	return rc;
2300	}
2301
2302
2303
2304	/**
2305	* Worker for GVMMR0SchedWakeUp and GVMMR0SchedWakeUpAndPokeCpus that wakes up
2306	* the a sleeping EMT.
2307	*
2308	* @retval VINF_SUCCESS if successfully woken up.
2309	* @retval VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
2310	*
2311	* @param pGVM The global (ring-0) VM structure.
2312	* @param pGVCpu The global (ring-0) VCPU structure.
2313	*/
2314	DECLINLINE(int) gvmmR0SchedWakeUpOne(PGVM pGVM, PGVMCPU pGVCpu)
2315	{
2316	pGVM->gvmm.s.StatsSched.cWakeUpCalls++;
2317
2318	/*
2319	* Signal the semaphore regardless of whether it's current blocked on it.
2320	*
2321	* The reason for this is that there is absolutely no way we can be 100%
2322	* certain that it isn't about go to go to sleep on it and just got
2323	* delayed a bit en route. So, we will always signal the semaphore when
2324	* the it is flagged as halted in the VMM.
2325	*/
2326	/** @todo we can optimize some of that by means of the pVCpu->enmState now. */
2327	int rc;
2328	if (pGVCpu->gvmm.s.u64HaltExpire)
2329	{
2330	rc = VINF_SUCCESS;
2331	ASMAtomicWriteU64(&pGVCpu->gvmm.s.u64HaltExpire, 0);
2332	}
2333	else
2334	{
2335	rc = VINF_GVM_NOT_BLOCKED;
2336	pGVM->gvmm.s.StatsSched.cWakeUpNotHalted++;
2337	}
2338
2339	int rc2 = RTSemEventMultiSignal(pGVCpu->gvmm.s.HaltEventMulti);
2340	AssertRC(rc2);
2341
2342	return rc;
2343	}
2344
2345
2346	/**
2347	* Wakes up the halted EMT thread so it can service a pending request.
2348	*
2349	* @returns VBox status code.
2350	* @retval VINF_SUCCESS if successfully woken up.
2351	* @retval VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
2352	*
2353	* @param pGVM The global (ring-0) VM structure.
2354	* @param idCpu The Virtual CPU ID of the EMT to wake up.
2355	* @param fTakeUsedLock Take the used lock or not
2356	* @thread Any but EMT(idCpu).
2357	*/
2358	GVMMR0DECL(int) GVMMR0SchedWakeUpEx(PGVM pGVM, VMCPUID idCpu, bool fTakeUsedLock)
2359	{
2360	GVMM_CHECK_SMAP_SETUP();
2361	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2362
2363	/*
2364	* Validate input and take the UsedLock.
2365	*/
2366	PGVMM pGVMM;
2367	int rc = gvmmR0ByGVM(pGVM, &pGVMM, fTakeUsedLock);
2368	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2369	if (RT_SUCCESS(rc))
2370	{
2371	if (idCpu < pGVM->cCpus)
2372	{
2373	/*
2374	* Do the actual job.
2375	*/
2376	rc = gvmmR0SchedWakeUpOne(pGVM, &pGVM->aCpus[idCpu]);
2377	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2378
2379	if (fTakeUsedLock && pGVMM->fDoEarlyWakeUps)
2380	{
2381	/*
2382	* While we're here, do a round of scheduling.
2383	*/
2384	Assert(ASMGetFlags() & X86_EFL_IF);
2385	const uint64_t u64Now = RTTimeNanoTS(); /* (GIP time) */
2386	pGVM->gvmm.s.StatsSched.cWakeUpWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64Now);
2387	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2388	}
2389	}
2390	else
2391	rc = VERR_INVALID_CPU_ID;
2392
2393	if (fTakeUsedLock)
2394	{
2395	int rc2 = GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2396	AssertRC(rc2);
2397	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2398	}
2399	}
2400
2401	LogFlow(("GVMMR0SchedWakeUpEx: returns %Rrc\n", rc));
2402	return rc;
2403	}
2404
2405
2406	/**
2407	* Wakes up the halted EMT thread so it can service a pending request.
2408	*
2409	* @returns VBox status code.
2410	* @retval VINF_SUCCESS if successfully woken up.
2411	* @retval VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
2412	*
2413	* @param pGVM The global (ring-0) VM structure.
2414	* @param idCpu The Virtual CPU ID of the EMT to wake up.
2415	* @thread Any but EMT(idCpu).
2416	*/
2417	GVMMR0DECL(int) GVMMR0SchedWakeUp(PGVM pGVM, VMCPUID idCpu)
2418	{
2419	return GVMMR0SchedWakeUpEx(pGVM, idCpu, true /* fTakeUsedLock */);
2420	}
2421
2422
2423	/**
2424	* Wakes up the halted EMT thread so it can service a pending request, no GVM
2425	* parameter and no used locking.
2426	*
2427	* @returns VBox status code.
2428	* @retval VINF_SUCCESS if successfully woken up.
2429	* @retval VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
2430	*
2431	* @param pGVM The global (ring-0) VM structure.
2432	* @param idCpu The Virtual CPU ID of the EMT to wake up.
2433	* @thread Any but EMT(idCpu).
2434	* @deprecated Don't use in new code if possible! Use the GVM variant.
2435	*/
2436	GVMMR0DECL(int) GVMMR0SchedWakeUpNoGVMNoLock(PGVM pGVM, VMCPUID idCpu)
2437	{
2438	GVMM_CHECK_SMAP_SETUP();
2439	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2440	PGVMM pGVMM;
2441	int rc = gvmmR0ByGVM(pGVM, &pGVMM, false /fTakeUsedLock/);
2442	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2443	if (RT_SUCCESS(rc))
2444	rc = GVMMR0SchedWakeUpEx(pGVM, idCpu, false /fTakeUsedLock/);
2445	return rc;
2446	}
2447
2448
2449	/**
2450	* Worker common to GVMMR0SchedPoke and GVMMR0SchedWakeUpAndPokeCpus that pokes
2451	* the Virtual CPU if it's still busy executing guest code.
2452	*
2453	* @returns VBox status code.
2454	* @retval VINF_SUCCESS if poked successfully.
2455	* @retval VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
2456	*
2457	* @param pGVM The global (ring-0) VM structure.
2458	* @param pVCpu The cross context virtual CPU structure.
2459	*/
2460	DECLINLINE(int) gvmmR0SchedPokeOne(PGVM pGVM, PVMCPUCC pVCpu)
2461	{
2462	pGVM->gvmm.s.StatsSched.cPokeCalls++;
2463
2464	RTCPUID idHostCpu = pVCpu->idHostCpu;
2465	if ( idHostCpu == NIL_RTCPUID
2466	\|\| VMCPU_GET_STATE(pVCpu) != VMCPUSTATE_STARTED_EXEC)
2467	{
2468	pGVM->gvmm.s.StatsSched.cPokeNotBusy++;
2469	return VINF_GVM_NOT_BUSY_IN_GC;
2470	}
2471
2472	/* Note: this function is not implemented on Darwin and Linux (kernel < 2.6.19) */
2473	RTMpPokeCpu(idHostCpu);
2474	return VINF_SUCCESS;
2475	}
2476
2477
2478	/**
2479	* Pokes an EMT if it's still busy running guest code.
2480	*
2481	* @returns VBox status code.
2482	* @retval VINF_SUCCESS if poked successfully.
2483	* @retval VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
2484	*
2485	* @param pGVM The global (ring-0) VM structure.
2486	* @param idCpu The ID of the virtual CPU to poke.
2487	* @param fTakeUsedLock Take the used lock or not
2488	*/
2489	GVMMR0DECL(int) GVMMR0SchedPokeEx(PGVM pGVM, VMCPUID idCpu, bool fTakeUsedLock)
2490	{
2491	/*
2492	* Validate input and take the UsedLock.
2493	*/
2494	PGVMM pGVMM;
2495	int rc = gvmmR0ByGVM(pGVM, &pGVMM, fTakeUsedLock);
2496	if (RT_SUCCESS(rc))
2497	{
2498	if (idCpu < pGVM->cCpus)
2499	rc = gvmmR0SchedPokeOne(pGVM, &pGVM->aCpus[idCpu]);
2500	else
2501	rc = VERR_INVALID_CPU_ID;
2502
2503	if (fTakeUsedLock)
2504	{
2505	int rc2 = GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2506	AssertRC(rc2);
2507	}
2508	}
2509
2510	LogFlow(("GVMMR0SchedWakeUpAndPokeCpus: returns %Rrc\n", rc));
2511	return rc;
2512	}
2513
2514
2515	/**
2516	* Pokes an EMT if it's still busy running guest code.
2517	*
2518	* @returns VBox status code.
2519	* @retval VINF_SUCCESS if poked successfully.
2520	* @retval VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
2521	*
2522	* @param pGVM The global (ring-0) VM structure.
2523	* @param idCpu The ID of the virtual CPU to poke.
2524	*/
2525	GVMMR0DECL(int) GVMMR0SchedPoke(PGVM pGVM, VMCPUID idCpu)
2526	{
2527	return GVMMR0SchedPokeEx(pGVM, idCpu, true /* fTakeUsedLock */);
2528	}
2529
2530
2531	/**
2532	* Pokes an EMT if it's still busy running guest code, no GVM parameter and no
2533	* used locking.
2534	*
2535	* @returns VBox status code.
2536	* @retval VINF_SUCCESS if poked successfully.
2537	* @retval VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
2538	*
2539	* @param pGVM The global (ring-0) VM structure.
2540	* @param idCpu The ID of the virtual CPU to poke.
2541	*
2542	* @deprecated Don't use in new code if possible! Use the GVM variant.
2543	*/
2544	GVMMR0DECL(int) GVMMR0SchedPokeNoGVMNoLock(PGVM pGVM, VMCPUID idCpu)
2545	{
2546	PGVMM pGVMM;
2547	int rc = gvmmR0ByGVM(pGVM, &pGVMM, false /fTakeUsedLock/);
2548	if (RT_SUCCESS(rc))
2549	{
2550	if (idCpu < pGVM->cCpus)
2551	rc = gvmmR0SchedPokeOne(pGVM, &pGVM->aCpus[idCpu]);
2552	else
2553	rc = VERR_INVALID_CPU_ID;
2554	}
2555	return rc;
2556	}
2557
2558
2559	/**
2560	* Wakes up a set of halted EMT threads so they can service pending request.
2561	*
2562	* @returns VBox status code, no informational stuff.
2563	*
2564	* @param pGVM The global (ring-0) VM structure.
2565	* @param pSleepSet The set of sleepers to wake up.
2566	* @param pPokeSet The set of CPUs to poke.
2567	*/
2568	GVMMR0DECL(int) GVMMR0SchedWakeUpAndPokeCpus(PGVM pGVM, PCVMCPUSET pSleepSet, PCVMCPUSET pPokeSet)
2569	{
2570	AssertPtrReturn(pSleepSet, VERR_INVALID_POINTER);
2571	AssertPtrReturn(pPokeSet, VERR_INVALID_POINTER);
2572	GVMM_CHECK_SMAP_SETUP();
2573	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2574	RTNATIVETHREAD hSelf = RTThreadNativeSelf();
2575
2576	/*
2577	* Validate input and take the UsedLock.
2578	*/
2579	PGVMM pGVMM;
2580	int rc = gvmmR0ByGVM(pGVM, &pGVMM, true /* fTakeUsedLock */);
2581	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2582	if (RT_SUCCESS(rc))
2583	{
2584	rc = VINF_SUCCESS;
2585	VMCPUID idCpu = pGVM->cCpus;
2586	while (idCpu-- > 0)
2587	{
2588	/* Don't try poke or wake up ourselves. */
2589	if (pGVM->aCpus[idCpu].hEMT == hSelf)
2590	continue;
2591
2592	/* just ignore errors for now. */
2593	if (VMCPUSET_IS_PRESENT(pSleepSet, idCpu))
2594	{
2595	gvmmR0SchedWakeUpOne(pGVM, &pGVM->aCpus[idCpu]);
2596	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2597	}
2598	else if (VMCPUSET_IS_PRESENT(pPokeSet, idCpu))
2599	{
2600	gvmmR0SchedPokeOne(pGVM, &pGVM->aCpus[idCpu]);
2601	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2602	}
2603	}
2604
2605	int rc2 = GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2606	AssertRC(rc2);
2607	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2608	}
2609
2610	LogFlow(("GVMMR0SchedWakeUpAndPokeCpus: returns %Rrc\n", rc));
2611	return rc;
2612	}
2613
2614
2615	/**
2616	* VMMR0 request wrapper for GVMMR0SchedWakeUpAndPokeCpus.
2617	*
2618	* @returns see GVMMR0SchedWakeUpAndPokeCpus.
2619	* @param pGVM The global (ring-0) VM structure.
2620	* @param pReq Pointer to the request packet.
2621	*/
2622	GVMMR0DECL(int) GVMMR0SchedWakeUpAndPokeCpusReq(PGVM pGVM, PGVMMSCHEDWAKEUPANDPOKECPUSREQ pReq)
2623	{
2624	/*
2625	* Validate input and pass it on.
2626	*/
2627	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
2628	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
2629
2630	return GVMMR0SchedWakeUpAndPokeCpus(pGVM, &pReq->SleepSet, &pReq->PokeSet);
2631	}
2632
2633
2634
2635	/**
2636	* Poll the schedule to see if someone else should get a chance to run.
2637	*
2638	* This is a bit hackish and will not work too well if the machine is
2639	* under heavy load from non-VM processes.
2640	*
2641	* @returns VINF_SUCCESS if not yielded.
2642	* VINF_GVM_YIELDED if an attempt to switch to a different VM task was made.
2643	* @param pGVM The global (ring-0) VM structure.
2644	* @param idCpu The Virtual CPU ID of the calling EMT.
2645	* @param fYield Whether to yield or not.
2646	* This is for when we're spinning in the halt loop.
2647	* @thread EMT(idCpu).
2648	*/
2649	GVMMR0DECL(int) GVMMR0SchedPoll(PGVM pGVM, VMCPUID idCpu, bool fYield)
2650	{
2651	/*
2652	* Validate input.
2653	*/
2654	PGVMM pGVMM;
2655	int rc = gvmmR0ByGVMandEMT(pGVM, idCpu, &pGVMM);
2656	if (RT_SUCCESS(rc))
2657	{
2658	/*
2659	* We currently only implement helping doing wakeups (fYield = false), so don't
2660	* bother taking the lock if gvmmR0SchedDoWakeUps is not going to do anything.
2661	*/
2662	if (!fYield && pGVMM->fDoEarlyWakeUps)
2663	{
2664	rc = GVMMR0_USED_SHARED_LOCK(pGVMM); AssertRC(rc);
2665	pGVM->gvmm.s.StatsSched.cPollCalls++;
2666
2667	Assert(ASMGetFlags() & X86_EFL_IF);
2668	const uint64_t u64Now = RTTimeNanoTS(); /* (GIP time) */
2669
2670	pGVM->gvmm.s.StatsSched.cPollWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64Now);
2671
2672	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2673	}
2674	/*
2675	* Not quite sure what we could do here...
2676	*/
2677	else if (fYield)
2678	rc = VERR_NOT_IMPLEMENTED; /** @todo implement this... */
2679	else
2680	rc = VINF_SUCCESS;
2681	}
2682
2683	LogFlow(("GVMMR0SchedWakeUp: returns %Rrc\n", rc));
2684	return rc;
2685	}
2686
2687
2688	#ifdef GVMM_SCHED_WITH_PPT
2689	/**
2690	* Timer callback for the periodic preemption timer.
2691	*
2692	* @param pTimer The timer handle.
2693	* @param pvUser Pointer to the per cpu structure.
2694	* @param iTick The current tick.
2695	*/
2696	static DECLCALLBACK(void) gvmmR0SchedPeriodicPreemptionTimerCallback(PRTTIMER pTimer, void *pvUser, uint64_t iTick)
2697	{
2698	PGVMMHOSTCPU pCpu = (PGVMMHOSTCPU)pvUser;
2699	NOREF(pTimer); NOREF(iTick);
2700
2701	/*
2702	* Termination check
2703	*/
2704	if (pCpu->u32Magic != GVMMHOSTCPU_MAGIC)
2705	return;
2706
2707	/*
2708	* Do the house keeping.
2709	*/
2710	RTSpinlockAcquire(pCpu->Ppt.hSpinlock);
2711
2712	if (++pCpu->Ppt.iTickHistorization >= pCpu->Ppt.cTicksHistoriziationInterval)
2713	{
2714	/*
2715	* Historicize the max frequency.
2716	*/
2717	uint32_t iHzHistory = ++pCpu->Ppt.iHzHistory % RT_ELEMENTS(pCpu->Ppt.aHzHistory);
2718	pCpu->Ppt.aHzHistory[iHzHistory] = pCpu->Ppt.uDesiredHz;
2719	pCpu->Ppt.iTickHistorization = 0;
2720	pCpu->Ppt.uDesiredHz = 0;
2721
2722	/*
2723	* Check if the current timer frequency.
2724	*/
2725	uint32_t uHistMaxHz = 0;
2726	for (uint32_t i = 0; i < RT_ELEMENTS(pCpu->Ppt.aHzHistory); i++)
2727	if (pCpu->Ppt.aHzHistory[i] > uHistMaxHz)
2728	uHistMaxHz = pCpu->Ppt.aHzHistory[i];
2729	if (uHistMaxHz == pCpu->Ppt.uTimerHz)
2730	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
2731	else if (uHistMaxHz)
2732	{
2733	/*
2734	* Reprogram it.
2735	*/
2736	pCpu->Ppt.cChanges++;
2737	pCpu->Ppt.iTickHistorization = 0;
2738	pCpu->Ppt.uTimerHz = uHistMaxHz;
2739	uint32_t const cNsInterval = RT_NS_1SEC / uHistMaxHz;
2740	pCpu->Ppt.cNsInterval = cNsInterval;
2741	if (cNsInterval < GVMMHOSTCPU_PPT_HIST_INTERVAL_NS)
2742	pCpu->Ppt.cTicksHistoriziationInterval = ( GVMMHOSTCPU_PPT_HIST_INTERVAL_NS
2743	+ GVMMHOSTCPU_PPT_HIST_INTERVAL_NS / 2 - 1)
2744	/ cNsInterval;
2745	else
2746	pCpu->Ppt.cTicksHistoriziationInterval = 1;
2747	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
2748
2749	/SUPR0Printf("Cpu%u: change to %u Hz / %u ns\n", pCpu->idxCpuSet, uHistMaxHz, cNsInterval);/
2750	RTTimerChangeInterval(pTimer, cNsInterval);
2751	}
2752	else
2753	{
2754	/*
2755	* Stop it.
2756	*/
2757	pCpu->Ppt.fStarted = false;
2758	pCpu->Ppt.uTimerHz = 0;
2759	pCpu->Ppt.cNsInterval = 0;
2760	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
2761
2762	/SUPR0Printf("Cpu%u: stopping (%u Hz)\n", pCpu->idxCpuSet, uHistMaxHz);/
2763	RTTimerStop(pTimer);
2764	}
2765	}
2766	else
2767	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
2768	}
2769	#endif /* GVMM_SCHED_WITH_PPT */
2770
2771
2772	/**
2773	* Updates the periodic preemption timer for the calling CPU.
2774	*
2775	* The caller must have disabled preemption!
2776	* The caller must check that the host can do high resolution timers.
2777	*
2778	* @param pGVM The global (ring-0) VM structure.
2779	* @param idHostCpu The current host CPU id.
2780	* @param uHz The desired frequency.
2781	*/
2782	GVMMR0DECL(void) GVMMR0SchedUpdatePeriodicPreemptionTimer(PGVM pGVM, RTCPUID idHostCpu, uint32_t uHz)
2783	{
2784	NOREF(pGVM);
2785	#ifdef GVMM_SCHED_WITH_PPT
2786	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2787	Assert(RTTimerCanDoHighResolution());
2788
2789	/*
2790	* Resolve the per CPU data.
2791	*/
2792	uint32_t iCpu = RTMpCpuIdToSetIndex(idHostCpu);
2793	PGVMM pGVMM = g_pGVMM;
2794	if ( !RT_VALID_PTR(pGVMM)
2795	\|\| pGVMM->u32Magic != GVMM_MAGIC)
2796	return;
2797	AssertMsgReturnVoid(iCpu < pGVMM->cHostCpus, ("iCpu=%d cHostCpus=%d\n", iCpu, pGVMM->cHostCpus));
2798	PGVMMHOSTCPU pCpu = &pGVMM->aHostCpus[iCpu];
2799	AssertMsgReturnVoid( pCpu->u32Magic == GVMMHOSTCPU_MAGIC
2800	&& pCpu->idCpu == idHostCpu,
2801	("u32Magic=%#x idCpu=% idHostCpu=%d\n", pCpu->u32Magic, pCpu->idCpu, idHostCpu));
2802
2803	/*
2804	* Check whether we need to do anything about the timer.
2805	* We have to be a little bit careful since we might be race the timer
2806	* callback here.
2807	*/
2808	if (uHz > 16384)
2809	uHz = 16384; /** @todo add a query method for this! */
2810	if (RT_UNLIKELY( uHz > ASMAtomicReadU32(&pCpu->Ppt.uDesiredHz)
2811	&& uHz >= pCpu->Ppt.uMinHz
2812	&& !pCpu->Ppt.fStarting /* solaris paranoia */))
2813	{
2814	RTSpinlockAcquire(pCpu->Ppt.hSpinlock);
2815
2816	pCpu->Ppt.uDesiredHz = uHz;
2817	uint32_t cNsInterval = 0;
2818	if (!pCpu->Ppt.fStarted)
2819	{
2820	pCpu->Ppt.cStarts++;
2821	pCpu->Ppt.fStarted = true;
2822	pCpu->Ppt.fStarting = true;
2823	pCpu->Ppt.iTickHistorization = 0;
2824	pCpu->Ppt.uTimerHz = uHz;
2825	pCpu->Ppt.cNsInterval = cNsInterval = RT_NS_1SEC / uHz;
2826	if (cNsInterval < GVMMHOSTCPU_PPT_HIST_INTERVAL_NS)
2827	pCpu->Ppt.cTicksHistoriziationInterval = ( GVMMHOSTCPU_PPT_HIST_INTERVAL_NS
2828	+ GVMMHOSTCPU_PPT_HIST_INTERVAL_NS / 2 - 1)
2829	/ cNsInterval;
2830	else
2831	pCpu->Ppt.cTicksHistoriziationInterval = 1;
2832	}
2833
2834	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
2835
2836	if (cNsInterval)
2837	{
2838	RTTimerChangeInterval(pCpu->Ppt.pTimer, cNsInterval);
2839	int rc = RTTimerStart(pCpu->Ppt.pTimer, cNsInterval);
2840	AssertRC(rc);
2841
2842	RTSpinlockAcquire(pCpu->Ppt.hSpinlock);
2843	if (RT_FAILURE(rc))
2844	pCpu->Ppt.fStarted = false;
2845	pCpu->Ppt.fStarting = false;
2846	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
2847	}
2848	}
2849	#else /* !GVMM_SCHED_WITH_PPT */
2850	NOREF(idHostCpu); NOREF(uHz);
2851	#endif /* !GVMM_SCHED_WITH_PPT */
2852	}
2853
2854
2855	/**
2856	* Calls @a pfnCallback for each VM in the system.
2857	*
2858	* This will enumerate the VMs while holding the global VM used list lock in
2859	* shared mode. So, only suitable for simple work. If more expensive work
2860	* needs doing, a different approach must be taken as using this API would
2861	* otherwise block VM creation and destruction.
2862	*
2863	* @returns VBox status code.
2864	* @param pfnCallback The callback function.
2865	* @param pvUser User argument to the callback.
2866	*/
2867	GVMMR0DECL(int) GVMMR0EnumVMs(PFNGVMMR0ENUMCALLBACK pfnCallback, void *pvUser)
2868	{
2869	PGVMM pGVMM;
2870	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
2871
2872	int rc = VINF_SUCCESS;
2873	GVMMR0_USED_SHARED_LOCK(pGVMM);
2874	for (unsigned i = pGVMM->iUsedHead, cLoops = 0;
2875	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
2876	i = pGVMM->aHandles[i].iNext, cLoops++)
2877	{
2878	PGVM pGVM = pGVMM->aHandles[i].pGVM;
2879	if ( RT_VALID_PTR(pGVM)
2880	&& RT_VALID_PTR(pGVMM->aHandles[i].pvObj)
2881	&& pGVM->u32Magic == GVM_MAGIC)
2882	{
2883	rc = pfnCallback(pGVM, pvUser);
2884	if (rc != VINF_SUCCESS)
2885	break;
2886	}
2887
2888	AssertBreak(cLoops < RT_ELEMENTS(pGVMM->aHandles) * 4); /* paranoia */
2889	}
2890	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2891	return rc;
2892	}
2893
2894
2895	/**
2896	* Retrieves the GVMM statistics visible to the caller.
2897	*
2898	* @returns VBox status code.
2899	*
2900	* @param pStats Where to put the statistics.
2901	* @param pSession The current session.
2902	* @param pGVM The GVM to obtain statistics for. Optional.
2903	*/
2904	GVMMR0DECL(int) GVMMR0QueryStatistics(PGVMMSTATS pStats, PSUPDRVSESSION pSession, PGVM pGVM)
2905	{
2906	LogFlow(("GVMMR0QueryStatistics: pStats=%p pSession=%p pGVM=%p\n", pStats, pSession, pGVM));
2907
2908	/*
2909	* Validate input.
2910	*/
2911	AssertPtrReturn(pSession, VERR_INVALID_POINTER);
2912	AssertPtrReturn(pStats, VERR_INVALID_POINTER);
2913	pStats->cVMs = 0; /* (crash before taking the sem...) */
2914
2915	/*
2916	* Take the lock and get the VM statistics.
2917	*/
2918	PGVMM pGVMM;
2919	if (pGVM)
2920	{
2921	int rc = gvmmR0ByGVM(pGVM, &pGVMM, true /fTakeUsedLock/);
2922	if (RT_FAILURE(rc))
2923	return rc;
2924	pStats->SchedVM = pGVM->gvmm.s.StatsSched;
2925	}
2926	else
2927	{
2928	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
2929	memset(&pStats->SchedVM, 0, sizeof(pStats->SchedVM));
2930
2931	int rc = GVMMR0_USED_SHARED_LOCK(pGVMM);
2932	AssertRCReturn(rc, rc);
2933	}
2934
2935	/*
2936	* Enumerate the VMs and add the ones visible to the statistics.
2937	*/
2938	pStats->cVMs = 0;
2939	pStats->cEMTs = 0;
2940	memset(&pStats->SchedSum, 0, sizeof(pStats->SchedSum));
2941
2942	for (unsigned i = pGVMM->iUsedHead;
2943	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
2944	i = pGVMM->aHandles[i].iNext)
2945	{
2946	PGVM pOtherGVM = pGVMM->aHandles[i].pGVM;
2947	void *pvObj = pGVMM->aHandles[i].pvObj;
2948	if ( RT_VALID_PTR(pvObj)
2949	&& RT_VALID_PTR(pOtherGVM)
2950	&& pOtherGVM->u32Magic == GVM_MAGIC
2951	&& RT_SUCCESS(SUPR0ObjVerifyAccess(pvObj, pSession, NULL)))
2952	{
2953	pStats->cVMs++;
2954	pStats->cEMTs += pOtherGVM->cCpus;
2955
2956	pStats->SchedSum.cHaltCalls += pOtherGVM->gvmm.s.StatsSched.cHaltCalls;
2957	pStats->SchedSum.cHaltBlocking += pOtherGVM->gvmm.s.StatsSched.cHaltBlocking;
2958	pStats->SchedSum.cHaltTimeouts += pOtherGVM->gvmm.s.StatsSched.cHaltTimeouts;
2959	pStats->SchedSum.cHaltNotBlocking += pOtherGVM->gvmm.s.StatsSched.cHaltNotBlocking;
2960	pStats->SchedSum.cHaltWakeUps += pOtherGVM->gvmm.s.StatsSched.cHaltWakeUps;
2961
2962	pStats->SchedSum.cWakeUpCalls += pOtherGVM->gvmm.s.StatsSched.cWakeUpCalls;
2963	pStats->SchedSum.cWakeUpNotHalted += pOtherGVM->gvmm.s.StatsSched.cWakeUpNotHalted;
2964	pStats->SchedSum.cWakeUpWakeUps += pOtherGVM->gvmm.s.StatsSched.cWakeUpWakeUps;
2965
2966	pStats->SchedSum.cPokeCalls += pOtherGVM->gvmm.s.StatsSched.cPokeCalls;
2967	pStats->SchedSum.cPokeNotBusy += pOtherGVM->gvmm.s.StatsSched.cPokeNotBusy;
2968
2969	pStats->SchedSum.cPollCalls += pOtherGVM->gvmm.s.StatsSched.cPollCalls;
2970	pStats->SchedSum.cPollHalts += pOtherGVM->gvmm.s.StatsSched.cPollHalts;
2971	pStats->SchedSum.cPollWakeUps += pOtherGVM->gvmm.s.StatsSched.cPollWakeUps;
2972	}
2973	}
2974
2975	/*
2976	* Copy out the per host CPU statistics.
2977	*/
2978	uint32_t iDstCpu = 0;
2979	uint32_t cSrcCpus = pGVMM->cHostCpus;
2980	for (uint32_t iSrcCpu = 0; iSrcCpu < cSrcCpus; iSrcCpu++)
2981	{
2982	if (pGVMM->aHostCpus[iSrcCpu].idCpu != NIL_RTCPUID)
2983	{
2984	pStats->aHostCpus[iDstCpu].idCpu = pGVMM->aHostCpus[iSrcCpu].idCpu;
2985	pStats->aHostCpus[iDstCpu].idxCpuSet = pGVMM->aHostCpus[iSrcCpu].idxCpuSet;
2986	#ifdef GVMM_SCHED_WITH_PPT
2987	pStats->aHostCpus[iDstCpu].uDesiredHz = pGVMM->aHostCpus[iSrcCpu].Ppt.uDesiredHz;
2988	pStats->aHostCpus[iDstCpu].uTimerHz = pGVMM->aHostCpus[iSrcCpu].Ppt.uTimerHz;
2989	pStats->aHostCpus[iDstCpu].cChanges = pGVMM->aHostCpus[iSrcCpu].Ppt.cChanges;
2990	pStats->aHostCpus[iDstCpu].cStarts = pGVMM->aHostCpus[iSrcCpu].Ppt.cStarts;
2991	#else
2992	pStats->aHostCpus[iDstCpu].uDesiredHz = 0;
2993	pStats->aHostCpus[iDstCpu].uTimerHz = 0;
2994	pStats->aHostCpus[iDstCpu].cChanges = 0;
2995	pStats->aHostCpus[iDstCpu].cStarts = 0;
2996	#endif
2997	iDstCpu++;
2998	if (iDstCpu >= RT_ELEMENTS(pStats->aHostCpus))
2999	break;
3000	}
3001	}
3002	pStats->cHostCpus = iDstCpu;
3003
3004	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
3005
3006	return VINF_SUCCESS;
3007	}
3008
3009
3010	/**
3011	* VMMR0 request wrapper for GVMMR0QueryStatistics.
3012	*
3013	* @returns see GVMMR0QueryStatistics.
3014	* @param pGVM The global (ring-0) VM structure. Optional.
3015	* @param pReq Pointer to the request packet.
3016	* @param pSession The current session.
3017	*/
3018	GVMMR0DECL(int) GVMMR0QueryStatisticsReq(PGVM pGVM, PGVMMQUERYSTATISTICSSREQ pReq, PSUPDRVSESSION pSession)
3019	{
3020	/*
3021	* Validate input and pass it on.
3022	*/
3023	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
3024	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
3025	AssertReturn(pReq->pSession == pSession, VERR_INVALID_PARAMETER);
3026
3027	return GVMMR0QueryStatistics(&pReq->Stats, pSession, pGVM);
3028	}
3029
3030
3031	/**
3032	* Resets the specified GVMM statistics.
3033	*
3034	* @returns VBox status code.
3035	*
3036	* @param pStats Which statistics to reset, that is, non-zero fields indicates which to reset.
3037	* @param pSession The current session.
3038	* @param pGVM The GVM to reset statistics for. Optional.
3039	*/
3040	GVMMR0DECL(int) GVMMR0ResetStatistics(PCGVMMSTATS pStats, PSUPDRVSESSION pSession, PGVM pGVM)
3041	{
3042	LogFlow(("GVMMR0ResetStatistics: pStats=%p pSession=%p pGVM=%p\n", pStats, pSession, pGVM));
3043
3044	/*
3045	* Validate input.
3046	*/
3047	AssertPtrReturn(pSession, VERR_INVALID_POINTER);
3048	AssertPtrReturn(pStats, VERR_INVALID_POINTER);
3049
3050	/*
3051	* Take the lock and get the VM statistics.
3052	*/
3053	PGVMM pGVMM;
3054	if (pGVM)
3055	{
3056	int rc = gvmmR0ByGVM(pGVM, &pGVMM, true /fTakeUsedLock/);
3057	if (RT_FAILURE(rc))
3058	return rc;
3059	# define MAYBE_RESET_FIELD(field) \
3060	do { if (pStats->SchedVM. field ) { pGVM->gvmm.s.StatsSched. field = 0; } } while (0)
3061	MAYBE_RESET_FIELD(cHaltCalls);
3062	MAYBE_RESET_FIELD(cHaltBlocking);
3063	MAYBE_RESET_FIELD(cHaltTimeouts);
3064	MAYBE_RESET_FIELD(cHaltNotBlocking);
3065	MAYBE_RESET_FIELD(cHaltWakeUps);
3066	MAYBE_RESET_FIELD(cWakeUpCalls);
3067	MAYBE_RESET_FIELD(cWakeUpNotHalted);
3068	MAYBE_RESET_FIELD(cWakeUpWakeUps);
3069	MAYBE_RESET_FIELD(cPokeCalls);
3070	MAYBE_RESET_FIELD(cPokeNotBusy);
3071	MAYBE_RESET_FIELD(cPollCalls);
3072	MAYBE_RESET_FIELD(cPollHalts);
3073	MAYBE_RESET_FIELD(cPollWakeUps);
3074	# undef MAYBE_RESET_FIELD
3075	}
3076	else
3077	{
3078	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
3079
3080	int rc = GVMMR0_USED_SHARED_LOCK(pGVMM);
3081	AssertRCReturn(rc, rc);
3082	}
3083
3084	/*
3085	* Enumerate the VMs and add the ones visible to the statistics.
3086	*/
3087	if (!ASMMemIsZero(&pStats->SchedSum, sizeof(pStats->SchedSum)))
3088	{
3089	for (unsigned i = pGVMM->iUsedHead;
3090	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
3091	i = pGVMM->aHandles[i].iNext)
3092	{
3093	PGVM pOtherGVM = pGVMM->aHandles[i].pGVM;
3094	void *pvObj = pGVMM->aHandles[i].pvObj;
3095	if ( RT_VALID_PTR(pvObj)
3096	&& RT_VALID_PTR(pOtherGVM)
3097	&& pOtherGVM->u32Magic == GVM_MAGIC
3098	&& RT_SUCCESS(SUPR0ObjVerifyAccess(pvObj, pSession, NULL)))
3099	{
3100	# define MAYBE_RESET_FIELD(field) \
3101	do { if (pStats->SchedSum. field ) { pOtherGVM->gvmm.s.StatsSched. field = 0; } } while (0)
3102	MAYBE_RESET_FIELD(cHaltCalls);
3103	MAYBE_RESET_FIELD(cHaltBlocking);
3104	MAYBE_RESET_FIELD(cHaltTimeouts);
3105	MAYBE_RESET_FIELD(cHaltNotBlocking);
3106	MAYBE_RESET_FIELD(cHaltWakeUps);
3107	MAYBE_RESET_FIELD(cWakeUpCalls);
3108	MAYBE_RESET_FIELD(cWakeUpNotHalted);
3109	MAYBE_RESET_FIELD(cWakeUpWakeUps);
3110	MAYBE_RESET_FIELD(cPokeCalls);
3111	MAYBE_RESET_FIELD(cPokeNotBusy);
3112	MAYBE_RESET_FIELD(cPollCalls);
3113	MAYBE_RESET_FIELD(cPollHalts);
3114	MAYBE_RESET_FIELD(cPollWakeUps);
3115	# undef MAYBE_RESET_FIELD
3116	}
3117	}
3118	}
3119
3120	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
3121
3122	return VINF_SUCCESS;
3123	}
3124
3125
3126	/**
3127	* VMMR0 request wrapper for GVMMR0ResetStatistics.
3128	*
3129	* @returns see GVMMR0ResetStatistics.
3130	* @param pGVM The global (ring-0) VM structure. Optional.
3131	* @param pReq Pointer to the request packet.
3132	* @param pSession The current session.
3133	*/
3134	GVMMR0DECL(int) GVMMR0ResetStatisticsReq(PGVM pGVM, PGVMMRESETSTATISTICSSREQ pReq, PSUPDRVSESSION pSession)
3135	{
3136	/*
3137	* Validate input and pass it on.
3138	*/
3139	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
3140	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
3141	AssertReturn(pReq->pSession == pSession, VERR_INVALID_PARAMETER);
3142
3143	return GVMMR0ResetStatistics(&pReq->Stats, pSession, pGVM);
3144	}
3145

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/GVMMR0.cpp@ 90597

Download in other formats: