GVMMR0.cpp@ 90976

Last change on this file since 90976 was 90897, checked in by vboxsync, 4 years ago
VMM: Initialize the ring-0 loggers as early as possible, so we can update their settings before doing vmmR0InitVM and similar where we'd like to have working log output. bugref:10086
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 111.0 KB

Line
1	/* $Id: GVMMR0.cpp 90897 2021-08-25 20:00:41Z vboxsync $ */
2	/** @file
3	* GVMM - Global VM Manager.
4	*/
5
6	/*
7	* Copyright (C) 2007-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/** @page pg_gvmm GVMM - The Global VM Manager
20	*
21	* The Global VM Manager lives in ring-0. Its main function at the moment is
22	* to manage a list of all running VMs, keep a ring-0 only structure (GVM) for
23	* each of them, and assign them unique identifiers (so GMM can track page
24	* owners). The GVMM also manage some of the host CPU resources, like the
25	* periodic preemption timer.
26	*
27	* The GVMM will create a ring-0 object for each VM when it is registered, this
28	* is both for session cleanup purposes and for having a point where it is
29	* possible to implement usage polices later (in SUPR0ObjRegister).
30	*
31	*
32	* @section sec_gvmm_ppt Periodic Preemption Timer (PPT)
33	*
34	* On system that sports a high resolution kernel timer API, we use per-cpu
35	* timers to generate interrupts that preempts VT-x, AMD-V and raw-mode guest
36	* execution. The timer frequency is calculating by taking the max
37	* TMCalcHostTimerFrequency for all VMs running on a CPU for the last ~160 ms
38	* (RT_ELEMENTS((PGVMMHOSTCPU)0, Ppt.aHzHistory) *
39	* GVMMHOSTCPU_PPT_HIST_INTERVAL_NS).
40	*
41	* The TMCalcHostTimerFrequency() part of the things gets its takes the max
42	* TMTimerSetFrequencyHint() value and adjusts by the current catch-up percent,
43	* warp drive percent and some fudge factors. VMMR0.cpp reports the result via
44	* GVMMR0SchedUpdatePeriodicPreemptionTimer() before switching to the VT-x,
45	* AMD-V and raw-mode execution environments.
46	*/
47
48
49	/*********************************************************************************************************************************
50	* Header Files *
51	*********************************************************************************************************************************/
52	#define LOG_GROUP LOG_GROUP_GVMM
53	#include <VBox/vmm/gvmm.h>
54	#include <VBox/vmm/gmm.h>
55	#include "GVMMR0Internal.h"
56	#include <VBox/vmm/dbgf.h>
57	#include <VBox/vmm/iom.h>
58	#include <VBox/vmm/pdm.h>
59	#include <VBox/vmm/pgm.h>
60	#include <VBox/vmm/vmm.h>
61	#ifdef VBOX_WITH_NEM_R0
62	# include <VBox/vmm/nem.h>
63	#endif
64	#include <VBox/vmm/vmcpuset.h>
65	#include <VBox/vmm/vmcc.h>
66	#include <VBox/param.h>
67	#include <VBox/err.h>
68
69	#include <iprt/asm.h>
70	#include <iprt/asm-amd64-x86.h>
71	#include <iprt/critsect.h>
72	#include <iprt/mem.h>
73	#include <iprt/semaphore.h>
74	#include <iprt/time.h>
75	#include <VBox/log.h>
76	#include <iprt/thread.h>
77	#include <iprt/process.h>
78	#include <iprt/param.h>
79	#include <iprt/string.h>
80	#include <iprt/assert.h>
81	#include <iprt/mem.h>
82	#include <iprt/memobj.h>
83	#include <iprt/mp.h>
84	#include <iprt/cpuset.h>
85	#include <iprt/spinlock.h>
86	#include <iprt/timer.h>
87
88	#include "dtrace/VBoxVMM.h"
89
90
91	/*********************************************************************************************************************************
92	* Defined Constants And Macros *
93	*********************************************************************************************************************************/
94	#if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS) \|\| defined(DOXYGEN_RUNNING)
95	/** Define this to enable the periodic preemption timer. */
96	# define GVMM_SCHED_WITH_PPT
97	#endif
98
99
100	/** @def GVMM_CHECK_SMAP_SETUP
101	* SMAP check setup. */
102	/** @def GVMM_CHECK_SMAP_CHECK
103	* Checks that the AC flag is set if SMAP is enabled. If AC is not set,
104	* it will be logged and @a a_BadExpr is executed. */
105	/** @def GVMM_CHECK_SMAP_CHECK2
106	* Checks that the AC flag is set if SMAP is enabled. If AC is not set, it will
107	* be logged, written to the VMs assertion text buffer, and @a a_BadExpr is
108	* executed. */
109	#if (defined(VBOX_STRICT) \|\| 1) && !defined(VBOX_WITH_RAM_IN_KERNEL)
110	# define GVMM_CHECK_SMAP_SETUP() uint32_t const fKernelFeatures = SUPR0GetKernelFeatures()
111	# define GVMM_CHECK_SMAP_CHECK(a_BadExpr) \
112	do { \
113	if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
114	{ \
115	RTCCUINTREG fEflCheck = ASMGetFlags(); \
116	if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
117	{ /* likely */ } \
118	else \
119	{ \
120	SUPR0Printf("%s, line %d: EFLAGS.AC is clear! (%#x)\n", __FUNCTION__, __LINE__, (uint32_t)fEflCheck); \
121	a_BadExpr; \
122	} \
123	} \
124	} while (0)
125	# define GVMM_CHECK_SMAP_CHECK2(a_pGVM, a_BadExpr) \
126	do { \
127	if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
128	{ \
129	RTCCUINTREG fEflCheck = ASMGetFlags(); \
130	if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
131	{ /* likely */ } \
132	else \
133	{ \
134	SUPR0BadContext((a_pGVM) ? (a_pGVM)->pSession : NULL, __FILE__, __LINE__, "EFLAGS.AC is zero!"); \
135	a_BadExpr; \
136	} \
137	} \
138	} while (0)
139	#else
140	# define GVMM_CHECK_SMAP_SETUP() uint32_t const fKernelFeatures = 0
141	# define GVMM_CHECK_SMAP_CHECK(a_BadExpr) NOREF(fKernelFeatures)
142	# define GVMM_CHECK_SMAP_CHECK2(a_pGVM, a_BadExpr) NOREF(fKernelFeatures)
143	#endif
144
145	/** Special value that GVMMR0DeregisterVCpu sets. */
146	#define GVMM_RTNATIVETHREAD_DESTROYED (~(RTNATIVETHREAD)1)
147	AssertCompile(GVMM_RTNATIVETHREAD_DESTROYED != NIL_RTNATIVETHREAD);
148
149
150	/*********************************************************************************************************************************
151	* Structures and Typedefs *
152	*********************************************************************************************************************************/
153
154	/**
155	* Global VM handle.
156	*/
157	typedef struct GVMHANDLE
158	{
159	/** The index of the next handle in the list (free or used). (0 is nil.) */
160	uint16_t volatile iNext;
161	/** Our own index / handle value. */
162	uint16_t iSelf;
163	/** The process ID of the handle owner.
164	* This is used for access checks. */
165	RTPROCESS ProcId;
166	/** The pointer to the ring-0 only (aka global) VM structure. */
167	PGVM pGVM;
168	/** The virtual machine object. */
169	void *pvObj;
170	/** The session this VM is associated with. */
171	PSUPDRVSESSION pSession;
172	/** The ring-0 handle of the EMT0 thread.
173	* This is used for ownership checks as well as looking up a VM handle by thread
174	* at times like assertions. */
175	RTNATIVETHREAD hEMT0;
176	} GVMHANDLE;
177	/** Pointer to a global VM handle. */
178	typedef GVMHANDLE *PGVMHANDLE;
179
180	/** Number of GVM handles (including the NIL handle). */
181	#if HC_ARCH_BITS == 64
182	# define GVMM_MAX_HANDLES 8192
183	#else
184	# define GVMM_MAX_HANDLES 128
185	#endif
186
187	/**
188	* Per host CPU GVMM data.
189	*/
190	typedef struct GVMMHOSTCPU
191	{
192	/** Magic number (GVMMHOSTCPU_MAGIC). */
193	uint32_t volatile u32Magic;
194	/** The CPU ID. */
195	RTCPUID idCpu;
196	/** The CPU set index. */
197	uint32_t idxCpuSet;
198
199	#ifdef GVMM_SCHED_WITH_PPT
200	/** Periodic preemption timer data. */
201	struct
202	{
203	/** The handle to the periodic preemption timer. */
204	PRTTIMER pTimer;
205	/** Spinlock protecting the data below. */
206	RTSPINLOCK hSpinlock;
207	/** The smalles Hz that we need to care about. (static) */
208	uint32_t uMinHz;
209	/** The number of ticks between each historization. */
210	uint32_t cTicksHistoriziationInterval;
211	/** The current historization tick (counting up to
212	* cTicksHistoriziationInterval and then resetting). */
213	uint32_t iTickHistorization;
214	/** The current timer interval. This is set to 0 when inactive. */
215	uint32_t cNsInterval;
216	/** The current timer frequency. This is set to 0 when inactive. */
217	uint32_t uTimerHz;
218	/** The current max frequency reported by the EMTs.
219	* This gets historicize and reset by the timer callback. This is
220	* read without holding the spinlock, so needs atomic updating. */
221	uint32_t volatile uDesiredHz;
222	/** Whether the timer was started or not. */
223	bool volatile fStarted;
224	/** Set if we're starting timer. */
225	bool volatile fStarting;
226	/** The index of the next history entry (mod it). */
227	uint32_t iHzHistory;
228	/** Historicized uDesiredHz values. The array wraps around, new entries
229	* are added at iHzHistory. This is updated approximately every
230	* GVMMHOSTCPU_PPT_HIST_INTERVAL_NS by the timer callback. */
231	uint32_t aHzHistory[8];
232	/** Statistics counter for recording the number of interval changes. */
233	uint32_t cChanges;
234	/** Statistics counter for recording the number of timer starts. */
235	uint32_t cStarts;
236	} Ppt;
237	#endif /* GVMM_SCHED_WITH_PPT */
238
239	} GVMMHOSTCPU;
240	/** Pointer to the per host CPU GVMM data. */
241	typedef GVMMHOSTCPU *PGVMMHOSTCPU;
242	/** The GVMMHOSTCPU::u32Magic value (Petra, Tanya & Rachel Haden). */
243	#define GVMMHOSTCPU_MAGIC UINT32_C(0x19711011)
244	/** The interval on history entry should cover (approximately) give in
245	* nanoseconds. */
246	#define GVMMHOSTCPU_PPT_HIST_INTERVAL_NS UINT32_C(20000000)
247
248
249	/**
250	* The GVMM instance data.
251	*/
252	typedef struct GVMM
253	{
254	/** Eyecatcher / magic. */
255	uint32_t u32Magic;
256	/** The index of the head of the free handle chain. (0 is nil.) */
257	uint16_t volatile iFreeHead;
258	/** The index of the head of the active handle chain. (0 is nil.) */
259	uint16_t volatile iUsedHead;
260	/** The number of VMs. */
261	uint16_t volatile cVMs;
262	/** Alignment padding. */
263	uint16_t u16Reserved;
264	/** The number of EMTs. */
265	uint32_t volatile cEMTs;
266	/** The number of EMTs that have halted in GVMMR0SchedHalt. */
267	uint32_t volatile cHaltedEMTs;
268	/** Mini lock for restricting early wake-ups to one thread. */
269	bool volatile fDoingEarlyWakeUps;
270	bool afPadding[3]; /*< explicit alignment padding. /
271	/** When the next halted or sleeping EMT will wake up.
272	* This is set to 0 when it needs recalculating and to UINT64_MAX when
273	* there are no halted or sleeping EMTs in the GVMM. */
274	uint64_t uNsNextEmtWakeup;
275	/** The lock used to serialize VM creation, destruction and associated events that
276	* isn't performance critical. Owners may acquire the list lock. */
277	RTCRITSECT CreateDestroyLock;
278	/** The lock used to serialize used list updates and accesses.
279	* This indirectly includes scheduling since the scheduler will have to walk the
280	* used list to examin running VMs. Owners may not acquire any other locks. */
281	RTCRITSECTRW UsedLock;
282	/** The handle array.
283	* The size of this array defines the maximum number of currently running VMs.
284	* The first entry is unused as it represents the NIL handle. */
285	GVMHANDLE aHandles[GVMM_MAX_HANDLES];
286
287	/** @gcfgm{/GVMM/cEMTsMeansCompany, 32-bit, 0, UINT32_MAX, 1}
288	* The number of EMTs that means we no longer consider ourselves alone on a
289	* CPU/Core.
290	*/
291	uint32_t cEMTsMeansCompany;
292	/** @gcfgm{/GVMM/MinSleepAlone,32-bit, 0, 100000000, 750000, ns}
293	* The minimum sleep time for when we're alone, in nano seconds.
294	*/
295	uint32_t nsMinSleepAlone;
296	/** @gcfgm{/GVMM/MinSleepCompany,32-bit,0, 100000000, 15000, ns}
297	* The minimum sleep time for when we've got company, in nano seconds.
298	*/
299	uint32_t nsMinSleepCompany;
300	/** @gcfgm{/GVMM/EarlyWakeUp1, 32-bit, 0, 100000000, 25000, ns}
301	* The limit for the first round of early wake-ups, given in nano seconds.
302	*/
303	uint32_t nsEarlyWakeUp1;
304	/** @gcfgm{/GVMM/EarlyWakeUp2, 32-bit, 0, 100000000, 50000, ns}
305	* The limit for the second round of early wake-ups, given in nano seconds.
306	*/
307	uint32_t nsEarlyWakeUp2;
308
309	/** Set if we're doing early wake-ups.
310	* This reflects nsEarlyWakeUp1 and nsEarlyWakeUp2. */
311	bool volatile fDoEarlyWakeUps;
312
313	/** The number of entries in the host CPU array (aHostCpus). */
314	uint32_t cHostCpus;
315	/** Per host CPU data (variable length). */
316	GVMMHOSTCPU aHostCpus[1];
317	} GVMM;
318	AssertCompileMemberAlignment(GVMM, CreateDestroyLock, 8);
319	AssertCompileMemberAlignment(GVMM, UsedLock, 8);
320	AssertCompileMemberAlignment(GVMM, uNsNextEmtWakeup, 8);
321	/** Pointer to the GVMM instance data. */
322	typedef GVMM *PGVMM;
323
324	/** The GVMM::u32Magic value (Charlie Haden). */
325	#define GVMM_MAGIC UINT32_C(0x19370806)
326
327
328
329	/*********************************************************************************************************************************
330	* Global Variables *
331	*********************************************************************************************************************************/
332	/** Pointer to the GVMM instance data.
333	* (Just my general dislike for global variables.) */
334	static PGVMM g_pGVMM = NULL;
335
336	/** Macro for obtaining and validating the g_pGVMM pointer.
337	* On failure it will return from the invoking function with the specified return value.
338	*
339	* @param pGVMM The name of the pGVMM variable.
340	* @param rc The return value on failure. Use VERR_GVMM_INSTANCE for VBox
341	* status codes.
342	*/
343	#define GVMM_GET_VALID_INSTANCE(pGVMM, rc) \
344	do { \
345	(pGVMM) = g_pGVMM;\
346	AssertPtrReturn((pGVMM), (rc)); \
347	AssertMsgReturn((pGVMM)->u32Magic == GVMM_MAGIC, ("%p - %#x\n", (pGVMM), (pGVMM)->u32Magic), (rc)); \
348	} while (0)
349
350	/** Macro for obtaining and validating the g_pGVMM pointer, void function variant.
351	* On failure it will return from the invoking function.
352	*
353	* @param pGVMM The name of the pGVMM variable.
354	*/
355	#define GVMM_GET_VALID_INSTANCE_VOID(pGVMM) \
356	do { \
357	(pGVMM) = g_pGVMM;\
358	AssertPtrReturnVoid((pGVMM)); \
359	AssertMsgReturnVoid((pGVMM)->u32Magic == GVMM_MAGIC, ("%p - %#x\n", (pGVMM), (pGVMM)->u32Magic)); \
360	} while (0)
361
362
363	/*********************************************************************************************************************************
364	* Internal Functions *
365	*********************************************************************************************************************************/
366	static void gvmmR0InitPerVMData(PGVM pGVM, int16_t hSelf, VMCPUID cCpus, PSUPDRVSESSION pSession);
367	static DECLCALLBACK(void) gvmmR0HandleObjDestructor(void pvObj, void pvGVMM, void *pvHandle);
368	static int gvmmR0ByGVM(PGVM pGVM, PGVMM *ppGVMM, bool fTakeUsedLock);
369	static int gvmmR0ByGVMandEMT(PGVM pGVM, VMCPUID idCpu, PGVMM *ppGVMM);
370
371	#ifdef GVMM_SCHED_WITH_PPT
372	static DECLCALLBACK(void) gvmmR0SchedPeriodicPreemptionTimerCallback(PRTTIMER pTimer, void *pvUser, uint64_t iTick);
373	#endif
374
375
376	/**
377	* Initializes the GVMM.
378	*
379	* This is called while owning the loader semaphore (see supdrvIOCtl_LdrLoad()).
380	*
381	* @returns VBox status code.
382	*/
383	GVMMR0DECL(int) GVMMR0Init(void)
384	{
385	LogFlow(("GVMMR0Init:\n"));
386
387	/*
388	* Allocate and initialize the instance data.
389	*/
390	uint32_t cHostCpus = RTMpGetArraySize();
391	AssertMsgReturn(cHostCpus > 0 && cHostCpus < _64K, ("%d", (int)cHostCpus), VERR_GVMM_HOST_CPU_RANGE);
392
393	PGVMM pGVMM = (PGVMM)RTMemAllocZ(RT_UOFFSETOF_DYN(GVMM, aHostCpus[cHostCpus]));
394	if (!pGVMM)
395	return VERR_NO_MEMORY;
396	int rc = RTCritSectInitEx(&pGVMM->CreateDestroyLock, 0, NIL_RTLOCKVALCLASS, RTLOCKVAL_SUB_CLASS_NONE,
397	"GVMM-CreateDestroyLock");
398	if (RT_SUCCESS(rc))
399	{
400	rc = RTCritSectRwInitEx(&pGVMM->UsedLock, 0, NIL_RTLOCKVALCLASS, RTLOCKVAL_SUB_CLASS_NONE, "GVMM-UsedLock");
401	if (RT_SUCCESS(rc))
402	{
403	pGVMM->u32Magic = GVMM_MAGIC;
404	pGVMM->iUsedHead = 0;
405	pGVMM->iFreeHead = 1;
406
407	/* the nil handle */
408	pGVMM->aHandles[0].iSelf = 0;
409	pGVMM->aHandles[0].iNext = 0;
410
411	/* the tail */
412	unsigned i = RT_ELEMENTS(pGVMM->aHandles) - 1;
413	pGVMM->aHandles[i].iSelf = i;
414	pGVMM->aHandles[i].iNext = 0; /* nil */
415
416	/* the rest */
417	while (i-- > 1)
418	{
419	pGVMM->aHandles[i].iSelf = i;
420	pGVMM->aHandles[i].iNext = i + 1;
421	}
422
423	/* The default configuration values. */
424	uint32_t cNsResolution = RTSemEventMultiGetResolution();
425	pGVMM->cEMTsMeansCompany = 1; /** @todo should be adjusted to relative to the cpu count or something... */
426	if (cNsResolution >= 5*RT_NS_100US)
427	{
428	pGVMM->nsMinSleepAlone = 750000 /* ns (0.750 ms) /; /* @todo this should be adjusted to be 75% (or something) of the scheduler granularity... */
429	pGVMM->nsMinSleepCompany = 15000 /* ns (0.015 ms) */;
430	pGVMM->nsEarlyWakeUp1 = 25000 /* ns (0.025 ms) */;
431	pGVMM->nsEarlyWakeUp2 = 50000 /* ns (0.050 ms) */;
432	}
433	else if (cNsResolution > RT_NS_100US)
434	{
435	pGVMM->nsMinSleepAlone = cNsResolution / 2;
436	pGVMM->nsMinSleepCompany = cNsResolution / 4;
437	pGVMM->nsEarlyWakeUp1 = 0;
438	pGVMM->nsEarlyWakeUp2 = 0;
439	}
440	else
441	{
442	pGVMM->nsMinSleepAlone = 2000;
443	pGVMM->nsMinSleepCompany = 2000;
444	pGVMM->nsEarlyWakeUp1 = 0;
445	pGVMM->nsEarlyWakeUp2 = 0;
446	}
447	pGVMM->fDoEarlyWakeUps = pGVMM->nsEarlyWakeUp1 > 0 && pGVMM->nsEarlyWakeUp2 > 0;
448
449	/* The host CPU data. */
450	pGVMM->cHostCpus = cHostCpus;
451	uint32_t iCpu = cHostCpus;
452	RTCPUSET PossibleSet;
453	RTMpGetSet(&PossibleSet);
454	while (iCpu-- > 0)
455	{
456	pGVMM->aHostCpus[iCpu].idxCpuSet = iCpu;
457	#ifdef GVMM_SCHED_WITH_PPT
458	pGVMM->aHostCpus[iCpu].Ppt.pTimer = NULL;
459	pGVMM->aHostCpus[iCpu].Ppt.hSpinlock = NIL_RTSPINLOCK;
460	pGVMM->aHostCpus[iCpu].Ppt.uMinHz = 5; /** @todo Add some API which figures this one out. (not that important) */
461	pGVMM->aHostCpus[iCpu].Ppt.cTicksHistoriziationInterval = 1;
462	//pGVMM->aHostCpus[iCpu].Ppt.iTickHistorization = 0;
463	//pGVMM->aHostCpus[iCpu].Ppt.cNsInterval = 0;
464	//pGVMM->aHostCpus[iCpu].Ppt.uTimerHz = 0;
465	//pGVMM->aHostCpus[iCpu].Ppt.uDesiredHz = 0;
466	//pGVMM->aHostCpus[iCpu].Ppt.fStarted = false;
467	//pGVMM->aHostCpus[iCpu].Ppt.fStarting = false;
468	//pGVMM->aHostCpus[iCpu].Ppt.iHzHistory = 0;
469	//pGVMM->aHostCpus[iCpu].Ppt.aHzHistory = {0};
470	#endif
471
472	if (RTCpuSetIsMember(&PossibleSet, iCpu))
473	{
474	pGVMM->aHostCpus[iCpu].idCpu = RTMpCpuIdFromSetIndex(iCpu);
475	pGVMM->aHostCpus[iCpu].u32Magic = GVMMHOSTCPU_MAGIC;
476
477	#ifdef GVMM_SCHED_WITH_PPT
478	rc = RTTimerCreateEx(&pGVMM->aHostCpus[iCpu].Ppt.pTimer,
479	5010001000 /* whatever */,
480	RTTIMER_FLAGS_CPU(iCpu) \| RTTIMER_FLAGS_HIGH_RES,
481	gvmmR0SchedPeriodicPreemptionTimerCallback,
482	&pGVMM->aHostCpus[iCpu]);
483	if (RT_SUCCESS(rc))
484	rc = RTSpinlockCreate(&pGVMM->aHostCpus[iCpu].Ppt.hSpinlock, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "GVMM/CPU");
485	if (RT_FAILURE(rc))
486	{
487	while (iCpu < cHostCpus)
488	{
489	RTTimerDestroy(pGVMM->aHostCpus[iCpu].Ppt.pTimer);
490	RTSpinlockDestroy(pGVMM->aHostCpus[iCpu].Ppt.hSpinlock);
491	pGVMM->aHostCpus[iCpu].Ppt.hSpinlock = NIL_RTSPINLOCK;
492	iCpu++;
493	}
494	break;
495	}
496	#endif
497	}
498	else
499	{
500	pGVMM->aHostCpus[iCpu].idCpu = NIL_RTCPUID;
501	pGVMM->aHostCpus[iCpu].u32Magic = 0;
502	}
503	}
504	if (RT_SUCCESS(rc))
505	{
506	g_pGVMM = pGVMM;
507	LogFlow(("GVMMR0Init: pGVMM=%p cHostCpus=%u\n", pGVMM, cHostCpus));
508	return VINF_SUCCESS;
509	}
510
511	/* bail out. */
512	RTCritSectRwDelete(&pGVMM->UsedLock);
513	}
514	RTCritSectDelete(&pGVMM->CreateDestroyLock);
515	}
516
517	RTMemFree(pGVMM);
518	return rc;
519	}
520
521
522	/**
523	* Terminates the GVM.
524	*
525	* This is called while owning the loader semaphore (see supdrvLdrFree()).
526	* And unless something is wrong, there should be absolutely no VMs
527	* registered at this point.
528	*/
529	GVMMR0DECL(void) GVMMR0Term(void)
530	{
531	LogFlow(("GVMMR0Term:\n"));
532
533	PGVMM pGVMM = g_pGVMM;
534	g_pGVMM = NULL;
535	if (RT_UNLIKELY(!RT_VALID_PTR(pGVMM)))
536	{
537	SUPR0Printf("GVMMR0Term: pGVMM=%RKv\n", pGVMM);
538	return;
539	}
540
541	/*
542	* First of all, stop all active timers.
543	*/
544	uint32_t cActiveTimers = 0;
545	uint32_t iCpu = pGVMM->cHostCpus;
546	while (iCpu-- > 0)
547	{
548	ASMAtomicWriteU32(&pGVMM->aHostCpus[iCpu].u32Magic, ~GVMMHOSTCPU_MAGIC);
549	#ifdef GVMM_SCHED_WITH_PPT
550	if ( pGVMM->aHostCpus[iCpu].Ppt.pTimer != NULL
551	&& RT_SUCCESS(RTTimerStop(pGVMM->aHostCpus[iCpu].Ppt.pTimer)))
552	cActiveTimers++;
553	#endif
554	}
555	if (cActiveTimers)
556	RTThreadSleep(1); /* fudge */
557
558	/*
559	* Invalidate the and free resources.
560	*/
561	pGVMM->u32Magic = ~GVMM_MAGIC;
562	RTCritSectRwDelete(&pGVMM->UsedLock);
563	RTCritSectDelete(&pGVMM->CreateDestroyLock);
564
565	pGVMM->iFreeHead = 0;
566	if (pGVMM->iUsedHead)
567	{
568	SUPR0Printf("GVMMR0Term: iUsedHead=%#x! (cVMs=%#x cEMTs=%#x)\n", pGVMM->iUsedHead, pGVMM->cVMs, pGVMM->cEMTs);
569	pGVMM->iUsedHead = 0;
570	}
571
572	#ifdef GVMM_SCHED_WITH_PPT
573	iCpu = pGVMM->cHostCpus;
574	while (iCpu-- > 0)
575	{
576	RTTimerDestroy(pGVMM->aHostCpus[iCpu].Ppt.pTimer);
577	pGVMM->aHostCpus[iCpu].Ppt.pTimer = NULL;
578	RTSpinlockDestroy(pGVMM->aHostCpus[iCpu].Ppt.hSpinlock);
579	pGVMM->aHostCpus[iCpu].Ppt.hSpinlock = NIL_RTSPINLOCK;
580	}
581	#endif
582
583	RTMemFree(pGVMM);
584	}
585
586
587	/**
588	* A quick hack for setting global config values.
589	*
590	* @returns VBox status code.
591	*
592	* @param pSession The session handle. Used for authentication.
593	* @param pszName The variable name.
594	* @param u64Value The new value.
595	*/
596	GVMMR0DECL(int) GVMMR0SetConfig(PSUPDRVSESSION pSession, const char *pszName, uint64_t u64Value)
597	{
598	/*
599	* Validate input.
600	*/
601	PGVMM pGVMM;
602	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
603	AssertPtrReturn(pSession, VERR_INVALID_HANDLE);
604	AssertPtrReturn(pszName, VERR_INVALID_POINTER);
605
606	/*
607	* String switch time!
608	*/
609	if (strncmp(pszName, RT_STR_TUPLE("/GVMM/")))
610	return VERR_CFGM_VALUE_NOT_FOUND; /* borrow status codes from CFGM... */
611	int rc = VINF_SUCCESS;
612	pszName += sizeof("/GVMM/") - 1;
613	if (!strcmp(pszName, "cEMTsMeansCompany"))
614	{
615	if (u64Value <= UINT32_MAX)
616	pGVMM->cEMTsMeansCompany = u64Value;
617	else
618	rc = VERR_OUT_OF_RANGE;
619	}
620	else if (!strcmp(pszName, "MinSleepAlone"))
621	{
622	if (u64Value <= RT_NS_100MS)
623	pGVMM->nsMinSleepAlone = u64Value;
624	else
625	rc = VERR_OUT_OF_RANGE;
626	}
627	else if (!strcmp(pszName, "MinSleepCompany"))
628	{
629	if (u64Value <= RT_NS_100MS)
630	pGVMM->nsMinSleepCompany = u64Value;
631	else
632	rc = VERR_OUT_OF_RANGE;
633	}
634	else if (!strcmp(pszName, "EarlyWakeUp1"))
635	{
636	if (u64Value <= RT_NS_100MS)
637	{
638	pGVMM->nsEarlyWakeUp1 = u64Value;
639	pGVMM->fDoEarlyWakeUps = pGVMM->nsEarlyWakeUp1 > 0 && pGVMM->nsEarlyWakeUp2 > 0;
640	}
641	else
642	rc = VERR_OUT_OF_RANGE;
643	}
644	else if (!strcmp(pszName, "EarlyWakeUp2"))
645	{
646	if (u64Value <= RT_NS_100MS)
647	{
648	pGVMM->nsEarlyWakeUp2 = u64Value;
649	pGVMM->fDoEarlyWakeUps = pGVMM->nsEarlyWakeUp1 > 0 && pGVMM->nsEarlyWakeUp2 > 0;
650	}
651	else
652	rc = VERR_OUT_OF_RANGE;
653	}
654	else
655	rc = VERR_CFGM_VALUE_NOT_FOUND;
656	return rc;
657	}
658
659
660	/**
661	* A quick hack for getting global config values.
662	*
663	* @returns VBox status code.
664	*
665	* @param pSession The session handle. Used for authentication.
666	* @param pszName The variable name.
667	* @param pu64Value Where to return the value.
668	*/
669	GVMMR0DECL(int) GVMMR0QueryConfig(PSUPDRVSESSION pSession, const char pszName, uint64_t pu64Value)
670	{
671	/*
672	* Validate input.
673	*/
674	PGVMM pGVMM;
675	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
676	AssertPtrReturn(pSession, VERR_INVALID_HANDLE);
677	AssertPtrReturn(pszName, VERR_INVALID_POINTER);
678	AssertPtrReturn(pu64Value, VERR_INVALID_POINTER);
679
680	/*
681	* String switch time!
682	*/
683	if (strncmp(pszName, RT_STR_TUPLE("/GVMM/")))
684	return VERR_CFGM_VALUE_NOT_FOUND; /* borrow status codes from CFGM... */
685	int rc = VINF_SUCCESS;
686	pszName += sizeof("/GVMM/") - 1;
687	if (!strcmp(pszName, "cEMTsMeansCompany"))
688	*pu64Value = pGVMM->cEMTsMeansCompany;
689	else if (!strcmp(pszName, "MinSleepAlone"))
690	*pu64Value = pGVMM->nsMinSleepAlone;
691	else if (!strcmp(pszName, "MinSleepCompany"))
692	*pu64Value = pGVMM->nsMinSleepCompany;
693	else if (!strcmp(pszName, "EarlyWakeUp1"))
694	*pu64Value = pGVMM->nsEarlyWakeUp1;
695	else if (!strcmp(pszName, "EarlyWakeUp2"))
696	*pu64Value = pGVMM->nsEarlyWakeUp2;
697	else
698	rc = VERR_CFGM_VALUE_NOT_FOUND;
699	return rc;
700	}
701
702
703	/**
704	* Acquire the 'used' lock in shared mode.
705	*
706	* This prevents destruction of the VM while we're in ring-0.
707	*
708	* @returns IPRT status code, see RTSemFastMutexRequest.
709	* @param a_pGVMM The GVMM instance data.
710	* @sa GVMMR0_USED_SHARED_UNLOCK, GVMMR0_USED_EXCLUSIVE_LOCK
711	*/
712	#define GVMMR0_USED_SHARED_LOCK(a_pGVMM) RTCritSectRwEnterShared(&(a_pGVMM)->UsedLock)
713
714	/**
715	* Release the 'used' lock in when owning it in shared mode.
716	*
717	* @returns IPRT status code, see RTSemFastMutexRequest.
718	* @param a_pGVMM The GVMM instance data.
719	* @sa GVMMR0_USED_SHARED_LOCK
720	*/
721	#define GVMMR0_USED_SHARED_UNLOCK(a_pGVMM) RTCritSectRwLeaveShared(&(a_pGVMM)->UsedLock)
722
723	/**
724	* Acquire the 'used' lock in exclusive mode.
725	*
726	* Only use this function when making changes to the used list.
727	*
728	* @returns IPRT status code, see RTSemFastMutexRequest.
729	* @param a_pGVMM The GVMM instance data.
730	* @sa GVMMR0_USED_EXCLUSIVE_UNLOCK
731	*/
732	#define GVMMR0_USED_EXCLUSIVE_LOCK(a_pGVMM) RTCritSectRwEnterExcl(&(a_pGVMM)->UsedLock)
733
734	/**
735	* Release the 'used' lock when owning it in exclusive mode.
736	*
737	* @returns IPRT status code, see RTSemFastMutexRelease.
738	* @param a_pGVMM The GVMM instance data.
739	* @sa GVMMR0_USED_EXCLUSIVE_LOCK, GVMMR0_USED_SHARED_UNLOCK
740	*/
741	#define GVMMR0_USED_EXCLUSIVE_UNLOCK(a_pGVMM) RTCritSectRwLeaveExcl(&(a_pGVMM)->UsedLock)
742
743
744	/**
745	* Try acquire the 'create & destroy' lock.
746	*
747	* @returns IPRT status code, see RTSemFastMutexRequest.
748	* @param pGVMM The GVMM instance data.
749	*/
750	DECLINLINE(int) gvmmR0CreateDestroyLock(PGVMM pGVMM)
751	{
752	LogFlow(("++gvmmR0CreateDestroyLock(%p)\n", pGVMM));
753	int rc = RTCritSectEnter(&pGVMM->CreateDestroyLock);
754	LogFlow(("gvmmR0CreateDestroyLock(%p)->%Rrc\n", pGVMM, rc));
755	return rc;
756	}
757
758
759	/**
760	* Release the 'create & destroy' lock.
761	*
762	* @returns IPRT status code, see RTSemFastMutexRequest.
763	* @param pGVMM The GVMM instance data.
764	*/
765	DECLINLINE(int) gvmmR0CreateDestroyUnlock(PGVMM pGVMM)
766	{
767	LogFlow(("--gvmmR0CreateDestroyUnlock(%p)\n", pGVMM));
768	int rc = RTCritSectLeave(&pGVMM->CreateDestroyLock);
769	AssertRC(rc);
770	return rc;
771	}
772
773
774	/**
775	* Request wrapper for the GVMMR0CreateVM API.
776	*
777	* @returns VBox status code.
778	* @param pReq The request buffer.
779	* @param pSession The session handle. The VM will be associated with this.
780	*/
781	GVMMR0DECL(int) GVMMR0CreateVMReq(PGVMMCREATEVMREQ pReq, PSUPDRVSESSION pSession)
782	{
783	/*
784	* Validate the request.
785	*/
786	if (!RT_VALID_PTR(pReq))
787	return VERR_INVALID_POINTER;
788	if (pReq->Hdr.cbReq != sizeof(*pReq))
789	return VERR_INVALID_PARAMETER;
790	if (pReq->pSession != pSession)
791	return VERR_INVALID_POINTER;
792
793	/*
794	* Execute it.
795	*/
796	PGVM pGVM;
797	pReq->pVMR0 = NULL;
798	pReq->pVMR3 = NIL_RTR3PTR;
799	int rc = GVMMR0CreateVM(pSession, pReq->cCpus, &pGVM);
800	if (RT_SUCCESS(rc))
801	{
802	pReq->pVMR0 = pGVM; /** @todo don't expose this to ring-3, use a unique random number instead. */
803	pReq->pVMR3 = pGVM->pVMR3;
804	}
805	return rc;
806	}
807
808
809	/**
810	* Allocates the VM structure and registers it with GVM.
811	*
812	* The caller will become the VM owner and there by the EMT.
813	*
814	* @returns VBox status code.
815	* @param pSession The support driver session.
816	* @param cCpus Number of virtual CPUs for the new VM.
817	* @param ppGVM Where to store the pointer to the VM structure.
818	*
819	* @thread EMT.
820	*/
821	GVMMR0DECL(int) GVMMR0CreateVM(PSUPDRVSESSION pSession, uint32_t cCpus, PGVM *ppGVM)
822	{
823	LogFlow(("GVMMR0CreateVM: pSession=%p\n", pSession));
824	PGVMM pGVMM;
825	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
826
827	AssertPtrReturn(ppGVM, VERR_INVALID_POINTER);
828	*ppGVM = NULL;
829
830	if ( cCpus == 0
831	\|\| cCpus > VMM_MAX_CPU_COUNT)
832	return VERR_INVALID_PARAMETER;
833
834	RTNATIVETHREAD hEMT0 = RTThreadNativeSelf();
835	AssertReturn(hEMT0 != NIL_RTNATIVETHREAD, VERR_GVMM_BROKEN_IPRT);
836	RTPROCESS ProcId = RTProcSelf();
837	AssertReturn(ProcId != NIL_RTPROCESS, VERR_GVMM_BROKEN_IPRT);
838
839	/*
840	* The whole allocation process is protected by the lock.
841	*/
842	int rc = gvmmR0CreateDestroyLock(pGVMM);
843	AssertRCReturn(rc, rc);
844
845	/*
846	* Only one VM per session.
847	*/
848	if (SUPR0GetSessionVM(pSession) != NULL)
849	{
850	gvmmR0CreateDestroyUnlock(pGVMM);
851	SUPR0Printf("GVMMR0CreateVM: The session %p already got a VM: %p\n", pSession, SUPR0GetSessionVM(pSession));
852	return VERR_ALREADY_EXISTS;
853	}
854
855	/*
856	* Allocate a handle first so we don't waste resources unnecessarily.
857	*/
858	uint16_t iHandle = pGVMM->iFreeHead;
859	if (iHandle)
860	{
861	PGVMHANDLE pHandle = &pGVMM->aHandles[iHandle];
862
863	/* consistency checks, a bit paranoid as always. */
864	if ( !pHandle->pGVM
865	&& !pHandle->pvObj
866	&& pHandle->iSelf == iHandle)
867	{
868	pHandle->pvObj = SUPR0ObjRegister(pSession, SUPDRVOBJTYPE_VM, gvmmR0HandleObjDestructor, pGVMM, pHandle);
869	if (pHandle->pvObj)
870	{
871	/*
872	* Move the handle from the free to used list and perform permission checks.
873	*/
874	rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
875	AssertRC(rc);
876
877	pGVMM->iFreeHead = pHandle->iNext;
878	pHandle->iNext = pGVMM->iUsedHead;
879	pGVMM->iUsedHead = iHandle;
880	pGVMM->cVMs++;
881
882	pHandle->pGVM = NULL;
883	pHandle->pSession = pSession;
884	pHandle->hEMT0 = NIL_RTNATIVETHREAD;
885	pHandle->ProcId = NIL_RTPROCESS;
886
887	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
888
889	rc = SUPR0ObjVerifyAccess(pHandle->pvObj, pSession, NULL);
890	if (RT_SUCCESS(rc))
891	{
892	/*
893	* Allocate memory for the VM structure (combined VM + GVM).
894	*/
895	const uint32_t cbVM = RT_UOFFSETOF_DYN(GVM, aCpus[cCpus]);
896	const uint32_t cPages = RT_ALIGN_32(cbVM, PAGE_SIZE) >> PAGE_SHIFT;
897	RTR0MEMOBJ hVMMemObj = NIL_RTR0MEMOBJ;
898	rc = RTR0MemObjAllocPage(&hVMMemObj, cPages << PAGE_SHIFT, false /* fExecutable */);
899	if (RT_SUCCESS(rc))
900	{
901	PGVM pGVM = (PGVM)RTR0MemObjAddress(hVMMemObj);
902	AssertPtr(pGVM);
903
904	/*
905	* Initialise the structure.
906	*/
907	RT_BZERO(pGVM, cPages << PAGE_SHIFT);
908	gvmmR0InitPerVMData(pGVM, iHandle, cCpus, pSession);
909	pGVM->gvmm.s.VMMemObj = hVMMemObj;
910	rc = GMMR0InitPerVMData(pGVM);
911	int rc2 = PGMR0InitPerVMData(pGVM);
912	int rc3 = VMMR0InitPerVMData(pGVM);
913	DBGFR0InitPerVMData(pGVM);
914	PDMR0InitPerVMData(pGVM);
915	IOMR0InitPerVMData(pGVM);
916	TMR0InitPerVMData(pGVM);
917	if (RT_SUCCESS(rc) && RT_SUCCESS(rc2) && RT_SUCCESS(rc3))
918	{
919	/*
920	* Allocate page array.
921	* This currently have to be made available to ring-3, but this is should change eventually.
922	*/
923	rc = RTR0MemObjAllocPage(&pGVM->gvmm.s.VMPagesMemObj, cPages * sizeof(SUPPAGE), false /* fExecutable */);
924	if (RT_SUCCESS(rc))
925	{
926	PSUPPAGE paPages = (PSUPPAGE)RTR0MemObjAddress(pGVM->gvmm.s.VMPagesMemObj); AssertPtr(paPages);
927	for (uint32_t iPage = 0; iPage < cPages; iPage++)
928	{
929	paPages[iPage].uReserved = 0;
930	paPages[iPage].Phys = RTR0MemObjGetPagePhysAddr(pGVM->gvmm.s.VMMemObj, iPage);
931	Assert(paPages[iPage].Phys != NIL_RTHCPHYS);
932	}
933
934	/*
935	* Map the page array, VM and VMCPU structures into ring-3.
936	*/
937	AssertCompileSizeAlignment(VM, PAGE_SIZE);
938	rc = RTR0MemObjMapUserEx(&pGVM->gvmm.s.VMMapObj, pGVM->gvmm.s.VMMemObj, (RTR3PTR)-1, 0,
939	RTMEM_PROT_READ \| RTMEM_PROT_WRITE, NIL_RTR0PROCESS,
940	0 /offSub/, sizeof(VM));
941	for (VMCPUID i = 0; i < cCpus && RT_SUCCESS(rc); i++)
942	{
943	AssertCompileSizeAlignment(VMCPU, PAGE_SIZE);
944	rc = RTR0MemObjMapUserEx(&pGVM->aCpus[i].gvmm.s.VMCpuMapObj, pGVM->gvmm.s.VMMemObj,
945	(RTR3PTR)-1, 0, RTMEM_PROT_READ \| RTMEM_PROT_WRITE, NIL_RTR0PROCESS,
946	RT_UOFFSETOF_DYN(GVM, aCpus[i]), sizeof(VMCPU));
947	}
948	if (RT_SUCCESS(rc))
949	rc = RTR0MemObjMapUser(&pGVM->gvmm.s.VMPagesMapObj, pGVM->gvmm.s.VMPagesMemObj, (RTR3PTR)-1,
950	0 /* uAlignment */, RTMEM_PROT_READ \| RTMEM_PROT_WRITE,
951	NIL_RTR0PROCESS);
952	if (RT_SUCCESS(rc))
953	{
954	/*
955	* Initialize all the VM pointers.
956	*/
957	PVMR3 pVMR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMMapObj);
958	AssertMsg(RTR0MemUserIsValidAddr(pVMR3) && pVMR3 != NIL_RTR3PTR, ("%p\n", pVMR3));
959
960	for (VMCPUID i = 0; i < cCpus; i++)
961	{
962	pGVM->aCpus[i].pVMR0 = pGVM;
963	pGVM->aCpus[i].pVMR3 = pVMR3;
964	pGVM->apCpusR3[i] = RTR0MemObjAddressR3(pGVM->aCpus[i].gvmm.s.VMCpuMapObj);
965	pGVM->aCpus[i].pVCpuR3 = pGVM->apCpusR3[i];
966	pGVM->apCpusR0[i] = &pGVM->aCpus[i];
967	AssertMsg(RTR0MemUserIsValidAddr(pGVM->apCpusR3[i]) && pGVM->apCpusR3[i] != NIL_RTR3PTR,
968	("apCpusR3[%u]=%p\n", i, pGVM->apCpusR3[i]));
969	}
970
971	pGVM->paVMPagesR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMPagesMapObj);
972	AssertMsg(RTR0MemUserIsValidAddr(pGVM->paVMPagesR3) && pGVM->paVMPagesR3 != NIL_RTR3PTR,
973	("%p\n", pGVM->paVMPagesR3));
974
975	/*
976	* Complete the handle - take the UsedLock sem just to be careful.
977	*/
978	rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
979	AssertRC(rc);
980
981	pHandle->pGVM = pGVM;
982	pHandle->hEMT0 = hEMT0;
983	pHandle->ProcId = ProcId;
984	pGVM->pVMR3 = pVMR3;
985	pGVM->pVMR3Unsafe = pVMR3;
986	pGVM->aCpus[0].hEMT = hEMT0;
987	pGVM->aCpus[0].hNativeThreadR0 = hEMT0;
988	pGVM->aCpus[0].cEmtHashCollisions = 0;
989	uint32_t const idxHash = GVMM_EMT_HASH_1(hEMT0);
990	pGVM->aCpus[0].gvmm.s.idxEmtHash = (uint16_t)idxHash;
991	pGVM->gvmm.s.aEmtHash[idxHash].hNativeEmt = hEMT0;
992	pGVM->gvmm.s.aEmtHash[idxHash].idVCpu = 0;
993	pGVMM->cEMTs += cCpus;
994
995	/* Associate it with the session and create the context hook for EMT0. */
996	rc = SUPR0SetSessionVM(pSession, pGVM, pGVM);
997	if (RT_SUCCESS(rc))
998	{
999	rc = VMMR0ThreadCtxHookCreateForEmt(&pGVM->aCpus[0]);
1000	if (RT_SUCCESS(rc))
1001	{
1002	/*
1003	* Done!
1004	*/
1005	VBOXVMM_R0_GVMM_VM_CREATED(pGVM, pGVM, ProcId, (void *)hEMT0, cCpus);
1006
1007	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1008	gvmmR0CreateDestroyUnlock(pGVMM);
1009
1010	CPUMR0RegisterVCpuThread(&pGVM->aCpus[0]);
1011
1012	*ppGVM = pGVM;
1013	Log(("GVMMR0CreateVM: pVMR3=%p pGVM=%p hGVM=%d\n", pVMR3, pGVM, iHandle));
1014	return VINF_SUCCESS;
1015	}
1016
1017	SUPR0SetSessionVM(pSession, NULL, NULL);
1018	}
1019	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1020	}
1021
1022	/* Cleanup mappings. */
1023	if (pGVM->gvmm.s.VMMapObj != NIL_RTR0MEMOBJ)
1024	{
1025	RTR0MemObjFree(pGVM->gvmm.s.VMMapObj, false /* fFreeMappings */);
1026	pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
1027	}
1028	for (VMCPUID i = 0; i < cCpus; i++)
1029	if (pGVM->aCpus[i].gvmm.s.VMCpuMapObj != NIL_RTR0MEMOBJ)
1030	{
1031	RTR0MemObjFree(pGVM->aCpus[i].gvmm.s.VMCpuMapObj, false /* fFreeMappings */);
1032	pGVM->aCpus[i].gvmm.s.VMCpuMapObj = NIL_RTR0MEMOBJ;
1033	}
1034	if (pGVM->gvmm.s.VMPagesMapObj != NIL_RTR0MEMOBJ)
1035	{
1036	RTR0MemObjFree(pGVM->gvmm.s.VMPagesMapObj, false /* fFreeMappings */);
1037	pGVM->gvmm.s.VMPagesMapObj = NIL_RTR0MEMOBJ;
1038	}
1039	}
1040	}
1041	else
1042	{
1043	if (RT_SUCCESS_NP(rc))
1044	rc = rc2;
1045	if (RT_SUCCESS_NP(rc))
1046	rc = rc3;
1047	}
1048	}
1049	}
1050	/* else: The user wasn't permitted to create this VM. */
1051
1052	/*
1053	* The handle will be freed by gvmmR0HandleObjDestructor as we release the
1054	* object reference here. A little extra mess because of non-recursive lock.
1055	*/
1056	void *pvObj = pHandle->pvObj;
1057	pHandle->pvObj = NULL;
1058	gvmmR0CreateDestroyUnlock(pGVMM);
1059
1060	SUPR0ObjRelease(pvObj, pSession);
1061
1062	SUPR0Printf("GVMMR0CreateVM: failed, rc=%Rrc\n", rc);
1063	return rc;
1064	}
1065
1066	rc = VERR_NO_MEMORY;
1067	}
1068	else
1069	rc = VERR_GVMM_IPE_1;
1070	}
1071	else
1072	rc = VERR_GVM_TOO_MANY_VMS;
1073
1074	gvmmR0CreateDestroyUnlock(pGVMM);
1075	return rc;
1076	}
1077
1078
1079	/**
1080	* Initializes the per VM data belonging to GVMM.
1081	*
1082	* @param pGVM Pointer to the global VM structure.
1083	* @param hSelf The handle.
1084	* @param cCpus The CPU count.
1085	* @param pSession The session this VM is associated with.
1086	*/
1087	static void gvmmR0InitPerVMData(PGVM pGVM, int16_t hSelf, VMCPUID cCpus, PSUPDRVSESSION pSession)
1088	{
1089	AssertCompile(RT_SIZEOFMEMB(GVM,gvmm.s) <= RT_SIZEOFMEMB(GVM,gvmm.padding));
1090	AssertCompile(RT_SIZEOFMEMB(GVMCPU,gvmm.s) <= RT_SIZEOFMEMB(GVMCPU,gvmm.padding));
1091	AssertCompileMemberAlignment(VM, cpum, 64);
1092	AssertCompileMemberAlignment(VM, tm, 64);
1093
1094	/* GVM: */
1095	pGVM->u32Magic = GVM_MAGIC;
1096	pGVM->hSelf = hSelf;
1097	pGVM->cCpus = cCpus;
1098	pGVM->pSession = pSession;
1099	pGVM->pSelf = pGVM;
1100
1101	/* VM: */
1102	pGVM->enmVMState = VMSTATE_CREATING;
1103	pGVM->hSelfUnsafe = hSelf;
1104	pGVM->pSessionUnsafe = pSession;
1105	pGVM->pVMR0ForCall = pGVM;
1106	pGVM->cCpusUnsafe = cCpus;
1107	pGVM->uCpuExecutionCap = 100; /* default is no cap. */
1108	pGVM->uStructVersion = 1;
1109	pGVM->cbSelf = sizeof(VM);
1110	pGVM->cbVCpu = sizeof(VMCPU);
1111
1112	/* GVMM: */
1113	pGVM->gvmm.s.VMMemObj = NIL_RTR0MEMOBJ;
1114	pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
1115	pGVM->gvmm.s.VMPagesMemObj = NIL_RTR0MEMOBJ;
1116	pGVM->gvmm.s.VMPagesMapObj = NIL_RTR0MEMOBJ;
1117	pGVM->gvmm.s.fDoneVMMR0Init = false;
1118	pGVM->gvmm.s.fDoneVMMR0Term = false;
1119	for (size_t i = 0; i < RT_ELEMENTS(pGVM->gvmm.s.aEmtHash); i++)
1120	{
1121	pGVM->gvmm.s.aEmtHash[i].hNativeEmt = NIL_RTNATIVETHREAD;
1122	pGVM->gvmm.s.aEmtHash[i].idVCpu = NIL_VMCPUID;
1123	}
1124
1125	/*
1126	* Per virtual CPU.
1127	*/
1128	for (VMCPUID i = 0; i < pGVM->cCpus; i++)
1129	{
1130	pGVM->aCpus[i].idCpu = i;
1131	pGVM->aCpus[i].idCpuUnsafe = i;
1132	pGVM->aCpus[i].gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
1133	pGVM->aCpus[i].gvmm.s.VMCpuMapObj = NIL_RTR0MEMOBJ;
1134	pGVM->aCpus[i].gvmm.s.idxEmtHash = UINT16_MAX;
1135	pGVM->aCpus[i].hEMT = NIL_RTNATIVETHREAD;
1136	pGVM->aCpus[i].pGVM = pGVM;
1137	pGVM->aCpus[i].idHostCpu = NIL_RTCPUID;
1138	pGVM->aCpus[i].iHostCpuSet = UINT32_MAX;
1139	pGVM->aCpus[i].hNativeThread = NIL_RTNATIVETHREAD;
1140	pGVM->aCpus[i].hNativeThreadR0 = NIL_RTNATIVETHREAD;
1141	pGVM->aCpus[i].enmState = VMCPUSTATE_STOPPED;
1142	pGVM->aCpus[i].pVCpuR0ForVtg = &pGVM->aCpus[i];
1143	}
1144	}
1145
1146
1147	/**
1148	* Does the VM initialization.
1149	*
1150	* @returns VBox status code.
1151	* @param pGVM The global (ring-0) VM structure.
1152	*/
1153	GVMMR0DECL(int) GVMMR0InitVM(PGVM pGVM)
1154	{
1155	LogFlow(("GVMMR0InitVM: pGVM=%p\n", pGVM));
1156
1157	int rc = VERR_INTERNAL_ERROR_3;
1158	if ( !pGVM->gvmm.s.fDoneVMMR0Init
1159	&& pGVM->aCpus[0].gvmm.s.HaltEventMulti == NIL_RTSEMEVENTMULTI)
1160	{
1161	for (VMCPUID i = 0; i < pGVM->cCpus; i++)
1162	{
1163	rc = RTSemEventMultiCreate(&pGVM->aCpus[i].gvmm.s.HaltEventMulti);
1164	if (RT_FAILURE(rc))
1165	{
1166	pGVM->aCpus[i].gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
1167	break;
1168	}
1169	}
1170	}
1171	else
1172	rc = VERR_WRONG_ORDER;
1173
1174	LogFlow(("GVMMR0InitVM: returns %Rrc\n", rc));
1175	return rc;
1176	}
1177
1178
1179	/**
1180	* Indicates that we're done with the ring-0 initialization
1181	* of the VM.
1182	*
1183	* @param pGVM The global (ring-0) VM structure.
1184	* @thread EMT(0)
1185	*/
1186	GVMMR0DECL(void) GVMMR0DoneInitVM(PGVM pGVM)
1187	{
1188	/* Set the indicator. */
1189	pGVM->gvmm.s.fDoneVMMR0Init = true;
1190	}
1191
1192
1193	/**
1194	* Indicates that we're doing the ring-0 termination of the VM.
1195	*
1196	* @returns true if termination hasn't been done already, false if it has.
1197	* @param pGVM Pointer to the global VM structure. Optional.
1198	* @thread EMT(0) or session cleanup thread.
1199	*/
1200	GVMMR0DECL(bool) GVMMR0DoingTermVM(PGVM pGVM)
1201	{
1202	/* Validate the VM structure, state and handle. */
1203	AssertPtrReturn(pGVM, false);
1204
1205	/* Set the indicator. */
1206	if (pGVM->gvmm.s.fDoneVMMR0Term)
1207	return false;
1208	pGVM->gvmm.s.fDoneVMMR0Term = true;
1209	return true;
1210	}
1211
1212
1213	/**
1214	* Destroys the VM, freeing all associated resources (the ring-0 ones anyway).
1215	*
1216	* This is call from the vmR3DestroyFinalBit and from a error path in VMR3Create,
1217	* and the caller is not the EMT thread, unfortunately. For security reasons, it
1218	* would've been nice if the caller was actually the EMT thread or that we somehow
1219	* could've associated the calling thread with the VM up front.
1220	*
1221	* @returns VBox status code.
1222	* @param pGVM The global (ring-0) VM structure.
1223	*
1224	* @thread EMT(0) if it's associated with the VM, otherwise any thread.
1225	*/
1226	GVMMR0DECL(int) GVMMR0DestroyVM(PGVM pGVM)
1227	{
1228	LogFlow(("GVMMR0DestroyVM: pGVM=%p\n", pGVM));
1229	PGVMM pGVMM;
1230	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
1231
1232	/*
1233	* Validate the VM structure, state and caller.
1234	*/
1235	AssertPtrReturn(pGVM, VERR_INVALID_POINTER);
1236	AssertReturn(!((uintptr_t)pGVM & PAGE_OFFSET_MASK), VERR_INVALID_POINTER);
1237	AssertMsgReturn(pGVM->enmVMState >= VMSTATE_CREATING && pGVM->enmVMState <= VMSTATE_TERMINATED, ("%d\n", pGVM->enmVMState),
1238	VERR_WRONG_ORDER);
1239
1240	uint32_t hGVM = pGVM->hSelf;
1241	ASMCompilerBarrier();
1242	AssertReturn(hGVM != NIL_GVM_HANDLE, VERR_INVALID_VM_HANDLE);
1243	AssertReturn(hGVM < RT_ELEMENTS(pGVMM->aHandles), VERR_INVALID_VM_HANDLE);
1244
1245	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1246	AssertReturn(pHandle->pGVM == pGVM, VERR_NOT_OWNER);
1247
1248	RTPROCESS ProcId = RTProcSelf();
1249	RTNATIVETHREAD hSelf = RTThreadNativeSelf();
1250	AssertReturn( ( pHandle->hEMT0 == hSelf
1251	&& pHandle->ProcId == ProcId)
1252	\|\| pHandle->hEMT0 == NIL_RTNATIVETHREAD, VERR_NOT_OWNER);
1253
1254	/*
1255	* Lookup the handle and destroy the object.
1256	* Since the lock isn't recursive and we'll have to leave it before dereferencing the
1257	* object, we take some precautions against racing callers just in case...
1258	*/
1259	int rc = gvmmR0CreateDestroyLock(pGVMM);
1260	AssertRC(rc);
1261
1262	/* Be careful here because we might theoretically be racing someone else cleaning up. */
1263	if ( pHandle->pGVM == pGVM
1264	&& ( ( pHandle->hEMT0 == hSelf
1265	&& pHandle->ProcId == ProcId)
1266	\|\| pHandle->hEMT0 == NIL_RTNATIVETHREAD)
1267	&& RT_VALID_PTR(pHandle->pvObj)
1268	&& RT_VALID_PTR(pHandle->pSession)
1269	&& RT_VALID_PTR(pHandle->pGVM)
1270	&& pHandle->pGVM->u32Magic == GVM_MAGIC)
1271	{
1272	/* Check that other EMTs have deregistered. */
1273	uint32_t cNotDeregistered = 0;
1274	for (VMCPUID idCpu = 1; idCpu < pGVM->cCpus; idCpu++)
1275	cNotDeregistered += pGVM->aCpus[idCpu].hEMT != GVMM_RTNATIVETHREAD_DESTROYED;
1276	if (cNotDeregistered == 0)
1277	{
1278	/* Grab the object pointer. */
1279	void *pvObj = pHandle->pvObj;
1280	pHandle->pvObj = NULL;
1281	gvmmR0CreateDestroyUnlock(pGVMM);
1282
1283	SUPR0ObjRelease(pvObj, pHandle->pSession);
1284	}
1285	else
1286	{
1287	gvmmR0CreateDestroyUnlock(pGVMM);
1288	rc = VERR_GVMM_NOT_ALL_EMTS_DEREGISTERED;
1289	}
1290	}
1291	else
1292	{
1293	SUPR0Printf("GVMMR0DestroyVM: pHandle=%RKv:{.pGVM=%p, .hEMT0=%p, .ProcId=%u, .pvObj=%p} pGVM=%p hSelf=%p\n",
1294	pHandle, pHandle->pGVM, pHandle->hEMT0, pHandle->ProcId, pHandle->pvObj, pGVM, hSelf);
1295	gvmmR0CreateDestroyUnlock(pGVMM);
1296	rc = VERR_GVMM_IPE_2;
1297	}
1298
1299	return rc;
1300	}
1301
1302
1303	/**
1304	* Performs VM cleanup task as part of object destruction.
1305	*
1306	* @param pGVM The GVM pointer.
1307	*/
1308	static void gvmmR0CleanupVM(PGVM pGVM)
1309	{
1310	if ( pGVM->gvmm.s.fDoneVMMR0Init
1311	&& !pGVM->gvmm.s.fDoneVMMR0Term)
1312	{
1313	if ( pGVM->gvmm.s.VMMemObj != NIL_RTR0MEMOBJ
1314	&& RTR0MemObjAddress(pGVM->gvmm.s.VMMemObj) == pGVM)
1315	{
1316	LogFlow(("gvmmR0CleanupVM: Calling VMMR0TermVM\n"));
1317	VMMR0TermVM(pGVM, NIL_VMCPUID);
1318	}
1319	else
1320	AssertMsgFailed(("gvmmR0CleanupVM: VMMemObj=%p pGVM=%p\n", pGVM->gvmm.s.VMMemObj, pGVM));
1321	}
1322
1323	GMMR0CleanupVM(pGVM);
1324	#ifdef VBOX_WITH_NEM_R0
1325	NEMR0CleanupVM(pGVM);
1326	#endif
1327	PDMR0CleanupVM(pGVM);
1328	IOMR0CleanupVM(pGVM);
1329	DBGFR0CleanupVM(pGVM);
1330	PGMR0CleanupVM(pGVM);
1331	TMR0CleanupVM(pGVM);
1332	VMMR0CleanupVM(pGVM);
1333	}
1334
1335
1336	/**
1337	* @callback_method_impl{FNSUPDRVDESTRUCTOR,VM handle destructor}
1338	*
1339	* pvUser1 is the GVM instance pointer.
1340	* pvUser2 is the handle pointer.
1341	*/
1342	static DECLCALLBACK(void) gvmmR0HandleObjDestructor(void pvObj, void pvUser1, void *pvUser2)
1343	{
1344	LogFlow(("gvmmR0HandleObjDestructor: %p %p %p\n", pvObj, pvUser1, pvUser2));
1345
1346	NOREF(pvObj);
1347
1348	/*
1349	* Some quick, paranoid, input validation.
1350	*/
1351	PGVMHANDLE pHandle = (PGVMHANDLE)pvUser2;
1352	AssertPtr(pHandle);
1353	PGVMM pGVMM = (PGVMM)pvUser1;
1354	Assert(pGVMM == g_pGVMM);
1355	const uint16_t iHandle = pHandle - &pGVMM->aHandles[0];
1356	if ( !iHandle
1357	\|\| iHandle >= RT_ELEMENTS(pGVMM->aHandles)
1358	\|\| iHandle != pHandle->iSelf)
1359	{
1360	SUPR0Printf("GVM: handle %d is out of range or corrupt (iSelf=%d)!\n", iHandle, pHandle->iSelf);
1361	return;
1362	}
1363
1364	int rc = gvmmR0CreateDestroyLock(pGVMM);
1365	AssertRC(rc);
1366	rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
1367	AssertRC(rc);
1368
1369	/*
1370	* This is a tad slow but a doubly linked list is too much hassle.
1371	*/
1372	if (RT_UNLIKELY(pHandle->iNext >= RT_ELEMENTS(pGVMM->aHandles)))
1373	{
1374	SUPR0Printf("GVM: used list index %d is out of range!\n", pHandle->iNext);
1375	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1376	gvmmR0CreateDestroyUnlock(pGVMM);
1377	return;
1378	}
1379
1380	if (pGVMM->iUsedHead == iHandle)
1381	pGVMM->iUsedHead = pHandle->iNext;
1382	else
1383	{
1384	uint16_t iPrev = pGVMM->iUsedHead;
1385	int c = RT_ELEMENTS(pGVMM->aHandles) + 2;
1386	while (iPrev)
1387	{
1388	if (RT_UNLIKELY(iPrev >= RT_ELEMENTS(pGVMM->aHandles)))
1389	{
1390	SUPR0Printf("GVM: used list index %d is out of range!\n", iPrev);
1391	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1392	gvmmR0CreateDestroyUnlock(pGVMM);
1393	return;
1394	}
1395	if (RT_UNLIKELY(c-- <= 0))
1396	{
1397	iPrev = 0;
1398	break;
1399	}
1400
1401	if (pGVMM->aHandles[iPrev].iNext == iHandle)
1402	break;
1403	iPrev = pGVMM->aHandles[iPrev].iNext;
1404	}
1405	if (!iPrev)
1406	{
1407	SUPR0Printf("GVM: can't find the handle previous previous of %d!\n", pHandle->iSelf);
1408	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1409	gvmmR0CreateDestroyUnlock(pGVMM);
1410	return;
1411	}
1412
1413	Assert(pGVMM->aHandles[iPrev].iNext == iHandle);
1414	pGVMM->aHandles[iPrev].iNext = pHandle->iNext;
1415	}
1416	pHandle->iNext = 0;
1417	pGVMM->cVMs--;
1418
1419	/*
1420	* Do the global cleanup round.
1421	*/
1422	PGVM pGVM = pHandle->pGVM;
1423	if ( RT_VALID_PTR(pGVM)
1424	&& pGVM->u32Magic == GVM_MAGIC)
1425	{
1426	pGVMM->cEMTs -= pGVM->cCpus;
1427
1428	if (pGVM->pSession)
1429	SUPR0SetSessionVM(pGVM->pSession, NULL, NULL);
1430
1431	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1432
1433	gvmmR0CleanupVM(pGVM);
1434
1435	/*
1436	* Do the GVMM cleanup - must be done last.
1437	*/
1438	/* The VM and VM pages mappings/allocations. */
1439	if (pGVM->gvmm.s.VMPagesMapObj != NIL_RTR0MEMOBJ)
1440	{
1441	rc = RTR0MemObjFree(pGVM->gvmm.s.VMPagesMapObj, false /* fFreeMappings */); AssertRC(rc);
1442	pGVM->gvmm.s.VMPagesMapObj = NIL_RTR0MEMOBJ;
1443	}
1444
1445	if (pGVM->gvmm.s.VMMapObj != NIL_RTR0MEMOBJ)
1446	{
1447	rc = RTR0MemObjFree(pGVM->gvmm.s.VMMapObj, false /* fFreeMappings */); AssertRC(rc);
1448	pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
1449	}
1450
1451	if (pGVM->gvmm.s.VMPagesMemObj != NIL_RTR0MEMOBJ)
1452	{
1453	rc = RTR0MemObjFree(pGVM->gvmm.s.VMPagesMemObj, false /* fFreeMappings */); AssertRC(rc);
1454	pGVM->gvmm.s.VMPagesMemObj = NIL_RTR0MEMOBJ;
1455	}
1456
1457	for (VMCPUID i = 0; i < pGVM->cCpus; i++)
1458	{
1459	if (pGVM->aCpus[i].gvmm.s.HaltEventMulti != NIL_RTSEMEVENTMULTI)
1460	{
1461	rc = RTSemEventMultiDestroy(pGVM->aCpus[i].gvmm.s.HaltEventMulti); AssertRC(rc);
1462	pGVM->aCpus[i].gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
1463	}
1464	if (pGVM->aCpus[i].gvmm.s.VMCpuMapObj != NIL_RTR0MEMOBJ)
1465	{
1466	rc = RTR0MemObjFree(pGVM->aCpus[i].gvmm.s.VMCpuMapObj, false /* fFreeMappings */); AssertRC(rc);
1467	pGVM->aCpus[i].gvmm.s.VMCpuMapObj = NIL_RTR0MEMOBJ;
1468	}
1469	}
1470
1471	/* the GVM structure itself. */
1472	pGVM->u32Magic \|= UINT32_C(0x80000000);
1473	Assert(pGVM->gvmm.s.VMMemObj != NIL_RTR0MEMOBJ);
1474	rc = RTR0MemObjFree(pGVM->gvmm.s.VMMemObj, true /fFreeMappings/); AssertRC(rc);
1475	pGVM = NULL;
1476
1477	/* Re-acquire the UsedLock before freeing the handle since we're updating handle fields. */
1478	rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
1479	AssertRC(rc);
1480	}
1481	/* else: GVMMR0CreateVM cleanup. */
1482
1483	/*
1484	* Free the handle.
1485	*/
1486	pHandle->iNext = pGVMM->iFreeHead;
1487	pGVMM->iFreeHead = iHandle;
1488	ASMAtomicWriteNullPtr(&pHandle->pGVM);
1489	ASMAtomicWriteNullPtr(&pHandle->pvObj);
1490	ASMAtomicWriteNullPtr(&pHandle->pSession);
1491	ASMAtomicWriteHandle(&pHandle->hEMT0, NIL_RTNATIVETHREAD);
1492	ASMAtomicWriteU32(&pHandle->ProcId, NIL_RTPROCESS);
1493
1494	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1495	gvmmR0CreateDestroyUnlock(pGVMM);
1496	LogFlow(("gvmmR0HandleObjDestructor: returns\n"));
1497	}
1498
1499
1500	/**
1501	* Registers the calling thread as the EMT of a Virtual CPU.
1502	*
1503	* Note that VCPU 0 is automatically registered during VM creation.
1504	*
1505	* @returns VBox status code
1506	* @param pGVM The global (ring-0) VM structure.
1507	* @param idCpu VCPU id to register the current thread as.
1508	*/
1509	GVMMR0DECL(int) GVMMR0RegisterVCpu(PGVM pGVM, VMCPUID idCpu)
1510	{
1511	AssertReturn(idCpu != 0, VERR_INVALID_FUNCTION);
1512
1513	/*
1514	* Validate the VM structure, state and handle.
1515	*/
1516	PGVMM pGVMM;
1517	int rc = gvmmR0ByGVM(pGVM, &pGVMM, false /* fTakeUsedLock */);
1518	if (RT_SUCCESS(rc))
1519	{
1520	if (idCpu < pGVM->cCpus)
1521	{
1522	RTNATIVETHREAD const hNativeSelf = RTThreadNativeSelf();
1523
1524	gvmmR0CreateDestroyLock(pGVMM); /** @todo per-VM lock? */
1525
1526	/* Check that the EMT isn't already assigned to a thread. */
1527	if (pGVM->aCpus[idCpu].hEMT == NIL_RTNATIVETHREAD)
1528	{
1529	Assert(pGVM->aCpus[idCpu].hNativeThreadR0 == NIL_RTNATIVETHREAD);
1530
1531	/* A thread may only be one EMT (this makes sure hNativeSelf isn't NIL). */
1532	for (VMCPUID iCpu = 0; iCpu < pGVM->cCpus; iCpu++)
1533	AssertBreakStmt(pGVM->aCpus[iCpu].hEMT != hNativeSelf, rc = VERR_INVALID_PARAMETER);
1534	if (RT_SUCCESS(rc))
1535	{
1536	/*
1537	* Do the assignment, then try setup the hook. Undo if that fails.
1538	*/
1539	unsigned cCollisions = 0;
1540	uint32_t idxHash = GVMM_EMT_HASH_1(hNativeSelf);
1541	if (pGVM->gvmm.s.aEmtHash[idxHash].hNativeEmt != NIL_RTNATIVETHREAD)
1542	{
1543	uint32_t const idxHash2 = GVMM_EMT_HASH_2(hNativeSelf);
1544	do
1545	{
1546	cCollisions++;
1547	Assert(cCollisions < GVMM_EMT_HASH_SIZE);
1548	idxHash = (idxHash + idxHash2) % GVMM_EMT_HASH_SIZE;
1549	} while (pGVM->gvmm.s.aEmtHash[idxHash].hNativeEmt != NIL_RTNATIVETHREAD);
1550	}
1551	pGVM->gvmm.s.aEmtHash[idxHash].hNativeEmt = hNativeSelf;
1552	pGVM->gvmm.s.aEmtHash[idxHash].idVCpu = idCpu;
1553	pGVM->aCpus[idCpu].hNativeThreadR0 = hNativeSelf;
1554	pGVM->aCpus[idCpu].hEMT = hNativeSelf;
1555	pGVM->aCpus[idCpu].cEmtHashCollisions = (uint8_t)cCollisions;
1556	pGVM->aCpus[idCpu].gvmm.s.idxEmtHash = (uint16_t)idxHash;
1557
1558	rc = VMMR0ThreadCtxHookCreateForEmt(&pGVM->aCpus[idCpu]);
1559	if (RT_SUCCESS(rc))
1560	CPUMR0RegisterVCpuThread(&pGVM->aCpus[idCpu]);
1561	else
1562	{
1563	pGVM->aCpus[idCpu].hNativeThreadR0 = NIL_RTNATIVETHREAD;
1564	pGVM->aCpus[idCpu].hEMT = NIL_RTNATIVETHREAD;
1565	pGVM->gvmm.s.aEmtHash[idxHash].hNativeEmt = NIL_RTNATIVETHREAD;
1566	pGVM->gvmm.s.aEmtHash[idxHash].idVCpu = NIL_VMCPUID;
1567	pGVM->aCpus[idCpu].gvmm.s.idxEmtHash = UINT16_MAX;
1568	}
1569	}
1570	}
1571	else
1572	rc = VERR_ACCESS_DENIED;
1573
1574	gvmmR0CreateDestroyUnlock(pGVMM);
1575	}
1576	else
1577	rc = VERR_INVALID_CPU_ID;
1578	}
1579	return rc;
1580	}
1581
1582
1583	/**
1584	* Deregisters the calling thread as the EMT of a Virtual CPU.
1585	*
1586	* Note that VCPU 0 shall call GVMMR0DestroyVM intead of this API.
1587	*
1588	* @returns VBox status code
1589	* @param pGVM The global (ring-0) VM structure.
1590	* @param idCpu VCPU id to register the current thread as.
1591	*/
1592	GVMMR0DECL(int) GVMMR0DeregisterVCpu(PGVM pGVM, VMCPUID idCpu)
1593	{
1594	AssertReturn(idCpu != 0, VERR_INVALID_FUNCTION);
1595
1596	/*
1597	* Validate the VM structure, state and handle.
1598	*/
1599	PGVMM pGVMM;
1600	int rc = gvmmR0ByGVMandEMT(pGVM, idCpu, &pGVMM);
1601	if (RT_SUCCESS(rc))
1602	{
1603	/*
1604	* Take the destruction lock and recheck the handle state to
1605	* prevent racing GVMMR0DestroyVM.
1606	*/
1607	gvmmR0CreateDestroyLock(pGVMM);
1608
1609	uint32_t hSelf = pGVM->hSelf;
1610	ASMCompilerBarrier();
1611	if ( hSelf < RT_ELEMENTS(pGVMM->aHandles)
1612	&& pGVMM->aHandles[hSelf].pvObj != NULL
1613	&& pGVMM->aHandles[hSelf].pGVM == pGVM)
1614	{
1615	/*
1616	* Do per-EMT cleanups.
1617	*/
1618	VMMR0ThreadCtxHookDestroyForEmt(&pGVM->aCpus[idCpu]);
1619
1620	/*
1621	* Invalidate hEMT. We don't use NIL here as that would allow
1622	* GVMMR0RegisterVCpu to be called again, and we don't want that.
1623	*/
1624	pGVM->aCpus[idCpu].hEMT = GVMM_RTNATIVETHREAD_DESTROYED;
1625	pGVM->aCpus[idCpu].hNativeThreadR0 = NIL_RTNATIVETHREAD;
1626
1627	uint32_t const idxHash = pGVM->aCpus[idCpu].gvmm.s.idxEmtHash;
1628	if (idxHash < RT_ELEMENTS(pGVM->gvmm.s.aEmtHash))
1629	pGVM->gvmm.s.aEmtHash[idxHash].hNativeEmt = GVMM_RTNATIVETHREAD_DESTROYED;
1630	}
1631
1632	gvmmR0CreateDestroyUnlock(pGVMM);
1633	}
1634	return rc;
1635	}
1636
1637
1638	/**
1639	* Lookup a GVM structure by its handle.
1640	*
1641	* @returns The GVM pointer on success, NULL on failure.
1642	* @param hGVM The global VM handle. Asserts on bad handle.
1643	*/
1644	GVMMR0DECL(PGVM) GVMMR0ByHandle(uint32_t hGVM)
1645	{
1646	PGVMM pGVMM;
1647	GVMM_GET_VALID_INSTANCE(pGVMM, NULL);
1648
1649	/*
1650	* Validate.
1651	*/
1652	AssertReturn(hGVM != NIL_GVM_HANDLE, NULL);
1653	AssertReturn(hGVM < RT_ELEMENTS(pGVMM->aHandles), NULL);
1654
1655	/*
1656	* Look it up.
1657	*/
1658	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1659	AssertPtrReturn(pHandle->pvObj, NULL);
1660	PGVM pGVM = pHandle->pGVM;
1661	AssertPtrReturn(pGVM, NULL);
1662
1663	return pGVM;
1664	}
1665
1666
1667	/**
1668	* Check that the given GVM and VM structures match up.
1669	*
1670	* The calling thread must be in the same process as the VM. All current lookups
1671	* are by threads inside the same process, so this will not be an issue.
1672	*
1673	* @returns VBox status code.
1674	* @param pGVM The global (ring-0) VM structure.
1675	* @param ppGVMM Where to store the pointer to the GVMM instance data.
1676	* @param fTakeUsedLock Whether to take the used lock or not. We take it in
1677	* shared mode when requested.
1678	*
1679	* Be very careful if not taking the lock as it's
1680	* possible that the VM will disappear then!
1681	*
1682	* @remark This will not assert on an invalid pGVM but try return silently.
1683	*/
1684	static int gvmmR0ByGVM(PGVM pGVM, PGVMM *ppGVMM, bool fTakeUsedLock)
1685	{
1686	/*
1687	* Check the pointers.
1688	*/
1689	int rc;
1690	if (RT_LIKELY( RT_VALID_PTR(pGVM)
1691	&& ((uintptr_t)pGVM & PAGE_OFFSET_MASK) == 0 ))
1692	{
1693	/*
1694	* Get the pGVMM instance and check the VM handle.
1695	*/
1696	PGVMM pGVMM;
1697	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
1698
1699	uint16_t hGVM = pGVM->hSelf;
1700	if (RT_LIKELY( hGVM != NIL_GVM_HANDLE
1701	&& hGVM < RT_ELEMENTS(pGVMM->aHandles)))
1702	{
1703	RTPROCESS const pidSelf = RTProcSelf();
1704	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1705	if (fTakeUsedLock)
1706	{
1707	rc = GVMMR0_USED_SHARED_LOCK(pGVMM);
1708	AssertRCReturn(rc, rc);
1709	}
1710
1711	if (RT_LIKELY( pHandle->pGVM == pGVM
1712	&& pHandle->ProcId == pidSelf
1713	&& RT_VALID_PTR(pHandle->pvObj)))
1714	{
1715	/*
1716	* Some more VM data consistency checks.
1717	*/
1718	if (RT_LIKELY( pGVM->cCpusUnsafe == pGVM->cCpus
1719	&& pGVM->hSelfUnsafe == hGVM
1720	&& pGVM->pSelf == pGVM))
1721	{
1722	if (RT_LIKELY( pGVM->enmVMState >= VMSTATE_CREATING
1723	&& pGVM->enmVMState <= VMSTATE_TERMINATED))
1724	{
1725	*ppGVMM = pGVMM;
1726	return VINF_SUCCESS;
1727	}
1728	rc = VERR_INCONSISTENT_VM_HANDLE;
1729	}
1730	else
1731	rc = VERR_INCONSISTENT_VM_HANDLE;
1732	}
1733	else
1734	rc = VERR_INVALID_VM_HANDLE;
1735
1736	if (fTakeUsedLock)
1737	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
1738	}
1739	else
1740	rc = VERR_INVALID_VM_HANDLE;
1741	}
1742	else
1743	rc = VERR_INVALID_POINTER;
1744	return rc;
1745	}
1746
1747
1748	/**
1749	* Validates a GVM/VM pair.
1750	*
1751	* @returns VBox status code.
1752	* @param pGVM The global (ring-0) VM structure.
1753	*/
1754	GVMMR0DECL(int) GVMMR0ValidateGVM(PGVM pGVM)
1755	{
1756	PGVMM pGVMM;
1757	return gvmmR0ByGVM(pGVM, &pGVMM, false /fTakeUsedLock/);
1758	}
1759
1760
1761	/**
1762	* Check that the given GVM and VM structures match up.
1763	*
1764	* The calling thread must be in the same process as the VM. All current lookups
1765	* are by threads inside the same process, so this will not be an issue.
1766	*
1767	* @returns VBox status code.
1768	* @param pGVM The global (ring-0) VM structure.
1769	* @param idCpu The (alleged) Virtual CPU ID of the calling EMT.
1770	* @param ppGVMM Where to store the pointer to the GVMM instance data.
1771	* @thread EMT
1772	*
1773	* @remarks This will assert in all failure paths.
1774	*/
1775	static int gvmmR0ByGVMandEMT(PGVM pGVM, VMCPUID idCpu, PGVMM *ppGVMM)
1776	{
1777	/*
1778	* Check the pointers.
1779	*/
1780	AssertPtrReturn(pGVM, VERR_INVALID_POINTER);
1781	AssertReturn(((uintptr_t)pGVM & PAGE_OFFSET_MASK) == 0, VERR_INVALID_POINTER);
1782
1783	/*
1784	* Get the pGVMM instance and check the VM handle.
1785	*/
1786	PGVMM pGVMM;
1787	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
1788
1789	uint16_t hGVM = pGVM->hSelf;
1790	ASMCompilerBarrier();
1791	AssertReturn( hGVM != NIL_GVM_HANDLE
1792	&& hGVM < RT_ELEMENTS(pGVMM->aHandles), VERR_INVALID_VM_HANDLE);
1793
1794	RTPROCESS const pidSelf = RTProcSelf();
1795	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1796	AssertReturn( pHandle->pGVM == pGVM
1797	&& pHandle->ProcId == pidSelf
1798	&& RT_VALID_PTR(pHandle->pvObj),
1799	VERR_INVALID_HANDLE);
1800
1801	/*
1802	* Check the EMT claim.
1803	*/
1804	RTNATIVETHREAD const hAllegedEMT = RTThreadNativeSelf();
1805	AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID);
1806	AssertReturn(pGVM->aCpus[idCpu].hEMT == hAllegedEMT, VERR_NOT_OWNER);
1807
1808	/*
1809	* Some more VM data consistency checks.
1810	*/
1811	AssertReturn(pGVM->cCpusUnsafe == pGVM->cCpus, VERR_INCONSISTENT_VM_HANDLE);
1812	AssertReturn(pGVM->hSelfUnsafe == hGVM, VERR_INCONSISTENT_VM_HANDLE);
1813	AssertReturn( pGVM->enmVMState >= VMSTATE_CREATING
1814	&& pGVM->enmVMState <= VMSTATE_TERMINATED, VERR_INCONSISTENT_VM_HANDLE);
1815
1816	*ppGVMM = pGVMM;
1817	return VINF_SUCCESS;
1818	}
1819
1820
1821	/**
1822	* Validates a GVM/EMT pair.
1823	*
1824	* @returns VBox status code.
1825	* @param pGVM The global (ring-0) VM structure.
1826	* @param idCpu The Virtual CPU ID of the calling EMT.
1827	* @thread EMT(idCpu)
1828	*/
1829	GVMMR0DECL(int) GVMMR0ValidateGVMandEMT(PGVM pGVM, VMCPUID idCpu)
1830	{
1831	PGVMM pGVMM;
1832	return gvmmR0ByGVMandEMT(pGVM, idCpu, &pGVMM);
1833	}
1834
1835
1836	/**
1837	* Looks up the VM belonging to the specified EMT thread.
1838	*
1839	* This is used by the assertion machinery in VMMR0.cpp to avoid causing
1840	* unnecessary kernel panics when the EMT thread hits an assertion. The
1841	* call may or not be an EMT thread.
1842	*
1843	* @returns Pointer to the VM on success, NULL on failure.
1844	* @param hEMT The native thread handle of the EMT.
1845	* NIL_RTNATIVETHREAD means the current thread
1846	*/
1847	GVMMR0DECL(PVMCC) GVMMR0GetVMByEMT(RTNATIVETHREAD hEMT)
1848	{
1849	/*
1850	* No Assertions here as we're usually called in a AssertMsgN or
1851	* RTAssert* context.
1852	*/
1853	PGVMM pGVMM = g_pGVMM;
1854	if ( !RT_VALID_PTR(pGVMM)
1855	\|\| pGVMM->u32Magic != GVMM_MAGIC)
1856	return NULL;
1857
1858	if (hEMT == NIL_RTNATIVETHREAD)
1859	hEMT = RTThreadNativeSelf();
1860	RTPROCESS ProcId = RTProcSelf();
1861
1862	/*
1863	* Search the handles in a linear fashion as we don't dare to take the lock (assert).
1864	*/
1865	/** @todo introduce some pid hash table here, please. */
1866	for (unsigned i = 1; i < RT_ELEMENTS(pGVMM->aHandles); i++)
1867	{
1868	if ( pGVMM->aHandles[i].iSelf == i
1869	&& pGVMM->aHandles[i].ProcId == ProcId
1870	&& RT_VALID_PTR(pGVMM->aHandles[i].pvObj)
1871	&& RT_VALID_PTR(pGVMM->aHandles[i].pGVM))
1872	{
1873	if (pGVMM->aHandles[i].hEMT0 == hEMT)
1874	return pGVMM->aHandles[i].pGVM;
1875
1876	/* This is fearly safe with the current process per VM approach. */
1877	PGVM pGVM = pGVMM->aHandles[i].pGVM;
1878	VMCPUID const cCpus = pGVM->cCpus;
1879	ASMCompilerBarrier();
1880	if ( cCpus < 1
1881	\|\| cCpus > VMM_MAX_CPU_COUNT)
1882	continue;
1883	for (VMCPUID idCpu = 1; idCpu < cCpus; idCpu++)
1884	if (pGVM->aCpus[idCpu].hEMT == hEMT)
1885	return pGVMM->aHandles[i].pGVM;
1886	}
1887	}
1888	return NULL;
1889	}
1890
1891
1892	/**
1893	* Looks up the GVMCPU belonging to the specified EMT thread.
1894	*
1895	* This is used by the assertion machinery in VMMR0.cpp to avoid causing
1896	* unnecessary kernel panics when the EMT thread hits an assertion. The
1897	* call may or not be an EMT thread.
1898	*
1899	* @returns Pointer to the VM on success, NULL on failure.
1900	* @param hEMT The native thread handle of the EMT.
1901	* NIL_RTNATIVETHREAD means the current thread
1902	*/
1903	GVMMR0DECL(PGVMCPU) GVMMR0GetGVCpuByEMT(RTNATIVETHREAD hEMT)
1904	{
1905	/*
1906	* No Assertions here as we're usually called in a AssertMsgN,
1907	* RTAssert*, Log and LogRel contexts.
1908	*/
1909	PGVMM pGVMM = g_pGVMM;
1910	if ( !RT_VALID_PTR(pGVMM)
1911	\|\| pGVMM->u32Magic != GVMM_MAGIC)
1912	return NULL;
1913
1914	if (hEMT == NIL_RTNATIVETHREAD)
1915	hEMT = RTThreadNativeSelf();
1916	RTPROCESS ProcId = RTProcSelf();
1917
1918	/*
1919	* Search the handles in a linear fashion as we don't dare to take the lock (assert).
1920	*/
1921	/** @todo introduce some pid hash table here, please. */
1922	for (unsigned i = 1; i < RT_ELEMENTS(pGVMM->aHandles); i++)
1923	{
1924	if ( pGVMM->aHandles[i].iSelf == i
1925	&& pGVMM->aHandles[i].ProcId == ProcId
1926	&& RT_VALID_PTR(pGVMM->aHandles[i].pvObj)
1927	&& RT_VALID_PTR(pGVMM->aHandles[i].pGVM))
1928	{
1929	PGVM pGVM = pGVMM->aHandles[i].pGVM;
1930	if (pGVMM->aHandles[i].hEMT0 == hEMT)
1931	return &pGVM->aCpus[0];
1932
1933	/* This is fearly safe with the current process per VM approach. */
1934	VMCPUID const cCpus = pGVM->cCpus;
1935	ASMCompilerBarrier();
1936	ASMCompilerBarrier();
1937	if ( cCpus < 1
1938	\|\| cCpus > VMM_MAX_CPU_COUNT)
1939	continue;
1940	for (VMCPUID idCpu = 1; idCpu < cCpus; idCpu++)
1941	if (pGVM->aCpus[idCpu].hEMT == hEMT)
1942	return &pGVM->aCpus[idCpu];
1943	}
1944	}
1945	return NULL;
1946	}
1947
1948
1949	/**
1950	* Get the GVMCPU structure for the given EMT.
1951	*
1952	* @returns The VCpu structure for @a hEMT, NULL if not an EMT.
1953	* @param pGVM The global (ring-0) VM structure.
1954	* @param hEMT The native thread handle of the EMT.
1955	* NIL_RTNATIVETHREAD means the current thread
1956	*/
1957	GVMMR0DECL(PGVMCPU) GVMMR0GetGVCpuByGVMandEMT(PGVM pGVM, RTNATIVETHREAD hEMT)
1958	{
1959	/*
1960	* Validate & adjust input.
1961	*/
1962	AssertPtr(pGVM);
1963	Assert(pGVM->u32Magic == GVM_MAGIC);
1964	if (hEMT == NIL_RTNATIVETHREAD /* likely */)
1965	{
1966	hEMT = RTThreadNativeSelf();
1967	AssertReturn(hEMT != NIL_RTNATIVETHREAD, NULL);
1968	}
1969
1970	/*
1971	* Find the matching hash table entry.
1972	*/
1973	uint32_t idxHash = GVMM_EMT_HASH_1(hEMT);
1974	if (pGVM->gvmm.s.aEmtHash[idxHash].hNativeEmt == hEMT)
1975	{ /* likely */ }
1976	else
1977	{
1978	#ifdef VBOX_STRICT
1979	unsigned cCollisions = 0;
1980	#endif
1981	uint32_t const idxHash2 = GVMM_EMT_HASH_2(hEMT);
1982	for (;;)
1983	{
1984	Assert(cCollisions++ < GVMM_EMT_HASH_SIZE);
1985	idxHash = (idxHash + idxHash2) % GVMM_EMT_HASH_SIZE;
1986	if (pGVM->gvmm.s.aEmtHash[idxHash].hNativeEmt == hEMT)
1987	break;
1988	if (pGVM->gvmm.s.aEmtHash[idxHash].hNativeEmt == NIL_RTNATIVETHREAD)
1989	{
1990	#ifdef VBOX_STRICT
1991	uint32_t idxCpu = pGVM->cCpus;
1992	AssertStmt(idxCpu < VMM_MAX_CPU_COUNT, idxCpu = VMM_MAX_CPU_COUNT);
1993	while (idxCpu-- > 0)
1994	Assert(pGVM->aCpus[idxCpu].hNativeThreadR0 != hEMT);
1995	#endif
1996	return NULL;
1997	}
1998	}
1999	}
2000
2001	/*
2002	* Validate the VCpu number and translate it into a pointer.
2003	*/
2004	VMCPUID const idCpu = pGVM->gvmm.s.aEmtHash[idxHash].idVCpu;
2005	AssertReturn(idCpu < pGVM->cCpus, NULL);
2006	PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
2007	Assert(pGVCpu->hNativeThreadR0 == hEMT);
2008	Assert(pGVCpu->gvmm.s.idxEmtHash == idxHash);
2009	return pGVCpu;
2010	}
2011
2012
2013	/**
2014	* This is will wake up expired and soon-to-be expired VMs.
2015	*
2016	* @returns Number of VMs that has been woken up.
2017	* @param pGVMM Pointer to the GVMM instance data.
2018	* @param u64Now The current time.
2019	*/
2020	static unsigned gvmmR0SchedDoWakeUps(PGVMM pGVMM, uint64_t u64Now)
2021	{
2022	/*
2023	* Skip this if we've got disabled because of high resolution wakeups or by
2024	* the user.
2025	*/
2026	if (!pGVMM->fDoEarlyWakeUps)
2027	return 0;
2028
2029	/** @todo Rewrite this algorithm. See performance defect XYZ. */
2030
2031	/*
2032	* A cheap optimization to stop wasting so much time here on big setups.
2033	*/
2034	const uint64_t uNsEarlyWakeUp2 = u64Now + pGVMM->nsEarlyWakeUp2;
2035	if ( pGVMM->cHaltedEMTs == 0
2036	\|\| uNsEarlyWakeUp2 > pGVMM->uNsNextEmtWakeup)
2037	return 0;
2038
2039	/*
2040	* Only one thread doing this at a time.
2041	*/
2042	if (!ASMAtomicCmpXchgBool(&pGVMM->fDoingEarlyWakeUps, true, false))
2043	return 0;
2044
2045	/*
2046	* The first pass will wake up VMs which have actually expired
2047	* and look for VMs that should be woken up in the 2nd and 3rd passes.
2048	*/
2049	const uint64_t uNsEarlyWakeUp1 = u64Now + pGVMM->nsEarlyWakeUp1;
2050	uint64_t u64Min = UINT64_MAX;
2051	unsigned cWoken = 0;
2052	unsigned cHalted = 0;
2053	unsigned cTodo2nd = 0;
2054	unsigned cTodo3rd = 0;
2055	for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
2056	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
2057	i = pGVMM->aHandles[i].iNext)
2058	{
2059	PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
2060	if ( RT_VALID_PTR(pCurGVM)
2061	&& pCurGVM->u32Magic == GVM_MAGIC)
2062	{
2063	for (VMCPUID idCpu = 0; idCpu < pCurGVM->cCpus; idCpu++)
2064	{
2065	PGVMCPU pCurGVCpu = &pCurGVM->aCpus[idCpu];
2066	uint64_t u64 = ASMAtomicUoReadU64(&pCurGVCpu->gvmm.s.u64HaltExpire);
2067	if (u64)
2068	{
2069	if (u64 <= u64Now)
2070	{
2071	if (ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0))
2072	{
2073	int rc = RTSemEventMultiSignal(pCurGVCpu->gvmm.s.HaltEventMulti);
2074	AssertRC(rc);
2075	cWoken++;
2076	}
2077	}
2078	else
2079	{
2080	cHalted++;
2081	if (u64 <= uNsEarlyWakeUp1)
2082	cTodo2nd++;
2083	else if (u64 <= uNsEarlyWakeUp2)
2084	cTodo3rd++;
2085	else if (u64 < u64Min)
2086	u64 = u64Min;
2087	}
2088	}
2089	}
2090	}
2091	AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
2092	}
2093
2094	if (cTodo2nd)
2095	{
2096	for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
2097	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
2098	i = pGVMM->aHandles[i].iNext)
2099	{
2100	PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
2101	if ( RT_VALID_PTR(pCurGVM)
2102	&& pCurGVM->u32Magic == GVM_MAGIC)
2103	{
2104	for (VMCPUID idCpu = 0; idCpu < pCurGVM->cCpus; idCpu++)
2105	{
2106	PGVMCPU pCurGVCpu = &pCurGVM->aCpus[idCpu];
2107	uint64_t u64 = ASMAtomicUoReadU64(&pCurGVCpu->gvmm.s.u64HaltExpire);
2108	if ( u64
2109	&& u64 <= uNsEarlyWakeUp1)
2110	{
2111	if (ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0))
2112	{
2113	int rc = RTSemEventMultiSignal(pCurGVCpu->gvmm.s.HaltEventMulti);
2114	AssertRC(rc);
2115	cWoken++;
2116	}
2117	}
2118	}
2119	}
2120	AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
2121	}
2122	}
2123
2124	if (cTodo3rd)
2125	{
2126	for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
2127	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
2128	i = pGVMM->aHandles[i].iNext)
2129	{
2130	PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
2131	if ( RT_VALID_PTR(pCurGVM)
2132	&& pCurGVM->u32Magic == GVM_MAGIC)
2133	{
2134	for (VMCPUID idCpu = 0; idCpu < pCurGVM->cCpus; idCpu++)
2135	{
2136	PGVMCPU pCurGVCpu = &pCurGVM->aCpus[idCpu];
2137	uint64_t u64 = ASMAtomicUoReadU64(&pCurGVCpu->gvmm.s.u64HaltExpire);
2138	if ( u64
2139	&& u64 <= uNsEarlyWakeUp2)
2140	{
2141	if (ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0))
2142	{
2143	int rc = RTSemEventMultiSignal(pCurGVCpu->gvmm.s.HaltEventMulti);
2144	AssertRC(rc);
2145	cWoken++;
2146	}
2147	}
2148	}
2149	}
2150	AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
2151	}
2152	}
2153
2154	/*
2155	* Set the minimum value.
2156	*/
2157	pGVMM->uNsNextEmtWakeup = u64Min;
2158
2159	ASMAtomicWriteBool(&pGVMM->fDoingEarlyWakeUps, false);
2160	return cWoken;
2161	}
2162
2163
2164	/**
2165	* Halt the EMT thread.
2166	*
2167	* @returns VINF_SUCCESS normal wakeup (timeout or kicked by other thread).
2168	* VERR_INTERRUPTED if a signal was scheduled for the thread.
2169	* @param pGVM The global (ring-0) VM structure.
2170	* @param pGVCpu The global (ring-0) CPU structure of the calling
2171	* EMT.
2172	* @param u64ExpireGipTime The time for the sleep to expire expressed as GIP time.
2173	* @thread EMT(pGVCpu).
2174	*/
2175	GVMMR0DECL(int) GVMMR0SchedHalt(PGVM pGVM, PGVMCPU pGVCpu, uint64_t u64ExpireGipTime)
2176	{
2177	LogFlow(("GVMMR0SchedHalt: pGVM=%p pGVCpu=%p(%d) u64ExpireGipTime=%#RX64\n",
2178	pGVM, pGVCpu, pGVCpu->idCpu, u64ExpireGipTime));
2179	GVMM_CHECK_SMAP_SETUP();
2180	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2181
2182	PGVMM pGVMM;
2183	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
2184
2185	pGVM->gvmm.s.StatsSched.cHaltCalls++;
2186	Assert(!pGVCpu->gvmm.s.u64HaltExpire);
2187
2188	/*
2189	* If we're doing early wake-ups, we must take the UsedList lock before we
2190	* start querying the current time.
2191	* Note! Interrupts must NOT be disabled at this point because we ask for GIP time!
2192	*/
2193	bool const fDoEarlyWakeUps = pGVMM->fDoEarlyWakeUps;
2194	if (fDoEarlyWakeUps)
2195	{
2196	int rc2 = GVMMR0_USED_SHARED_LOCK(pGVMM); AssertRC(rc2);
2197	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2198	}
2199
2200	pGVCpu->gvmm.s.iCpuEmt = ASMGetApicId();
2201
2202	/* GIP hack: We might are frequently sleeping for short intervals where the
2203	difference between GIP and system time matters on systems with high resolution
2204	system time. So, convert the input from GIP to System time in that case. */
2205	Assert(ASMGetFlags() & X86_EFL_IF);
2206	const uint64_t u64NowSys = RTTimeSystemNanoTS();
2207	const uint64_t u64NowGip = RTTimeNanoTS();
2208	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2209
2210	if (fDoEarlyWakeUps)
2211	{
2212	pGVM->gvmm.s.StatsSched.cHaltWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64NowGip);
2213	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2214	}
2215
2216	/*
2217	* Go to sleep if we must...
2218	* Cap the sleep time to 1 second to be on the safe side.
2219	*/
2220	int rc;
2221	uint64_t cNsInterval = u64ExpireGipTime - u64NowGip;
2222	if ( u64NowGip < u64ExpireGipTime
2223	&& cNsInterval >= (pGVMM->cEMTs > pGVMM->cEMTsMeansCompany
2224	? pGVMM->nsMinSleepCompany
2225	: pGVMM->nsMinSleepAlone))
2226	{
2227	pGVM->gvmm.s.StatsSched.cHaltBlocking++;
2228	if (cNsInterval > RT_NS_1SEC)
2229	u64ExpireGipTime = u64NowGip + RT_NS_1SEC;
2230	ASMAtomicWriteU64(&pGVCpu->gvmm.s.u64HaltExpire, u64ExpireGipTime);
2231	ASMAtomicIncU32(&pGVMM->cHaltedEMTs);
2232	if (fDoEarlyWakeUps)
2233	{
2234	if (u64ExpireGipTime < pGVMM->uNsNextEmtWakeup)
2235	pGVMM->uNsNextEmtWakeup = u64ExpireGipTime;
2236	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2237	}
2238	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2239
2240	rc = RTSemEventMultiWaitEx(pGVCpu->gvmm.s.HaltEventMulti,
2241	RTSEMWAIT_FLAGS_ABSOLUTE \| RTSEMWAIT_FLAGS_NANOSECS \| RTSEMWAIT_FLAGS_INTERRUPTIBLE,
2242	u64NowGip > u64NowSys ? u64ExpireGipTime : u64NowSys + cNsInterval);
2243	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2244
2245	ASMAtomicWriteU64(&pGVCpu->gvmm.s.u64HaltExpire, 0);
2246	ASMAtomicDecU32(&pGVMM->cHaltedEMTs);
2247
2248	/* Reset the semaphore to try prevent a few false wake-ups. */
2249	if (rc == VINF_SUCCESS)
2250	{
2251	RTSemEventMultiReset(pGVCpu->gvmm.s.HaltEventMulti);
2252	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2253	}
2254	else if (rc == VERR_TIMEOUT)
2255	{
2256	pGVM->gvmm.s.StatsSched.cHaltTimeouts++;
2257	rc = VINF_SUCCESS;
2258	}
2259	}
2260	else
2261	{
2262	pGVM->gvmm.s.StatsSched.cHaltNotBlocking++;
2263	if (fDoEarlyWakeUps)
2264	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2265	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2266	RTSemEventMultiReset(pGVCpu->gvmm.s.HaltEventMulti);
2267	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2268	rc = VINF_SUCCESS;
2269	}
2270
2271	return rc;
2272	}
2273
2274
2275	/**
2276	* Halt the EMT thread.
2277	*
2278	* @returns VINF_SUCCESS normal wakeup (timeout or kicked by other thread).
2279	* VERR_INTERRUPTED if a signal was scheduled for the thread.
2280	* @param pGVM The global (ring-0) VM structure.
2281	* @param idCpu The Virtual CPU ID of the calling EMT.
2282	* @param u64ExpireGipTime The time for the sleep to expire expressed as GIP time.
2283	* @thread EMT(idCpu).
2284	*/
2285	GVMMR0DECL(int) GVMMR0SchedHaltReq(PGVM pGVM, VMCPUID idCpu, uint64_t u64ExpireGipTime)
2286	{
2287	GVMM_CHECK_SMAP_SETUP();
2288	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2289	PGVMM pGVMM;
2290	int rc = gvmmR0ByGVMandEMT(pGVM, idCpu, &pGVMM);
2291	if (RT_SUCCESS(rc))
2292	{
2293	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2294	rc = GVMMR0SchedHalt(pGVM, &pGVM->aCpus[idCpu], u64ExpireGipTime);
2295	}
2296	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2297	return rc;
2298	}
2299
2300
2301
2302	/**
2303	* Worker for GVMMR0SchedWakeUp and GVMMR0SchedWakeUpAndPokeCpus that wakes up
2304	* the a sleeping EMT.
2305	*
2306	* @retval VINF_SUCCESS if successfully woken up.
2307	* @retval VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
2308	*
2309	* @param pGVM The global (ring-0) VM structure.
2310	* @param pGVCpu The global (ring-0) VCPU structure.
2311	*/
2312	DECLINLINE(int) gvmmR0SchedWakeUpOne(PGVM pGVM, PGVMCPU pGVCpu)
2313	{
2314	pGVM->gvmm.s.StatsSched.cWakeUpCalls++;
2315
2316	/*
2317	* Signal the semaphore regardless of whether it's current blocked on it.
2318	*
2319	* The reason for this is that there is absolutely no way we can be 100%
2320	* certain that it isn't about go to go to sleep on it and just got
2321	* delayed a bit en route. So, we will always signal the semaphore when
2322	* the it is flagged as halted in the VMM.
2323	*/
2324	/** @todo we can optimize some of that by means of the pVCpu->enmState now. */
2325	int rc;
2326	if (pGVCpu->gvmm.s.u64HaltExpire)
2327	{
2328	rc = VINF_SUCCESS;
2329	ASMAtomicWriteU64(&pGVCpu->gvmm.s.u64HaltExpire, 0);
2330	}
2331	else
2332	{
2333	rc = VINF_GVM_NOT_BLOCKED;
2334	pGVM->gvmm.s.StatsSched.cWakeUpNotHalted++;
2335	}
2336
2337	int rc2 = RTSemEventMultiSignal(pGVCpu->gvmm.s.HaltEventMulti);
2338	AssertRC(rc2);
2339
2340	return rc;
2341	}
2342
2343
2344	/**
2345	* Wakes up the halted EMT thread so it can service a pending request.
2346	*
2347	* @returns VBox status code.
2348	* @retval VINF_SUCCESS if successfully woken up.
2349	* @retval VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
2350	*
2351	* @param pGVM The global (ring-0) VM structure.
2352	* @param idCpu The Virtual CPU ID of the EMT to wake up.
2353	* @param fTakeUsedLock Take the used lock or not
2354	* @thread Any but EMT(idCpu).
2355	*/
2356	GVMMR0DECL(int) GVMMR0SchedWakeUpEx(PGVM pGVM, VMCPUID idCpu, bool fTakeUsedLock)
2357	{
2358	GVMM_CHECK_SMAP_SETUP();
2359	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2360
2361	/*
2362	* Validate input and take the UsedLock.
2363	*/
2364	PGVMM pGVMM;
2365	int rc = gvmmR0ByGVM(pGVM, &pGVMM, fTakeUsedLock);
2366	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2367	if (RT_SUCCESS(rc))
2368	{
2369	if (idCpu < pGVM->cCpus)
2370	{
2371	/*
2372	* Do the actual job.
2373	*/
2374	rc = gvmmR0SchedWakeUpOne(pGVM, &pGVM->aCpus[idCpu]);
2375	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2376
2377	if (fTakeUsedLock && pGVMM->fDoEarlyWakeUps)
2378	{
2379	/*
2380	* While we're here, do a round of scheduling.
2381	*/
2382	Assert(ASMGetFlags() & X86_EFL_IF);
2383	const uint64_t u64Now = RTTimeNanoTS(); /* (GIP time) */
2384	pGVM->gvmm.s.StatsSched.cWakeUpWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64Now);
2385	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2386	}
2387	}
2388	else
2389	rc = VERR_INVALID_CPU_ID;
2390
2391	if (fTakeUsedLock)
2392	{
2393	int rc2 = GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2394	AssertRC(rc2);
2395	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2396	}
2397	}
2398
2399	LogFlow(("GVMMR0SchedWakeUpEx: returns %Rrc\n", rc));
2400	return rc;
2401	}
2402
2403
2404	/**
2405	* Wakes up the halted EMT thread so it can service a pending request.
2406	*
2407	* @returns VBox status code.
2408	* @retval VINF_SUCCESS if successfully woken up.
2409	* @retval VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
2410	*
2411	* @param pGVM The global (ring-0) VM structure.
2412	* @param idCpu The Virtual CPU ID of the EMT to wake up.
2413	* @thread Any but EMT(idCpu).
2414	*/
2415	GVMMR0DECL(int) GVMMR0SchedWakeUp(PGVM pGVM, VMCPUID idCpu)
2416	{
2417	return GVMMR0SchedWakeUpEx(pGVM, idCpu, true /* fTakeUsedLock */);
2418	}
2419
2420
2421	/**
2422	* Wakes up the halted EMT thread so it can service a pending request, no GVM
2423	* parameter and no used locking.
2424	*
2425	* @returns VBox status code.
2426	* @retval VINF_SUCCESS if successfully woken up.
2427	* @retval VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
2428	*
2429	* @param pGVM The global (ring-0) VM structure.
2430	* @param idCpu The Virtual CPU ID of the EMT to wake up.
2431	* @thread Any but EMT(idCpu).
2432	* @deprecated Don't use in new code if possible! Use the GVM variant.
2433	*/
2434	GVMMR0DECL(int) GVMMR0SchedWakeUpNoGVMNoLock(PGVM pGVM, VMCPUID idCpu)
2435	{
2436	GVMM_CHECK_SMAP_SETUP();
2437	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2438	PGVMM pGVMM;
2439	int rc = gvmmR0ByGVM(pGVM, &pGVMM, false /fTakeUsedLock/);
2440	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2441	if (RT_SUCCESS(rc))
2442	rc = GVMMR0SchedWakeUpEx(pGVM, idCpu, false /fTakeUsedLock/);
2443	return rc;
2444	}
2445
2446
2447	/**
2448	* Worker common to GVMMR0SchedPoke and GVMMR0SchedWakeUpAndPokeCpus that pokes
2449	* the Virtual CPU if it's still busy executing guest code.
2450	*
2451	* @returns VBox status code.
2452	* @retval VINF_SUCCESS if poked successfully.
2453	* @retval VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
2454	*
2455	* @param pGVM The global (ring-0) VM structure.
2456	* @param pVCpu The cross context virtual CPU structure.
2457	*/
2458	DECLINLINE(int) gvmmR0SchedPokeOne(PGVM pGVM, PVMCPUCC pVCpu)
2459	{
2460	pGVM->gvmm.s.StatsSched.cPokeCalls++;
2461
2462	RTCPUID idHostCpu = pVCpu->idHostCpu;
2463	if ( idHostCpu == NIL_RTCPUID
2464	\|\| VMCPU_GET_STATE(pVCpu) != VMCPUSTATE_STARTED_EXEC)
2465	{
2466	pGVM->gvmm.s.StatsSched.cPokeNotBusy++;
2467	return VINF_GVM_NOT_BUSY_IN_GC;
2468	}
2469
2470	/* Note: this function is not implemented on Darwin and Linux (kernel < 2.6.19) */
2471	RTMpPokeCpu(idHostCpu);
2472	return VINF_SUCCESS;
2473	}
2474
2475
2476	/**
2477	* Pokes an EMT if it's still busy running guest code.
2478	*
2479	* @returns VBox status code.
2480	* @retval VINF_SUCCESS if poked successfully.
2481	* @retval VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
2482	*
2483	* @param pGVM The global (ring-0) VM structure.
2484	* @param idCpu The ID of the virtual CPU to poke.
2485	* @param fTakeUsedLock Take the used lock or not
2486	*/
2487	GVMMR0DECL(int) GVMMR0SchedPokeEx(PGVM pGVM, VMCPUID idCpu, bool fTakeUsedLock)
2488	{
2489	/*
2490	* Validate input and take the UsedLock.
2491	*/
2492	PGVMM pGVMM;
2493	int rc = gvmmR0ByGVM(pGVM, &pGVMM, fTakeUsedLock);
2494	if (RT_SUCCESS(rc))
2495	{
2496	if (idCpu < pGVM->cCpus)
2497	rc = gvmmR0SchedPokeOne(pGVM, &pGVM->aCpus[idCpu]);
2498	else
2499	rc = VERR_INVALID_CPU_ID;
2500
2501	if (fTakeUsedLock)
2502	{
2503	int rc2 = GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2504	AssertRC(rc2);
2505	}
2506	}
2507
2508	LogFlow(("GVMMR0SchedWakeUpAndPokeCpus: returns %Rrc\n", rc));
2509	return rc;
2510	}
2511
2512
2513	/**
2514	* Pokes an EMT if it's still busy running guest code.
2515	*
2516	* @returns VBox status code.
2517	* @retval VINF_SUCCESS if poked successfully.
2518	* @retval VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
2519	*
2520	* @param pGVM The global (ring-0) VM structure.
2521	* @param idCpu The ID of the virtual CPU to poke.
2522	*/
2523	GVMMR0DECL(int) GVMMR0SchedPoke(PGVM pGVM, VMCPUID idCpu)
2524	{
2525	return GVMMR0SchedPokeEx(pGVM, idCpu, true /* fTakeUsedLock */);
2526	}
2527
2528
2529	/**
2530	* Pokes an EMT if it's still busy running guest code, no GVM parameter and no
2531	* used locking.
2532	*
2533	* @returns VBox status code.
2534	* @retval VINF_SUCCESS if poked successfully.
2535	* @retval VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
2536	*
2537	* @param pGVM The global (ring-0) VM structure.
2538	* @param idCpu The ID of the virtual CPU to poke.
2539	*
2540	* @deprecated Don't use in new code if possible! Use the GVM variant.
2541	*/
2542	GVMMR0DECL(int) GVMMR0SchedPokeNoGVMNoLock(PGVM pGVM, VMCPUID idCpu)
2543	{
2544	PGVMM pGVMM;
2545	int rc = gvmmR0ByGVM(pGVM, &pGVMM, false /fTakeUsedLock/);
2546	if (RT_SUCCESS(rc))
2547	{
2548	if (idCpu < pGVM->cCpus)
2549	rc = gvmmR0SchedPokeOne(pGVM, &pGVM->aCpus[idCpu]);
2550	else
2551	rc = VERR_INVALID_CPU_ID;
2552	}
2553	return rc;
2554	}
2555
2556
2557	/**
2558	* Wakes up a set of halted EMT threads so they can service pending request.
2559	*
2560	* @returns VBox status code, no informational stuff.
2561	*
2562	* @param pGVM The global (ring-0) VM structure.
2563	* @param pSleepSet The set of sleepers to wake up.
2564	* @param pPokeSet The set of CPUs to poke.
2565	*/
2566	GVMMR0DECL(int) GVMMR0SchedWakeUpAndPokeCpus(PGVM pGVM, PCVMCPUSET pSleepSet, PCVMCPUSET pPokeSet)
2567	{
2568	AssertPtrReturn(pSleepSet, VERR_INVALID_POINTER);
2569	AssertPtrReturn(pPokeSet, VERR_INVALID_POINTER);
2570	GVMM_CHECK_SMAP_SETUP();
2571	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2572	RTNATIVETHREAD hSelf = RTThreadNativeSelf();
2573
2574	/*
2575	* Validate input and take the UsedLock.
2576	*/
2577	PGVMM pGVMM;
2578	int rc = gvmmR0ByGVM(pGVM, &pGVMM, true /* fTakeUsedLock */);
2579	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2580	if (RT_SUCCESS(rc))
2581	{
2582	rc = VINF_SUCCESS;
2583	VMCPUID idCpu = pGVM->cCpus;
2584	while (idCpu-- > 0)
2585	{
2586	/* Don't try poke or wake up ourselves. */
2587	if (pGVM->aCpus[idCpu].hEMT == hSelf)
2588	continue;
2589
2590	/* just ignore errors for now. */
2591	if (VMCPUSET_IS_PRESENT(pSleepSet, idCpu))
2592	{
2593	gvmmR0SchedWakeUpOne(pGVM, &pGVM->aCpus[idCpu]);
2594	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2595	}
2596	else if (VMCPUSET_IS_PRESENT(pPokeSet, idCpu))
2597	{
2598	gvmmR0SchedPokeOne(pGVM, &pGVM->aCpus[idCpu]);
2599	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2600	}
2601	}
2602
2603	int rc2 = GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2604	AssertRC(rc2);
2605	GVMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
2606	}
2607
2608	LogFlow(("GVMMR0SchedWakeUpAndPokeCpus: returns %Rrc\n", rc));
2609	return rc;
2610	}
2611
2612
2613	/**
2614	* VMMR0 request wrapper for GVMMR0SchedWakeUpAndPokeCpus.
2615	*
2616	* @returns see GVMMR0SchedWakeUpAndPokeCpus.
2617	* @param pGVM The global (ring-0) VM structure.
2618	* @param pReq Pointer to the request packet.
2619	*/
2620	GVMMR0DECL(int) GVMMR0SchedWakeUpAndPokeCpusReq(PGVM pGVM, PGVMMSCHEDWAKEUPANDPOKECPUSREQ pReq)
2621	{
2622	/*
2623	* Validate input and pass it on.
2624	*/
2625	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
2626	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
2627
2628	return GVMMR0SchedWakeUpAndPokeCpus(pGVM, &pReq->SleepSet, &pReq->PokeSet);
2629	}
2630
2631
2632
2633	/**
2634	* Poll the schedule to see if someone else should get a chance to run.
2635	*
2636	* This is a bit hackish and will not work too well if the machine is
2637	* under heavy load from non-VM processes.
2638	*
2639	* @returns VINF_SUCCESS if not yielded.
2640	* VINF_GVM_YIELDED if an attempt to switch to a different VM task was made.
2641	* @param pGVM The global (ring-0) VM structure.
2642	* @param idCpu The Virtual CPU ID of the calling EMT.
2643	* @param fYield Whether to yield or not.
2644	* This is for when we're spinning in the halt loop.
2645	* @thread EMT(idCpu).
2646	*/
2647	GVMMR0DECL(int) GVMMR0SchedPoll(PGVM pGVM, VMCPUID idCpu, bool fYield)
2648	{
2649	/*
2650	* Validate input.
2651	*/
2652	PGVMM pGVMM;
2653	int rc = gvmmR0ByGVMandEMT(pGVM, idCpu, &pGVMM);
2654	if (RT_SUCCESS(rc))
2655	{
2656	/*
2657	* We currently only implement helping doing wakeups (fYield = false), so don't
2658	* bother taking the lock if gvmmR0SchedDoWakeUps is not going to do anything.
2659	*/
2660	if (!fYield && pGVMM->fDoEarlyWakeUps)
2661	{
2662	rc = GVMMR0_USED_SHARED_LOCK(pGVMM); AssertRC(rc);
2663	pGVM->gvmm.s.StatsSched.cPollCalls++;
2664
2665	Assert(ASMGetFlags() & X86_EFL_IF);
2666	const uint64_t u64Now = RTTimeNanoTS(); /* (GIP time) */
2667
2668	pGVM->gvmm.s.StatsSched.cPollWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64Now);
2669
2670	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2671	}
2672	/*
2673	* Not quite sure what we could do here...
2674	*/
2675	else if (fYield)
2676	rc = VERR_NOT_IMPLEMENTED; /** @todo implement this... */
2677	else
2678	rc = VINF_SUCCESS;
2679	}
2680
2681	LogFlow(("GVMMR0SchedWakeUp: returns %Rrc\n", rc));
2682	return rc;
2683	}
2684
2685
2686	#ifdef GVMM_SCHED_WITH_PPT
2687	/**
2688	* Timer callback for the periodic preemption timer.
2689	*
2690	* @param pTimer The timer handle.
2691	* @param pvUser Pointer to the per cpu structure.
2692	* @param iTick The current tick.
2693	*/
2694	static DECLCALLBACK(void) gvmmR0SchedPeriodicPreemptionTimerCallback(PRTTIMER pTimer, void *pvUser, uint64_t iTick)
2695	{
2696	PGVMMHOSTCPU pCpu = (PGVMMHOSTCPU)pvUser;
2697	NOREF(pTimer); NOREF(iTick);
2698
2699	/*
2700	* Termination check
2701	*/
2702	if (pCpu->u32Magic != GVMMHOSTCPU_MAGIC)
2703	return;
2704
2705	/*
2706	* Do the house keeping.
2707	*/
2708	RTSpinlockAcquire(pCpu->Ppt.hSpinlock);
2709
2710	if (++pCpu->Ppt.iTickHistorization >= pCpu->Ppt.cTicksHistoriziationInterval)
2711	{
2712	/*
2713	* Historicize the max frequency.
2714	*/
2715	uint32_t iHzHistory = ++pCpu->Ppt.iHzHistory % RT_ELEMENTS(pCpu->Ppt.aHzHistory);
2716	pCpu->Ppt.aHzHistory[iHzHistory] = pCpu->Ppt.uDesiredHz;
2717	pCpu->Ppt.iTickHistorization = 0;
2718	pCpu->Ppt.uDesiredHz = 0;
2719
2720	/*
2721	* Check if the current timer frequency.
2722	*/
2723	uint32_t uHistMaxHz = 0;
2724	for (uint32_t i = 0; i < RT_ELEMENTS(pCpu->Ppt.aHzHistory); i++)
2725	if (pCpu->Ppt.aHzHistory[i] > uHistMaxHz)
2726	uHistMaxHz = pCpu->Ppt.aHzHistory[i];
2727	if (uHistMaxHz == pCpu->Ppt.uTimerHz)
2728	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
2729	else if (uHistMaxHz)
2730	{
2731	/*
2732	* Reprogram it.
2733	*/
2734	pCpu->Ppt.cChanges++;
2735	pCpu->Ppt.iTickHistorization = 0;
2736	pCpu->Ppt.uTimerHz = uHistMaxHz;
2737	uint32_t const cNsInterval = RT_NS_1SEC / uHistMaxHz;
2738	pCpu->Ppt.cNsInterval = cNsInterval;
2739	if (cNsInterval < GVMMHOSTCPU_PPT_HIST_INTERVAL_NS)
2740	pCpu->Ppt.cTicksHistoriziationInterval = ( GVMMHOSTCPU_PPT_HIST_INTERVAL_NS
2741	+ GVMMHOSTCPU_PPT_HIST_INTERVAL_NS / 2 - 1)
2742	/ cNsInterval;
2743	else
2744	pCpu->Ppt.cTicksHistoriziationInterval = 1;
2745	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
2746
2747	/SUPR0Printf("Cpu%u: change to %u Hz / %u ns\n", pCpu->idxCpuSet, uHistMaxHz, cNsInterval);/
2748	RTTimerChangeInterval(pTimer, cNsInterval);
2749	}
2750	else
2751	{
2752	/*
2753	* Stop it.
2754	*/
2755	pCpu->Ppt.fStarted = false;
2756	pCpu->Ppt.uTimerHz = 0;
2757	pCpu->Ppt.cNsInterval = 0;
2758	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
2759
2760	/SUPR0Printf("Cpu%u: stopping (%u Hz)\n", pCpu->idxCpuSet, uHistMaxHz);/
2761	RTTimerStop(pTimer);
2762	}
2763	}
2764	else
2765	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
2766	}
2767	#endif /* GVMM_SCHED_WITH_PPT */
2768
2769
2770	/**
2771	* Updates the periodic preemption timer for the calling CPU.
2772	*
2773	* The caller must have disabled preemption!
2774	* The caller must check that the host can do high resolution timers.
2775	*
2776	* @param pGVM The global (ring-0) VM structure.
2777	* @param idHostCpu The current host CPU id.
2778	* @param uHz The desired frequency.
2779	*/
2780	GVMMR0DECL(void) GVMMR0SchedUpdatePeriodicPreemptionTimer(PGVM pGVM, RTCPUID idHostCpu, uint32_t uHz)
2781	{
2782	NOREF(pGVM);
2783	#ifdef GVMM_SCHED_WITH_PPT
2784	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2785	Assert(RTTimerCanDoHighResolution());
2786
2787	/*
2788	* Resolve the per CPU data.
2789	*/
2790	uint32_t iCpu = RTMpCpuIdToSetIndex(idHostCpu);
2791	PGVMM pGVMM = g_pGVMM;
2792	if ( !RT_VALID_PTR(pGVMM)
2793	\|\| pGVMM->u32Magic != GVMM_MAGIC)
2794	return;
2795	AssertMsgReturnVoid(iCpu < pGVMM->cHostCpus, ("iCpu=%d cHostCpus=%d\n", iCpu, pGVMM->cHostCpus));
2796	PGVMMHOSTCPU pCpu = &pGVMM->aHostCpus[iCpu];
2797	AssertMsgReturnVoid( pCpu->u32Magic == GVMMHOSTCPU_MAGIC
2798	&& pCpu->idCpu == idHostCpu,
2799	("u32Magic=%#x idCpu=% idHostCpu=%d\n", pCpu->u32Magic, pCpu->idCpu, idHostCpu));
2800
2801	/*
2802	* Check whether we need to do anything about the timer.
2803	* We have to be a little bit careful since we might be race the timer
2804	* callback here.
2805	*/
2806	if (uHz > 16384)
2807	uHz = 16384; /** @todo add a query method for this! */
2808	if (RT_UNLIKELY( uHz > ASMAtomicReadU32(&pCpu->Ppt.uDesiredHz)
2809	&& uHz >= pCpu->Ppt.uMinHz
2810	&& !pCpu->Ppt.fStarting /* solaris paranoia */))
2811	{
2812	RTSpinlockAcquire(pCpu->Ppt.hSpinlock);
2813
2814	pCpu->Ppt.uDesiredHz = uHz;
2815	uint32_t cNsInterval = 0;
2816	if (!pCpu->Ppt.fStarted)
2817	{
2818	pCpu->Ppt.cStarts++;
2819	pCpu->Ppt.fStarted = true;
2820	pCpu->Ppt.fStarting = true;
2821	pCpu->Ppt.iTickHistorization = 0;
2822	pCpu->Ppt.uTimerHz = uHz;
2823	pCpu->Ppt.cNsInterval = cNsInterval = RT_NS_1SEC / uHz;
2824	if (cNsInterval < GVMMHOSTCPU_PPT_HIST_INTERVAL_NS)
2825	pCpu->Ppt.cTicksHistoriziationInterval = ( GVMMHOSTCPU_PPT_HIST_INTERVAL_NS
2826	+ GVMMHOSTCPU_PPT_HIST_INTERVAL_NS / 2 - 1)
2827	/ cNsInterval;
2828	else
2829	pCpu->Ppt.cTicksHistoriziationInterval = 1;
2830	}
2831
2832	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
2833
2834	if (cNsInterval)
2835	{
2836	RTTimerChangeInterval(pCpu->Ppt.pTimer, cNsInterval);
2837	int rc = RTTimerStart(pCpu->Ppt.pTimer, cNsInterval);
2838	AssertRC(rc);
2839
2840	RTSpinlockAcquire(pCpu->Ppt.hSpinlock);
2841	if (RT_FAILURE(rc))
2842	pCpu->Ppt.fStarted = false;
2843	pCpu->Ppt.fStarting = false;
2844	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
2845	}
2846	}
2847	#else /* !GVMM_SCHED_WITH_PPT */
2848	NOREF(idHostCpu); NOREF(uHz);
2849	#endif /* !GVMM_SCHED_WITH_PPT */
2850	}
2851
2852
2853	/**
2854	* Calls @a pfnCallback for each VM in the system.
2855	*
2856	* This will enumerate the VMs while holding the global VM used list lock in
2857	* shared mode. So, only suitable for simple work. If more expensive work
2858	* needs doing, a different approach must be taken as using this API would
2859	* otherwise block VM creation and destruction.
2860	*
2861	* @returns VBox status code.
2862	* @param pfnCallback The callback function.
2863	* @param pvUser User argument to the callback.
2864	*/
2865	GVMMR0DECL(int) GVMMR0EnumVMs(PFNGVMMR0ENUMCALLBACK pfnCallback, void *pvUser)
2866	{
2867	PGVMM pGVMM;
2868	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
2869
2870	int rc = VINF_SUCCESS;
2871	GVMMR0_USED_SHARED_LOCK(pGVMM);
2872	for (unsigned i = pGVMM->iUsedHead, cLoops = 0;
2873	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
2874	i = pGVMM->aHandles[i].iNext, cLoops++)
2875	{
2876	PGVM pGVM = pGVMM->aHandles[i].pGVM;
2877	if ( RT_VALID_PTR(pGVM)
2878	&& RT_VALID_PTR(pGVMM->aHandles[i].pvObj)
2879	&& pGVM->u32Magic == GVM_MAGIC)
2880	{
2881	rc = pfnCallback(pGVM, pvUser);
2882	if (rc != VINF_SUCCESS)
2883	break;
2884	}
2885
2886	AssertBreak(cLoops < RT_ELEMENTS(pGVMM->aHandles) * 4); /* paranoia */
2887	}
2888	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2889	return rc;
2890	}
2891
2892
2893	/**
2894	* Retrieves the GVMM statistics visible to the caller.
2895	*
2896	* @returns VBox status code.
2897	*
2898	* @param pStats Where to put the statistics.
2899	* @param pSession The current session.
2900	* @param pGVM The GVM to obtain statistics for. Optional.
2901	*/
2902	GVMMR0DECL(int) GVMMR0QueryStatistics(PGVMMSTATS pStats, PSUPDRVSESSION pSession, PGVM pGVM)
2903	{
2904	LogFlow(("GVMMR0QueryStatistics: pStats=%p pSession=%p pGVM=%p\n", pStats, pSession, pGVM));
2905
2906	/*
2907	* Validate input.
2908	*/
2909	AssertPtrReturn(pSession, VERR_INVALID_POINTER);
2910	AssertPtrReturn(pStats, VERR_INVALID_POINTER);
2911	pStats->cVMs = 0; /* (crash before taking the sem...) */
2912
2913	/*
2914	* Take the lock and get the VM statistics.
2915	*/
2916	PGVMM pGVMM;
2917	if (pGVM)
2918	{
2919	int rc = gvmmR0ByGVM(pGVM, &pGVMM, true /fTakeUsedLock/);
2920	if (RT_FAILURE(rc))
2921	return rc;
2922	pStats->SchedVM = pGVM->gvmm.s.StatsSched;
2923	}
2924	else
2925	{
2926	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
2927	memset(&pStats->SchedVM, 0, sizeof(pStats->SchedVM));
2928
2929	int rc = GVMMR0_USED_SHARED_LOCK(pGVMM);
2930	AssertRCReturn(rc, rc);
2931	}
2932
2933	/*
2934	* Enumerate the VMs and add the ones visible to the statistics.
2935	*/
2936	pStats->cVMs = 0;
2937	pStats->cEMTs = 0;
2938	memset(&pStats->SchedSum, 0, sizeof(pStats->SchedSum));
2939
2940	for (unsigned i = pGVMM->iUsedHead;
2941	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
2942	i = pGVMM->aHandles[i].iNext)
2943	{
2944	PGVM pOtherGVM = pGVMM->aHandles[i].pGVM;
2945	void *pvObj = pGVMM->aHandles[i].pvObj;
2946	if ( RT_VALID_PTR(pvObj)
2947	&& RT_VALID_PTR(pOtherGVM)
2948	&& pOtherGVM->u32Magic == GVM_MAGIC
2949	&& RT_SUCCESS(SUPR0ObjVerifyAccess(pvObj, pSession, NULL)))
2950	{
2951	pStats->cVMs++;
2952	pStats->cEMTs += pOtherGVM->cCpus;
2953
2954	pStats->SchedSum.cHaltCalls += pOtherGVM->gvmm.s.StatsSched.cHaltCalls;
2955	pStats->SchedSum.cHaltBlocking += pOtherGVM->gvmm.s.StatsSched.cHaltBlocking;
2956	pStats->SchedSum.cHaltTimeouts += pOtherGVM->gvmm.s.StatsSched.cHaltTimeouts;
2957	pStats->SchedSum.cHaltNotBlocking += pOtherGVM->gvmm.s.StatsSched.cHaltNotBlocking;
2958	pStats->SchedSum.cHaltWakeUps += pOtherGVM->gvmm.s.StatsSched.cHaltWakeUps;
2959
2960	pStats->SchedSum.cWakeUpCalls += pOtherGVM->gvmm.s.StatsSched.cWakeUpCalls;
2961	pStats->SchedSum.cWakeUpNotHalted += pOtherGVM->gvmm.s.StatsSched.cWakeUpNotHalted;
2962	pStats->SchedSum.cWakeUpWakeUps += pOtherGVM->gvmm.s.StatsSched.cWakeUpWakeUps;
2963
2964	pStats->SchedSum.cPokeCalls += pOtherGVM->gvmm.s.StatsSched.cPokeCalls;
2965	pStats->SchedSum.cPokeNotBusy += pOtherGVM->gvmm.s.StatsSched.cPokeNotBusy;
2966
2967	pStats->SchedSum.cPollCalls += pOtherGVM->gvmm.s.StatsSched.cPollCalls;
2968	pStats->SchedSum.cPollHalts += pOtherGVM->gvmm.s.StatsSched.cPollHalts;
2969	pStats->SchedSum.cPollWakeUps += pOtherGVM->gvmm.s.StatsSched.cPollWakeUps;
2970	}
2971	}
2972
2973	/*
2974	* Copy out the per host CPU statistics.
2975	*/
2976	uint32_t iDstCpu = 0;
2977	uint32_t cSrcCpus = pGVMM->cHostCpus;
2978	for (uint32_t iSrcCpu = 0; iSrcCpu < cSrcCpus; iSrcCpu++)
2979	{
2980	if (pGVMM->aHostCpus[iSrcCpu].idCpu != NIL_RTCPUID)
2981	{
2982	pStats->aHostCpus[iDstCpu].idCpu = pGVMM->aHostCpus[iSrcCpu].idCpu;
2983	pStats->aHostCpus[iDstCpu].idxCpuSet = pGVMM->aHostCpus[iSrcCpu].idxCpuSet;
2984	#ifdef GVMM_SCHED_WITH_PPT
2985	pStats->aHostCpus[iDstCpu].uDesiredHz = pGVMM->aHostCpus[iSrcCpu].Ppt.uDesiredHz;
2986	pStats->aHostCpus[iDstCpu].uTimerHz = pGVMM->aHostCpus[iSrcCpu].Ppt.uTimerHz;
2987	pStats->aHostCpus[iDstCpu].cChanges = pGVMM->aHostCpus[iSrcCpu].Ppt.cChanges;
2988	pStats->aHostCpus[iDstCpu].cStarts = pGVMM->aHostCpus[iSrcCpu].Ppt.cStarts;
2989	#else
2990	pStats->aHostCpus[iDstCpu].uDesiredHz = 0;
2991	pStats->aHostCpus[iDstCpu].uTimerHz = 0;
2992	pStats->aHostCpus[iDstCpu].cChanges = 0;
2993	pStats->aHostCpus[iDstCpu].cStarts = 0;
2994	#endif
2995	iDstCpu++;
2996	if (iDstCpu >= RT_ELEMENTS(pStats->aHostCpus))
2997	break;
2998	}
2999	}
3000	pStats->cHostCpus = iDstCpu;
3001
3002	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
3003
3004	return VINF_SUCCESS;
3005	}
3006
3007
3008	/**
3009	* VMMR0 request wrapper for GVMMR0QueryStatistics.
3010	*
3011	* @returns see GVMMR0QueryStatistics.
3012	* @param pGVM The global (ring-0) VM structure. Optional.
3013	* @param pReq Pointer to the request packet.
3014	* @param pSession The current session.
3015	*/
3016	GVMMR0DECL(int) GVMMR0QueryStatisticsReq(PGVM pGVM, PGVMMQUERYSTATISTICSSREQ pReq, PSUPDRVSESSION pSession)
3017	{
3018	/*
3019	* Validate input and pass it on.
3020	*/
3021	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
3022	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
3023	AssertReturn(pReq->pSession == pSession, VERR_INVALID_PARAMETER);
3024
3025	return GVMMR0QueryStatistics(&pReq->Stats, pSession, pGVM);
3026	}
3027
3028
3029	/**
3030	* Resets the specified GVMM statistics.
3031	*
3032	* @returns VBox status code.
3033	*
3034	* @param pStats Which statistics to reset, that is, non-zero fields indicates which to reset.
3035	* @param pSession The current session.
3036	* @param pGVM The GVM to reset statistics for. Optional.
3037	*/
3038	GVMMR0DECL(int) GVMMR0ResetStatistics(PCGVMMSTATS pStats, PSUPDRVSESSION pSession, PGVM pGVM)
3039	{
3040	LogFlow(("GVMMR0ResetStatistics: pStats=%p pSession=%p pGVM=%p\n", pStats, pSession, pGVM));
3041
3042	/*
3043	* Validate input.
3044	*/
3045	AssertPtrReturn(pSession, VERR_INVALID_POINTER);
3046	AssertPtrReturn(pStats, VERR_INVALID_POINTER);
3047
3048	/*
3049	* Take the lock and get the VM statistics.
3050	*/
3051	PGVMM pGVMM;
3052	if (pGVM)
3053	{
3054	int rc = gvmmR0ByGVM(pGVM, &pGVMM, true /fTakeUsedLock/);
3055	if (RT_FAILURE(rc))
3056	return rc;
3057	# define MAYBE_RESET_FIELD(field) \
3058	do { if (pStats->SchedVM. field ) { pGVM->gvmm.s.StatsSched. field = 0; } } while (0)
3059	MAYBE_RESET_FIELD(cHaltCalls);
3060	MAYBE_RESET_FIELD(cHaltBlocking);
3061	MAYBE_RESET_FIELD(cHaltTimeouts);
3062	MAYBE_RESET_FIELD(cHaltNotBlocking);
3063	MAYBE_RESET_FIELD(cHaltWakeUps);
3064	MAYBE_RESET_FIELD(cWakeUpCalls);
3065	MAYBE_RESET_FIELD(cWakeUpNotHalted);
3066	MAYBE_RESET_FIELD(cWakeUpWakeUps);
3067	MAYBE_RESET_FIELD(cPokeCalls);
3068	MAYBE_RESET_FIELD(cPokeNotBusy);
3069	MAYBE_RESET_FIELD(cPollCalls);
3070	MAYBE_RESET_FIELD(cPollHalts);
3071	MAYBE_RESET_FIELD(cPollWakeUps);
3072	# undef MAYBE_RESET_FIELD
3073	}
3074	else
3075	{
3076	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
3077
3078	int rc = GVMMR0_USED_SHARED_LOCK(pGVMM);
3079	AssertRCReturn(rc, rc);
3080	}
3081
3082	/*
3083	* Enumerate the VMs and add the ones visible to the statistics.
3084	*/
3085	if (!ASMMemIsZero(&pStats->SchedSum, sizeof(pStats->SchedSum)))
3086	{
3087	for (unsigned i = pGVMM->iUsedHead;
3088	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
3089	i = pGVMM->aHandles[i].iNext)
3090	{
3091	PGVM pOtherGVM = pGVMM->aHandles[i].pGVM;
3092	void *pvObj = pGVMM->aHandles[i].pvObj;
3093	if ( RT_VALID_PTR(pvObj)
3094	&& RT_VALID_PTR(pOtherGVM)
3095	&& pOtherGVM->u32Magic == GVM_MAGIC
3096	&& RT_SUCCESS(SUPR0ObjVerifyAccess(pvObj, pSession, NULL)))
3097	{
3098	# define MAYBE_RESET_FIELD(field) \
3099	do { if (pStats->SchedSum. field ) { pOtherGVM->gvmm.s.StatsSched. field = 0; } } while (0)
3100	MAYBE_RESET_FIELD(cHaltCalls);
3101	MAYBE_RESET_FIELD(cHaltBlocking);
3102	MAYBE_RESET_FIELD(cHaltTimeouts);
3103	MAYBE_RESET_FIELD(cHaltNotBlocking);
3104	MAYBE_RESET_FIELD(cHaltWakeUps);
3105	MAYBE_RESET_FIELD(cWakeUpCalls);
3106	MAYBE_RESET_FIELD(cWakeUpNotHalted);
3107	MAYBE_RESET_FIELD(cWakeUpWakeUps);
3108	MAYBE_RESET_FIELD(cPokeCalls);
3109	MAYBE_RESET_FIELD(cPokeNotBusy);
3110	MAYBE_RESET_FIELD(cPollCalls);
3111	MAYBE_RESET_FIELD(cPollHalts);
3112	MAYBE_RESET_FIELD(cPollWakeUps);
3113	# undef MAYBE_RESET_FIELD
3114	}
3115	}
3116	}
3117
3118	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
3119
3120	return VINF_SUCCESS;
3121	}
3122
3123
3124	/**
3125	* VMMR0 request wrapper for GVMMR0ResetStatistics.
3126	*
3127	* @returns see GVMMR0ResetStatistics.
3128	* @param pGVM The global (ring-0) VM structure. Optional.
3129	* @param pReq Pointer to the request packet.
3130	* @param pSession The current session.
3131	*/
3132	GVMMR0DECL(int) GVMMR0ResetStatisticsReq(PGVM pGVM, PGVMMRESETSTATISTICSSREQ pReq, PSUPDRVSESSION pSession)
3133	{
3134	/*
3135	* Validate input and pass it on.
3136	*/
3137	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
3138	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
3139	AssertReturn(pReq->pSession == pSession, VERR_INVALID_PARAMETER);
3140
3141	return GVMMR0ResetStatistics(&pReq->Stats, pSession, pGVM);
3142	}
3143

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/GVMMR0.cpp@ 90976

Download in other formats: