GVMMR0.cpp@ 78927

Last change on this file since 78927 was 78431, checked in by vboxsync, 6 years ago
VMM: Started refactoring GVM & VM structures for bugref:9217
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 121.5 KB

Line
1	/* $Id: GVMMR0.cpp 78431 2019-05-07 14:01:45Z vboxsync $ */
2	/** @file
3	* GVMM - Global VM Manager.
4	*/
5
6	/*
7	* Copyright (C) 2007-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/** @page pg_gvmm GVMM - The Global VM Manager
20	*
21	* The Global VM Manager lives in ring-0. Its main function at the moment is
22	* to manage a list of all running VMs, keep a ring-0 only structure (GVM) for
23	* each of them, and assign them unique identifiers (so GMM can track page
24	* owners). The GVMM also manage some of the host CPU resources, like the
25	* periodic preemption timer.
26	*
27	* The GVMM will create a ring-0 object for each VM when it is registered, this
28	* is both for session cleanup purposes and for having a point where it is
29	* possible to implement usage polices later (in SUPR0ObjRegister).
30	*
31	*
32	* @section sec_gvmm_ppt Periodic Preemption Timer (PPT)
33	*
34	* On system that sports a high resolution kernel timer API, we use per-cpu
35	* timers to generate interrupts that preempts VT-x, AMD-V and raw-mode guest
36	* execution. The timer frequency is calculating by taking the max
37	* TMCalcHostTimerFrequency for all VMs running on a CPU for the last ~160 ms
38	* (RT_ELEMENTS((PGVMMHOSTCPU)0, Ppt.aHzHistory) *
39	* GVMMHOSTCPU_PPT_HIST_INTERVAL_NS).
40	*
41	* The TMCalcHostTimerFrequency() part of the things gets its takes the max
42	* TMTimerSetFrequencyHint() value and adjusts by the current catch-up percent,
43	* warp drive percent and some fudge factors. VMMR0.cpp reports the result via
44	* GVMMR0SchedUpdatePeriodicPreemptionTimer() before switching to the VT-x,
45	* AMD-V and raw-mode execution environments.
46	*/
47
48
49	/*********************************************************************************************************************************
50	* Header Files *
51	*********************************************************************************************************************************/
52	#define LOG_GROUP LOG_GROUP_GVMM
53	#include <VBox/vmm/gvmm.h>
54	#include <VBox/vmm/gmm.h>
55	#include "GVMMR0Internal.h"
56	#include <VBox/vmm/gvm.h>
57	#include <VBox/vmm/vm.h>
58	#include <VBox/vmm/vmcpuset.h>
59	#include <VBox/vmm/vmm.h>
60	#ifdef VBOX_WITH_NEM_R0
61	# include <VBox/vmm/nem.h>
62	#endif
63	#include <VBox/param.h>
64	#include <VBox/err.h>
65
66	#include <iprt/asm.h>
67	#include <iprt/asm-amd64-x86.h>
68	#include <iprt/critsect.h>
69	#include <iprt/mem.h>
70	#include <iprt/semaphore.h>
71	#include <iprt/time.h>
72	#include <VBox/log.h>
73	#include <iprt/thread.h>
74	#include <iprt/process.h>
75	#include <iprt/param.h>
76	#include <iprt/string.h>
77	#include <iprt/assert.h>
78	#include <iprt/mem.h>
79	#include <iprt/memobj.h>
80	#include <iprt/mp.h>
81	#include <iprt/cpuset.h>
82	#include <iprt/spinlock.h>
83	#include <iprt/timer.h>
84
85	#include "dtrace/VBoxVMM.h"
86
87
88	/*********************************************************************************************************************************
89	* Defined Constants And Macros *
90	*********************************************************************************************************************************/
91	#if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS) \|\| defined(DOXYGEN_RUNNING)
92	/** Define this to enable the periodic preemption timer. */
93	# define GVMM_SCHED_WITH_PPT
94	#endif
95
96
97	/** @def GVMM_CHECK_SMAP_SETUP
98	* SMAP check setup. */
99	/** @def GVMM_CHECK_SMAP_CHECK
100	* Checks that the AC flag is set if SMAP is enabled. If AC is not set,
101	* it will be logged and @a a_BadExpr is executed. */
102	/** @def GVMM_CHECK_SMAP_CHECK2
103	* Checks that the AC flag is set if SMAP is enabled. If AC is not set, it will
104	* be logged, written to the VMs assertion text buffer, and @a a_BadExpr is
105	* executed. */
106	#if defined(VBOX_STRICT) \|\| 1
107	# define GVMM_CHECK_SMAP_SETUP() uint32_t const fKernelFeatures = SUPR0GetKernelFeatures()
108	# define GVMM_CHECK_SMAP_CHECK(a_BadExpr) \
109	do { \
110	if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
111	{ \
112	RTCCUINTREG fEflCheck = ASMGetFlags(); \
113	if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
114	{ /* likely */ } \
115	else \
116	{ \
117	SUPR0Printf("%s, line %d: EFLAGS.AC is clear! (%#x)\n", __FUNCTION__, __LINE__, (uint32_t)fEflCheck); \
118	a_BadExpr; \
119	} \
120	} \
121	} while (0)
122	# define GVMM_CHECK_SMAP_CHECK2(a_pVM, a_BadExpr) \
123	do { \
124	if (fKernelFeatures & SUPKERNELFEATURES_SMAP) \
125	{ \
126	RTCCUINTREG fEflCheck = ASMGetFlags(); \
127	if (RT_LIKELY(fEflCheck & X86_EFL_AC)) \
128	{ /* likely */ } \
129	else \
130	{ \
131	SUPR0BadContext((a_pVM) ? (a_pVM)->pSession : NULL, __FILE__, __LINE__, "EFLAGS.AC is zero!"); \
132	a_BadExpr; \
133	} \
134	} \
135	} while (0)
136	#else
137	# define GVMM_CHECK_SMAP_SETUP() uint32_t const fKernelFeatures = 0
138	# define GVMM_CHECK_SMAP_CHECK(a_BadExpr) NOREF(fKernelFeatures)
139	# define GVMM_CHECK_SMAP_CHECK2(a_pVM, a_BadExpr) NOREF(fKernelFeatures)
140	#endif
141
142
143
144	/*********************************************************************************************************************************
145	* Structures and Typedefs *
146	*********************************************************************************************************************************/
147
148	/**
149	* Global VM handle.
150	*/
151	typedef struct GVMHANDLE
152	{
153	/** The index of the next handle in the list (free or used). (0 is nil.) */
154	uint16_t volatile iNext;
155	/** Our own index / handle value. */
156	uint16_t iSelf;
157	/** The process ID of the handle owner.
158	* This is used for access checks. */
159	RTPROCESS ProcId;
160	/** The pointer to the ring-0 only (aka global) VM structure. */
161	PGVM pGVM;
162	/** The ring-0 mapping of the shared VM instance data. */
163	PVM pVM;
164	/** The virtual machine object. */
165	void *pvObj;
166	/** The session this VM is associated with. */
167	PSUPDRVSESSION pSession;
168	/** The ring-0 handle of the EMT0 thread.
169	* This is used for ownership checks as well as looking up a VM handle by thread
170	* at times like assertions. */
171	RTNATIVETHREAD hEMT0;
172	} GVMHANDLE;
173	/** Pointer to a global VM handle. */
174	typedef GVMHANDLE *PGVMHANDLE;
175
176	/** Number of GVM handles (including the NIL handle). */
177	#if HC_ARCH_BITS == 64
178	# define GVMM_MAX_HANDLES 8192
179	#else
180	# define GVMM_MAX_HANDLES 128
181	#endif
182
183	/**
184	* Per host CPU GVMM data.
185	*/
186	typedef struct GVMMHOSTCPU
187	{
188	/** Magic number (GVMMHOSTCPU_MAGIC). */
189	uint32_t volatile u32Magic;
190	/** The CPU ID. */
191	RTCPUID idCpu;
192	/** The CPU set index. */
193	uint32_t idxCpuSet;
194
195	#ifdef GVMM_SCHED_WITH_PPT
196	/** Periodic preemption timer data. */
197	struct
198	{
199	/** The handle to the periodic preemption timer. */
200	PRTTIMER pTimer;
201	/** Spinlock protecting the data below. */
202	RTSPINLOCK hSpinlock;
203	/** The smalles Hz that we need to care about. (static) */
204	uint32_t uMinHz;
205	/** The number of ticks between each historization. */
206	uint32_t cTicksHistoriziationInterval;
207	/** The current historization tick (counting up to
208	* cTicksHistoriziationInterval and then resetting). */
209	uint32_t iTickHistorization;
210	/** The current timer interval. This is set to 0 when inactive. */
211	uint32_t cNsInterval;
212	/** The current timer frequency. This is set to 0 when inactive. */
213	uint32_t uTimerHz;
214	/** The current max frequency reported by the EMTs.
215	* This gets historicize and reset by the timer callback. This is
216	* read without holding the spinlock, so needs atomic updating. */
217	uint32_t volatile uDesiredHz;
218	/** Whether the timer was started or not. */
219	bool volatile fStarted;
220	/** Set if we're starting timer. */
221	bool volatile fStarting;
222	/** The index of the next history entry (mod it). */
223	uint32_t iHzHistory;
224	/** Historicized uDesiredHz values. The array wraps around, new entries
225	* are added at iHzHistory. This is updated approximately every
226	* GVMMHOSTCPU_PPT_HIST_INTERVAL_NS by the timer callback. */
227	uint32_t aHzHistory[8];
228	/** Statistics counter for recording the number of interval changes. */
229	uint32_t cChanges;
230	/** Statistics counter for recording the number of timer starts. */
231	uint32_t cStarts;
232	} Ppt;
233	#endif /* GVMM_SCHED_WITH_PPT */
234
235	} GVMMHOSTCPU;
236	/** Pointer to the per host CPU GVMM data. */
237	typedef GVMMHOSTCPU *PGVMMHOSTCPU;
238	/** The GVMMHOSTCPU::u32Magic value (Petra, Tanya & Rachel Haden). */
239	#define GVMMHOSTCPU_MAGIC UINT32_C(0x19711011)
240	/** The interval on history entry should cover (approximately) give in
241	* nanoseconds. */
242	#define GVMMHOSTCPU_PPT_HIST_INTERVAL_NS UINT32_C(20000000)
243
244
245	/**
246	* The GVMM instance data.
247	*/
248	typedef struct GVMM
249	{
250	/** Eyecatcher / magic. */
251	uint32_t u32Magic;
252	/** The index of the head of the free handle chain. (0 is nil.) */
253	uint16_t volatile iFreeHead;
254	/** The index of the head of the active handle chain. (0 is nil.) */
255	uint16_t volatile iUsedHead;
256	/** The number of VMs. */
257	uint16_t volatile cVMs;
258	/** Alignment padding. */
259	uint16_t u16Reserved;
260	/** The number of EMTs. */
261	uint32_t volatile cEMTs;
262	/** The number of EMTs that have halted in GVMMR0SchedHalt. */
263	uint32_t volatile cHaltedEMTs;
264	/** Mini lock for restricting early wake-ups to one thread. */
265	bool volatile fDoingEarlyWakeUps;
266	bool afPadding[3]; /*< explicit alignment padding. /
267	/** When the next halted or sleeping EMT will wake up.
268	* This is set to 0 when it needs recalculating and to UINT64_MAX when
269	* there are no halted or sleeping EMTs in the GVMM. */
270	uint64_t uNsNextEmtWakeup;
271	/** The lock used to serialize VM creation, destruction and associated events that
272	* isn't performance critical. Owners may acquire the list lock. */
273	RTCRITSECT CreateDestroyLock;
274	/** The lock used to serialize used list updates and accesses.
275	* This indirectly includes scheduling since the scheduler will have to walk the
276	* used list to examin running VMs. Owners may not acquire any other locks. */
277	RTCRITSECTRW UsedLock;
278	/** The handle array.
279	* The size of this array defines the maximum number of currently running VMs.
280	* The first entry is unused as it represents the NIL handle. */
281	GVMHANDLE aHandles[GVMM_MAX_HANDLES];
282
283	/** @gcfgm{/GVMM/cEMTsMeansCompany, 32-bit, 0, UINT32_MAX, 1}
284	* The number of EMTs that means we no longer consider ourselves alone on a
285	* CPU/Core.
286	*/
287	uint32_t cEMTsMeansCompany;
288	/** @gcfgm{/GVMM/MinSleepAlone,32-bit, 0, 100000000, 750000, ns}
289	* The minimum sleep time for when we're alone, in nano seconds.
290	*/
291	uint32_t nsMinSleepAlone;
292	/** @gcfgm{/GVMM/MinSleepCompany,32-bit,0, 100000000, 15000, ns}
293	* The minimum sleep time for when we've got company, in nano seconds.
294	*/
295	uint32_t nsMinSleepCompany;
296	/** @gcfgm{/GVMM/EarlyWakeUp1, 32-bit, 0, 100000000, 25000, ns}
297	* The limit for the first round of early wake-ups, given in nano seconds.
298	*/
299	uint32_t nsEarlyWakeUp1;
300	/** @gcfgm{/GVMM/EarlyWakeUp2, 32-bit, 0, 100000000, 50000, ns}
301	* The limit for the second round of early wake-ups, given in nano seconds.
302	*/
303	uint32_t nsEarlyWakeUp2;
304
305	/** Set if we're doing early wake-ups.
306	* This reflects nsEarlyWakeUp1 and nsEarlyWakeUp2. */
307	bool volatile fDoEarlyWakeUps;
308
309	/** The number of entries in the host CPU array (aHostCpus). */
310	uint32_t cHostCpus;
311	/** Per host CPU data (variable length). */
312	GVMMHOSTCPU aHostCpus[1];
313	} GVMM;
314	AssertCompileMemberAlignment(GVMM, CreateDestroyLock, 8);
315	AssertCompileMemberAlignment(GVMM, UsedLock, 8);
316	AssertCompileMemberAlignment(GVMM, uNsNextEmtWakeup, 8);
317	/** Pointer to the GVMM instance data. */
318	typedef GVMM *PGVMM;
319
320	/** The GVMM::u32Magic value (Charlie Haden). */
321	#define GVMM_MAGIC UINT32_C(0x19370806)
322
323
324
325	/*********************************************************************************************************************************
326	* Global Variables *
327	*********************************************************************************************************************************/
328	/** Pointer to the GVMM instance data.
329	* (Just my general dislike for global variables.) */
330	static PGVMM g_pGVMM = NULL;
331
332	/** Macro for obtaining and validating the g_pGVMM pointer.
333	* On failure it will return from the invoking function with the specified return value.
334	*
335	* @param pGVMM The name of the pGVMM variable.
336	* @param rc The return value on failure. Use VERR_GVMM_INSTANCE for VBox
337	* status codes.
338	*/
339	#define GVMM_GET_VALID_INSTANCE(pGVMM, rc) \
340	do { \
341	(pGVMM) = g_pGVMM;\
342	AssertPtrReturn((pGVMM), (rc)); \
343	AssertMsgReturn((pGVMM)->u32Magic == GVMM_MAGIC, ("%p - %#x\n", (pGVMM), (pGVMM)->u32Magic), (rc)); \
344	} while (0)
345
346	/** Macro for obtaining and validating the g_pGVMM pointer, void function variant.
347	* On failure it will return from the invoking function.
348	*
349	* @param pGVMM The name of the pGVMM variable.
350	*/
351	#define GVMM_GET_VALID_INSTANCE_VOID(pGVMM) \
352	do { \
353	(pGVMM) = g_pGVMM;\
354	AssertPtrReturnVoid((pGVMM)); \
355	AssertMsgReturnVoid((pGVMM)->u32Magic == GVMM_MAGIC, ("%p - %#x\n", (pGVMM), (pGVMM)->u32Magic)); \
356	} while (0)
357
358
359	/*********************************************************************************************************************************
360	* Internal Functions *
361	*********************************************************************************************************************************/
362	#ifdef VBOX_BUGREF_9217
363	static void gvmmR0InitPerVMData(PGVM pGVM, int16_t hSelf, VMCPUID cCpus, PSUPDRVSESSION pSession);
364	#else
365	static void gvmmR0InitPerVMData(PGVM pGVM);
366	#endif
367	static DECLCALLBACK(void) gvmmR0HandleObjDestructor(void pvObj, void pvGVMM, void *pvHandle);
368	static int gvmmR0ByGVMandVM(PGVM pGVM, PVM pVM, PGVMM *ppGVMM, bool fTakeUsedLock);
369	static int gvmmR0ByGVMandVMandEMT(PGVM pGVM, PVM pVM, VMCPUID idCpu, PGVMM *ppGVMM);
370
371	#ifdef GVMM_SCHED_WITH_PPT
372	static DECLCALLBACK(void) gvmmR0SchedPeriodicPreemptionTimerCallback(PRTTIMER pTimer, void *pvUser, uint64_t iTick);
373	#endif
374
375
376	/**
377	* Initializes the GVMM.
378	*
379	* This is called while owning the loader semaphore (see supdrvIOCtl_LdrLoad()).
380	*
381	* @returns VBox status code.
382	*/
383	GVMMR0DECL(int) GVMMR0Init(void)
384	{
385	LogFlow(("GVMMR0Init:\n"));
386
387	/*
388	* Allocate and initialize the instance data.
389	*/
390	uint32_t cHostCpus = RTMpGetArraySize();
391	AssertMsgReturn(cHostCpus > 0 && cHostCpus < _64K, ("%d", (int)cHostCpus), VERR_GVMM_HOST_CPU_RANGE);
392
393	PGVMM pGVMM = (PGVMM)RTMemAllocZ(RT_UOFFSETOF_DYN(GVMM, aHostCpus[cHostCpus]));
394	if (!pGVMM)
395	return VERR_NO_MEMORY;
396	int rc = RTCritSectInitEx(&pGVMM->CreateDestroyLock, 0, NIL_RTLOCKVALCLASS, RTLOCKVAL_SUB_CLASS_NONE,
397	"GVMM-CreateDestroyLock");
398	if (RT_SUCCESS(rc))
399	{
400	rc = RTCritSectRwInitEx(&pGVMM->UsedLock, 0, NIL_RTLOCKVALCLASS, RTLOCKVAL_SUB_CLASS_NONE, "GVMM-UsedLock");
401	if (RT_SUCCESS(rc))
402	{
403	pGVMM->u32Magic = GVMM_MAGIC;
404	pGVMM->iUsedHead = 0;
405	pGVMM->iFreeHead = 1;
406
407	/* the nil handle */
408	pGVMM->aHandles[0].iSelf = 0;
409	pGVMM->aHandles[0].iNext = 0;
410
411	/* the tail */
412	unsigned i = RT_ELEMENTS(pGVMM->aHandles) - 1;
413	pGVMM->aHandles[i].iSelf = i;
414	pGVMM->aHandles[i].iNext = 0; /* nil */
415
416	/* the rest */
417	while (i-- > 1)
418	{
419	pGVMM->aHandles[i].iSelf = i;
420	pGVMM->aHandles[i].iNext = i + 1;
421	}
422
423	/* The default configuration values. */
424	uint32_t cNsResolution = RTSemEventMultiGetResolution();
425	pGVMM->cEMTsMeansCompany = 1; /** @todo should be adjusted to relative to the cpu count or something... */
426	if (cNsResolution >= 5*RT_NS_100US)
427	{
428	pGVMM->nsMinSleepAlone = 750000 /* ns (0.750 ms) /; /* @todo this should be adjusted to be 75% (or something) of the scheduler granularity... */
429	pGVMM->nsMinSleepCompany = 15000 /* ns (0.015 ms) */;
430	pGVMM->nsEarlyWakeUp1 = 25000 /* ns (0.025 ms) */;
431	pGVMM->nsEarlyWakeUp2 = 50000 /* ns (0.050 ms) */;
432	}
433	else if (cNsResolution > RT_NS_100US)
434	{
435	pGVMM->nsMinSleepAlone = cNsResolution / 2;
436	pGVMM->nsMinSleepCompany = cNsResolution / 4;
437	pGVMM->nsEarlyWakeUp1 = 0;
438	pGVMM->nsEarlyWakeUp2 = 0;
439	}
440	else
441	{
442	pGVMM->nsMinSleepAlone = 2000;
443	pGVMM->nsMinSleepCompany = 2000;
444	pGVMM->nsEarlyWakeUp1 = 0;
445	pGVMM->nsEarlyWakeUp2 = 0;
446	}
447	pGVMM->fDoEarlyWakeUps = pGVMM->nsEarlyWakeUp1 > 0 && pGVMM->nsEarlyWakeUp2 > 0;
448
449	/* The host CPU data. */
450	pGVMM->cHostCpus = cHostCpus;
451	uint32_t iCpu = cHostCpus;
452	RTCPUSET PossibleSet;
453	RTMpGetSet(&PossibleSet);
454	while (iCpu-- > 0)
455	{
456	pGVMM->aHostCpus[iCpu].idxCpuSet = iCpu;
457	#ifdef GVMM_SCHED_WITH_PPT
458	pGVMM->aHostCpus[iCpu].Ppt.pTimer = NULL;
459	pGVMM->aHostCpus[iCpu].Ppt.hSpinlock = NIL_RTSPINLOCK;
460	pGVMM->aHostCpus[iCpu].Ppt.uMinHz = 5; /** @todo Add some API which figures this one out. (not that important) */
461	pGVMM->aHostCpus[iCpu].Ppt.cTicksHistoriziationInterval = 1;
462	//pGVMM->aHostCpus[iCpu].Ppt.iTickHistorization = 0;
463	//pGVMM->aHostCpus[iCpu].Ppt.cNsInterval = 0;
464	//pGVMM->aHostCpus[iCpu].Ppt.uTimerHz = 0;
465	//pGVMM->aHostCpus[iCpu].Ppt.uDesiredHz = 0;
466	//pGVMM->aHostCpus[iCpu].Ppt.fStarted = false;
467	//pGVMM->aHostCpus[iCpu].Ppt.fStarting = false;
468	//pGVMM->aHostCpus[iCpu].Ppt.iHzHistory = 0;
469	//pGVMM->aHostCpus[iCpu].Ppt.aHzHistory = {0};
470	#endif
471
472	if (RTCpuSetIsMember(&PossibleSet, iCpu))
473	{
474	pGVMM->aHostCpus[iCpu].idCpu = RTMpCpuIdFromSetIndex(iCpu);
475	pGVMM->aHostCpus[iCpu].u32Magic = GVMMHOSTCPU_MAGIC;
476
477	#ifdef GVMM_SCHED_WITH_PPT
478	rc = RTTimerCreateEx(&pGVMM->aHostCpus[iCpu].Ppt.pTimer,
479	5010001000 /* whatever */,
480	RTTIMER_FLAGS_CPU(iCpu) \| RTTIMER_FLAGS_HIGH_RES,
481	gvmmR0SchedPeriodicPreemptionTimerCallback,
482	&pGVMM->aHostCpus[iCpu]);
483	if (RT_SUCCESS(rc))
484	rc = RTSpinlockCreate(&pGVMM->aHostCpus[iCpu].Ppt.hSpinlock, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "GVMM/CPU");
485	if (RT_FAILURE(rc))
486	{
487	while (iCpu < cHostCpus)
488	{
489	RTTimerDestroy(pGVMM->aHostCpus[iCpu].Ppt.pTimer);
490	RTSpinlockDestroy(pGVMM->aHostCpus[iCpu].Ppt.hSpinlock);
491	pGVMM->aHostCpus[iCpu].Ppt.hSpinlock = NIL_RTSPINLOCK;
492	iCpu++;
493	}
494	break;
495	}
496	#endif
497	}
498	else
499	{
500	pGVMM->aHostCpus[iCpu].idCpu = NIL_RTCPUID;
501	pGVMM->aHostCpus[iCpu].u32Magic = 0;
502	}
503	}
504	if (RT_SUCCESS(rc))
505	{
506	g_pGVMM = pGVMM;
507	LogFlow(("GVMMR0Init: pGVMM=%p cHostCpus=%u\n", pGVMM, cHostCpus));
508	return VINF_SUCCESS;
509	}
510
511	/* bail out. */
512	RTCritSectRwDelete(&pGVMM->UsedLock);
513	}
514	RTCritSectDelete(&pGVMM->CreateDestroyLock);
515	}
516
517	RTMemFree(pGVMM);
518	return rc;
519	}
520
521
522	/**
523	* Terminates the GVM.
524	*
525	* This is called while owning the loader semaphore (see supdrvLdrFree()).
526	* And unless something is wrong, there should be absolutely no VMs
527	* registered at this point.
528	*/
529	GVMMR0DECL(void) GVMMR0Term(void)
530	{
531	LogFlow(("GVMMR0Term:\n"));
532
533	PGVMM pGVMM = g_pGVMM;
534	g_pGVMM = NULL;
535	if (RT_UNLIKELY(!VALID_PTR(pGVMM)))
536	{
537	SUPR0Printf("GVMMR0Term: pGVMM=%RKv\n", pGVMM);
538	return;
539	}
540
541	/*
542	* First of all, stop all active timers.
543	*/
544	uint32_t cActiveTimers = 0;
545	uint32_t iCpu = pGVMM->cHostCpus;
546	while (iCpu-- > 0)
547	{
548	ASMAtomicWriteU32(&pGVMM->aHostCpus[iCpu].u32Magic, ~GVMMHOSTCPU_MAGIC);
549	#ifdef GVMM_SCHED_WITH_PPT
550	if ( pGVMM->aHostCpus[iCpu].Ppt.pTimer != NULL
551	&& RT_SUCCESS(RTTimerStop(pGVMM->aHostCpus[iCpu].Ppt.pTimer)))
552	cActiveTimers++;
553	#endif
554	}
555	if (cActiveTimers)
556	RTThreadSleep(1); /* fudge */
557
558	/*
559	* Invalidate the and free resources.
560	*/
561	pGVMM->u32Magic = ~GVMM_MAGIC;
562	RTCritSectRwDelete(&pGVMM->UsedLock);
563	RTCritSectDelete(&pGVMM->CreateDestroyLock);
564
565	pGVMM->iFreeHead = 0;
566	if (pGVMM->iUsedHead)
567	{
568	SUPR0Printf("GVMMR0Term: iUsedHead=%#x! (cVMs=%#x cEMTs=%#x)\n", pGVMM->iUsedHead, pGVMM->cVMs, pGVMM->cEMTs);
569	pGVMM->iUsedHead = 0;
570	}
571
572	#ifdef GVMM_SCHED_WITH_PPT
573	iCpu = pGVMM->cHostCpus;
574	while (iCpu-- > 0)
575	{
576	RTTimerDestroy(pGVMM->aHostCpus[iCpu].Ppt.pTimer);
577	pGVMM->aHostCpus[iCpu].Ppt.pTimer = NULL;
578	RTSpinlockDestroy(pGVMM->aHostCpus[iCpu].Ppt.hSpinlock);
579	pGVMM->aHostCpus[iCpu].Ppt.hSpinlock = NIL_RTSPINLOCK;
580	}
581	#endif
582
583	RTMemFree(pGVMM);
584	}
585
586
587	/**
588	* A quick hack for setting global config values.
589	*
590	* @returns VBox status code.
591	*
592	* @param pSession The session handle. Used for authentication.
593	* @param pszName The variable name.
594	* @param u64Value The new value.
595	*/
596	GVMMR0DECL(int) GVMMR0SetConfig(PSUPDRVSESSION pSession, const char *pszName, uint64_t u64Value)
597	{
598	/*
599	* Validate input.
600	*/
601	PGVMM pGVMM;
602	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
603	AssertPtrReturn(pSession, VERR_INVALID_HANDLE);
604	AssertPtrReturn(pszName, VERR_INVALID_POINTER);
605
606	/*
607	* String switch time!
608	*/
609	if (strncmp(pszName, RT_STR_TUPLE("/GVMM/")))
610	return VERR_CFGM_VALUE_NOT_FOUND; /* borrow status codes from CFGM... */
611	int rc = VINF_SUCCESS;
612	pszName += sizeof("/GVMM/") - 1;
613	if (!strcmp(pszName, "cEMTsMeansCompany"))
614	{
615	if (u64Value <= UINT32_MAX)
616	pGVMM->cEMTsMeansCompany = u64Value;
617	else
618	rc = VERR_OUT_OF_RANGE;
619	}
620	else if (!strcmp(pszName, "MinSleepAlone"))
621	{
622	if (u64Value <= RT_NS_100MS)
623	pGVMM->nsMinSleepAlone = u64Value;
624	else
625	rc = VERR_OUT_OF_RANGE;
626	}
627	else if (!strcmp(pszName, "MinSleepCompany"))
628	{
629	if (u64Value <= RT_NS_100MS)
630	pGVMM->nsMinSleepCompany = u64Value;
631	else
632	rc = VERR_OUT_OF_RANGE;
633	}
634	else if (!strcmp(pszName, "EarlyWakeUp1"))
635	{
636	if (u64Value <= RT_NS_100MS)
637	{
638	pGVMM->nsEarlyWakeUp1 = u64Value;
639	pGVMM->fDoEarlyWakeUps = pGVMM->nsEarlyWakeUp1 > 0 && pGVMM->nsEarlyWakeUp2 > 0;
640	}
641	else
642	rc = VERR_OUT_OF_RANGE;
643	}
644	else if (!strcmp(pszName, "EarlyWakeUp2"))
645	{
646	if (u64Value <= RT_NS_100MS)
647	{
648	pGVMM->nsEarlyWakeUp2 = u64Value;
649	pGVMM->fDoEarlyWakeUps = pGVMM->nsEarlyWakeUp1 > 0 && pGVMM->nsEarlyWakeUp2 > 0;
650	}
651	else
652	rc = VERR_OUT_OF_RANGE;
653	}
654	else
655	rc = VERR_CFGM_VALUE_NOT_FOUND;
656	return rc;
657	}
658
659
660	/**
661	* A quick hack for getting global config values.
662	*
663	* @returns VBox status code.
664	*
665	* @param pSession The session handle. Used for authentication.
666	* @param pszName The variable name.
667	* @param pu64Value Where to return the value.
668	*/
669	GVMMR0DECL(int) GVMMR0QueryConfig(PSUPDRVSESSION pSession, const char pszName, uint64_t pu64Value)
670	{
671	/*
672	* Validate input.
673	*/
674	PGVMM pGVMM;
675	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
676	AssertPtrReturn(pSession, VERR_INVALID_HANDLE);
677	AssertPtrReturn(pszName, VERR_INVALID_POINTER);
678	AssertPtrReturn(pu64Value, VERR_INVALID_POINTER);
679
680	/*
681	* String switch time!
682	*/
683	if (strncmp(pszName, RT_STR_TUPLE("/GVMM/")))
684	return VERR_CFGM_VALUE_NOT_FOUND; /* borrow status codes from CFGM... */
685	int rc = VINF_SUCCESS;
686	pszName += sizeof("/GVMM/") - 1;
687	if (!strcmp(pszName, "cEMTsMeansCompany"))
688	*pu64Value = pGVMM->cEMTsMeansCompany;
689	else if (!strcmp(pszName, "MinSleepAlone"))
690	*pu64Value = pGVMM->nsMinSleepAlone;
691	else if (!strcmp(pszName, "MinSleepCompany"))
692	*pu64Value = pGVMM->nsMinSleepCompany;
693	else if (!strcmp(pszName, "EarlyWakeUp1"))
694	*pu64Value = pGVMM->nsEarlyWakeUp1;
695	else if (!strcmp(pszName, "EarlyWakeUp2"))
696	*pu64Value = pGVMM->nsEarlyWakeUp2;
697	else
698	rc = VERR_CFGM_VALUE_NOT_FOUND;
699	return rc;
700	}
701
702
703	/**
704	* Acquire the 'used' lock in shared mode.
705	*
706	* This prevents destruction of the VM while we're in ring-0.
707	*
708	* @returns IPRT status code, see RTSemFastMutexRequest.
709	* @param a_pGVMM The GVMM instance data.
710	* @sa GVMMR0_USED_SHARED_UNLOCK, GVMMR0_USED_EXCLUSIVE_LOCK
711	*/
712	#define GVMMR0_USED_SHARED_LOCK(a_pGVMM) RTCritSectRwEnterShared(&(a_pGVMM)->UsedLock)
713
714	/**
715	* Release the 'used' lock in when owning it in shared mode.
716	*
717	* @returns IPRT status code, see RTSemFastMutexRequest.
718	* @param a_pGVMM The GVMM instance data.
719	* @sa GVMMR0_USED_SHARED_LOCK
720	*/
721	#define GVMMR0_USED_SHARED_UNLOCK(a_pGVMM) RTCritSectRwLeaveShared(&(a_pGVMM)->UsedLock)
722
723	/**
724	* Acquire the 'used' lock in exclusive mode.
725	*
726	* Only use this function when making changes to the used list.
727	*
728	* @returns IPRT status code, see RTSemFastMutexRequest.
729	* @param a_pGVMM The GVMM instance data.
730	* @sa GVMMR0_USED_EXCLUSIVE_UNLOCK
731	*/
732	#define GVMMR0_USED_EXCLUSIVE_LOCK(a_pGVMM) RTCritSectRwEnterExcl(&(a_pGVMM)->UsedLock)
733
734	/**
735	* Release the 'used' lock when owning it in exclusive mode.
736	*
737	* @returns IPRT status code, see RTSemFastMutexRelease.
738	* @param a_pGVMM The GVMM instance data.
739	* @sa GVMMR0_USED_EXCLUSIVE_LOCK, GVMMR0_USED_SHARED_UNLOCK
740	*/
741	#define GVMMR0_USED_EXCLUSIVE_UNLOCK(a_pGVMM) RTCritSectRwLeaveExcl(&(a_pGVMM)->UsedLock)
742
743
744	/**
745	* Try acquire the 'create & destroy' lock.
746	*
747	* @returns IPRT status code, see RTSemFastMutexRequest.
748	* @param pGVMM The GVMM instance data.
749	*/
750	DECLINLINE(int) gvmmR0CreateDestroyLock(PGVMM pGVMM)
751	{
752	LogFlow(("++gvmmR0CreateDestroyLock(%p)\n", pGVMM));
753	int rc = RTCritSectEnter(&pGVMM->CreateDestroyLock);
754	LogFlow(("gvmmR0CreateDestroyLock(%p)->%Rrc\n", pGVMM, rc));
755	return rc;
756	}
757
758
759	/**
760	* Release the 'create & destroy' lock.
761	*
762	* @returns IPRT status code, see RTSemFastMutexRequest.
763	* @param pGVMM The GVMM instance data.
764	*/
765	DECLINLINE(int) gvmmR0CreateDestroyUnlock(PGVMM pGVMM)
766	{
767	LogFlow(("--gvmmR0CreateDestroyUnlock(%p)\n", pGVMM));
768	int rc = RTCritSectLeave(&pGVMM->CreateDestroyLock);
769	AssertRC(rc);
770	return rc;
771	}
772
773
774	/**
775	* Request wrapper for the GVMMR0CreateVM API.
776	*
777	* @returns VBox status code.
778	* @param pReq The request buffer.
779	* @param pSession The session handle. The VM will be associated with this.
780	*/
781	GVMMR0DECL(int) GVMMR0CreateVMReq(PGVMMCREATEVMREQ pReq, PSUPDRVSESSION pSession)
782	{
783	/*
784	* Validate the request.
785	*/
786	if (!VALID_PTR(pReq))
787	return VERR_INVALID_POINTER;
788	if (pReq->Hdr.cbReq != sizeof(*pReq))
789	return VERR_INVALID_PARAMETER;
790	if (pReq->pSession != pSession)
791	return VERR_INVALID_POINTER;
792
793	/*
794	* Execute it.
795	*/
796	PVM pVM;
797	pReq->pVMR0 = NULL;
798	pReq->pVMR3 = NIL_RTR3PTR;
799	int rc = GVMMR0CreateVM(pSession, pReq->cCpus, &pVM);
800	if (RT_SUCCESS(rc))
801	{
802	pReq->pVMR0 = pVM;
803	pReq->pVMR3 = pVM->pVMR3;
804	}
805	return rc;
806	}
807
808
809	/**
810	* Allocates the VM structure and registers it with GVM.
811	*
812	* The caller will become the VM owner and there by the EMT.
813	*
814	* @returns VBox status code.
815	* @param pSession The support driver session.
816	* @param cCpus Number of virtual CPUs for the new VM.
817	* @param ppVM Where to store the pointer to the VM structure.
818	*
819	* @thread EMT.
820	*/
821	GVMMR0DECL(int) GVMMR0CreateVM(PSUPDRVSESSION pSession, uint32_t cCpus, PVM *ppVM)
822	{
823	LogFlow(("GVMMR0CreateVM: pSession=%p\n", pSession));
824	PGVMM pGVMM;
825	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
826
827	AssertPtrReturn(ppVM, VERR_INVALID_POINTER);
828	*ppVM = NULL;
829
830	if ( cCpus == 0
831	\|\| cCpus > VMM_MAX_CPU_COUNT)
832	return VERR_INVALID_PARAMETER;
833
834	RTNATIVETHREAD hEMT0 = RTThreadNativeSelf();
835	AssertReturn(hEMT0 != NIL_RTNATIVETHREAD, VERR_GVMM_BROKEN_IPRT);
836	RTPROCESS ProcId = RTProcSelf();
837	AssertReturn(ProcId != NIL_RTPROCESS, VERR_GVMM_BROKEN_IPRT);
838
839	/*
840	* The whole allocation process is protected by the lock.
841	*/
842	int rc = gvmmR0CreateDestroyLock(pGVMM);
843	AssertRCReturn(rc, rc);
844
845	/*
846	* Only one VM per session.
847	*/
848	if (SUPR0GetSessionVM(pSession) != NULL)
849	{
850	gvmmR0CreateDestroyUnlock(pGVMM);
851	SUPR0Printf("GVMMR0CreateVM: The session %p already got a VM: %p\n", pSession, SUPR0GetSessionVM(pSession));
852	return VERR_ALREADY_EXISTS;
853	}
854
855	/*
856	* Allocate a handle first so we don't waste resources unnecessarily.
857	*/
858	uint16_t iHandle = pGVMM->iFreeHead;
859	if (iHandle)
860	{
861	PGVMHANDLE pHandle = &pGVMM->aHandles[iHandle];
862
863	/* consistency checks, a bit paranoid as always. */
864	if ( !pHandle->pVM
865	&& !pHandle->pGVM
866	&& !pHandle->pvObj
867	&& pHandle->iSelf == iHandle)
868	{
869	pHandle->pvObj = SUPR0ObjRegister(pSession, SUPDRVOBJTYPE_VM, gvmmR0HandleObjDestructor, pGVMM, pHandle);
870	if (pHandle->pvObj)
871	{
872	/*
873	* Move the handle from the free to used list and perform permission checks.
874	*/
875	rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
876	AssertRC(rc);
877
878	pGVMM->iFreeHead = pHandle->iNext;
879	pHandle->iNext = pGVMM->iUsedHead;
880	pGVMM->iUsedHead = iHandle;
881	pGVMM->cVMs++;
882
883	pHandle->pVM = NULL;
884	pHandle->pGVM = NULL;
885	pHandle->pSession = pSession;
886	pHandle->hEMT0 = NIL_RTNATIVETHREAD;
887	pHandle->ProcId = NIL_RTPROCESS;
888
889	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
890
891	rc = SUPR0ObjVerifyAccess(pHandle->pvObj, pSession, NULL);
892	if (RT_SUCCESS(rc))
893	{
894	#ifdef VBOX_BUGREF_9217
895	/*
896	* Allocate memory for the VM structure (combined VM + GVM).
897	*/
898	const uint32_t cbVM = RT_UOFFSETOF_DYN(GVM, aCpus[cCpus]);
899	const uint32_t cPages = RT_ALIGN_32(cbVM, PAGE_SIZE) >> PAGE_SHIFT;
900	RTR0MEMOBJ hVMMemObj = NIL_RTR0MEMOBJ;
901	# if defined(VBOX_WITH_RAW_MODE) \|\| HC_ARCH_BITS == 32
902	rc = RTR0MemObjAllocLow(&hVMMemObj, cPages << PAGE_SHIFT, false /* fExecutable */);
903	# else
904	rc = RTR0MemObjAllocPage(&hVMMemObj, cPages << PAGE_SHIFT, false /* fExecutable */);
905	# endif
906	if (RT_SUCCESS(rc))
907	{
908	PGVM pGVM = (PGVM)RTR0MemObjAddress(hVMMemObj);
909	AssertPtr(pGVM);
910
911	/*
912	* Initialise the structure.
913	*/
914	RT_BZERO(pGVM, cPages << PAGE_SHIFT);
915	gvmmR0InitPerVMData(pGVM, iHandle, cCpus, pSession);
916	GMMR0InitPerVMData(pGVM);
917	pGVM->gvmm.s.VMMemObj = hVMMemObj;
918
919	/*
920	* Allocate page array.
921	* This currently have to be made available to ring-3, but this is should change eventually.
922	*/
923	rc = RTR0MemObjAllocPage(&pGVM->gvmm.s.VMPagesMemObj, cPages * sizeof(SUPPAGE), false /* fExecutable */);
924	if (RT_SUCCESS(rc))
925	{
926	PSUPPAGE paPages = (PSUPPAGE)RTR0MemObjAddress(pGVM->gvmm.s.VMPagesMemObj); AssertPtr(paPages);
927	for (uint32_t iPage = 0; iPage < cPages; iPage++)
928	{
929	paPages[iPage].uReserved = 0;
930	paPages[iPage].Phys = RTR0MemObjGetPagePhysAddr(pGVM->gvmm.s.VMMemObj, iPage);
931	Assert(paPages[iPage].Phys != NIL_RTHCPHYS);
932	}
933
934	/*
935	* Map the page array, VM and VMCPU structures into ring-3.
936	*/
937	AssertCompileSizeAlignment(VM, PAGE_SIZE);
938	rc = RTR0MemObjMapUserEx(&pGVM->gvmm.s.VMMapObj, pGVM->gvmm.s.VMMemObj, (RTR3PTR)-1, 0,
939	RTMEM_PROT_READ \| RTMEM_PROT_WRITE, NIL_RTR0PROCESS,
940	0 /offSub/, sizeof(VM));
941	for (VMCPUID i = 0; i < cCpus && RT_SUCCESS(rc); i++)
942	{
943	AssertCompileSizeAlignment(VMCPU, PAGE_SIZE);
944	rc = RTR0MemObjMapUserEx(&pGVM->aCpus[i].gvmm.s.VMCpuMapObj, pGVM->gvmm.s.VMMemObj,
945	(RTR3PTR)-1, 0, RTMEM_PROT_READ \| RTMEM_PROT_WRITE, NIL_RTR0PROCESS,
946	RT_UOFFSETOF_DYN(GVM, aCpus[i]), sizeof(VMCPU));
947	}
948	if (RT_SUCCESS(rc))
949	rc = RTR0MemObjMapUser(&pGVM->gvmm.s.VMPagesMapObj, pGVM->gvmm.s.VMPagesMemObj, (RTR3PTR)-1,
950	0 /* uAlignment */, RTMEM_PROT_READ \| RTMEM_PROT_WRITE,
951	NIL_RTR0PROCESS);
952	if (RT_SUCCESS(rc))
953	{
954	/*
955	* Initialize all the VM pointer.
956	*/
957	PVMR3 pVMR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMMapObj);
958	AssertPtr((void *)pVMR3);
959
960	for (VMCPUID i = 0; i < cCpus; i++)
961	{
962	pGVM->aCpus[i].pVMR0 = pGVM;
963	pGVM->aCpus[i].pVMR3 = pVMR3;
964	pGVM->apCpus[i] = RTR0MemObjAddressR3(pGVM->aCpus[i].gvmm.s.VMCpuMapObj);
965	AssertPtr((void *)pGVM->apCpus[i]);
966	}
967
968	pGVM->paVMPagesR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMPagesMapObj);
969	AssertPtr((void *)pGVM->paVMPagesR3);
970
971	/*
972	* Complete the handle - take the UsedLock sem just to be careful.
973	*/
974	rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
975	AssertRC(rc);
976
977	pHandle->pVM = pGVM;
978	pHandle->pGVM = pGVM;
979	pHandle->hEMT0 = hEMT0;
980	pHandle->ProcId = ProcId;
981	pGVM->pVMR3 = pVMR3;
982	pGVM->aCpus[0].hEMT = hEMT0;
983	pGVM->aCpus[0].hNativeThreadR0 = hEMT0;
984	pGVMM->cEMTs += cCpus;
985
986	/* Associate it with the session and create the context hook for EMT0. */
987	rc = SUPR0SetSessionVM(pSession, pGVM, pGVM);
988	if (RT_SUCCESS(rc))
989	{
990	rc = VMMR0ThreadCtxHookCreateForEmt(&pGVM->aCpus[0]);
991	if (RT_SUCCESS(rc))
992	{
993	/*
994	* Done!
995	*/
996	VBOXVMM_R0_GVMM_VM_CREATED(pGVM, pGVM, ProcId, (void *)hEMT0, cCpus);
997
998	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
999	gvmmR0CreateDestroyUnlock(pGVMM);
1000
1001	CPUMR0RegisterVCpuThread(&pGVM->aCpus[0]);
1002
1003	*ppVM = pGVM;
1004	Log(("GVMMR0CreateVM: pVMR3=%p pGVM=%p hGVM=%d\n", pVMR3, pGVM, iHandle));
1005	return VINF_SUCCESS;
1006	}
1007
1008	SUPR0SetSessionVM(pSession, NULL, NULL);
1009	}
1010	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1011	}
1012
1013	/* Cleanup mappings. */
1014	if (pGVM->gvmm.s.VMMapObj != NIL_RTR0MEMOBJ)
1015	{
1016	RTR0MemObjFree(pGVM->gvmm.s.VMMapObj, false /* fFreeMappings */);
1017	pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
1018	}
1019	for (VMCPUID i = 0; i < cCpus; i++)
1020	if (pGVM->aCpus[i].gvmm.s.VMCpuMapObj != NIL_RTR0MEMOBJ)
1021	{
1022	RTR0MemObjFree(pGVM->aCpus[i].gvmm.s.VMCpuMapObj, false /* fFreeMappings */);
1023	pGVM->aCpus[i].gvmm.s.VMCpuMapObj = NIL_RTR0MEMOBJ;
1024	}
1025	if (pGVM->gvmm.s.VMPagesMapObj != NIL_RTR0MEMOBJ)
1026	{
1027	RTR0MemObjFree(pGVM->gvmm.s.VMPagesMapObj, false /* fFreeMappings */);
1028	pGVM->gvmm.s.VMPagesMapObj = NIL_RTR0MEMOBJ;
1029	}
1030	}
1031	}
1032
1033	#else
1034	/*
1035	* Allocate the global VM structure (GVM) and initialize it.
1036	*/
1037	PGVM pGVM = (PGVM)RTMemAllocZ(RT_UOFFSETOF_DYN(GVM, aCpus[cCpus]));
1038	if (pGVM)
1039	{
1040	pGVM->u32Magic = GVM_MAGIC;
1041	pGVM->hSelf = iHandle;
1042	pGVM->pVM = NULL;
1043	pGVM->cCpus = cCpus;
1044	pGVM->pSession = pSession;
1045
1046	gvmmR0InitPerVMData(pGVM);
1047	GMMR0InitPerVMData(pGVM);
1048
1049	/*
1050	* Allocate the shared VM structure and associated page array.
1051	*/
1052	const uint32_t cbVM = RT_UOFFSETOF_DYN(VM, aCpus[cCpus]);
1053	const uint32_t cPages = RT_ALIGN_32(cbVM, PAGE_SIZE) >> PAGE_SHIFT;
1054	rc = RTR0MemObjAllocLow(&pGVM->gvmm.s.VMMemObj, cPages << PAGE_SHIFT, false /* fExecutable */);
1055	if (RT_SUCCESS(rc))
1056	{
1057	PVM pVM = (PVM)RTR0MemObjAddress(pGVM->gvmm.s.VMMemObj); AssertPtr(pVM);
1058	memset(pVM, 0, cPages << PAGE_SHIFT);
1059	pVM->enmVMState = VMSTATE_CREATING;
1060	pVM->pVMR0 = pVM;
1061	pVM->pSession = pSession;
1062	pVM->hSelf = iHandle;
1063	pVM->cbSelf = cbVM;
1064	pVM->cCpus = cCpus;
1065	pVM->uCpuExecutionCap = 100; /* default is no cap. */
1066	# ifdef VBOX_WITH_RAW_MODE
1067	pVM->offVMCPU = RT_UOFFSETOF_DYN(VM, aCpus);
1068	# endif
1069	AssertCompileMemberAlignment(VM, cpum, 64);
1070	AssertCompileMemberAlignment(VM, tm, 64);
1071	AssertCompileMemberAlignment(VM, aCpus, PAGE_SIZE);
1072
1073	rc = RTR0MemObjAllocPage(&pGVM->gvmm.s.VMPagesMemObj, cPages * sizeof(SUPPAGE), false /* fExecutable */);
1074	if (RT_SUCCESS(rc))
1075	{
1076	PSUPPAGE paPages = (PSUPPAGE)RTR0MemObjAddress(pGVM->gvmm.s.VMPagesMemObj); AssertPtr(paPages);
1077	for (uint32_t iPage = 0; iPage < cPages; iPage++)
1078	{
1079	paPages[iPage].uReserved = 0;
1080	paPages[iPage].Phys = RTR0MemObjGetPagePhysAddr(pGVM->gvmm.s.VMMemObj, iPage);
1081	Assert(paPages[iPage].Phys != NIL_RTHCPHYS);
1082	}
1083
1084	/*
1085	* Map them into ring-3.
1086	*/
1087	rc = RTR0MemObjMapUser(&pGVM->gvmm.s.VMMapObj, pGVM->gvmm.s.VMMemObj, (RTR3PTR)-1, 0,
1088	RTMEM_PROT_READ \| RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
1089	if (RT_SUCCESS(rc))
1090	{
1091	PVMR3 pVMR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMMapObj);
1092	pVM->pVMR3 = pVMR3;
1093	AssertPtr((void *)pVMR3);
1094
1095	/* Initialize all the VM pointers. */
1096	for (VMCPUID i = 0; i < cCpus; i++)
1097	{
1098	pVM->aCpus[i].idCpu = i;
1099	pVM->aCpus[i].pVMR0 = pVM;
1100	pVM->aCpus[i].pVMR3 = pVMR3;
1101	pVM->aCpus[i].idHostCpu = NIL_RTCPUID;
1102	pVM->aCpus[i].hNativeThreadR0 = NIL_RTNATIVETHREAD;
1103	}
1104
1105	rc = RTR0MemObjMapUser(&pGVM->gvmm.s.VMPagesMapObj, pGVM->gvmm.s.VMPagesMemObj, (RTR3PTR)-1,
1106	0 /* uAlignment */, RTMEM_PROT_READ \| RTMEM_PROT_WRITE,
1107	NIL_RTR0PROCESS);
1108	if (RT_SUCCESS(rc))
1109	{
1110	pVM->paVMPagesR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMPagesMapObj);
1111	AssertPtr((void *)pVM->paVMPagesR3);
1112
1113	/* complete the handle - take the UsedLock sem just to be careful. */
1114	rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
1115	AssertRC(rc);
1116
1117	pHandle->pVM = pVM;
1118	pHandle->pGVM = pGVM;
1119	pHandle->hEMT0 = hEMT0;
1120	pHandle->ProcId = ProcId;
1121	pGVM->pVM = pVM;
1122	pGVM->pVMR3 = pVMR3;
1123	pGVM->aCpus[0].hEMT = hEMT0;
1124	pVM->aCpus[0].hNativeThreadR0 = hEMT0;
1125	pGVMM->cEMTs += cCpus;
1126
1127	for (VMCPUID i = 0; i < cCpus; i++)
1128	{
1129	pGVM->aCpus[i].pVCpu = &pVM->aCpus[i];
1130	pGVM->aCpus[i].pVM = pVM;
1131	}
1132
1133	/* Associate it with the session and create the context hook for EMT0. */
1134	rc = SUPR0SetSessionVM(pSession, pGVM, pVM);
1135	if (RT_SUCCESS(rc))
1136	{
1137	rc = VMMR0ThreadCtxHookCreateForEmt(&pVM->aCpus[0]);
1138	if (RT_SUCCESS(rc))
1139	{
1140	/*
1141	* Done!
1142	*/
1143	VBOXVMM_R0_GVMM_VM_CREATED(pGVM, pVM, ProcId, (void *)hEMT0, cCpus);
1144
1145	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1146	gvmmR0CreateDestroyUnlock(pGVMM);
1147
1148	CPUMR0RegisterVCpuThread(&pVM->aCpus[0]);
1149
1150	*ppVM = pVM;
1151	Log(("GVMMR0CreateVM: pVM=%p pVMR3=%p pGVM=%p hGVM=%d\n", pVM, pVMR3, pGVM, iHandle));
1152	return VINF_SUCCESS;
1153	}
1154
1155	SUPR0SetSessionVM(pSession, NULL, NULL);
1156	}
1157	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1158	}
1159
1160	RTR0MemObjFree(pGVM->gvmm.s.VMMapObj, false /* fFreeMappings */);
1161	pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
1162	}
1163	RTR0MemObjFree(pGVM->gvmm.s.VMPagesMemObj, false /* fFreeMappings */);
1164	pGVM->gvmm.s.VMPagesMemObj = NIL_RTR0MEMOBJ;
1165	}
1166	RTR0MemObjFree(pGVM->gvmm.s.VMMemObj, false /* fFreeMappings */);
1167	pGVM->gvmm.s.VMMemObj = NIL_RTR0MEMOBJ;
1168	}
1169	}
1170	#endif
1171	}
1172	/* else: The user wasn't permitted to create this VM. */
1173
1174	/*
1175	* The handle will be freed by gvmmR0HandleObjDestructor as we release the
1176	* object reference here. A little extra mess because of non-recursive lock.
1177	*/
1178	void *pvObj = pHandle->pvObj;
1179	pHandle->pvObj = NULL;
1180	gvmmR0CreateDestroyUnlock(pGVMM);
1181
1182	SUPR0ObjRelease(pvObj, pSession);
1183
1184	SUPR0Printf("GVMMR0CreateVM: failed, rc=%Rrc\n", rc);
1185	return rc;
1186	}
1187
1188	rc = VERR_NO_MEMORY;
1189	}
1190	else
1191	rc = VERR_GVMM_IPE_1;
1192	}
1193	else
1194	rc = VERR_GVM_TOO_MANY_VMS;
1195
1196	gvmmR0CreateDestroyUnlock(pGVMM);
1197	return rc;
1198	}
1199
1200
1201	#ifdef VBOX_BUGREF_9217
1202	/**
1203	* Initializes the per VM data belonging to GVMM.
1204	*
1205	* @param pGVM Pointer to the global VM structure.
1206	*/
1207	static void gvmmR0InitPerVMData(PGVM pGVM, int16_t hSelf, VMCPUID cCpus, PSUPDRVSESSION pSession)
1208	#else
1209	/**
1210	* Initializes the per VM data belonging to GVMM.
1211	*
1212	* @param pGVM Pointer to the global VM structure.
1213	*/
1214	static void gvmmR0InitPerVMData(PGVM pGVM)
1215	#endif
1216	{
1217	AssertCompile(RT_SIZEOFMEMB(GVM,gvmm.s) <= RT_SIZEOFMEMB(GVM,gvmm.padding));
1218	AssertCompile(RT_SIZEOFMEMB(GVMCPU,gvmm.s) <= RT_SIZEOFMEMB(GVMCPU,gvmm.padding));
1219	#ifdef VBOX_BUGREF_9217
1220	AssertCompileMemberAlignment(VM, cpum, 64);
1221	AssertCompileMemberAlignment(VM, tm, 64);
1222
1223	/* GVM: */
1224	pGVM->u32Magic = GVM_MAGIC;
1225	pGVM->hSelfSafe = hSelf;
1226	pGVM->cCpusSafe = cCpus;
1227	pGVM->pSessionSafe = pSession;
1228
1229	/* VM: */
1230	pGVM->enmVMState = VMSTATE_CREATING;
1231	pGVM->pVMR0 = pGVM;
1232	pGVM->pSession = pSession;
1233	pGVM->hSelf = hSelf;
1234	pGVM->cCpus = cCpus;
1235	pGVM->uCpuExecutionCap = 100; /* default is no cap. */
1236	pGVM->uStructVersion = 1;
1237	pGVM->cbSelf = sizeof(VM);
1238	pGVM->cbVCpu = sizeof(VMCPU);
1239	# ifdef VBOX_WITH_RAW_MODE
1240	pGVM->offVMCPU = RT_UOFFSETOF_DYN(GVM, aCpus); /** @todo set this when mapping the VM structure into raw-mode context */
1241	# endif
1242	#endif
1243
1244	/* GVMM: */
1245	pGVM->gvmm.s.VMMemObj = NIL_RTR0MEMOBJ;
1246	pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
1247	pGVM->gvmm.s.VMPagesMemObj = NIL_RTR0MEMOBJ;
1248	pGVM->gvmm.s.VMPagesMapObj = NIL_RTR0MEMOBJ;
1249	pGVM->gvmm.s.fDoneVMMR0Init = false;
1250	pGVM->gvmm.s.fDoneVMMR0Term = false;
1251
1252	/*
1253	* Per virtual CPU.
1254	*/
1255	for (VMCPUID i = 0; i < pGVM->cCpus; i++)
1256	{
1257	pGVM->aCpus[i].idCpu = i;
1258	#ifdef VBOX_BUGREF_9217
1259	pGVM->aCpus[i].idCpuSafe = i;
1260	#endif
1261	pGVM->aCpus[i].gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
1262	#ifdef VBOX_BUGREF_9217
1263	pGVM->aCpus[i].gvmm.s.VMCpuMapObj = NIL_RTR0MEMOBJ;
1264	#endif
1265	pGVM->aCpus[i].hEMT = NIL_RTNATIVETHREAD;
1266	pGVM->aCpus[i].pGVM = pGVM;
1267	#ifndef VBOX_BUGREF_9217
1268	pGVM->aCpus[i].pVCpu = NULL;
1269	pGVM->aCpus[i].pVM = NULL;
1270	#endif
1271	#ifdef VBOX_BUGREF_9217
1272	pGVM->aCpus[i].idHostCpu = NIL_RTCPUID;
1273	pGVM->aCpus[i].iHostCpuSet = UINT32_MAX;
1274	pGVM->aCpus[i].hNativeThread = NIL_RTNATIVETHREAD;
1275	pGVM->aCpus[i].hNativeThreadR0 = NIL_RTNATIVETHREAD;
1276	pGVM->aCpus[i].enmState = VMCPUSTATE_STOPPED;
1277	#endif
1278	}
1279	}
1280
1281
1282	/**
1283	* Does the VM initialization.
1284	*
1285	* @returns VBox status code.
1286	* @param pGVM The global (ring-0) VM structure.
1287	*/
1288	GVMMR0DECL(int) GVMMR0InitVM(PGVM pGVM)
1289	{
1290	LogFlow(("GVMMR0InitVM: pGVM=%p\n", pGVM));
1291
1292	int rc = VERR_INTERNAL_ERROR_3;
1293	if ( !pGVM->gvmm.s.fDoneVMMR0Init
1294	&& pGVM->aCpus[0].gvmm.s.HaltEventMulti == NIL_RTSEMEVENTMULTI)
1295	{
1296	for (VMCPUID i = 0; i < pGVM->cCpus; i++)
1297	{
1298	rc = RTSemEventMultiCreate(&pGVM->aCpus[i].gvmm.s.HaltEventMulti);
1299	if (RT_FAILURE(rc))
1300	{
1301	pGVM->aCpus[i].gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
1302	break;
1303	}
1304	}
1305	}
1306	else
1307	rc = VERR_WRONG_ORDER;
1308
1309	LogFlow(("GVMMR0InitVM: returns %Rrc\n", rc));
1310	return rc;
1311	}
1312
1313
1314	/**
1315	* Indicates that we're done with the ring-0 initialization
1316	* of the VM.
1317	*
1318	* @param pGVM The global (ring-0) VM structure.
1319	* @thread EMT(0)
1320	*/
1321	GVMMR0DECL(void) GVMMR0DoneInitVM(PGVM pGVM)
1322	{
1323	/* Set the indicator. */
1324	pGVM->gvmm.s.fDoneVMMR0Init = true;
1325	}
1326
1327
1328	/**
1329	* Indicates that we're doing the ring-0 termination of the VM.
1330	*
1331	* @returns true if termination hasn't been done already, false if it has.
1332	* @param pGVM Pointer to the global VM structure. Optional.
1333	* @thread EMT(0) or session cleanup thread.
1334	*/
1335	GVMMR0DECL(bool) GVMMR0DoingTermVM(PGVM pGVM)
1336	{
1337	/* Validate the VM structure, state and handle. */
1338	AssertPtrReturn(pGVM, false);
1339
1340	/* Set the indicator. */
1341	if (pGVM->gvmm.s.fDoneVMMR0Term)
1342	return false;
1343	pGVM->gvmm.s.fDoneVMMR0Term = true;
1344	return true;
1345	}
1346
1347
1348	/**
1349	* Destroys the VM, freeing all associated resources (the ring-0 ones anyway).
1350	*
1351	* This is call from the vmR3DestroyFinalBit and from a error path in VMR3Create,
1352	* and the caller is not the EMT thread, unfortunately. For security reasons, it
1353	* would've been nice if the caller was actually the EMT thread or that we somehow
1354	* could've associated the calling thread with the VM up front.
1355	*
1356	* @returns VBox status code.
1357	* @param pGVM The global (ring-0) VM structure.
1358	* @param pVM The cross context VM structure.
1359	*
1360	* @thread EMT(0) if it's associated with the VM, otherwise any thread.
1361	*/
1362	GVMMR0DECL(int) GVMMR0DestroyVM(PGVM pGVM, PVM pVM)
1363	{
1364	LogFlow(("GVMMR0DestroyVM: pGVM=%p pVM=%p\n", pGVM, pVM));
1365	PGVMM pGVMM;
1366	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
1367
1368	/*
1369	* Validate the VM structure, state and caller.
1370	*/
1371	AssertPtrReturn(pGVM, VERR_INVALID_POINTER);
1372	AssertPtrReturn(pVM, VERR_INVALID_POINTER);
1373	AssertReturn(!((uintptr_t)pVM & PAGE_OFFSET_MASK), VERR_INVALID_POINTER);
1374	#ifdef VBOX_BUGREF_9217
1375	AssertReturn(pGVM == pVM, VERR_INVALID_POINTER);
1376	#else
1377	AssertReturn(pGVM->pVM == pVM, VERR_INVALID_POINTER);
1378	#endif
1379	AssertMsgReturn(pVM->enmVMState >= VMSTATE_CREATING && pVM->enmVMState <= VMSTATE_TERMINATED, ("%d\n", pVM->enmVMState),
1380	VERR_WRONG_ORDER);
1381
1382	uint32_t hGVM = pGVM->hSelf;
1383	ASMCompilerBarrier();
1384	AssertReturn(hGVM != NIL_GVM_HANDLE, VERR_INVALID_VM_HANDLE);
1385	AssertReturn(hGVM < RT_ELEMENTS(pGVMM->aHandles), VERR_INVALID_VM_HANDLE);
1386
1387	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1388	AssertReturn(pHandle->pVM == pVM, VERR_NOT_OWNER);
1389
1390	RTPROCESS ProcId = RTProcSelf();
1391	RTNATIVETHREAD hSelf = RTThreadNativeSelf();
1392	AssertReturn( ( pHandle->hEMT0 == hSelf
1393	&& pHandle->ProcId == ProcId)
1394	\|\| pHandle->hEMT0 == NIL_RTNATIVETHREAD, VERR_NOT_OWNER);
1395
1396	/*
1397	* Lookup the handle and destroy the object.
1398	* Since the lock isn't recursive and we'll have to leave it before dereferencing the
1399	* object, we take some precautions against racing callers just in case...
1400	*/
1401	int rc = gvmmR0CreateDestroyLock(pGVMM);
1402	AssertRC(rc);
1403
1404	/* Be careful here because we might theoretically be racing someone else cleaning up. */
1405	if ( pHandle->pVM == pVM
1406	&& ( ( pHandle->hEMT0 == hSelf
1407	&& pHandle->ProcId == ProcId)
1408	\|\| pHandle->hEMT0 == NIL_RTNATIVETHREAD)
1409	&& VALID_PTR(pHandle->pvObj)
1410	&& VALID_PTR(pHandle->pSession)
1411	&& VALID_PTR(pHandle->pGVM)
1412	&& pHandle->pGVM->u32Magic == GVM_MAGIC)
1413	{
1414	/* Check that other EMTs have deregistered. */
1415	uint32_t cNotDeregistered = 0;
1416	for (VMCPUID idCpu = 1; idCpu < pGVM->cCpus; idCpu++)
1417	cNotDeregistered += pGVM->aCpus[idCpu].hEMT != ~(RTNATIVETHREAD)1; /* see GVMMR0DeregisterVCpu for the value */
1418	if (cNotDeregistered == 0)
1419	{
1420	/* Grab the object pointer. */
1421	void *pvObj = pHandle->pvObj;
1422	pHandle->pvObj = NULL;
1423	gvmmR0CreateDestroyUnlock(pGVMM);
1424
1425	SUPR0ObjRelease(pvObj, pHandle->pSession);
1426	}
1427	else
1428	{
1429	gvmmR0CreateDestroyUnlock(pGVMM);
1430	rc = VERR_GVMM_NOT_ALL_EMTS_DEREGISTERED;
1431	}
1432	}
1433	else
1434	{
1435	SUPR0Printf("GVMMR0DestroyVM: pHandle=%RKv:{.pVM=%p, .hEMT0=%p, .ProcId=%u, .pvObj=%p} pVM=%p hSelf=%p\n",
1436	pHandle, pHandle->pVM, pHandle->hEMT0, pHandle->ProcId, pHandle->pvObj, pVM, hSelf);
1437	gvmmR0CreateDestroyUnlock(pGVMM);
1438	rc = VERR_GVMM_IPE_2;
1439	}
1440
1441	return rc;
1442	}
1443
1444
1445	/**
1446	* Performs VM cleanup task as part of object destruction.
1447	*
1448	* @param pGVM The GVM pointer.
1449	*/
1450	static void gvmmR0CleanupVM(PGVM pGVM)
1451	{
1452	if ( pGVM->gvmm.s.fDoneVMMR0Init
1453	&& !pGVM->gvmm.s.fDoneVMMR0Term)
1454	{
1455	if ( pGVM->gvmm.s.VMMemObj != NIL_RTR0MEMOBJ
1456	#ifdef VBOX_BUGREF_9217
1457	&& RTR0MemObjAddress(pGVM->gvmm.s.VMMemObj) == pGVM
1458	#else
1459	&& RTR0MemObjAddress(pGVM->gvmm.s.VMMemObj) == pGVM->pVM
1460	#endif
1461	)
1462	{
1463	LogFlow(("gvmmR0CleanupVM: Calling VMMR0TermVM\n"));
1464	#ifdef VBOX_BUGREF_9217
1465	VMMR0TermVM(pGVM, pGVM, NIL_VMCPUID);
1466	#else
1467	VMMR0TermVM(pGVM, pGVM->pVM, NIL_VMCPUID);
1468	#endif
1469	}
1470	else
1471	#ifdef VBOX_BUGREF_9217
1472	AssertMsgFailed(("gvmmR0CleanupVM: VMMemObj=%p pGVM=%p\n", pGVM->gvmm.s.VMMemObj, pGVM));
1473	#else
1474	AssertMsgFailed(("gvmmR0CleanupVM: VMMemObj=%p pVM=%p\n", pGVM->gvmm.s.VMMemObj, pGVM->pVM));
1475	#endif
1476	}
1477
1478	GMMR0CleanupVM(pGVM);
1479	#ifdef VBOX_WITH_NEM_R0
1480	NEMR0CleanupVM(pGVM);
1481	#endif
1482
1483	AssertCompile(NIL_RTTHREADCTXHOOK == (RTTHREADCTXHOOK)0); /* Depends on zero initialized memory working for NIL at the moment. */
1484	#ifdef VBOX_BUGREF_9217
1485	for (VMCPUID idCpu = 0; idCpu < pGVM->cCpusSafe; idCpu++)
1486	#else
1487	for (VMCPUID idCpu = 0; idCpu < pGVM->cCpus; idCpu++)
1488	#endif
1489	{
1490	/** @todo Can we busy wait here for all thread-context hooks to be
1491	* deregistered before releasing (destroying) it? Only until we find a
1492	* solution for not deregistering hooks everytime we're leaving HMR0
1493	* context. */
1494	#ifdef VBOX_BUGREF_9217
1495	VMMR0ThreadCtxHookDestroyForEmt(&pGVM->aCpus[idCpu]);
1496	#else
1497	VMMR0ThreadCtxHookDestroyForEmt(&pGVM->pVM->aCpus[idCpu]);
1498	#endif
1499	}
1500	}
1501
1502
1503	/**
1504	* @callback_method_impl{FNSUPDRVDESTRUCTOR,VM handle destructor}
1505	*
1506	* pvUser1 is the GVM instance pointer.
1507	* pvUser2 is the handle pointer.
1508	*/
1509	static DECLCALLBACK(void) gvmmR0HandleObjDestructor(void pvObj, void pvUser1, void *pvUser2)
1510	{
1511	LogFlow(("gvmmR0HandleObjDestructor: %p %p %p\n", pvObj, pvUser1, pvUser2));
1512
1513	NOREF(pvObj);
1514
1515	/*
1516	* Some quick, paranoid, input validation.
1517	*/
1518	PGVMHANDLE pHandle = (PGVMHANDLE)pvUser2;
1519	AssertPtr(pHandle);
1520	PGVMM pGVMM = (PGVMM)pvUser1;
1521	Assert(pGVMM == g_pGVMM);
1522	const uint16_t iHandle = pHandle - &pGVMM->aHandles[0];
1523	if ( !iHandle
1524	\|\| iHandle >= RT_ELEMENTS(pGVMM->aHandles)
1525	\|\| iHandle != pHandle->iSelf)
1526	{
1527	SUPR0Printf("GVM: handle %d is out of range or corrupt (iSelf=%d)!\n", iHandle, pHandle->iSelf);
1528	return;
1529	}
1530
1531	int rc = gvmmR0CreateDestroyLock(pGVMM);
1532	AssertRC(rc);
1533	rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
1534	AssertRC(rc);
1535
1536	/*
1537	* This is a tad slow but a doubly linked list is too much hassle.
1538	*/
1539	if (RT_UNLIKELY(pHandle->iNext >= RT_ELEMENTS(pGVMM->aHandles)))
1540	{
1541	SUPR0Printf("GVM: used list index %d is out of range!\n", pHandle->iNext);
1542	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1543	gvmmR0CreateDestroyUnlock(pGVMM);
1544	return;
1545	}
1546
1547	if (pGVMM->iUsedHead == iHandle)
1548	pGVMM->iUsedHead = pHandle->iNext;
1549	else
1550	{
1551	uint16_t iPrev = pGVMM->iUsedHead;
1552	int c = RT_ELEMENTS(pGVMM->aHandles) + 2;
1553	while (iPrev)
1554	{
1555	if (RT_UNLIKELY(iPrev >= RT_ELEMENTS(pGVMM->aHandles)))
1556	{
1557	SUPR0Printf("GVM: used list index %d is out of range!\n", iPrev);
1558	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1559	gvmmR0CreateDestroyUnlock(pGVMM);
1560	return;
1561	}
1562	if (RT_UNLIKELY(c-- <= 0))
1563	{
1564	iPrev = 0;
1565	break;
1566	}
1567
1568	if (pGVMM->aHandles[iPrev].iNext == iHandle)
1569	break;
1570	iPrev = pGVMM->aHandles[iPrev].iNext;
1571	}
1572	if (!iPrev)
1573	{
1574	SUPR0Printf("GVM: can't find the handle previous previous of %d!\n", pHandle->iSelf);
1575	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1576	gvmmR0CreateDestroyUnlock(pGVMM);
1577	return;
1578	}
1579
1580	Assert(pGVMM->aHandles[iPrev].iNext == iHandle);
1581	pGVMM->aHandles[iPrev].iNext = pHandle->iNext;
1582	}
1583	pHandle->iNext = 0;
1584	pGVMM->cVMs--;
1585
1586	/*
1587	* Do the global cleanup round.
1588	*/
1589	PGVM pGVM = pHandle->pGVM;
1590	if ( VALID_PTR(pGVM)
1591	&& pGVM->u32Magic == GVM_MAGIC)
1592	{
1593	pGVMM->cEMTs -= pGVM->cCpus;
1594
1595	if (pGVM->pSession)
1596	SUPR0SetSessionVM(pGVM->pSession, NULL, NULL);
1597
1598	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1599
1600	gvmmR0CleanupVM(pGVM);
1601
1602	/*
1603	* Do the GVMM cleanup - must be done last.
1604	*/
1605	/* The VM and VM pages mappings/allocations. */
1606	if (pGVM->gvmm.s.VMPagesMapObj != NIL_RTR0MEMOBJ)
1607	{
1608	rc = RTR0MemObjFree(pGVM->gvmm.s.VMPagesMapObj, false /* fFreeMappings */); AssertRC(rc);
1609	pGVM->gvmm.s.VMPagesMapObj = NIL_RTR0MEMOBJ;
1610	}
1611
1612	if (pGVM->gvmm.s.VMMapObj != NIL_RTR0MEMOBJ)
1613	{
1614	rc = RTR0MemObjFree(pGVM->gvmm.s.VMMapObj, false /* fFreeMappings */); AssertRC(rc);
1615	pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
1616	}
1617
1618	if (pGVM->gvmm.s.VMPagesMemObj != NIL_RTR0MEMOBJ)
1619	{
1620	rc = RTR0MemObjFree(pGVM->gvmm.s.VMPagesMemObj, false /* fFreeMappings */); AssertRC(rc);
1621	pGVM->gvmm.s.VMPagesMemObj = NIL_RTR0MEMOBJ;
1622	}
1623
1624	#ifndef VBOX_BUGREF_9217
1625	if (pGVM->gvmm.s.VMMemObj != NIL_RTR0MEMOBJ)
1626	{
1627	rc = RTR0MemObjFree(pGVM->gvmm.s.VMMemObj, false /* fFreeMappings */); AssertRC(rc);
1628	pGVM->gvmm.s.VMMemObj = NIL_RTR0MEMOBJ;
1629	}
1630	#endif
1631
1632	for (VMCPUID i = 0; i < pGVM->cCpus; i++)
1633	{
1634	if (pGVM->aCpus[i].gvmm.s.HaltEventMulti != NIL_RTSEMEVENTMULTI)
1635	{
1636	rc = RTSemEventMultiDestroy(pGVM->aCpus[i].gvmm.s.HaltEventMulti); AssertRC(rc);
1637	pGVM->aCpus[i].gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
1638	}
1639	#ifdef VBOX_BUGREF_9217
1640	if (pGVM->aCpus[i].gvmm.s.VMCpuMapObj != NIL_RTR0MEMOBJ)
1641	{
1642	rc = RTR0MemObjFree(pGVM->aCpus[i].gvmm.s.VMCpuMapObj, false /* fFreeMappings */); AssertRC(rc);
1643	pGVM->aCpus[i].gvmm.s.VMCpuMapObj = NIL_RTR0MEMOBJ;
1644	}
1645	#endif
1646	}
1647
1648	/* the GVM structure itself. */
1649	pGVM->u32Magic \|= UINT32_C(0x80000000);
1650	#ifdef VBOX_BUGREF_9217
1651	Assert(pGVM->gvmm.s.VMMemObj != NIL_RTR0MEMOBJ);
1652	rc = RTR0MemObjFree(pGVM->gvmm.s.VMMemObj, true /fFreeMappings/); AssertRC(rc);
1653	#else
1654	RTMemFree(pGVM);
1655	#endif
1656	pGVM = NULL;
1657
1658	/* Re-acquire the UsedLock before freeing the handle since we're updating handle fields. */
1659	rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
1660	AssertRC(rc);
1661	}
1662	/* else: GVMMR0CreateVM cleanup. */
1663
1664	/*
1665	* Free the handle.
1666	*/
1667	pHandle->iNext = pGVMM->iFreeHead;
1668	pGVMM->iFreeHead = iHandle;
1669	ASMAtomicWriteNullPtr(&pHandle->pGVM);
1670	ASMAtomicWriteNullPtr(&pHandle->pVM);
1671	ASMAtomicWriteNullPtr(&pHandle->pvObj);
1672	ASMAtomicWriteNullPtr(&pHandle->pSession);
1673	ASMAtomicWriteHandle(&pHandle->hEMT0, NIL_RTNATIVETHREAD);
1674	ASMAtomicWriteU32(&pHandle->ProcId, NIL_RTPROCESS);
1675
1676	GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
1677	gvmmR0CreateDestroyUnlock(pGVMM);
1678	LogFlow(("gvmmR0HandleObjDestructor: returns\n"));
1679	}
1680
1681
1682	/**
1683	* Registers the calling thread as the EMT of a Virtual CPU.
1684	*
1685	* Note that VCPU 0 is automatically registered during VM creation.
1686	*
1687	* @returns VBox status code
1688	* @param pGVM The global (ring-0) VM structure.
1689	* @param pVM The cross context VM structure.
1690	* @param idCpu VCPU id to register the current thread as.
1691	*/
1692	GVMMR0DECL(int) GVMMR0RegisterVCpu(PGVM pGVM, PVM pVM, VMCPUID idCpu)
1693	{
1694	AssertReturn(idCpu != 0, VERR_INVALID_FUNCTION);
1695
1696	/*
1697	* Validate the VM structure, state and handle.
1698	*/
1699	PGVMM pGVMM;
1700	int rc = gvmmR0ByGVMandVM(pGVM, pVM, &pGVMM, false /* fTakeUsedLock /); /* @todo take lock here. */
1701	if (RT_SUCCESS(rc))
1702	{
1703	if (idCpu < pGVM->cCpus)
1704	{
1705	/* Check that the EMT isn't already assigned to a thread. */
1706	if (pGVM->aCpus[idCpu].hEMT == NIL_RTNATIVETHREAD)
1707	{
1708	#ifdef VBOX_BUGREF_9217
1709	Assert(pGVM->aCpus[idCpu].hNativeThreadR0 == NIL_RTNATIVETHREAD);
1710	#else
1711	Assert(pVM->aCpus[idCpu].hNativeThreadR0 == NIL_RTNATIVETHREAD);
1712	#endif
1713
1714	/* A thread may only be one EMT. */
1715	RTNATIVETHREAD const hNativeSelf = RTThreadNativeSelf();
1716	for (VMCPUID iCpu = 0; iCpu < pGVM->cCpus; iCpu++)
1717	AssertBreakStmt(pGVM->aCpus[iCpu].hEMT != hNativeSelf, rc = VERR_INVALID_PARAMETER);
1718	if (RT_SUCCESS(rc))
1719	{
1720	/*
1721	* Do the assignment, then try setup the hook. Undo if that fails.
1722	*/
1723	#ifdef VBOX_BUGREF_9217
1724	pGVM->aCpus[idCpu].hNativeThreadR0 = pGVM->aCpus[idCpu].hEMT = RTThreadNativeSelf();
1725
1726	rc = VMMR0ThreadCtxHookCreateForEmt(&pGVM->aCpus[idCpu]);
1727	if (RT_SUCCESS(rc))
1728	CPUMR0RegisterVCpuThread(&pGVM->aCpus[idCpu]);
1729	else
1730	pGVM->aCpus[idCpu].hNativeThreadR0 = pGVM->aCpus[idCpu].hEMT = NIL_RTNATIVETHREAD;
1731	#else
1732	pVM->aCpus[idCpu].hNativeThreadR0 = pGVM->aCpus[idCpu].hEMT = RTThreadNativeSelf();
1733
1734	rc = VMMR0ThreadCtxHookCreateForEmt(&pVM->aCpus[idCpu]);
1735	if (RT_SUCCESS(rc))
1736	CPUMR0RegisterVCpuThread(&pVM->aCpus[idCpu]);
1737	else
1738	pVM->aCpus[idCpu].hNativeThreadR0 = pGVM->aCpus[idCpu].hEMT = NIL_RTNATIVETHREAD;
1739	#endif
1740	}
1741	}
1742	else
1743	rc = VERR_ACCESS_DENIED;
1744	}
1745	else
1746	rc = VERR_INVALID_CPU_ID;
1747	}
1748	return rc;
1749	}
1750
1751
1752	/**
1753	* Deregisters the calling thread as the EMT of a Virtual CPU.
1754	*
1755	* Note that VCPU 0 shall call GVMMR0DestroyVM intead of this API.
1756	*
1757	* @returns VBox status code
1758	* @param pGVM The global (ring-0) VM structure.
1759	* @param pVM The cross context VM structure.
1760	* @param idCpu VCPU id to register the current thread as.
1761	*/
1762	GVMMR0DECL(int) GVMMR0DeregisterVCpu(PGVM pGVM, PVM pVM, VMCPUID idCpu)
1763	{
1764	AssertReturn(idCpu != 0, VERR_INVALID_FUNCTION);
1765
1766	/*
1767	* Validate the VM structure, state and handle.
1768	*/
1769	PGVMM pGVMM;
1770	int rc = gvmmR0ByGVMandVMandEMT(pGVM, pVM, idCpu, &pGVMM);
1771	if (RT_SUCCESS(rc))
1772	{
1773	/*
1774	* Take the destruction lock and recheck the handle state to
1775	* prevent racing GVMMR0DestroyVM.
1776	*/
1777	gvmmR0CreateDestroyLock(pGVMM);
1778	uint32_t hSelf = pGVM->hSelf;
1779	ASMCompilerBarrier();
1780	if ( hSelf < RT_ELEMENTS(pGVMM->aHandles)
1781	&& pGVMM->aHandles[hSelf].pvObj != NULL
1782	&& pGVMM->aHandles[hSelf].pGVM == pGVM)
1783	{
1784	/*
1785	* Do per-EMT cleanups.
1786	*/
1787	#ifdef VBOX_BUGREF_9217
1788	VMMR0ThreadCtxHookDestroyForEmt(&pGVM->aCpus[idCpu]);
1789	#else
1790	VMMR0ThreadCtxHookDestroyForEmt(&pVM->aCpus[idCpu]);
1791	#endif
1792
1793	/*
1794	* Invalidate hEMT. We don't use NIL here as that would allow
1795	* GVMMR0RegisterVCpu to be called again, and we don't want that.
1796	*/
1797	AssertCompile(~(RTNATIVETHREAD)1 != NIL_RTNATIVETHREAD);
1798	pGVM->aCpus[idCpu].hEMT = ~(RTNATIVETHREAD)1;
1799	#ifdef VBOX_BUGREF_9217
1800	pGVM->aCpus[idCpu].hNativeThreadR0 = NIL_RTNATIVETHREAD;
1801	#else
1802	pVM->aCpus[idCpu].hNativeThreadR0 = NIL_RTNATIVETHREAD;
1803	#endif
1804	}
1805
1806	gvmmR0CreateDestroyUnlock(pGVMM);
1807	}
1808	return rc;
1809	}
1810
1811
1812	/**
1813	* Lookup a GVM structure by its handle.
1814	*
1815	* @returns The GVM pointer on success, NULL on failure.
1816	* @param hGVM The global VM handle. Asserts on bad handle.
1817	*/
1818	GVMMR0DECL(PGVM) GVMMR0ByHandle(uint32_t hGVM)
1819	{
1820	PGVMM pGVMM;
1821	GVMM_GET_VALID_INSTANCE(pGVMM, NULL);
1822
1823	/*
1824	* Validate.
1825	*/
1826	AssertReturn(hGVM != NIL_GVM_HANDLE, NULL);
1827	AssertReturn(hGVM < RT_ELEMENTS(pGVMM->aHandles), NULL);
1828
1829	/*
1830	* Look it up.
1831	*/
1832	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1833	AssertPtrReturn(pHandle->pVM, NULL);
1834	AssertPtrReturn(pHandle->pvObj, NULL);
1835	PGVM pGVM = pHandle->pGVM;
1836	AssertPtrReturn(pGVM, NULL);
1837	#ifdef VBOX_BUGREF_9217
1838	AssertReturn(pGVM == pHandle->pVM, NULL);
1839	#else
1840	AssertReturn(pGVM->pVM == pHandle->pVM, NULL);
1841	#endif
1842
1843	return pHandle->pGVM;
1844	}
1845
1846
1847	/**
1848	* Lookup a GVM structure by the shared VM structure.
1849	*
1850	* The calling thread must be in the same process as the VM. All current lookups
1851	* are by threads inside the same process, so this will not be an issue.
1852	*
1853	* @returns VBox status code.
1854	* @param pVM The cross context VM structure.
1855	* @param ppGVM Where to store the GVM pointer.
1856	* @param ppGVMM Where to store the pointer to the GVMM instance data.
1857	* @param fTakeUsedLock Whether to take the used lock or not. We take it in
1858	* shared mode when requested.
1859	*
1860	* Be very careful if not taking the lock as it's
1861	* possible that the VM will disappear then!
1862	*
1863	* @remark This will not assert on an invalid pVM but try return silently.
1864	*/
1865	static int gvmmR0ByVM(PVM pVM, PGVM ppGVM, PGVMM ppGVMM, bool fTakeUsedLock)
1866	{
1867	RTPROCESS ProcId = RTProcSelf();
1868	PGVMM pGVMM;
1869	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
1870
1871	/*
1872	* Validate.
1873	*/
1874	if (RT_UNLIKELY( !VALID_PTR(pVM)
1875	\|\| ((uintptr_t)pVM & PAGE_OFFSET_MASK)))
1876	return VERR_INVALID_POINTER;
1877	if (RT_UNLIKELY( pVM->enmVMState < VMSTATE_CREATING
1878	\|\| pVM->enmVMState >= VMSTATE_TERMINATED))
1879	return VERR_INVALID_POINTER;
1880
1881	uint16_t hGVM = pVM->hSelf;
1882	ASMCompilerBarrier();
1883	if (RT_UNLIKELY( hGVM == NIL_GVM_HANDLE
1884	\|\| hGVM >= RT_ELEMENTS(pGVMM->aHandles)))
1885	return VERR_INVALID_HANDLE;
1886
1887	/*
1888	* Look it up.
1889	*/
1890	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1891	PGVM pGVM;
1892	if (fTakeUsedLock)
1893	{
1894	int rc = GVMMR0_USED_SHARED_LOCK(pGVMM);
1895	AssertRCReturn(rc, rc);
1896
1897	pGVM = pHandle->pGVM;
1898	#ifdef VBOX_BUGREF_9217
1899	if (RT_UNLIKELY( pHandle->pVM != pVM
1900	\|\| pHandle->ProcId != ProcId
1901	\|\| !VALID_PTR(pHandle->pvObj)
1902	\|\| !VALID_PTR(pGVM)
1903	\|\| pGVM != pVM))
1904	#else
1905	if (RT_UNLIKELY( pHandle->pVM != pVM
1906	\|\| pHandle->ProcId != ProcId
1907	\|\| !VALID_PTR(pHandle->pvObj)
1908	\|\| !VALID_PTR(pGVM)
1909	\|\| pGVM->pVM != pVM))
1910	#endif
1911	{
1912	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
1913	return VERR_INVALID_HANDLE;
1914	}
1915	}
1916	else
1917	{
1918	if (RT_UNLIKELY(pHandle->pVM != pVM))
1919	return VERR_INVALID_HANDLE;
1920	if (RT_UNLIKELY(pHandle->ProcId != ProcId))
1921	return VERR_INVALID_HANDLE;
1922	if (RT_UNLIKELY(!VALID_PTR(pHandle->pvObj)))
1923	return VERR_INVALID_HANDLE;
1924
1925	pGVM = pHandle->pGVM;
1926	if (RT_UNLIKELY(!VALID_PTR(pGVM)))
1927	return VERR_INVALID_HANDLE;
1928	#ifdef VBOX_BUGREF_9217
1929	if (RT_UNLIKELY(pGVM != pVM))
1930	#else
1931	if (RT_UNLIKELY(pGVM->pVM != pVM))
1932	#endif
1933	return VERR_INVALID_HANDLE;
1934	}
1935
1936	*ppGVM = pGVM;
1937	*ppGVMM = pGVMM;
1938	return VINF_SUCCESS;
1939	}
1940
1941
1942	/**
1943	* Fast look up a GVM structure by the cross context VM structure.
1944	*
1945	* This is mainly used a glue function, so performance is .
1946	*
1947	* @returns GVM on success, NULL on failure.
1948	* @param pVM The cross context VM structure. ASSUMES to be
1949	* reasonably valid, so we can do fewer checks than in
1950	* gvmmR0ByVM.
1951	*
1952	* @note Do not use this on pVM structures from userland!
1953	*/
1954	GVMMR0DECL(PGVM) GVMMR0FastGetGVMByVM(PVM pVM)
1955	{
1956	AssertPtr(pVM);
1957	Assert(!((uintptr_t)pVM & PAGE_OFFSET_MASK));
1958
1959	PGVMM pGVMM;
1960	GVMM_GET_VALID_INSTANCE(pGVMM, NULL);
1961
1962	/*
1963	* Validate.
1964	*/
1965	uint16_t hGVM = pVM->hSelf;
1966	ASMCompilerBarrier();
1967	AssertReturn(hGVM != NIL_GVM_HANDLE && hGVM < RT_ELEMENTS(pGVMM->aHandles), NULL);
1968
1969	/*
1970	* Look it up and check pVM against the value in the handle and GVM structures.
1971	*/
1972	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1973	AssertReturn(pHandle->pVM == pVM, NULL);
1974
1975	PGVM pGVM = pHandle->pGVM;
1976	AssertPtrReturn(pGVM, NULL);
1977	#ifdef VBOX_BUGREF_9217
1978	AssertReturn(pGVM == pVM, NULL);
1979	#else
1980	AssertReturn(pGVM->pVM == pVM, NULL);
1981	#endif
1982
1983	return pGVM;
1984	}
1985
1986
1987	/**
1988	* Check that the given GVM and VM structures match up.
1989	*
1990	* The calling thread must be in the same process as the VM. All current lookups
1991	* are by threads inside the same process, so this will not be an issue.
1992	*
1993	* @returns VBox status code.
1994	* @param pGVM The global (ring-0) VM structure.
1995	* @param pVM The cross context VM structure.
1996	* @param ppGVMM Where to store the pointer to the GVMM instance data.
1997	* @param fTakeUsedLock Whether to take the used lock or not. We take it in
1998	* shared mode when requested.
1999	*
2000	* Be very careful if not taking the lock as it's
2001	* possible that the VM will disappear then!
2002	*
2003	* @remark This will not assert on an invalid pVM but try return silently.
2004	*/
2005	static int gvmmR0ByGVMandVM(PGVM pGVM, PVM pVM, PGVMM *ppGVMM, bool fTakeUsedLock)
2006	{
2007	/*
2008	* Check the pointers.
2009	*/
2010	int rc;
2011	if (RT_LIKELY(RT_VALID_PTR(pGVM)))
2012	{
2013	if (RT_LIKELY( RT_VALID_PTR(pVM)
2014	&& ((uintptr_t)pVM & PAGE_OFFSET_MASK) == 0))
2015	{
2016	#ifdef VBOX_BUGREF_9217
2017	if (RT_LIKELY(pGVM == pVM))
2018	#else
2019	if (RT_LIKELY(pGVM->pVM == pVM))
2020	#endif
2021	{
2022	/*
2023	* Get the pGVMM instance and check the VM handle.
2024	*/
2025	PGVMM pGVMM;
2026	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
2027
2028	uint16_t hGVM = pGVM->hSelf;
2029	if (RT_LIKELY( hGVM != NIL_GVM_HANDLE
2030	&& hGVM < RT_ELEMENTS(pGVMM->aHandles)))
2031	{
2032	RTPROCESS const pidSelf = RTProcSelf();
2033	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
2034	if (fTakeUsedLock)
2035	{
2036	rc = GVMMR0_USED_SHARED_LOCK(pGVMM);
2037	AssertRCReturn(rc, rc);
2038	}
2039
2040	if (RT_LIKELY( pHandle->pGVM == pGVM
2041	&& pHandle->pVM == pVM
2042	&& pHandle->ProcId == pidSelf
2043	&& RT_VALID_PTR(pHandle->pvObj)))
2044	{
2045	/*
2046	* Some more VM data consistency checks.
2047	*/
2048	if (RT_LIKELY( pVM->cCpus == pGVM->cCpus
2049	&& pVM->hSelf == hGVM
2050	&& pVM->enmVMState >= VMSTATE_CREATING
2051	&& pVM->enmVMState <= VMSTATE_TERMINATED
2052	&& pVM->pVMR0 == pVM))
2053	{
2054	*ppGVMM = pGVMM;
2055	return VINF_SUCCESS;
2056	}
2057	}
2058
2059	if (fTakeUsedLock)
2060	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2061	}
2062	}
2063	rc = VERR_INVALID_VM_HANDLE;
2064	}
2065	else
2066	rc = VERR_INVALID_POINTER;
2067	}
2068	else
2069	rc = VERR_INVALID_POINTER;
2070	return rc;
2071	}
2072
2073
2074	/**
2075	* Check that the given GVM and VM structures match up.
2076	*
2077	* The calling thread must be in the same process as the VM. All current lookups
2078	* are by threads inside the same process, so this will not be an issue.
2079	*
2080	* @returns VBox status code.
2081	* @param pGVM The global (ring-0) VM structure.
2082	* @param pVM The cross context VM structure.
2083	* @param idCpu The (alleged) Virtual CPU ID of the calling EMT.
2084	* @param ppGVMM Where to store the pointer to the GVMM instance data.
2085	* @thread EMT
2086	*
2087	* @remarks This will assert in all failure paths.
2088	*/
2089	static int gvmmR0ByGVMandVMandEMT(PGVM pGVM, PVM pVM, VMCPUID idCpu, PGVMM *ppGVMM)
2090	{
2091	/*
2092	* Check the pointers.
2093	*/
2094	AssertPtrReturn(pGVM, VERR_INVALID_POINTER);
2095
2096	AssertPtrReturn(pVM, VERR_INVALID_POINTER);
2097	AssertReturn(((uintptr_t)pVM & PAGE_OFFSET_MASK) == 0, VERR_INVALID_POINTER);
2098	#ifdef VBOX_BUGREF_9217
2099	AssertReturn(pGVM == pVM, VERR_INVALID_VM_HANDLE);
2100	#else
2101	AssertReturn(pGVM->pVM == pVM, VERR_INVALID_VM_HANDLE);
2102	#endif
2103
2104
2105	/*
2106	* Get the pGVMM instance and check the VM handle.
2107	*/
2108	PGVMM pGVMM;
2109	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
2110
2111	uint16_t hGVM = pGVM->hSelf;
2112	ASMCompilerBarrier();
2113	AssertReturn( hGVM != NIL_GVM_HANDLE
2114	&& hGVM < RT_ELEMENTS(pGVMM->aHandles), VERR_INVALID_VM_HANDLE);
2115
2116	RTPROCESS const pidSelf = RTProcSelf();
2117	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
2118	AssertReturn( pHandle->pGVM == pGVM
2119	&& pHandle->pVM == pVM
2120	&& pHandle->ProcId == pidSelf
2121	&& RT_VALID_PTR(pHandle->pvObj),
2122	VERR_INVALID_HANDLE);
2123
2124	/*
2125	* Check the EMT claim.
2126	*/
2127	RTNATIVETHREAD const hAllegedEMT = RTThreadNativeSelf();
2128	AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID);
2129	AssertReturn(pGVM->aCpus[idCpu].hEMT == hAllegedEMT, VERR_NOT_OWNER);
2130
2131	/*
2132	* Some more VM data consistency checks.
2133	*/
2134	AssertReturn(pVM->cCpus == pGVM->cCpus, VERR_INCONSISTENT_VM_HANDLE);
2135	AssertReturn(pVM->hSelf == hGVM, VERR_INCONSISTENT_VM_HANDLE);
2136	AssertReturn(pVM->pVMR0 == pVM, VERR_INCONSISTENT_VM_HANDLE);
2137	AssertReturn( pVM->enmVMState >= VMSTATE_CREATING
2138	&& pVM->enmVMState <= VMSTATE_TERMINATED, VERR_INCONSISTENT_VM_HANDLE);
2139
2140	*ppGVMM = pGVMM;
2141	return VINF_SUCCESS;
2142	}
2143
2144
2145	/**
2146	* Validates a GVM/VM pair.
2147	*
2148	* @returns VBox status code.
2149	* @param pGVM The global (ring-0) VM structure.
2150	* @param pVM The cross context VM structure.
2151	*/
2152	GVMMR0DECL(int) GVMMR0ValidateGVMandVM(PGVM pGVM, PVM pVM)
2153	{
2154	PGVMM pGVMM;
2155	return gvmmR0ByGVMandVM(pGVM, pVM, &pGVMM, false /fTakeUsedLock/);
2156	}
2157
2158
2159
2160	/**
2161	* Validates a GVM/VM/EMT combo.
2162	*
2163	* @returns VBox status code.
2164	* @param pGVM The global (ring-0) VM structure.
2165	* @param pVM The cross context VM structure.
2166	* @param idCpu The Virtual CPU ID of the calling EMT.
2167	* @thread EMT(idCpu)
2168	*/
2169	GVMMR0DECL(int) GVMMR0ValidateGVMandVMandEMT(PGVM pGVM, PVM pVM, VMCPUID idCpu)
2170	{
2171	PGVMM pGVMM;
2172	return gvmmR0ByGVMandVMandEMT(pGVM, pVM, idCpu, &pGVMM);
2173	}
2174
2175
2176	/**
2177	* Looks up the VM belonging to the specified EMT thread.
2178	*
2179	* This is used by the assertion machinery in VMMR0.cpp to avoid causing
2180	* unnecessary kernel panics when the EMT thread hits an assertion. The
2181	* call may or not be an EMT thread.
2182	*
2183	* @returns Pointer to the VM on success, NULL on failure.
2184	* @param hEMT The native thread handle of the EMT.
2185	* NIL_RTNATIVETHREAD means the current thread
2186	*/
2187	GVMMR0DECL(PVM) GVMMR0GetVMByEMT(RTNATIVETHREAD hEMT)
2188	{
2189	/*
2190	* No Assertions here as we're usually called in a AssertMsgN or
2191	* RTAssert* context.
2192	*/
2193	PGVMM pGVMM = g_pGVMM;
2194	if ( !VALID_PTR(pGVMM)
2195	\|\| pGVMM->u32Magic != GVMM_MAGIC)
2196	return NULL;
2197
2198	if (hEMT == NIL_RTNATIVETHREAD)
2199	hEMT = RTThreadNativeSelf();
2200	RTPROCESS ProcId = RTProcSelf();
2201
2202	/*
2203	* Search the handles in a linear fashion as we don't dare to take the lock (assert).
2204	*/
2205	/** @todo introduce some pid hash table here, please. */
2206	for (unsigned i = 1; i < RT_ELEMENTS(pGVMM->aHandles); i++)
2207	{
2208	if ( pGVMM->aHandles[i].iSelf == i
2209	&& pGVMM->aHandles[i].ProcId == ProcId
2210	&& VALID_PTR(pGVMM->aHandles[i].pvObj)
2211	&& VALID_PTR(pGVMM->aHandles[i].pVM)
2212	&& VALID_PTR(pGVMM->aHandles[i].pGVM))
2213	{
2214	if (pGVMM->aHandles[i].hEMT0 == hEMT)
2215	return pGVMM->aHandles[i].pVM;
2216
2217	/* This is fearly safe with the current process per VM approach. */
2218	PGVM pGVM = pGVMM->aHandles[i].pGVM;
2219	VMCPUID const cCpus = pGVM->cCpus;
2220	ASMCompilerBarrier();
2221	if ( cCpus < 1
2222	\|\| cCpus > VMM_MAX_CPU_COUNT)
2223	continue;
2224	for (VMCPUID idCpu = 1; idCpu < cCpus; idCpu++)
2225	if (pGVM->aCpus[idCpu].hEMT == hEMT)
2226	return pGVMM->aHandles[i].pVM;
2227	}
2228	}
2229	return NULL;
2230	}
2231
2232
2233	/**
2234	* Looks up the GVMCPU belonging to the specified EMT thread.
2235	*
2236	* This is used by the assertion machinery in VMMR0.cpp to avoid causing
2237	* unnecessary kernel panics when the EMT thread hits an assertion. The
2238	* call may or not be an EMT thread.
2239	*
2240	* @returns Pointer to the VM on success, NULL on failure.
2241	* @param hEMT The native thread handle of the EMT.
2242	* NIL_RTNATIVETHREAD means the current thread
2243	*/
2244	GVMMR0DECL(PGVMCPU) GVMMR0GetGVCpuByEMT(RTNATIVETHREAD hEMT)
2245	{
2246	/*
2247	* No Assertions here as we're usually called in a AssertMsgN,
2248	* RTAssert*, Log and LogRel contexts.
2249	*/
2250	PGVMM pGVMM = g_pGVMM;
2251	if ( !VALID_PTR(pGVMM)
2252	\|\| pGVMM->u32Magic != GVMM_MAGIC)
2253	return NULL;
2254
2255	if (hEMT == NIL_RTNATIVETHREAD)
2256	hEMT = RTThreadNativeSelf();
2257	RTPROCESS ProcId = RTProcSelf();
2258
2259	/*
2260	* Search the handles in a linear fashion as we don't dare to take the lock (assert).
2261	*/
2262	/** @todo introduce some pid hash table here, please. */
2263	for (unsigned i = 1; i < RT_ELEMENTS(pGVMM->aHandles); i++)
2264	{
2265	if ( pGVMM->aHandles[i].iSelf == i
2266	&& pGVMM->aHandles[i].ProcId == ProcId
2267	&& VALID_PTR(pGVMM->aHandles[i].pvObj)
2268	&& VALID_PTR(pGVMM->aHandles[i].pVM)
2269	&& VALID_PTR(pGVMM->aHandles[i].pGVM))
2270	{
2271	PGVM pGVM = pGVMM->aHandles[i].pGVM;
2272	if (pGVMM->aHandles[i].hEMT0 == hEMT)
2273	return &pGVM->aCpus[0];
2274
2275	/* This is fearly safe with the current process per VM approach. */
2276	VMCPUID const cCpus = pGVM->cCpus;
2277	ASMCompilerBarrier();
2278	ASMCompilerBarrier();
2279	if ( cCpus < 1
2280	\|\| cCpus > VMM_MAX_CPU_COUNT)
2281	continue;
2282	for (VMCPUID idCpu = 1; idCpu < cCpus; idCpu++)
2283	if (pGVM->aCpus[idCpu].hEMT == hEMT)
2284	return &pGVM->aCpus[idCpu];
2285	}
2286	}
2287	return NULL;
2288	}
2289
2290
2291	/**
2292	* This is will wake up expired and soon-to-be expired VMs.
2293	*
2294	* @returns Number of VMs that has been woken up.
2295	* @param pGVMM Pointer to the GVMM instance data.
2296	* @param u64Now The current time.
2297	*/
2298	static unsigned gvmmR0SchedDoWakeUps(PGVMM pGVMM, uint64_t u64Now)
2299	{
2300	/*
2301	* Skip this if we've got disabled because of high resolution wakeups or by
2302	* the user.
2303	*/
2304	if (!pGVMM->fDoEarlyWakeUps)
2305	return 0;
2306
2307	/** @todo Rewrite this algorithm. See performance defect XYZ. */
2308
2309	/*
2310	* A cheap optimization to stop wasting so much time here on big setups.
2311	*/
2312	const uint64_t uNsEarlyWakeUp2 = u64Now + pGVMM->nsEarlyWakeUp2;
2313	if ( pGVMM->cHaltedEMTs == 0
2314	\|\| uNsEarlyWakeUp2 > pGVMM->uNsNextEmtWakeup)
2315	return 0;
2316
2317	/*
2318	* Only one thread doing this at a time.
2319	*/
2320	if (!ASMAtomicCmpXchgBool(&pGVMM->fDoingEarlyWakeUps, true, false))
2321	return 0;
2322
2323	/*
2324	* The first pass will wake up VMs which have actually expired
2325	* and look for VMs that should be woken up in the 2nd and 3rd passes.
2326	*/
2327	const uint64_t uNsEarlyWakeUp1 = u64Now + pGVMM->nsEarlyWakeUp1;
2328	uint64_t u64Min = UINT64_MAX;
2329	unsigned cWoken = 0;
2330	unsigned cHalted = 0;
2331	unsigned cTodo2nd = 0;
2332	unsigned cTodo3rd = 0;
2333	for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
2334	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
2335	i = pGVMM->aHandles[i].iNext)
2336	{
2337	PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
2338	if ( VALID_PTR(pCurGVM)
2339	&& pCurGVM->u32Magic == GVM_MAGIC)
2340	{
2341	for (VMCPUID idCpu = 0; idCpu < pCurGVM->cCpus; idCpu++)
2342	{
2343	PGVMCPU pCurGVCpu = &pCurGVM->aCpus[idCpu];
2344	uint64_t u64 = ASMAtomicUoReadU64(&pCurGVCpu->gvmm.s.u64HaltExpire);
2345	if (u64)
2346	{
2347	if (u64 <= u64Now)
2348	{
2349	if (ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0))
2350	{
2351	int rc = RTSemEventMultiSignal(pCurGVCpu->gvmm.s.HaltEventMulti);
2352	AssertRC(rc);
2353	cWoken++;
2354	}
2355	}
2356	else
2357	{
2358	cHalted++;
2359	if (u64 <= uNsEarlyWakeUp1)
2360	cTodo2nd++;
2361	else if (u64 <= uNsEarlyWakeUp2)
2362	cTodo3rd++;
2363	else if (u64 < u64Min)
2364	u64 = u64Min;
2365	}
2366	}
2367	}
2368	}
2369	AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
2370	}
2371
2372	if (cTodo2nd)
2373	{
2374	for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
2375	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
2376	i = pGVMM->aHandles[i].iNext)
2377	{
2378	PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
2379	if ( VALID_PTR(pCurGVM)
2380	&& pCurGVM->u32Magic == GVM_MAGIC)
2381	{
2382	for (VMCPUID idCpu = 0; idCpu < pCurGVM->cCpus; idCpu++)
2383	{
2384	PGVMCPU pCurGVCpu = &pCurGVM->aCpus[idCpu];
2385	uint64_t u64 = ASMAtomicUoReadU64(&pCurGVCpu->gvmm.s.u64HaltExpire);
2386	if ( u64
2387	&& u64 <= uNsEarlyWakeUp1)
2388	{
2389	if (ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0))
2390	{
2391	int rc = RTSemEventMultiSignal(pCurGVCpu->gvmm.s.HaltEventMulti);
2392	AssertRC(rc);
2393	cWoken++;
2394	}
2395	}
2396	}
2397	}
2398	AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
2399	}
2400	}
2401
2402	if (cTodo3rd)
2403	{
2404	for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
2405	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
2406	i = pGVMM->aHandles[i].iNext)
2407	{
2408	PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
2409	if ( VALID_PTR(pCurGVM)
2410	&& pCurGVM->u32Magic == GVM_MAGIC)
2411	{
2412	for (VMCPUID idCpu = 0; idCpu < pCurGVM->cCpus; idCpu++)
2413	{
2414	PGVMCPU pCurGVCpu = &pCurGVM->aCpus[idCpu];
2415	uint64_t u64 = ASMAtomicUoReadU64(&pCurGVCpu->gvmm.s.u64HaltExpire);
2416	if ( u64
2417	&& u64 <= uNsEarlyWakeUp2)
2418	{
2419	if (ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0))
2420	{
2421	int rc = RTSemEventMultiSignal(pCurGVCpu->gvmm.s.HaltEventMulti);
2422	AssertRC(rc);
2423	cWoken++;
2424	}
2425	}
2426	}
2427	}
2428	AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
2429	}
2430	}
2431
2432	/*
2433	* Set the minimum value.
2434	*/
2435	pGVMM->uNsNextEmtWakeup = u64Min;
2436
2437	ASMAtomicWriteBool(&pGVMM->fDoingEarlyWakeUps, false);
2438	return cWoken;
2439	}
2440
2441
2442	/**
2443	* Halt the EMT thread.
2444	*
2445	* @returns VINF_SUCCESS normal wakeup (timeout or kicked by other thread).
2446	* VERR_INTERRUPTED if a signal was scheduled for the thread.
2447	* @param pGVM The global (ring-0) VM structure.
2448	* @param pVM The cross context VM structure.
2449	* @param pGVCpu The global (ring-0) CPU structure of the calling
2450	* EMT.
2451	* @param u64ExpireGipTime The time for the sleep to expire expressed as GIP time.
2452	* @thread EMT(pGVCpu).
2453	*/
2454	GVMMR0DECL(int) GVMMR0SchedHalt(PGVM pGVM, PVM pVM, PGVMCPU pGVCpu, uint64_t u64ExpireGipTime)
2455	{
2456	LogFlow(("GVMMR0SchedHalt: pGVM=%p pVM=%p pGVCpu=%p(%d) u64ExpireGipTime=%#RX64\n",
2457	pGVM, pVM, pGVCpu, pGVCpu->idCpu, u64ExpireGipTime));
2458	GVMM_CHECK_SMAP_SETUP();
2459	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2460
2461	PGVMM pGVMM;
2462	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
2463
2464	pGVM->gvmm.s.StatsSched.cHaltCalls++;
2465	Assert(!pGVCpu->gvmm.s.u64HaltExpire);
2466
2467	/*
2468	* If we're doing early wake-ups, we must take the UsedList lock before we
2469	* start querying the current time.
2470	* Note! Interrupts must NOT be disabled at this point because we ask for GIP time!
2471	*/
2472	bool const fDoEarlyWakeUps = pGVMM->fDoEarlyWakeUps;
2473	if (fDoEarlyWakeUps)
2474	{
2475	int rc2 = GVMMR0_USED_SHARED_LOCK(pGVMM); AssertRC(rc2);
2476	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2477	}
2478
2479	pGVCpu->gvmm.s.iCpuEmt = ASMGetApicId();
2480
2481	/* GIP hack: We might are frequently sleeping for short intervals where the
2482	difference between GIP and system time matters on systems with high resolution
2483	system time. So, convert the input from GIP to System time in that case. */
2484	Assert(ASMGetFlags() & X86_EFL_IF);
2485	const uint64_t u64NowSys = RTTimeSystemNanoTS();
2486	const uint64_t u64NowGip = RTTimeNanoTS();
2487	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2488
2489	if (fDoEarlyWakeUps)
2490	{
2491	pGVM->gvmm.s.StatsSched.cHaltWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64NowGip);
2492	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2493	}
2494
2495	/*
2496	* Go to sleep if we must...
2497	* Cap the sleep time to 1 second to be on the safe side.
2498	*/
2499	int rc;
2500	uint64_t cNsInterval = u64ExpireGipTime - u64NowGip;
2501	if ( u64NowGip < u64ExpireGipTime
2502	&& cNsInterval >= (pGVMM->cEMTs > pGVMM->cEMTsMeansCompany
2503	? pGVMM->nsMinSleepCompany
2504	: pGVMM->nsMinSleepAlone))
2505	{
2506	pGVM->gvmm.s.StatsSched.cHaltBlocking++;
2507	if (cNsInterval > RT_NS_1SEC)
2508	u64ExpireGipTime = u64NowGip + RT_NS_1SEC;
2509	ASMAtomicWriteU64(&pGVCpu->gvmm.s.u64HaltExpire, u64ExpireGipTime);
2510	ASMAtomicIncU32(&pGVMM->cHaltedEMTs);
2511	if (fDoEarlyWakeUps)
2512	{
2513	if (u64ExpireGipTime < pGVMM->uNsNextEmtWakeup)
2514	pGVMM->uNsNextEmtWakeup = u64ExpireGipTime;
2515	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2516	}
2517	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2518
2519	rc = RTSemEventMultiWaitEx(pGVCpu->gvmm.s.HaltEventMulti,
2520	RTSEMWAIT_FLAGS_ABSOLUTE \| RTSEMWAIT_FLAGS_NANOSECS \| RTSEMWAIT_FLAGS_INTERRUPTIBLE,
2521	u64NowGip > u64NowSys ? u64ExpireGipTime : u64NowSys + cNsInterval);
2522	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2523
2524	ASMAtomicWriteU64(&pGVCpu->gvmm.s.u64HaltExpire, 0);
2525	ASMAtomicDecU32(&pGVMM->cHaltedEMTs);
2526
2527	/* Reset the semaphore to try prevent a few false wake-ups. */
2528	if (rc == VINF_SUCCESS)
2529	{
2530	RTSemEventMultiReset(pGVCpu->gvmm.s.HaltEventMulti);
2531	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2532	}
2533	else if (rc == VERR_TIMEOUT)
2534	{
2535	pGVM->gvmm.s.StatsSched.cHaltTimeouts++;
2536	rc = VINF_SUCCESS;
2537	}
2538	}
2539	else
2540	{
2541	pGVM->gvmm.s.StatsSched.cHaltNotBlocking++;
2542	if (fDoEarlyWakeUps)
2543	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2544	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2545	RTSemEventMultiReset(pGVCpu->gvmm.s.HaltEventMulti);
2546	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2547	rc = VINF_SUCCESS;
2548	}
2549
2550	return rc;
2551	}
2552
2553
2554	/**
2555	* Halt the EMT thread.
2556	*
2557	* @returns VINF_SUCCESS normal wakeup (timeout or kicked by other thread).
2558	* VERR_INTERRUPTED if a signal was scheduled for the thread.
2559	* @param pGVM The global (ring-0) VM structure.
2560	* @param pVM The cross context VM structure.
2561	* @param idCpu The Virtual CPU ID of the calling EMT.
2562	* @param u64ExpireGipTime The time for the sleep to expire expressed as GIP time.
2563	* @thread EMT(idCpu).
2564	*/
2565	GVMMR0DECL(int) GVMMR0SchedHaltReq(PGVM pGVM, PVM pVM, VMCPUID idCpu, uint64_t u64ExpireGipTime)
2566	{
2567	GVMM_CHECK_SMAP_SETUP();
2568	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2569	PGVMM pGVMM;
2570	int rc = gvmmR0ByGVMandVMandEMT(pGVM, pVM, idCpu, &pGVMM);
2571	if (RT_SUCCESS(rc))
2572	{
2573	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2574	rc = GVMMR0SchedHalt(pGVM, pVM, &pGVM->aCpus[idCpu], u64ExpireGipTime);
2575	}
2576	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2577	return rc;
2578	}
2579
2580
2581
2582	/**
2583	* Worker for GVMMR0SchedWakeUp and GVMMR0SchedWakeUpAndPokeCpus that wakes up
2584	* the a sleeping EMT.
2585	*
2586	* @retval VINF_SUCCESS if successfully woken up.
2587	* @retval VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
2588	*
2589	* @param pGVM The global (ring-0) VM structure.
2590	* @param pGVCpu The global (ring-0) VCPU structure.
2591	*/
2592	DECLINLINE(int) gvmmR0SchedWakeUpOne(PGVM pGVM, PGVMCPU pGVCpu)
2593	{
2594	pGVM->gvmm.s.StatsSched.cWakeUpCalls++;
2595
2596	/*
2597	* Signal the semaphore regardless of whether it's current blocked on it.
2598	*
2599	* The reason for this is that there is absolutely no way we can be 100%
2600	* certain that it isn't about go to go to sleep on it and just got
2601	* delayed a bit en route. So, we will always signal the semaphore when
2602	* the it is flagged as halted in the VMM.
2603	*/
2604	/** @todo we can optimize some of that by means of the pVCpu->enmState now. */
2605	int rc;
2606	if (pGVCpu->gvmm.s.u64HaltExpire)
2607	{
2608	rc = VINF_SUCCESS;
2609	ASMAtomicWriteU64(&pGVCpu->gvmm.s.u64HaltExpire, 0);
2610	}
2611	else
2612	{
2613	rc = VINF_GVM_NOT_BLOCKED;
2614	pGVM->gvmm.s.StatsSched.cWakeUpNotHalted++;
2615	}
2616
2617	int rc2 = RTSemEventMultiSignal(pGVCpu->gvmm.s.HaltEventMulti);
2618	AssertRC(rc2);
2619
2620	return rc;
2621	}
2622
2623
2624	/**
2625	* Wakes up the halted EMT thread so it can service a pending request.
2626	*
2627	* @returns VBox status code.
2628	* @retval VINF_SUCCESS if successfully woken up.
2629	* @retval VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
2630	*
2631	* @param pGVM The global (ring-0) VM structure.
2632	* @param pVM The cross context VM structure.
2633	* @param idCpu The Virtual CPU ID of the EMT to wake up.
2634	* @param fTakeUsedLock Take the used lock or not
2635	* @thread Any but EMT(idCpu).
2636	*/
2637	GVMMR0DECL(int) GVMMR0SchedWakeUpEx(PGVM pGVM, PVM pVM, VMCPUID idCpu, bool fTakeUsedLock)
2638	{
2639	GVMM_CHECK_SMAP_SETUP();
2640	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2641
2642	/*
2643	* Validate input and take the UsedLock.
2644	*/
2645	PGVMM pGVMM;
2646	int rc = gvmmR0ByGVMandVM(pGVM, pVM, &pGVMM, fTakeUsedLock);
2647	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2648	if (RT_SUCCESS(rc))
2649	{
2650	if (idCpu < pGVM->cCpus)
2651	{
2652	/*
2653	* Do the actual job.
2654	*/
2655	rc = gvmmR0SchedWakeUpOne(pGVM, &pGVM->aCpus[idCpu]);
2656	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2657
2658	if (fTakeUsedLock && pGVMM->fDoEarlyWakeUps)
2659	{
2660	/*
2661	* While we're here, do a round of scheduling.
2662	*/
2663	Assert(ASMGetFlags() & X86_EFL_IF);
2664	const uint64_t u64Now = RTTimeNanoTS(); /* (GIP time) */
2665	pGVM->gvmm.s.StatsSched.cWakeUpWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64Now);
2666	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2667	}
2668	}
2669	else
2670	rc = VERR_INVALID_CPU_ID;
2671
2672	if (fTakeUsedLock)
2673	{
2674	int rc2 = GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2675	AssertRC(rc2);
2676	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2677	}
2678	}
2679
2680	LogFlow(("GVMMR0SchedWakeUpEx: returns %Rrc\n", rc));
2681	return rc;
2682	}
2683
2684
2685	/**
2686	* Wakes up the halted EMT thread so it can service a pending request.
2687	*
2688	* @returns VBox status code.
2689	* @retval VINF_SUCCESS if successfully woken up.
2690	* @retval VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
2691	*
2692	* @param pGVM The global (ring-0) VM structure.
2693	* @param pVM The cross context VM structure.
2694	* @param idCpu The Virtual CPU ID of the EMT to wake up.
2695	* @thread Any but EMT(idCpu).
2696	*/
2697	GVMMR0DECL(int) GVMMR0SchedWakeUp(PGVM pGVM, PVM pVM, VMCPUID idCpu)
2698	{
2699	return GVMMR0SchedWakeUpEx(pGVM, pVM, idCpu, true /* fTakeUsedLock */);
2700	}
2701
2702
2703	/**
2704	* Wakes up the halted EMT thread so it can service a pending request, no GVM
2705	* parameter and no used locking.
2706	*
2707	* @returns VBox status code.
2708	* @retval VINF_SUCCESS if successfully woken up.
2709	* @retval VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
2710	*
2711	* @param pVM The cross context VM structure.
2712	* @param idCpu The Virtual CPU ID of the EMT to wake up.
2713	* @thread Any but EMT(idCpu).
2714	* @deprecated Don't use in new code if possible! Use the GVM variant.
2715	*/
2716	GVMMR0DECL(int) GVMMR0SchedWakeUpNoGVMNoLock(PVM pVM, VMCPUID idCpu)
2717	{
2718	GVMM_CHECK_SMAP_SETUP();
2719	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2720	PGVM pGVM;
2721	PGVMM pGVMM;
2722	int rc = gvmmR0ByVM(pVM, &pGVM, &pGVMM, false /fTakeUsedLock/);
2723	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2724	if (RT_SUCCESS(rc))
2725	rc = GVMMR0SchedWakeUpEx(pGVM, pVM, idCpu, false /fTakeUsedLock/);
2726	return rc;
2727	}
2728
2729
2730	/**
2731	* Worker common to GVMMR0SchedPoke and GVMMR0SchedWakeUpAndPokeCpus that pokes
2732	* the Virtual CPU if it's still busy executing guest code.
2733	*
2734	* @returns VBox status code.
2735	* @retval VINF_SUCCESS if poked successfully.
2736	* @retval VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
2737	*
2738	* @param pGVM The global (ring-0) VM structure.
2739	* @param pVCpu The cross context virtual CPU structure.
2740	*/
2741	DECLINLINE(int) gvmmR0SchedPokeOne(PGVM pGVM, PVMCPU pVCpu)
2742	{
2743	pGVM->gvmm.s.StatsSched.cPokeCalls++;
2744
2745	RTCPUID idHostCpu = pVCpu->idHostCpu;
2746	if ( idHostCpu == NIL_RTCPUID
2747	\|\| VMCPU_GET_STATE(pVCpu) != VMCPUSTATE_STARTED_EXEC)
2748	{
2749	pGVM->gvmm.s.StatsSched.cPokeNotBusy++;
2750	return VINF_GVM_NOT_BUSY_IN_GC;
2751	}
2752
2753	/* Note: this function is not implemented on Darwin and Linux (kernel < 2.6.19) */
2754	RTMpPokeCpu(idHostCpu);
2755	return VINF_SUCCESS;
2756	}
2757
2758
2759	/**
2760	* Pokes an EMT if it's still busy running guest code.
2761	*
2762	* @returns VBox status code.
2763	* @retval VINF_SUCCESS if poked successfully.
2764	* @retval VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
2765	*
2766	* @param pGVM The global (ring-0) VM structure.
2767	* @param pVM The cross context VM structure.
2768	* @param idCpu The ID of the virtual CPU to poke.
2769	* @param fTakeUsedLock Take the used lock or not
2770	*/
2771	GVMMR0DECL(int) GVMMR0SchedPokeEx(PGVM pGVM, PVM pVM, VMCPUID idCpu, bool fTakeUsedLock)
2772	{
2773	/*
2774	* Validate input and take the UsedLock.
2775	*/
2776	PGVMM pGVMM;
2777	int rc = gvmmR0ByGVMandVM(pGVM, pVM, &pGVMM, fTakeUsedLock);
2778	if (RT_SUCCESS(rc))
2779	{
2780	if (idCpu < pGVM->cCpus)
2781	#ifdef VBOX_BUGREF_9217
2782	rc = gvmmR0SchedPokeOne(pGVM, &pGVM->aCpus[idCpu]);
2783	#else
2784	rc = gvmmR0SchedPokeOne(pGVM, &pVM->aCpus[idCpu]);
2785	#endif
2786	else
2787	rc = VERR_INVALID_CPU_ID;
2788
2789	if (fTakeUsedLock)
2790	{
2791	int rc2 = GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2792	AssertRC(rc2);
2793	}
2794	}
2795
2796	LogFlow(("GVMMR0SchedWakeUpAndPokeCpus: returns %Rrc\n", rc));
2797	return rc;
2798	}
2799
2800
2801	/**
2802	* Pokes an EMT if it's still busy running guest code.
2803	*
2804	* @returns VBox status code.
2805	* @retval VINF_SUCCESS if poked successfully.
2806	* @retval VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
2807	*
2808	* @param pGVM The global (ring-0) VM structure.
2809	* @param pVM The cross context VM structure.
2810	* @param idCpu The ID of the virtual CPU to poke.
2811	*/
2812	GVMMR0DECL(int) GVMMR0SchedPoke(PGVM pGVM, PVM pVM, VMCPUID idCpu)
2813	{
2814	return GVMMR0SchedPokeEx(pGVM, pVM, idCpu, true /* fTakeUsedLock */);
2815	}
2816
2817
2818	/**
2819	* Pokes an EMT if it's still busy running guest code, no GVM parameter and no
2820	* used locking.
2821	*
2822	* @returns VBox status code.
2823	* @retval VINF_SUCCESS if poked successfully.
2824	* @retval VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
2825	*
2826	* @param pVM The cross context VM structure.
2827	* @param idCpu The ID of the virtual CPU to poke.
2828	*
2829	* @deprecated Don't use in new code if possible! Use the GVM variant.
2830	*/
2831	GVMMR0DECL(int) GVMMR0SchedPokeNoGVMNoLock(PVM pVM, VMCPUID idCpu)
2832	{
2833	PGVM pGVM;
2834	PGVMM pGVMM;
2835	int rc = gvmmR0ByVM(pVM, &pGVM, &pGVMM, false /fTakeUsedLock/);
2836	if (RT_SUCCESS(rc))
2837	{
2838	if (idCpu < pGVM->cCpus)
2839	#ifdef VBOX_BUGREF_9217
2840	rc = gvmmR0SchedPokeOne(pGVM, &pGVM->aCpus[idCpu]);
2841	#else
2842	rc = gvmmR0SchedPokeOne(pGVM, &pVM->aCpus[idCpu]);
2843	#endif
2844	else
2845	rc = VERR_INVALID_CPU_ID;
2846	}
2847	return rc;
2848	}
2849
2850
2851	/**
2852	* Wakes up a set of halted EMT threads so they can service pending request.
2853	*
2854	* @returns VBox status code, no informational stuff.
2855	*
2856	* @param pGVM The global (ring-0) VM structure.
2857	* @param pVM The cross context VM structure.
2858	* @param pSleepSet The set of sleepers to wake up.
2859	* @param pPokeSet The set of CPUs to poke.
2860	*/
2861	GVMMR0DECL(int) GVMMR0SchedWakeUpAndPokeCpus(PGVM pGVM, PVM pVM, PCVMCPUSET pSleepSet, PCVMCPUSET pPokeSet)
2862	{
2863	AssertPtrReturn(pSleepSet, VERR_INVALID_POINTER);
2864	AssertPtrReturn(pPokeSet, VERR_INVALID_POINTER);
2865	GVMM_CHECK_SMAP_SETUP();
2866	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2867	RTNATIVETHREAD hSelf = RTThreadNativeSelf();
2868
2869	/*
2870	* Validate input and take the UsedLock.
2871	*/
2872	PGVMM pGVMM;
2873	int rc = gvmmR0ByGVMandVM(pGVM, pVM, &pGVMM, true /* fTakeUsedLock */);
2874	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2875	if (RT_SUCCESS(rc))
2876	{
2877	rc = VINF_SUCCESS;
2878	VMCPUID idCpu = pGVM->cCpus;
2879	while (idCpu-- > 0)
2880	{
2881	/* Don't try poke or wake up ourselves. */
2882	if (pGVM->aCpus[idCpu].hEMT == hSelf)
2883	continue;
2884
2885	/* just ignore errors for now. */
2886	if (VMCPUSET_IS_PRESENT(pSleepSet, idCpu))
2887	{
2888	gvmmR0SchedWakeUpOne(pGVM, &pGVM->aCpus[idCpu]);
2889	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2890	}
2891	else if (VMCPUSET_IS_PRESENT(pPokeSet, idCpu))
2892	{
2893	#ifdef VBOX_BUGREF_9217
2894	gvmmR0SchedPokeOne(pGVM, &pGVM->aCpus[idCpu]);
2895	#else
2896	gvmmR0SchedPokeOne(pGVM, &pVM->aCpus[idCpu]);
2897	#endif
2898	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2899	}
2900	}
2901
2902	int rc2 = GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2903	AssertRC(rc2);
2904	GVMM_CHECK_SMAP_CHECK2(pVM, RT_NOTHING);
2905	}
2906
2907	LogFlow(("GVMMR0SchedWakeUpAndPokeCpus: returns %Rrc\n", rc));
2908	return rc;
2909	}
2910
2911
2912	/**
2913	* VMMR0 request wrapper for GVMMR0SchedWakeUpAndPokeCpus.
2914	*
2915	* @returns see GVMMR0SchedWakeUpAndPokeCpus.
2916	* @param pGVM The global (ring-0) VM structure.
2917	* @param pVM The cross context VM structure.
2918	* @param pReq Pointer to the request packet.
2919	*/
2920	GVMMR0DECL(int) GVMMR0SchedWakeUpAndPokeCpusReq(PGVM pGVM, PVM pVM, PGVMMSCHEDWAKEUPANDPOKECPUSREQ pReq)
2921	{
2922	/*
2923	* Validate input and pass it on.
2924	*/
2925	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
2926	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
2927
2928	return GVMMR0SchedWakeUpAndPokeCpus(pGVM, pVM, &pReq->SleepSet, &pReq->PokeSet);
2929	}
2930
2931
2932
2933	/**
2934	* Poll the schedule to see if someone else should get a chance to run.
2935	*
2936	* This is a bit hackish and will not work too well if the machine is
2937	* under heavy load from non-VM processes.
2938	*
2939	* @returns VINF_SUCCESS if not yielded.
2940	* VINF_GVM_YIELDED if an attempt to switch to a different VM task was made.
2941	* @param pGVM The global (ring-0) VM structure.
2942	* @param pVM The cross context VM structure.
2943	* @param idCpu The Virtual CPU ID of the calling EMT.
2944	* @param fYield Whether to yield or not.
2945	* This is for when we're spinning in the halt loop.
2946	* @thread EMT(idCpu).
2947	*/
2948	GVMMR0DECL(int) GVMMR0SchedPoll(PGVM pGVM, PVM pVM, VMCPUID idCpu, bool fYield)
2949	{
2950	/*
2951	* Validate input.
2952	*/
2953	PGVMM pGVMM;
2954	int rc = gvmmR0ByGVMandVMandEMT(pGVM, pVM, idCpu, &pGVMM);
2955	if (RT_SUCCESS(rc))
2956	{
2957	/*
2958	* We currently only implement helping doing wakeups (fYield = false), so don't
2959	* bother taking the lock if gvmmR0SchedDoWakeUps is not going to do anything.
2960	*/
2961	if (!fYield && pGVMM->fDoEarlyWakeUps)
2962	{
2963	rc = GVMMR0_USED_SHARED_LOCK(pGVMM); AssertRC(rc);
2964	pGVM->gvmm.s.StatsSched.cPollCalls++;
2965
2966	Assert(ASMGetFlags() & X86_EFL_IF);
2967	const uint64_t u64Now = RTTimeNanoTS(); /* (GIP time) */
2968
2969	pGVM->gvmm.s.StatsSched.cPollWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64Now);
2970
2971	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
2972	}
2973	/*
2974	* Not quite sure what we could do here...
2975	*/
2976	else if (fYield)
2977	rc = VERR_NOT_IMPLEMENTED; /** @todo implement this... */
2978	else
2979	rc = VINF_SUCCESS;
2980	}
2981
2982	LogFlow(("GVMMR0SchedWakeUp: returns %Rrc\n", rc));
2983	return rc;
2984	}
2985
2986
2987	#ifdef GVMM_SCHED_WITH_PPT
2988	/**
2989	* Timer callback for the periodic preemption timer.
2990	*
2991	* @param pTimer The timer handle.
2992	* @param pvUser Pointer to the per cpu structure.
2993	* @param iTick The current tick.
2994	*/
2995	static DECLCALLBACK(void) gvmmR0SchedPeriodicPreemptionTimerCallback(PRTTIMER pTimer, void *pvUser, uint64_t iTick)
2996	{
2997	PGVMMHOSTCPU pCpu = (PGVMMHOSTCPU)pvUser;
2998	NOREF(pTimer); NOREF(iTick);
2999
3000	/*
3001	* Termination check
3002	*/
3003	if (pCpu->u32Magic != GVMMHOSTCPU_MAGIC)
3004	return;
3005
3006	/*
3007	* Do the house keeping.
3008	*/
3009	RTSpinlockAcquire(pCpu->Ppt.hSpinlock);
3010
3011	if (++pCpu->Ppt.iTickHistorization >= pCpu->Ppt.cTicksHistoriziationInterval)
3012	{
3013	/*
3014	* Historicize the max frequency.
3015	*/
3016	uint32_t iHzHistory = ++pCpu->Ppt.iHzHistory % RT_ELEMENTS(pCpu->Ppt.aHzHistory);
3017	pCpu->Ppt.aHzHistory[iHzHistory] = pCpu->Ppt.uDesiredHz;
3018	pCpu->Ppt.iTickHistorization = 0;
3019	pCpu->Ppt.uDesiredHz = 0;
3020
3021	/*
3022	* Check if the current timer frequency.
3023	*/
3024	uint32_t uHistMaxHz = 0;
3025	for (uint32_t i = 0; i < RT_ELEMENTS(pCpu->Ppt.aHzHistory); i++)
3026	if (pCpu->Ppt.aHzHistory[i] > uHistMaxHz)
3027	uHistMaxHz = pCpu->Ppt.aHzHistory[i];
3028	if (uHistMaxHz == pCpu->Ppt.uTimerHz)
3029	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
3030	else if (uHistMaxHz)
3031	{
3032	/*
3033	* Reprogram it.
3034	*/
3035	pCpu->Ppt.cChanges++;
3036	pCpu->Ppt.iTickHistorization = 0;
3037	pCpu->Ppt.uTimerHz = uHistMaxHz;
3038	uint32_t const cNsInterval = RT_NS_1SEC / uHistMaxHz;
3039	pCpu->Ppt.cNsInterval = cNsInterval;
3040	if (cNsInterval < GVMMHOSTCPU_PPT_HIST_INTERVAL_NS)
3041	pCpu->Ppt.cTicksHistoriziationInterval = ( GVMMHOSTCPU_PPT_HIST_INTERVAL_NS
3042	+ GVMMHOSTCPU_PPT_HIST_INTERVAL_NS / 2 - 1)
3043	/ cNsInterval;
3044	else
3045	pCpu->Ppt.cTicksHistoriziationInterval = 1;
3046	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
3047
3048	/SUPR0Printf("Cpu%u: change to %u Hz / %u ns\n", pCpu->idxCpuSet, uHistMaxHz, cNsInterval);/
3049	RTTimerChangeInterval(pTimer, cNsInterval);
3050	}
3051	else
3052	{
3053	/*
3054	* Stop it.
3055	*/
3056	pCpu->Ppt.fStarted = false;
3057	pCpu->Ppt.uTimerHz = 0;
3058	pCpu->Ppt.cNsInterval = 0;
3059	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
3060
3061	/SUPR0Printf("Cpu%u: stopping (%u Hz)\n", pCpu->idxCpuSet, uHistMaxHz);/
3062	RTTimerStop(pTimer);
3063	}
3064	}
3065	else
3066	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
3067	}
3068	#endif /* GVMM_SCHED_WITH_PPT */
3069
3070
3071	/**
3072	* Updates the periodic preemption timer for the calling CPU.
3073	*
3074	* The caller must have disabled preemption!
3075	* The caller must check that the host can do high resolution timers.
3076	*
3077	* @param pVM The cross context VM structure.
3078	* @param idHostCpu The current host CPU id.
3079	* @param uHz The desired frequency.
3080	*/
3081	GVMMR0DECL(void) GVMMR0SchedUpdatePeriodicPreemptionTimer(PVM pVM, RTCPUID idHostCpu, uint32_t uHz)
3082	{
3083	NOREF(pVM);
3084	#ifdef GVMM_SCHED_WITH_PPT
3085	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3086	Assert(RTTimerCanDoHighResolution());
3087
3088	/*
3089	* Resolve the per CPU data.
3090	*/
3091	uint32_t iCpu = RTMpCpuIdToSetIndex(idHostCpu);
3092	PGVMM pGVMM = g_pGVMM;
3093	if ( !VALID_PTR(pGVMM)
3094	\|\| pGVMM->u32Magic != GVMM_MAGIC)
3095	return;
3096	AssertMsgReturnVoid(iCpu < pGVMM->cHostCpus, ("iCpu=%d cHostCpus=%d\n", iCpu, pGVMM->cHostCpus));
3097	PGVMMHOSTCPU pCpu = &pGVMM->aHostCpus[iCpu];
3098	AssertMsgReturnVoid( pCpu->u32Magic == GVMMHOSTCPU_MAGIC
3099	&& pCpu->idCpu == idHostCpu,
3100	("u32Magic=%#x idCpu=% idHostCpu=%d\n", pCpu->u32Magic, pCpu->idCpu, idHostCpu));
3101
3102	/*
3103	* Check whether we need to do anything about the timer.
3104	* We have to be a little bit careful since we might be race the timer
3105	* callback here.
3106	*/
3107	if (uHz > 16384)
3108	uHz = 16384; /** @todo add a query method for this! */
3109	if (RT_UNLIKELY( uHz > ASMAtomicReadU32(&pCpu->Ppt.uDesiredHz)
3110	&& uHz >= pCpu->Ppt.uMinHz
3111	&& !pCpu->Ppt.fStarting /* solaris paranoia */))
3112	{
3113	RTSpinlockAcquire(pCpu->Ppt.hSpinlock);
3114
3115	pCpu->Ppt.uDesiredHz = uHz;
3116	uint32_t cNsInterval = 0;
3117	if (!pCpu->Ppt.fStarted)
3118	{
3119	pCpu->Ppt.cStarts++;
3120	pCpu->Ppt.fStarted = true;
3121	pCpu->Ppt.fStarting = true;
3122	pCpu->Ppt.iTickHistorization = 0;
3123	pCpu->Ppt.uTimerHz = uHz;
3124	pCpu->Ppt.cNsInterval = cNsInterval = RT_NS_1SEC / uHz;
3125	if (cNsInterval < GVMMHOSTCPU_PPT_HIST_INTERVAL_NS)
3126	pCpu->Ppt.cTicksHistoriziationInterval = ( GVMMHOSTCPU_PPT_HIST_INTERVAL_NS
3127	+ GVMMHOSTCPU_PPT_HIST_INTERVAL_NS / 2 - 1)
3128	/ cNsInterval;
3129	else
3130	pCpu->Ppt.cTicksHistoriziationInterval = 1;
3131	}
3132
3133	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
3134
3135	if (cNsInterval)
3136	{
3137	RTTimerChangeInterval(pCpu->Ppt.pTimer, cNsInterval);
3138	int rc = RTTimerStart(pCpu->Ppt.pTimer, cNsInterval);
3139	AssertRC(rc);
3140
3141	RTSpinlockAcquire(pCpu->Ppt.hSpinlock);
3142	if (RT_FAILURE(rc))
3143	pCpu->Ppt.fStarted = false;
3144	pCpu->Ppt.fStarting = false;
3145	RTSpinlockRelease(pCpu->Ppt.hSpinlock);
3146	}
3147	}
3148	#else /* !GVMM_SCHED_WITH_PPT */
3149	NOREF(idHostCpu); NOREF(uHz);
3150	#endif /* !GVMM_SCHED_WITH_PPT */
3151	}
3152
3153
3154	/**
3155	* Retrieves the GVMM statistics visible to the caller.
3156	*
3157	* @returns VBox status code.
3158	*
3159	* @param pStats Where to put the statistics.
3160	* @param pSession The current session.
3161	* @param pGVM The GVM to obtain statistics for. Optional.
3162	* @param pVM The VM structure corresponding to @a pGVM.
3163	*/
3164	GVMMR0DECL(int) GVMMR0QueryStatistics(PGVMMSTATS pStats, PSUPDRVSESSION pSession, PGVM pGVM, PVM pVM)
3165	{
3166	LogFlow(("GVMMR0QueryStatistics: pStats=%p pSession=%p pGVM=%p pVM=%p\n", pStats, pSession, pGVM, pVM));
3167
3168	/*
3169	* Validate input.
3170	*/
3171	AssertPtrReturn(pSession, VERR_INVALID_POINTER);
3172	AssertPtrReturn(pStats, VERR_INVALID_POINTER);
3173	pStats->cVMs = 0; /* (crash before taking the sem...) */
3174
3175	/*
3176	* Take the lock and get the VM statistics.
3177	*/
3178	PGVMM pGVMM;
3179	if (pGVM)
3180	{
3181	int rc = gvmmR0ByGVMandVM(pGVM, pVM, &pGVMM, true /fTakeUsedLock/);
3182	if (RT_FAILURE(rc))
3183	return rc;
3184	pStats->SchedVM = pGVM->gvmm.s.StatsSched;
3185	}
3186	else
3187	{
3188	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
3189	memset(&pStats->SchedVM, 0, sizeof(pStats->SchedVM));
3190
3191	int rc = GVMMR0_USED_SHARED_LOCK(pGVMM);
3192	AssertRCReturn(rc, rc);
3193	}
3194
3195	/*
3196	* Enumerate the VMs and add the ones visible to the statistics.
3197	*/
3198	pStats->cVMs = 0;
3199	pStats->cEMTs = 0;
3200	memset(&pStats->SchedSum, 0, sizeof(pStats->SchedSum));
3201
3202	for (unsigned i = pGVMM->iUsedHead;
3203	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
3204	i = pGVMM->aHandles[i].iNext)
3205	{
3206	PGVM pOtherGVM = pGVMM->aHandles[i].pGVM;
3207	void *pvObj = pGVMM->aHandles[i].pvObj;
3208	if ( VALID_PTR(pvObj)
3209	&& VALID_PTR(pOtherGVM)
3210	&& pOtherGVM->u32Magic == GVM_MAGIC
3211	&& RT_SUCCESS(SUPR0ObjVerifyAccess(pvObj, pSession, NULL)))
3212	{
3213	pStats->cVMs++;
3214	pStats->cEMTs += pOtherGVM->cCpus;
3215
3216	pStats->SchedSum.cHaltCalls += pOtherGVM->gvmm.s.StatsSched.cHaltCalls;
3217	pStats->SchedSum.cHaltBlocking += pOtherGVM->gvmm.s.StatsSched.cHaltBlocking;
3218	pStats->SchedSum.cHaltTimeouts += pOtherGVM->gvmm.s.StatsSched.cHaltTimeouts;
3219	pStats->SchedSum.cHaltNotBlocking += pOtherGVM->gvmm.s.StatsSched.cHaltNotBlocking;
3220	pStats->SchedSum.cHaltWakeUps += pOtherGVM->gvmm.s.StatsSched.cHaltWakeUps;
3221
3222	pStats->SchedSum.cWakeUpCalls += pOtherGVM->gvmm.s.StatsSched.cWakeUpCalls;
3223	pStats->SchedSum.cWakeUpNotHalted += pOtherGVM->gvmm.s.StatsSched.cWakeUpNotHalted;
3224	pStats->SchedSum.cWakeUpWakeUps += pOtherGVM->gvmm.s.StatsSched.cWakeUpWakeUps;
3225
3226	pStats->SchedSum.cPokeCalls += pOtherGVM->gvmm.s.StatsSched.cPokeCalls;
3227	pStats->SchedSum.cPokeNotBusy += pOtherGVM->gvmm.s.StatsSched.cPokeNotBusy;
3228
3229	pStats->SchedSum.cPollCalls += pOtherGVM->gvmm.s.StatsSched.cPollCalls;
3230	pStats->SchedSum.cPollHalts += pOtherGVM->gvmm.s.StatsSched.cPollHalts;
3231	pStats->SchedSum.cPollWakeUps += pOtherGVM->gvmm.s.StatsSched.cPollWakeUps;
3232	}
3233	}
3234
3235	/*
3236	* Copy out the per host CPU statistics.
3237	*/
3238	uint32_t iDstCpu = 0;
3239	uint32_t cSrcCpus = pGVMM->cHostCpus;
3240	for (uint32_t iSrcCpu = 0; iSrcCpu < cSrcCpus; iSrcCpu++)
3241	{
3242	if (pGVMM->aHostCpus[iSrcCpu].idCpu != NIL_RTCPUID)
3243	{
3244	pStats->aHostCpus[iDstCpu].idCpu = pGVMM->aHostCpus[iSrcCpu].idCpu;
3245	pStats->aHostCpus[iDstCpu].idxCpuSet = pGVMM->aHostCpus[iSrcCpu].idxCpuSet;
3246	#ifdef GVMM_SCHED_WITH_PPT
3247	pStats->aHostCpus[iDstCpu].uDesiredHz = pGVMM->aHostCpus[iSrcCpu].Ppt.uDesiredHz;
3248	pStats->aHostCpus[iDstCpu].uTimerHz = pGVMM->aHostCpus[iSrcCpu].Ppt.uTimerHz;
3249	pStats->aHostCpus[iDstCpu].cChanges = pGVMM->aHostCpus[iSrcCpu].Ppt.cChanges;
3250	pStats->aHostCpus[iDstCpu].cStarts = pGVMM->aHostCpus[iSrcCpu].Ppt.cStarts;
3251	#else
3252	pStats->aHostCpus[iDstCpu].uDesiredHz = 0;
3253	pStats->aHostCpus[iDstCpu].uTimerHz = 0;
3254	pStats->aHostCpus[iDstCpu].cChanges = 0;
3255	pStats->aHostCpus[iDstCpu].cStarts = 0;
3256	#endif
3257	iDstCpu++;
3258	if (iDstCpu >= RT_ELEMENTS(pStats->aHostCpus))
3259	break;
3260	}
3261	}
3262	pStats->cHostCpus = iDstCpu;
3263
3264	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
3265
3266	return VINF_SUCCESS;
3267	}
3268
3269
3270	/**
3271	* VMMR0 request wrapper for GVMMR0QueryStatistics.
3272	*
3273	* @returns see GVMMR0QueryStatistics.
3274	* @param pGVM The global (ring-0) VM structure. Optional.
3275	* @param pVM The cross context VM structure. Optional.
3276	* @param pReq Pointer to the request packet.
3277	* @param pSession The current session.
3278	*/
3279	GVMMR0DECL(int) GVMMR0QueryStatisticsReq(PGVM pGVM, PVM pVM, PGVMMQUERYSTATISTICSSREQ pReq, PSUPDRVSESSION pSession)
3280	{
3281	/*
3282	* Validate input and pass it on.
3283	*/
3284	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
3285	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
3286	AssertReturn(pReq->pSession == pSession, VERR_INVALID_PARAMETER);
3287
3288	return GVMMR0QueryStatistics(&pReq->Stats, pSession, pGVM, pVM);
3289	}
3290
3291
3292	/**
3293	* Resets the specified GVMM statistics.
3294	*
3295	* @returns VBox status code.
3296	*
3297	* @param pStats Which statistics to reset, that is, non-zero fields indicates which to reset.
3298	* @param pSession The current session.
3299	* @param pGVM The GVM to reset statistics for. Optional.
3300	* @param pVM The VM structure corresponding to @a pGVM.
3301	*/
3302	GVMMR0DECL(int) GVMMR0ResetStatistics(PCGVMMSTATS pStats, PSUPDRVSESSION pSession, PGVM pGVM, PVM pVM)
3303	{
3304	LogFlow(("GVMMR0ResetStatistics: pStats=%p pSession=%p pGVM=%p pVM=%p\n", pStats, pSession, pGVM, pVM));
3305
3306	/*
3307	* Validate input.
3308	*/
3309	AssertPtrReturn(pSession, VERR_INVALID_POINTER);
3310	AssertPtrReturn(pStats, VERR_INVALID_POINTER);
3311
3312	/*
3313	* Take the lock and get the VM statistics.
3314	*/
3315	PGVMM pGVMM;
3316	if (pGVM)
3317	{
3318	int rc = gvmmR0ByGVMandVM(pGVM, pVM, &pGVMM, true /fTakeUsedLock/);
3319	if (RT_FAILURE(rc))
3320	return rc;
3321	# define MAYBE_RESET_FIELD(field) \
3322	do { if (pStats->SchedVM. field ) { pGVM->gvmm.s.StatsSched. field = 0; } } while (0)
3323	MAYBE_RESET_FIELD(cHaltCalls);
3324	MAYBE_RESET_FIELD(cHaltBlocking);
3325	MAYBE_RESET_FIELD(cHaltTimeouts);
3326	MAYBE_RESET_FIELD(cHaltNotBlocking);
3327	MAYBE_RESET_FIELD(cHaltWakeUps);
3328	MAYBE_RESET_FIELD(cWakeUpCalls);
3329	MAYBE_RESET_FIELD(cWakeUpNotHalted);
3330	MAYBE_RESET_FIELD(cWakeUpWakeUps);
3331	MAYBE_RESET_FIELD(cPokeCalls);
3332	MAYBE_RESET_FIELD(cPokeNotBusy);
3333	MAYBE_RESET_FIELD(cPollCalls);
3334	MAYBE_RESET_FIELD(cPollHalts);
3335	MAYBE_RESET_FIELD(cPollWakeUps);
3336	# undef MAYBE_RESET_FIELD
3337	}
3338	else
3339	{
3340	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_GVMM_INSTANCE);
3341
3342	int rc = GVMMR0_USED_SHARED_LOCK(pGVMM);
3343	AssertRCReturn(rc, rc);
3344	}
3345
3346	/*
3347	* Enumerate the VMs and add the ones visible to the statistics.
3348	*/
3349	if (!ASMMemIsZero(&pStats->SchedSum, sizeof(pStats->SchedSum)))
3350	{
3351	for (unsigned i = pGVMM->iUsedHead;
3352	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
3353	i = pGVMM->aHandles[i].iNext)
3354	{
3355	PGVM pOtherGVM = pGVMM->aHandles[i].pGVM;
3356	void *pvObj = pGVMM->aHandles[i].pvObj;
3357	if ( VALID_PTR(pvObj)
3358	&& VALID_PTR(pOtherGVM)
3359	&& pOtherGVM->u32Magic == GVM_MAGIC
3360	&& RT_SUCCESS(SUPR0ObjVerifyAccess(pvObj, pSession, NULL)))
3361	{
3362	# define MAYBE_RESET_FIELD(field) \
3363	do { if (pStats->SchedSum. field ) { pOtherGVM->gvmm.s.StatsSched. field = 0; } } while (0)
3364	MAYBE_RESET_FIELD(cHaltCalls);
3365	MAYBE_RESET_FIELD(cHaltBlocking);
3366	MAYBE_RESET_FIELD(cHaltTimeouts);
3367	MAYBE_RESET_FIELD(cHaltNotBlocking);
3368	MAYBE_RESET_FIELD(cHaltWakeUps);
3369	MAYBE_RESET_FIELD(cWakeUpCalls);
3370	MAYBE_RESET_FIELD(cWakeUpNotHalted);
3371	MAYBE_RESET_FIELD(cWakeUpWakeUps);
3372	MAYBE_RESET_FIELD(cPokeCalls);
3373	MAYBE_RESET_FIELD(cPokeNotBusy);
3374	MAYBE_RESET_FIELD(cPollCalls);
3375	MAYBE_RESET_FIELD(cPollHalts);
3376	MAYBE_RESET_FIELD(cPollWakeUps);
3377	# undef MAYBE_RESET_FIELD
3378	}
3379	}
3380	}
3381
3382	GVMMR0_USED_SHARED_UNLOCK(pGVMM);
3383
3384	return VINF_SUCCESS;
3385	}
3386
3387
3388	/**
3389	* VMMR0 request wrapper for GVMMR0ResetStatistics.
3390	*
3391	* @returns see GVMMR0ResetStatistics.
3392	* @param pGVM The global (ring-0) VM structure. Optional.
3393	* @param pVM The cross context VM structure. Optional.
3394	* @param pReq Pointer to the request packet.
3395	* @param pSession The current session.
3396	*/
3397	GVMMR0DECL(int) GVMMR0ResetStatisticsReq(PGVM pGVM, PVM pVM, PGVMMRESETSTATISTICSSREQ pReq, PSUPDRVSESSION pSession)
3398	{
3399	/*
3400	* Validate input and pass it on.
3401	*/
3402	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
3403	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
3404	AssertReturn(pReq->pSession == pSession, VERR_INVALID_PARAMETER);
3405
3406	return GVMMR0ResetStatistics(&pReq->Stats, pSession, pGVM, pVM);
3407	}
3408

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/GVMMR0.cpp@ 78927

Download in other formats: