GVMMR0.cpp@ 19606

Last change on this file since 19606 was 19454, checked in by vboxsync, 16 years ago
VMM++: More on poking. Fixed broken R0 stats (wrong way of calling into VMMR0), use NIL_VMCPUID instead of 0 to VMMR0EntryEx when it is supposed to be irrellevant. Use VMCPUID. Allow for and check NIL_VMCPUID. Fixed a few missing/wrong idCpu checks (paranoia mostly).
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 66.4 KB

Line
1	/* $Id: GVMMR0.cpp 19454 2009-05-06 19:20:18Z vboxsync $ */
2	/** @file
3	* GVMM - Global VM Manager.
4	*/
5
6	/*
7	* Copyright (C) 2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/** @page pg_gvmm GVMM - The Global VM Manager
24	*
25	* The Global VM Manager lives in ring-0. It's main function at the moment
26	* is to manage a list of all running VMs, keep a ring-0 only structure (GVM)
27	* for each of them, and assign them unique identifiers (so GMM can track
28	* page owners). The idea for the future is to add an idle priority kernel
29	* thread that can take care of tasks like page sharing.
30	*
31	* The GVMM will create a ring-0 object for each VM when it's registered,
32	* this is both for session cleanup purposes and for having a point where
33	* it's possible to implement usage polices later (in SUPR0ObjRegister).
34	*/
35
36
37	/*******************************************************************************
38	* Header Files *
39	*******************************************************************************/
40	#define LOG_GROUP LOG_GROUP_GVMM
41	#include <VBox/gvmm.h>
42	#include <VBox/gmm.h>
43	#include "GVMMR0Internal.h"
44	#include <VBox/gvm.h>
45	#include <VBox/vm.h>
46	#include <VBox/vmm.h>
47	#include <VBox/param.h>
48	#include <VBox/err.h>
49	#include <iprt/alloc.h>
50	#include <iprt/semaphore.h>
51	#include <iprt/time.h>
52	#include <VBox/log.h>
53	#include <iprt/thread.h>
54	#include <iprt/process.h>
55	#include <iprt/param.h>
56	#include <iprt/string.h>
57	#include <iprt/assert.h>
58	#include <iprt/mem.h>
59	#include <iprt/memobj.h>
60	#include <iprt/mp.h>
61
62
63	/*******************************************************************************
64	* Structures and Typedefs *
65	*******************************************************************************/
66
67	/**
68	* Global VM handle.
69	*/
70	typedef struct GVMHANDLE
71	{
72	/** The index of the next handle in the list (free or used). (0 is nil.) */
73	uint16_t volatile iNext;
74	/** Our own index / handle value. */
75	uint16_t iSelf;
76	/** The pointer to the ring-0 only (aka global) VM structure. */
77	PGVM pGVM;
78	/** The ring-0 mapping of the shared VM instance data. */
79	PVM pVM;
80	/** The virtual machine object. */
81	void *pvObj;
82	/** The session this VM is associated with. */
83	PSUPDRVSESSION pSession;
84	/** The ring-0 handle of the EMT0 thread.
85	* This is used for ownership checks as well as looking up a VM handle by thread
86	* at times like assertions. */
87	RTNATIVETHREAD hEMT0;
88	/** The process ID of the handle owner.
89	* This is used for access checks. */
90	RTPROCESS ProcId;
91	} GVMHANDLE;
92	/** Pointer to a global VM handle. */
93	typedef GVMHANDLE *PGVMHANDLE;
94
95	/** Number of GVM handles (including the NIL handle). */
96	#if HC_ARCH_BITS == 64
97	# define GVMM_MAX_HANDLES 1024
98	#else
99	# define GVMM_MAX_HANDLES 128
100	#endif
101
102	/**
103	* The GVMM instance data.
104	*/
105	typedef struct GVMM
106	{
107	/** Eyecatcher / magic. */
108	uint32_t u32Magic;
109	/** The index of the head of the free handle chain. (0 is nil.) */
110	uint16_t volatile iFreeHead;
111	/** The index of the head of the active handle chain. (0 is nil.) */
112	uint16_t volatile iUsedHead;
113	/** The number of VMs. */
114	uint16_t volatile cVMs;
115	// /** The number of halted EMT threads. */
116	// uint16_t volatile cHaltedEMTs;
117	/** The lock used to serialize VM creation, destruction and associated events that
118	* isn't performance critical. Owners may acquire the list lock. */
119	RTSEMFASTMUTEX CreateDestroyLock;
120	/** The lock used to serialize used list updates and accesses.
121	* This indirectly includes scheduling since the scheduler will have to walk the
122	* used list to examin running VMs. Owners may not acquire any other locks. */
123	RTSEMFASTMUTEX UsedLock;
124	/** The handle array.
125	* The size of this array defines the maximum number of currently running VMs.
126	* The first entry is unused as it represents the NIL handle. */
127	GVMHANDLE aHandles[GVMM_MAX_HANDLES];
128
129	/** @gcfgm{/GVMM/cVMsMeansCompany, 32-bit, 0, UINT32_MAX, 1}
130	* The number of VMs that means we no longer consider ourselves alone on a CPU/Core.
131	*/
132	uint32_t cVMsMeansCompany;
133	/** @gcfgm{/GVMM/MinSleepAlone,32-bit, 0, 100000000, 750000, ns}
134	* The minimum sleep time for when we're alone, in nano seconds.
135	*/
136	uint32_t nsMinSleepAlone;
137	/** @gcfgm{/GVMM/MinSleepCompany,32-bit,0, 100000000, 15000, ns}
138	* The minimum sleep time for when we've got company, in nano seconds.
139	*/
140	uint32_t nsMinSleepCompany;
141	/** @gcfgm{/GVMM/EarlyWakeUp1, 32-bit, 0, 100000000, 25000, ns}
142	* The limit for the first round of early wakeups, given in nano seconds.
143	*/
144	uint32_t nsEarlyWakeUp1;
145	/** @gcfgm{/GVMM/EarlyWakeUp2, 32-bit, 0, 100000000, 50000, ns}
146	* The limit for the second round of early wakeups, given in nano seconds.
147	*/
148	uint32_t nsEarlyWakeUp2;
149	} GVMM;
150	/** Pointer to the GVMM instance data. */
151	typedef GVMM *PGVMM;
152
153	/** The GVMM::u32Magic value (Charlie Haden). */
154	#define GVMM_MAGIC 0x19370806
155
156
157
158	/*******************************************************************************
159	* Global Variables *
160	*******************************************************************************/
161	/** Pointer to the GVMM instance data.
162	* (Just my general dislike for global variables.) */
163	static PGVMM g_pGVMM = NULL;
164
165	/** Macro for obtaining and validating the g_pGVMM pointer.
166	* On failure it will return from the invoking function with the specified return value.
167	*
168	* @param pGVMM The name of the pGVMM variable.
169	* @param rc The return value on failure. Use VERR_INTERNAL_ERROR for
170	* VBox status codes.
171	*/
172	#define GVMM_GET_VALID_INSTANCE(pGVMM, rc) \
173	do { \
174	(pGVMM) = g_pGVMM;\
175	AssertPtrReturn((pGVMM), (rc)); \
176	AssertMsgReturn((pGVMM)->u32Magic == GVMM_MAGIC, ("%p - %#x\n", (pGVMM), (pGVMM)->u32Magic), (rc)); \
177	} while (0)
178
179	/** Macro for obtaining and validating the g_pGVMM pointer, void function variant.
180	* On failure it will return from the invoking function.
181	*
182	* @param pGVMM The name of the pGVMM variable.
183	*/
184	#define GVMM_GET_VALID_INSTANCE_VOID(pGVMM) \
185	do { \
186	(pGVMM) = g_pGVMM;\
187	AssertPtrReturnVoid((pGVMM)); \
188	AssertMsgReturnVoid((pGVMM)->u32Magic == GVMM_MAGIC, ("%p - %#x\n", (pGVMM), (pGVMM)->u32Magic)); \
189	} while (0)
190
191
192	/*******************************************************************************
193	* Internal Functions *
194	*******************************************************************************/
195	static void gvmmR0InitPerVMData(PGVM pGVM);
196	static DECLCALLBACK(void) gvmmR0HandleObjDestructor(void pvObj, void pvGVMM, void *pvHandle);
197	static int gvmmR0ByVM(PVM pVM, PGVM ppGVM, PGVMM ppGVMM, bool fTakeUsedLock);
198	static int gvmmR0ByVMAndEMT(PVM pVM, VMCPUID idCpu, PGVM ppGVM, PGVMM ppGVMM);
199
200
201	/**
202	* Initializes the GVMM.
203	*
204	* This is called while owninng the loader sempahore (see supdrvIOCtl_LdrLoad()).
205	*
206	* @returns VBox status code.
207	*/
208	GVMMR0DECL(int) GVMMR0Init(void)
209	{
210	LogFlow(("GVMMR0Init:\n"));
211
212	/*
213	* Allocate and initialize the instance data.
214	*/
215	PGVMM pGVMM = (PGVMM)RTMemAllocZ(sizeof(*pGVMM));
216	if (!pGVMM)
217	return VERR_NO_MEMORY;
218	int rc = RTSemFastMutexCreate(&pGVMM->CreateDestroyLock);
219	if (RT_SUCCESS(rc))
220	{
221	rc = RTSemFastMutexCreate(&pGVMM->UsedLock);
222	if (RT_SUCCESS(rc))
223	{
224	pGVMM->u32Magic = GVMM_MAGIC;
225	pGVMM->iUsedHead = 0;
226	pGVMM->iFreeHead = 1;
227
228	/* the nil handle */
229	pGVMM->aHandles[0].iSelf = 0;
230	pGVMM->aHandles[0].iNext = 0;
231
232	/* the tail */
233	unsigned i = RT_ELEMENTS(pGVMM->aHandles) - 1;
234	pGVMM->aHandles[i].iSelf = i;
235	pGVMM->aHandles[i].iNext = 0; /* nil */
236
237	/* the rest */
238	while (i-- > 1)
239	{
240	pGVMM->aHandles[i].iSelf = i;
241	pGVMM->aHandles[i].iNext = i + 1;
242	}
243
244	/* The default configuration values. */
245	pGVMM->cVMsMeansCompany = 1; /** @todo should be adjusted to relative to the cpu count or something... */
246	pGVMM->nsMinSleepAlone = 750000 /* ns (0.750 ms) /; /* @todo this should be adjusted to be 75% (or something) of the scheduler granularity... */
247	pGVMM->nsMinSleepCompany = 15000 /* ns (0.015 ms) */;
248	pGVMM->nsEarlyWakeUp1 = 25000 /* ns (0.025 ms) */;
249	pGVMM->nsEarlyWakeUp2 = 50000 /* ns (0.050 ms) */;
250
251	g_pGVMM = pGVMM;
252	LogFlow(("GVMMR0Init: pGVMM=%p\n", pGVMM));
253	return VINF_SUCCESS;
254	}
255
256	RTSemFastMutexDestroy(pGVMM->CreateDestroyLock);
257	}
258
259	RTMemFree(pGVMM);
260	return rc;
261	}
262
263
264	/**
265	* Terminates the GVM.
266	*
267	* This is called while owning the loader semaphore (see supdrvLdrFree()).
268	* And unless something is wrong, there should be absolutely no VMs
269	* registered at this point.
270	*/
271	GVMMR0DECL(void) GVMMR0Term(void)
272	{
273	LogFlow(("GVMMR0Term:\n"));
274
275	PGVMM pGVMM = g_pGVMM;
276	g_pGVMM = NULL;
277	if (RT_UNLIKELY(!VALID_PTR(pGVMM)))
278	{
279	SUPR0Printf("GVMMR0Term: pGVMM=%p\n", pGVMM);
280	return;
281	}
282
283	pGVMM->u32Magic++;
284
285	RTSemFastMutexDestroy(pGVMM->UsedLock);
286	pGVMM->UsedLock = NIL_RTSEMFASTMUTEX;
287	RTSemFastMutexDestroy(pGVMM->CreateDestroyLock);
288	pGVMM->CreateDestroyLock = NIL_RTSEMFASTMUTEX;
289
290	pGVMM->iFreeHead = 0;
291	if (pGVMM->iUsedHead)
292	{
293	SUPR0Printf("GVMMR0Term: iUsedHead=%#x! (cVMs=%#x)\n", pGVMM->iUsedHead, pGVMM->cVMs);
294	pGVMM->iUsedHead = 0;
295	}
296
297	RTMemFree(pGVMM);
298	}
299
300
301	/**
302	* A quick hack for setting global config values.
303	*
304	* @returns VBox status code.
305	*
306	* @param pSession The session handle. Used for authentication.
307	* @param pszName The variable name.
308	* @param u64Value The new value.
309	*/
310	GVMMR0DECL(int) GVMMR0SetConfig(PSUPDRVSESSION pSession, const char *pszName, uint64_t u64Value)
311	{
312	/*
313	* Validate input.
314	*/
315	PGVMM pGVMM;
316	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_INTERNAL_ERROR);
317	AssertPtrReturn(pSession, VERR_INVALID_HANDLE);
318	AssertPtrReturn(pszName, VERR_INVALID_POINTER);
319
320	/*
321	* String switch time!
322	*/
323	if (strncmp(pszName, "/GVMM/", sizeof("/GVMM/") - 1))
324	return VERR_CFGM_VALUE_NOT_FOUND; /* borrow status codes from CFGM... */
325	int rc = VINF_SUCCESS;
326	pszName += sizeof("/GVMM/") - 1;
327	if (!strcmp(pszName, "cVMsMeansCompany"))
328	{
329	if (u64Value <= UINT32_MAX)
330	pGVMM->cVMsMeansCompany = u64Value;
331	else
332	rc = VERR_OUT_OF_RANGE;
333	}
334	else if (!strcmp(pszName, "MinSleepAlone"))
335	{
336	if (u64Value <= 100000000)
337	pGVMM->nsMinSleepAlone = u64Value;
338	else
339	rc = VERR_OUT_OF_RANGE;
340	}
341	else if (!strcmp(pszName, "MinSleepCompany"))
342	{
343	if (u64Value <= 100000000)
344	pGVMM->nsMinSleepCompany = u64Value;
345	else
346	rc = VERR_OUT_OF_RANGE;
347	}
348	else if (!strcmp(pszName, "EarlyWakeUp1"))
349	{
350	if (u64Value <= 100000000)
351	pGVMM->nsEarlyWakeUp1 = u64Value;
352	else
353	rc = VERR_OUT_OF_RANGE;
354	}
355	else if (!strcmp(pszName, "EarlyWakeUp2"))
356	{
357	if (u64Value <= 100000000)
358	pGVMM->nsEarlyWakeUp2 = u64Value;
359	else
360	rc = VERR_OUT_OF_RANGE;
361	}
362	else
363	rc = VERR_CFGM_VALUE_NOT_FOUND;
364	return rc;
365	}
366
367
368	/**
369	* A quick hack for getting global config values.
370	*
371	* @returns VBox status code.
372	*
373	* @param pSession The session handle. Used for authentication.
374	* @param pszName The variable name.
375	* @param u64Value The new value.
376	*/
377	GVMMR0DECL(int) GVMMR0QueryConfig(PSUPDRVSESSION pSession, const char pszName, uint64_t pu64Value)
378	{
379	/*
380	* Validate input.
381	*/
382	PGVMM pGVMM;
383	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_INTERNAL_ERROR);
384	AssertPtrReturn(pSession, VERR_INVALID_HANDLE);
385	AssertPtrReturn(pszName, VERR_INVALID_POINTER);
386	AssertPtrReturn(pu64Value, VERR_INVALID_POINTER);
387
388	/*
389	* String switch time!
390	*/
391	if (strncmp(pszName, "/GVMM/", sizeof("/GVMM/") - 1))
392	return VERR_CFGM_VALUE_NOT_FOUND; /* borrow status codes from CFGM... */
393	int rc = VINF_SUCCESS;
394	pszName += sizeof("/GVMM/") - 1;
395	if (!strcmp(pszName, "cVMsMeansCompany"))
396	*pu64Value = pGVMM->cVMsMeansCompany;
397	else if (!strcmp(pszName, "MinSleepAlone"))
398	*pu64Value = pGVMM->nsMinSleepAlone;
399	else if (!strcmp(pszName, "MinSleepCompany"))
400	*pu64Value = pGVMM->nsMinSleepCompany;
401	else if (!strcmp(pszName, "EarlyWakeUp1"))
402	*pu64Value = pGVMM->nsEarlyWakeUp1;
403	else if (!strcmp(pszName, "EarlyWakeUp2"))
404	*pu64Value = pGVMM->nsEarlyWakeUp2;
405	else
406	rc = VERR_CFGM_VALUE_NOT_FOUND;
407	return rc;
408	}
409
410
411	/**
412	* Try acquire the 'used' lock.
413	*
414	* @returns IPRT status code, see RTSemFastMutexRequest.
415	* @param pGVMM The GVMM instance data.
416	*/
417	DECLINLINE(int) gvmmR0UsedLock(PGVMM pGVMM)
418	{
419	LogFlow(("++gvmmR0UsedLock(%p)\n", pGVMM));
420	int rc = RTSemFastMutexRequest(pGVMM->UsedLock);
421	LogFlow(("gvmmR0UsedLock(%p)->%Rrc\n", pGVMM, rc));
422	return rc;
423	}
424
425
426	/**
427	* Release the 'used' lock.
428	*
429	* @returns IPRT status code, see RTSemFastMutexRelease.
430	* @param pGVMM The GVMM instance data.
431	*/
432	DECLINLINE(int) gvmmR0UsedUnlock(PGVMM pGVMM)
433	{
434	LogFlow(("--gvmmR0UsedUnlock(%p)\n", pGVMM));
435	int rc = RTSemFastMutexRelease(pGVMM->UsedLock);
436	AssertRC(rc);
437	return rc;
438	}
439
440
441	/**
442	* Try acquire the 'create & destroy' lock.
443	*
444	* @returns IPRT status code, see RTSemFastMutexRequest.
445	* @param pGVMM The GVMM instance data.
446	*/
447	DECLINLINE(int) gvmmR0CreateDestroyLock(PGVMM pGVMM)
448	{
449	LogFlow(("++gvmmR0CreateDestroyLock(%p)\n", pGVMM));
450	int rc = RTSemFastMutexRequest(pGVMM->CreateDestroyLock);
451	LogFlow(("gvmmR0CreateDestroyLock(%p)->%Rrc\n", pGVMM, rc));
452	return rc;
453	}
454
455
456	/**
457	* Release the 'create & destroy' lock.
458	*
459	* @returns IPRT status code, see RTSemFastMutexRequest.
460	* @param pGVMM The GVMM instance data.
461	*/
462	DECLINLINE(int) gvmmR0CreateDestroyUnlock(PGVMM pGVMM)
463	{
464	LogFlow(("--gvmmR0CreateDestroyUnlock(%p)\n", pGVMM));
465	int rc = RTSemFastMutexRelease(pGVMM->CreateDestroyLock);
466	AssertRC(rc);
467	return rc;
468	}
469
470
471	/**
472	* Request wrapper for the GVMMR0CreateVM API.
473	*
474	* @returns VBox status code.
475	* @param pReq The request buffer.
476	*/
477	GVMMR0DECL(int) GVMMR0CreateVMReq(PGVMMCREATEVMREQ pReq)
478	{
479	/*
480	* Validate the request.
481	*/
482	if (!VALID_PTR(pReq))
483	return VERR_INVALID_POINTER;
484	if (pReq->Hdr.cbReq != sizeof(*pReq))
485	return VERR_INVALID_PARAMETER;
486	if (!VALID_PTR(pReq->pSession))
487	return VERR_INVALID_POINTER;
488
489	/*
490	* Execute it.
491	*/
492	PVM pVM;
493	pReq->pVMR0 = NULL;
494	pReq->pVMR3 = NIL_RTR3PTR;
495	int rc = GVMMR0CreateVM(pReq->pSession, pReq->cCpus, &pVM);
496	if (RT_SUCCESS(rc))
497	{
498	pReq->pVMR0 = pVM;
499	pReq->pVMR3 = pVM->pVMR3;
500	}
501	return rc;
502	}
503
504
505	/**
506	* Allocates the VM structure and registers it with GVM.
507	*
508	* The caller will become the VM owner and there by the EMT.
509	*
510	* @returns VBox status code.
511	* @param pSession The support driver session.
512	* @param cCpus Number of virtual CPUs for the new VM.
513	* @param ppVM Where to store the pointer to the VM structure.
514	*
515	* @thread EMT.
516	*/
517	GVMMR0DECL(int) GVMMR0CreateVM(PSUPDRVSESSION pSession, uint32_t cCpus, PVM *ppVM)
518	{
519	LogFlow(("GVMMR0CreateVM: pSession=%p\n", pSession));
520	PGVMM pGVMM;
521	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_INTERNAL_ERROR);
522
523	AssertPtrReturn(ppVM, VERR_INVALID_POINTER);
524	*ppVM = NULL;
525
526	if ( cCpus == 0
527	\|\| cCpus > VMM_MAX_CPU_COUNT)
528	return VERR_INVALID_PARAMETER;
529
530	RTNATIVETHREAD hEMT0 = RTThreadNativeSelf();
531	AssertReturn(hEMT0 != NIL_RTNATIVETHREAD, VERR_INTERNAL_ERROR);
532	RTNATIVETHREAD ProcId = RTProcSelf();
533	AssertReturn(ProcId != NIL_RTPROCESS, VERR_INTERNAL_ERROR);
534
535	/*
536	* The whole allocation process is protected by the lock.
537	*/
538	int rc = gvmmR0CreateDestroyLock(pGVMM);
539	AssertRCReturn(rc, rc);
540
541	/*
542	* Allocate a handle first so we don't waste resources unnecessarily.
543	*/
544	uint16_t iHandle = pGVMM->iFreeHead;
545	if (iHandle)
546	{
547	PGVMHANDLE pHandle = &pGVMM->aHandles[iHandle];
548
549	/* consistency checks, a bit paranoid as always. */
550	if ( !pHandle->pVM
551	&& !pHandle->pGVM
552	&& !pHandle->pvObj
553	&& pHandle->iSelf == iHandle)
554	{
555	pHandle->pvObj = SUPR0ObjRegister(pSession, SUPDRVOBJTYPE_VM, gvmmR0HandleObjDestructor, pGVMM, pHandle);
556	if (pHandle->pvObj)
557	{
558	/*
559	* Move the handle from the free to used list and perform permission checks.
560	*/
561	rc = gvmmR0UsedLock(pGVMM);
562	AssertRC(rc);
563
564	pGVMM->iFreeHead = pHandle->iNext;
565	pHandle->iNext = pGVMM->iUsedHead;
566	pGVMM->iUsedHead = iHandle;
567	pGVMM->cVMs++;
568
569	pHandle->pVM = NULL;
570	pHandle->pGVM = NULL;
571	pHandle->pSession = pSession;
572	pHandle->hEMT0 = NIL_RTNATIVETHREAD;
573	pHandle->ProcId = NIL_RTPROCESS;
574
575	gvmmR0UsedUnlock(pGVMM);
576
577	rc = SUPR0ObjVerifyAccess(pHandle->pvObj, pSession, NULL);
578	if (RT_SUCCESS(rc))
579	{
580	/*
581	* Allocate the global VM structure (GVM) and initialize it.
582	*/
583	PGVM pGVM = (PGVM)RTMemAllocZ(RT_UOFFSETOF(GVM, aCpus[cCpus]));
584	if (pGVM)
585	{
586	pGVM->u32Magic = GVM_MAGIC;
587	pGVM->hSelf = iHandle;
588	pGVM->pVM = NULL;
589	pGVM->cCpus = cCpus;
590
591	gvmmR0InitPerVMData(pGVM);
592	GMMR0InitPerVMData(pGVM);
593
594	/*
595	* Allocate the shared VM structure and associated page array.
596	*/
597	const uint32_t cbVM = RT_UOFFSETOF(VM, aCpus[cCpus]);
598	const uint32_t cPages = RT_ALIGN_32(cbVM, PAGE_SIZE) >> PAGE_SHIFT;
599	rc = RTR0MemObjAllocLow(&pGVM->gvmm.s.VMMemObj, cPages << PAGE_SHIFT, false /* fExecutable */);
600	if (RT_SUCCESS(rc))
601	{
602	PVM pVM = (PVM)RTR0MemObjAddress(pGVM->gvmm.s.VMMemObj); AssertPtr(pVM);
603	memset(pVM, 0, cPages << PAGE_SHIFT);
604	pVM->enmVMState = VMSTATE_CREATING;
605	pVM->pVMR0 = pVM;
606	pVM->pSession = pSession;
607	pVM->hSelf = iHandle;
608	pVM->cbSelf = cbVM;
609	pVM->cCPUs = cCpus;
610	pVM->offVMCPU = RT_UOFFSETOF(VM, aCpus);
611
612	rc = RTR0MemObjAllocPage(&pGVM->gvmm.s.VMPagesMemObj, cPages * sizeof(SUPPAGE), false /* fExecutable */);
613	if (RT_SUCCESS(rc))
614	{
615	PSUPPAGE paPages = (PSUPPAGE)RTR0MemObjAddress(pGVM->gvmm.s.VMPagesMemObj); AssertPtr(paPages);
616	for (uint32_t iPage = 0; iPage < cPages; iPage++)
617	{
618	paPages[iPage].uReserved = 0;
619	paPages[iPage].Phys = RTR0MemObjGetPagePhysAddr(pGVM->gvmm.s.VMMemObj, iPage);
620	Assert(paPages[iPage].Phys != NIL_RTHCPHYS);
621	}
622
623	/*
624	* Map them into ring-3.
625	*/
626	rc = RTR0MemObjMapUser(&pGVM->gvmm.s.VMMapObj, pGVM->gvmm.s.VMMemObj, (RTR3PTR)-1, 0,
627	RTMEM_PROT_READ \| RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
628	if (RT_SUCCESS(rc))
629	{
630	pVM->pVMR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMMapObj);
631	AssertPtr((void *)pVM->pVMR3);
632
633	/* Initialize all the VM pointers. */
634	for (uint32_t i = 0; i < cCpus; i++)
635	{
636	pVM->aCpus[i].pVMR0 = pVM;
637	pVM->aCpus[i].pVMR3 = pVM->pVMR3;
638	}
639
640	rc = RTR0MemObjMapUser(&pGVM->gvmm.s.VMPagesMapObj, pGVM->gvmm.s.VMPagesMemObj, (RTR3PTR)-1, 0,
641	RTMEM_PROT_READ \| RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
642	if (RT_SUCCESS(rc))
643	{
644	pVM->paVMPagesR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMPagesMapObj);
645	AssertPtr((void *)pVM->paVMPagesR3);
646
647	/* complete the handle - take the UsedLock sem just to be careful. */
648	rc = gvmmR0UsedLock(pGVMM);
649	AssertRC(rc);
650
651	pHandle->pVM = pVM;
652	pHandle->pGVM = pGVM;
653	pHandle->hEMT0 = hEMT0;
654	pHandle->ProcId = ProcId;
655	pGVM->pVM = pVM;
656	pGVM->aCpus[0].hEMT = hEMT0;
657
658	gvmmR0UsedUnlock(pGVMM);
659	gvmmR0CreateDestroyUnlock(pGVMM);
660
661	*ppVM = pVM;
662	Log(("GVMMR0CreateVM: pVM=%p pVMR3=%p pGVM=%p hGVM=%d\n", pVM, pVM->pVMR3, pGVM, iHandle));
663	return VINF_SUCCESS;
664	}
665
666	RTR0MemObjFree(pGVM->gvmm.s.VMMapObj, false /* fFreeMappings */);
667	pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
668	}
669	RTR0MemObjFree(pGVM->gvmm.s.VMPagesMemObj, false /* fFreeMappings */);
670	pGVM->gvmm.s.VMPagesMemObj = NIL_RTR0MEMOBJ;
671	}
672	RTR0MemObjFree(pGVM->gvmm.s.VMMemObj, false /* fFreeMappings */);
673	pGVM->gvmm.s.VMMemObj = NIL_RTR0MEMOBJ;
674	}
675	}
676	}
677	/* else: The user wasn't permitted to create this VM. */
678
679	/*
680	* The handle will be freed by gvmmR0HandleObjDestructor as we release the
681	* object reference here. A little extra mess because of non-recursive lock.
682	*/
683	void *pvObj = pHandle->pvObj;
684	pHandle->pvObj = NULL;
685	gvmmR0CreateDestroyUnlock(pGVMM);
686
687	SUPR0ObjRelease(pvObj, pSession);
688
689	SUPR0Printf("GVMMR0CreateVM: failed, rc=%d\n", rc);
690	return rc;
691	}
692
693	rc = VERR_NO_MEMORY;
694	}
695	else
696	rc = VERR_INTERNAL_ERROR;
697	}
698	else
699	rc = VERR_GVM_TOO_MANY_VMS;
700
701	gvmmR0CreateDestroyUnlock(pGVMM);
702	return rc;
703	}
704
705
706	/**
707	* Initializes the per VM data belonging to GVMM.
708	*
709	* @param pGVM Pointer to the global VM structure.
710	*/
711	static void gvmmR0InitPerVMData(PGVM pGVM)
712	{
713	AssertCompile(RT_SIZEOFMEMB(GVM,gvmm.s) <= RT_SIZEOFMEMB(GVM,gvmm.padding));
714	AssertCompile(RT_SIZEOFMEMB(GVMCPU,gvmm.s) <= RT_SIZEOFMEMB(GVMCPU,gvmm.padding));
715	pGVM->gvmm.s.VMMemObj = NIL_RTR0MEMOBJ;
716	pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
717	pGVM->gvmm.s.VMPagesMemObj = NIL_RTR0MEMOBJ;
718	pGVM->gvmm.s.VMPagesMapObj = NIL_RTR0MEMOBJ;
719	pGVM->gvmm.s.fDoneVMMR0Init = false;
720	pGVM->gvmm.s.fDoneVMMR0Term = false;
721
722	for (VMCPUID i = 0; i < pGVM->cCpus; i++)
723	{
724	pGVM->aCpus[i].gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
725	pGVM->aCpus[i].hEMT = NIL_RTNATIVETHREAD;
726	}
727	}
728
729
730	/**
731	* Does the VM initialization.
732	*
733	* @returns VBox status code.
734	* @param pVM Pointer to the shared VM structure.
735	*/
736	GVMMR0DECL(int) GVMMR0InitVM(PVM pVM)
737	{
738	LogFlow(("GVMMR0InitVM: pVM=%p\n", pVM));
739
740	/*
741	* Validate the VM structure, state and handle.
742	*/
743	PGVM pGVM;
744	PGVMM pGVMM;
745	int rc = gvmmR0ByVMAndEMT(pVM, 0 /* idCpu */, &pGVM, &pGVMM);
746	if (RT_SUCCESS(rc))
747	{
748	if ( !pGVM->gvmm.s.fDoneVMMR0Init
749	&& pGVM->aCpus[0].gvmm.s.HaltEventMulti == NIL_RTSEMEVENTMULTI)
750	{
751	for (VMCPUID i = 0; i < pGVM->cCpus; i++)
752	{
753	rc = RTSemEventMultiCreate(&pGVM->aCpus[i].gvmm.s.HaltEventMulti);
754	if (RT_FAILURE(rc))
755	{
756	pGVM->aCpus[i].gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
757	break;
758	}
759	}
760	}
761	else
762	rc = VERR_WRONG_ORDER;
763	}
764
765	LogFlow(("GVMMR0InitVM: returns %Rrc\n", rc));
766	return rc;
767	}
768
769
770	/**
771	* Indicates that we're done with the ring-0 initialization
772	* of the VM.
773	*
774	* @param pVM Pointer to the shared VM structure.
775	* @thread EMT(0)
776	*/
777	GVMMR0DECL(void) GVMMR0DoneInitVM(PVM pVM)
778	{
779	/* Validate the VM structure, state and handle. */
780	PGVM pGVM;
781	PGVMM pGVMM;
782	int rc = gvmmR0ByVMAndEMT(pVM, 0 /* idCpu */, &pGVM, &pGVMM);
783	AssertRCReturnVoid(rc);
784
785	/* Set the indicator. */
786	pGVM->gvmm.s.fDoneVMMR0Init = true;
787	}
788
789
790	/**
791	* Indicates that we're doing the ring-0 termination of the VM.
792	*
793	* @returns true if termination hasn't been done already, false if it has.
794	* @param pVM Pointer to the shared VM structure.
795	* @param pGVM Pointer to the global VM structure. Optional.
796	* @thread EMT(0)
797	*/
798	GVMMR0DECL(bool) GVMMR0DoingTermVM(PVM pVM, PGVM pGVM)
799	{
800	/* Validate the VM structure, state and handle. */
801	AssertPtrNullReturn(pGVM, false);
802	AssertReturn(!pGVM \|\| pGVM->u32Magic == GVM_MAGIC, false);
803	if (!pGVM)
804	{
805	PGVMM pGVMM;
806	int rc = gvmmR0ByVMAndEMT(pVM, 0 /* idCpu */, &pGVM, &pGVMM);
807	AssertRCReturn(rc, false);
808	}
809
810	/* Set the indicator. */
811	if (pGVM->gvmm.s.fDoneVMMR0Term)
812	return false;
813	pGVM->gvmm.s.fDoneVMMR0Term = true;
814	return true;
815	}
816
817
818	/**
819	* Destroys the VM, freeing all associated resources (the ring-0 ones anyway).
820	*
821	* This is call from the vmR3DestroyFinalBit and from a error path in VMR3Create,
822	* and the caller is not the EMT thread, unfortunately. For security reasons, it
823	* would've been nice if the caller was actually the EMT thread or that we somehow
824	* could've associated the calling thread with the VM up front.
825	*
826	* @returns VBox status code.
827	* @param pVM Where to store the pointer to the VM structure.
828	*
829	* @thread EMT(0) if it's associated with the VM, otherwise any thread.
830	*/
831	GVMMR0DECL(int) GVMMR0DestroyVM(PVM pVM)
832	{
833	LogFlow(("GVMMR0DestroyVM: pVM=%p\n", pVM));
834	PGVMM pGVMM;
835	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_INTERNAL_ERROR);
836
837
838	/*
839	* Validate the VM structure, state and caller.
840	*/
841	AssertPtrReturn(pVM, VERR_INVALID_POINTER);
842	AssertReturn(!((uintptr_t)pVM & PAGE_OFFSET_MASK), VERR_INVALID_POINTER);
843	AssertMsgReturn(pVM->enmVMState >= VMSTATE_CREATING && pVM->enmVMState <= VMSTATE_TERMINATED, ("%d\n", pVM->enmVMState), VERR_WRONG_ORDER);
844
845	uint32_t hGVM = pVM->hSelf;
846	AssertReturn(hGVM != NIL_GVM_HANDLE, VERR_INVALID_HANDLE);
847	AssertReturn(hGVM < RT_ELEMENTS(pGVMM->aHandles), VERR_INVALID_HANDLE);
848
849	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
850	AssertReturn(pHandle->pVM == pVM, VERR_NOT_OWNER);
851
852	RTPROCESS ProcId = RTProcSelf();
853	RTNATIVETHREAD hSelf = RTThreadNativeSelf();
854	AssertReturn( ( pHandle->hEMT0 == hSelf
855	&& pHandle->ProcId == ProcId)
856	\|\| pHandle->hEMT0 == NIL_RTNATIVETHREAD, VERR_NOT_OWNER);
857
858	/*
859	* Lookup the handle and destroy the object.
860	* Since the lock isn't recursive and we'll have to leave it before dereferencing the
861	* object, we take some precautions against racing callers just in case...
862	*/
863	int rc = gvmmR0CreateDestroyLock(pGVMM);
864	AssertRC(rc);
865
866	/* be careful here because we might theoretically be racing someone else cleaning up. */
867	if ( pHandle->pVM == pVM
868	&& ( ( pHandle->hEMT0 == hSelf
869	&& pHandle->ProcId == ProcId)
870	\|\| pHandle->hEMT0 == NIL_RTNATIVETHREAD)
871	&& VALID_PTR(pHandle->pvObj)
872	&& VALID_PTR(pHandle->pSession)
873	&& VALID_PTR(pHandle->pGVM)
874	&& pHandle->pGVM->u32Magic == GVM_MAGIC)
875	{
876	void *pvObj = pHandle->pvObj;
877	pHandle->pvObj = NULL;
878	gvmmR0CreateDestroyUnlock(pGVMM);
879
880	SUPR0ObjRelease(pvObj, pHandle->pSession);
881	}
882	else
883	{
884	SUPR0Printf("GVMMR0DestroyVM: pHandle=%p:{.pVM=%p, .hEMT0=%p, .ProcId=%u, .pvObj=%p} pVM=%p hSelf=%p\n",
885	pHandle, pHandle->pVM, pHandle->hEMT0, pHandle->ProcId, pHandle->pvObj, pVM, hSelf);
886	gvmmR0CreateDestroyUnlock(pGVMM);
887	rc = VERR_INTERNAL_ERROR;
888	}
889
890	return rc;
891	}
892
893
894	/**
895	* Performs VM cleanup task as part of object destruction.
896	*
897	* @param pGVM The GVM pointer.
898	*/
899	static void gvmmR0CleanupVM(PGVM pGVM)
900	{
901	if ( pGVM->gvmm.s.fDoneVMMR0Init
902	&& !pGVM->gvmm.s.fDoneVMMR0Term)
903	{
904	if ( pGVM->gvmm.s.VMMemObj != NIL_RTR0MEMOBJ
905	&& RTR0MemObjAddress(pGVM->gvmm.s.VMMemObj) == pGVM->pVM)
906	{
907	LogFlow(("gvmmR0CleanupVM: Calling VMMR0TermVM\n"));
908	VMMR0TermVM(pGVM->pVM, pGVM);
909	}
910	else
911	AssertMsgFailed(("gvmmR0CleanupVM: VMMemObj=%p pVM=%p\n", pGVM->gvmm.s.VMMemObj, pGVM->pVM));
912	}
913
914	GMMR0CleanupVM(pGVM);
915	}
916
917
918	/**
919	* Handle destructor.
920	*
921	* @param pvGVMM The GVM instance pointer.
922	* @param pvHandle The handle pointer.
923	*/
924	static DECLCALLBACK(void) gvmmR0HandleObjDestructor(void pvObj, void pvGVMM, void *pvHandle)
925	{
926	LogFlow(("gvmmR0HandleObjDestructor: %p %p %p\n", pvObj, pvGVMM, pvHandle));
927
928	/*
929	* Some quick, paranoid, input validation.
930	*/
931	PGVMHANDLE pHandle = (PGVMHANDLE)pvHandle;
932	AssertPtr(pHandle);
933	PGVMM pGVMM = (PGVMM)pvGVMM;
934	Assert(pGVMM == g_pGVMM);
935	const uint16_t iHandle = pHandle - &pGVMM->aHandles[0];
936	if ( !iHandle
937	\|\| iHandle >= RT_ELEMENTS(pGVMM->aHandles)
938	\|\| iHandle != pHandle->iSelf)
939	{
940	SUPR0Printf("GVM: handle %d is out of range or corrupt (iSelf=%d)!\n", iHandle, pHandle->iSelf);
941	return;
942	}
943
944	int rc = gvmmR0CreateDestroyLock(pGVMM);
945	AssertRC(rc);
946	rc = gvmmR0UsedLock(pGVMM);
947	AssertRC(rc);
948
949	/*
950	* This is a tad slow but a doubly linked list is too much hazzle.
951	*/
952	if (RT_UNLIKELY(pHandle->iNext >= RT_ELEMENTS(pGVMM->aHandles)))
953	{
954	SUPR0Printf("GVM: used list index %d is out of range!\n", pHandle->iNext);
955	gvmmR0UsedUnlock(pGVMM);
956	gvmmR0CreateDestroyUnlock(pGVMM);
957	return;
958	}
959
960	if (pGVMM->iUsedHead == iHandle)
961	pGVMM->iUsedHead = pHandle->iNext;
962	else
963	{
964	uint16_t iPrev = pGVMM->iUsedHead;
965	int c = RT_ELEMENTS(pGVMM->aHandles) + 2;
966	while (iPrev)
967	{
968	if (RT_UNLIKELY(iPrev >= RT_ELEMENTS(pGVMM->aHandles)))
969	{
970	SUPR0Printf("GVM: used list index %d is out of range!\n");
971	gvmmR0UsedUnlock(pGVMM);
972	gvmmR0CreateDestroyUnlock(pGVMM);
973	return;
974	}
975	if (RT_UNLIKELY(c-- <= 0))
976	{
977	iPrev = 0;
978	break;
979	}
980
981	if (pGVMM->aHandles[iPrev].iNext == iHandle)
982	break;
983	iPrev = pGVMM->aHandles[iPrev].iNext;
984	}
985	if (!iPrev)
986	{
987	SUPR0Printf("GVM: can't find the handle previous previous of %d!\n", pHandle->iSelf);
988	gvmmR0UsedUnlock(pGVMM);
989	gvmmR0CreateDestroyUnlock(pGVMM);
990	return;
991	}
992
993	Assert(pGVMM->aHandles[iPrev].iNext == iHandle);
994	pGVMM->aHandles[iPrev].iNext = pHandle->iNext;
995	}
996	pHandle->iNext = 0;
997	pGVMM->cVMs--;
998
999	gvmmR0UsedUnlock(pGVMM);
1000
1001	/*
1002	* Do the global cleanup round.
1003	*/
1004	PGVM pGVM = pHandle->pGVM;
1005	if ( VALID_PTR(pGVM)
1006	&& pGVM->u32Magic == GVM_MAGIC)
1007	{
1008	gvmmR0CleanupVM(pGVM);
1009
1010	/*
1011	* Do the GVMM cleanup - must be done last.
1012	*/
1013	/* The VM and VM pages mappings/allocations. */
1014	if (pGVM->gvmm.s.VMPagesMapObj != NIL_RTR0MEMOBJ)
1015	{
1016	rc = RTR0MemObjFree(pGVM->gvmm.s.VMPagesMapObj, false /* fFreeMappings */); AssertRC(rc);
1017	pGVM->gvmm.s.VMPagesMapObj = NIL_RTR0MEMOBJ;
1018	}
1019
1020	if (pGVM->gvmm.s.VMMapObj != NIL_RTR0MEMOBJ)
1021	{
1022	rc = RTR0MemObjFree(pGVM->gvmm.s.VMMapObj, false /* fFreeMappings */); AssertRC(rc);
1023	pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
1024	}
1025
1026	if (pGVM->gvmm.s.VMPagesMemObj != NIL_RTR0MEMOBJ)
1027	{
1028	rc = RTR0MemObjFree(pGVM->gvmm.s.VMPagesMemObj, false /* fFreeMappings */); AssertRC(rc);
1029	pGVM->gvmm.s.VMPagesMemObj = NIL_RTR0MEMOBJ;
1030	}
1031
1032	if (pGVM->gvmm.s.VMMemObj != NIL_RTR0MEMOBJ)
1033	{
1034	rc = RTR0MemObjFree(pGVM->gvmm.s.VMMemObj, false /* fFreeMappings */); AssertRC(rc);
1035	pGVM->gvmm.s.VMMemObj = NIL_RTR0MEMOBJ;
1036	}
1037
1038	for (VMCPUID i = 0; i < pGVM->cCpus; i++)
1039	{
1040	if (pGVM->aCpus[i].gvmm.s.HaltEventMulti != NIL_RTSEMEVENTMULTI)
1041	{
1042	rc = RTSemEventMultiDestroy(pGVM->aCpus[i].gvmm.s.HaltEventMulti); AssertRC(rc);
1043	pGVM->aCpus[i].gvmm.s.HaltEventMulti = NIL_RTSEMEVENTMULTI;
1044	}
1045	}
1046
1047	/* the GVM structure itself. */
1048	pGVM->u32Magic \|= UINT32_C(0x80000000);
1049	RTMemFree(pGVM);
1050	}
1051	/* else: GVMMR0CreateVM cleanup. */
1052
1053	/*
1054	* Free the handle.
1055	* Reacquire the UsedLock here to since we're updating handle fields.
1056	*/
1057	rc = gvmmR0UsedLock(pGVMM);
1058	AssertRC(rc);
1059
1060	pHandle->iNext = pGVMM->iFreeHead;
1061	pGVMM->iFreeHead = iHandle;
1062	ASMAtomicXchgPtr((void * volatile *)&pHandle->pGVM, NULL);
1063	ASMAtomicXchgPtr((void * volatile *)&pHandle->pVM, NULL);
1064	ASMAtomicXchgPtr((void * volatile *)&pHandle->pvObj, NULL);
1065	ASMAtomicXchgPtr((void * volatile *)&pHandle->pSession, NULL);
1066	ASMAtomicXchgSize(&pHandle->hEMT0, NIL_RTNATIVETHREAD);
1067	ASMAtomicXchgSize(&pHandle->ProcId, NIL_RTPROCESS);
1068
1069	gvmmR0UsedUnlock(pGVMM);
1070	gvmmR0CreateDestroyUnlock(pGVMM);
1071	LogFlow(("gvmmR0HandleObjDestructor: returns\n"));
1072	}
1073
1074
1075	/**
1076	* Registers the calling thread as the EMT of a Virtual CPU.
1077	*
1078	* Note that VCPU 0 is automatically registered during VM creation.
1079	*
1080	* @returns VBox status code
1081	* @param pVM The shared VM structure (the ring-0 mapping).
1082	* @param idCpu VCPU id.
1083	*/
1084	GVMMR0DECL(int) GVMMR0RegisterVCpu(PVM pVM, VMCPUID idCpu)
1085	{
1086	AssertReturn(idCpu != 0, VERR_NOT_OWNER);
1087
1088	/*
1089	* Validate the VM structure, state and handle.
1090	*/
1091	PGVM pGVM;
1092	PGVMM pGVMM;
1093	int rc = gvmmR0ByVM(pVM, &pGVM, &pGVMM, false /* fTakeUsedLock */);
1094	if (RT_FAILURE(rc))
1095	return rc;
1096
1097	AssertReturn(idCpu < pVM->cCPUs, VERR_INVALID_CPU_ID);
1098	AssertReturn(pGVM->aCpus[idCpu].hEMT == NIL_RTNATIVETHREAD, VERR_ACCESS_DENIED);
1099
1100	pGVM->aCpus[idCpu].hEMT = RTThreadNativeSelf();
1101	return VINF_SUCCESS;
1102	}
1103
1104
1105	/**
1106	* Lookup a GVM structure by its handle.
1107	*
1108	* @returns The GVM pointer on success, NULL on failure.
1109	* @param hGVM The global VM handle. Asserts on bad handle.
1110	*/
1111	GVMMR0DECL(PGVM) GVMMR0ByHandle(uint32_t hGVM)
1112	{
1113	PGVMM pGVMM;
1114	GVMM_GET_VALID_INSTANCE(pGVMM, NULL);
1115
1116	/*
1117	* Validate.
1118	*/
1119	AssertReturn(hGVM != NIL_GVM_HANDLE, NULL);
1120	AssertReturn(hGVM < RT_ELEMENTS(pGVMM->aHandles), NULL);
1121
1122	/*
1123	* Look it up.
1124	*/
1125	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1126	AssertPtrReturn(pHandle->pVM, NULL);
1127	AssertPtrReturn(pHandle->pvObj, NULL);
1128	PGVM pGVM = pHandle->pGVM;
1129	AssertPtrReturn(pGVM, NULL);
1130	AssertReturn(pGVM->pVM == pHandle->pVM, NULL);
1131
1132	return pHandle->pGVM;
1133	}
1134
1135
1136	/**
1137	* Lookup a GVM structure by the shared VM structure.
1138	*
1139	* The calling thread must be in the same process as the VM. All current lookups
1140	* are by threads inside the same process, so this will not be an issue.
1141	*
1142	* @returns VBox status code.
1143	* @param pVM The shared VM structure (the ring-0 mapping).
1144	* @param ppGVM Where to store the GVM pointer.
1145	* @param ppGVMM Where to store the pointer to the GVMM instance data.
1146	* @param fTakeUsedLock Whether to take the used lock or not.
1147	* Be very careful if not taking the lock as it's possible that
1148	* the VM will disappear then.
1149	*
1150	* @remark This will not assert on an invalid pVM but try return sliently.
1151	*/
1152	static int gvmmR0ByVM(PVM pVM, PGVM ppGVM, PGVMM ppGVMM, bool fTakeUsedLock)
1153	{
1154	RTPROCESS ProcId = RTProcSelf();
1155	PGVMM pGVMM;
1156	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_INTERNAL_ERROR);
1157
1158	/*
1159	* Validate.
1160	*/
1161	if (RT_UNLIKELY( !VALID_PTR(pVM)
1162	\|\| ((uintptr_t)pVM & PAGE_OFFSET_MASK)))
1163	return VERR_INVALID_POINTER;
1164	if (RT_UNLIKELY( pVM->enmVMState < VMSTATE_CREATING
1165	\|\| pVM->enmVMState >= VMSTATE_TERMINATED))
1166	return VERR_INVALID_POINTER;
1167
1168	uint16_t hGVM = pVM->hSelf;
1169	if (RT_UNLIKELY( hGVM == NIL_GVM_HANDLE
1170	\|\| hGVM >= RT_ELEMENTS(pGVMM->aHandles)))
1171	return VERR_INVALID_HANDLE;
1172
1173	/*
1174	* Look it up.
1175	*/
1176	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1177	PGVM pGVM;
1178	if (fTakeUsedLock)
1179	{
1180	int rc = gvmmR0UsedLock(pGVMM);
1181	AssertRCReturn(rc, rc);
1182
1183	pGVM = pHandle->pGVM;
1184	if (RT_UNLIKELY( pHandle->pVM != pVM
1185	\|\| pHandle->ProcId != ProcId
1186	\|\| !VALID_PTR(pHandle->pvObj)
1187	\|\| !VALID_PTR(pGVM)
1188	\|\| pGVM->pVM != pVM))
1189	{
1190	gvmmR0UsedUnlock(pGVMM);
1191	return VERR_INVALID_HANDLE;
1192	}
1193	}
1194	else
1195	{
1196	if (RT_UNLIKELY(pHandle->pVM != pVM))
1197	return VERR_INVALID_HANDLE;
1198	if (RT_UNLIKELY(pHandle->ProcId != ProcId))
1199	return VERR_INVALID_HANDLE;
1200	if (RT_UNLIKELY(!VALID_PTR(pHandle->pvObj)))
1201	return VERR_INVALID_HANDLE;
1202
1203	pGVM = pHandle->pGVM;
1204	if (RT_UNLIKELY(!VALID_PTR(pGVM)))
1205	return VERR_INVALID_HANDLE;
1206	if (RT_UNLIKELY(pGVM->pVM != pVM))
1207	return VERR_INVALID_HANDLE;
1208	}
1209
1210	*ppGVM = pGVM;
1211	*ppGVMM = pGVMM;
1212	return VINF_SUCCESS;
1213	}
1214
1215
1216	/**
1217	* Lookup a GVM structure by the shared VM structure.
1218	*
1219	* @returns The GVM pointer on success, NULL on failure.
1220	* @param pVM The shared VM structure (the ring-0 mapping).
1221	*
1222	* @remark This will not take the 'used'-lock because it doesn't do
1223	* nesting and this function will be used from under the lock.
1224	*/
1225	GVMMR0DECL(PGVM) GVMMR0ByVM(PVM pVM)
1226	{
1227	PGVM pGVM;
1228	PGVMM pGVMM;
1229	int rc = gvmmR0ByVM(pVM, &pGVM, &pGVMM, false /* fTakeUsedLock */);
1230	if (RT_SUCCESS(rc))
1231	return pGVM;
1232	AssertRC(rc);
1233	return NULL;
1234	}
1235
1236
1237	/**
1238	* Lookup a GVM structure by the shared VM structure and ensuring that the
1239	* caller is an EMT thread.
1240	*
1241	* @returns VBox status code.
1242	* @param pVM The shared VM structure (the ring-0 mapping).
1243	* @param idCpu The Virtual CPU ID of the calling EMT.
1244	* @param ppGVM Where to store the GVM pointer.
1245	* @param ppGVMM Where to store the pointer to the GVMM instance data.
1246	* @thread EMT
1247	*
1248	* @remark This will assert in all failure paths.
1249	*/
1250	static int gvmmR0ByVMAndEMT(PVM pVM, VMCPUID idCpu, PGVM ppGVM, PGVMM ppGVMM)
1251	{
1252	PGVMM pGVMM;
1253	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_INTERNAL_ERROR);
1254
1255	/*
1256	* Validate.
1257	*/
1258	AssertPtrReturn(pVM, VERR_INVALID_POINTER);
1259	AssertReturn(!((uintptr_t)pVM & PAGE_OFFSET_MASK), VERR_INVALID_POINTER);
1260
1261	uint16_t hGVM = pVM->hSelf;
1262	AssertReturn(hGVM != NIL_GVM_HANDLE, VERR_INVALID_HANDLE);
1263	AssertReturn(hGVM < RT_ELEMENTS(pGVMM->aHandles), VERR_INVALID_HANDLE);
1264
1265	/*
1266	* Look it up.
1267	*/
1268	PGVMHANDLE pHandle = &pGVMM->aHandles[hGVM];
1269	AssertReturn(pHandle->pVM == pVM, VERR_NOT_OWNER);
1270	RTPROCESS ProcId = RTProcSelf();
1271	AssertReturn(pHandle->ProcId == ProcId, VERR_NOT_OWNER);
1272	AssertPtrReturn(pHandle->pvObj, VERR_INTERNAL_ERROR);
1273
1274	PGVM pGVM = pHandle->pGVM;
1275	AssertPtrReturn(pGVM, VERR_INTERNAL_ERROR);
1276	AssertReturn(pGVM->pVM == pVM, VERR_INTERNAL_ERROR);
1277	RTNATIVETHREAD hAllegedEMT = RTThreadNativeSelf();
1278	AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID);
1279	AssertReturn(pGVM->aCpus[idCpu].hEMT == hAllegedEMT, VERR_INTERNAL_ERROR);
1280
1281	*ppGVM = pGVM;
1282	*ppGVMM = pGVMM;
1283	return VINF_SUCCESS;
1284	}
1285
1286
1287	/**
1288	* Lookup a GVM structure by the shared VM structure
1289	* and ensuring that the caller is the EMT thread.
1290	*
1291	* @returns VBox status code.
1292	* @param pVM The shared VM structure (the ring-0 mapping).
1293	* @param idCpu The Virtual CPU ID of the calling EMT.
1294	* @param ppGVM Where to store the GVM pointer.
1295	* @thread EMT
1296	*/
1297	GVMMR0DECL(int) GVMMR0ByVMAndEMT(PVM pVM, VMCPUID idCpu, PGVM *ppGVM)
1298	{
1299	AssertPtrReturn(ppGVM, VERR_INVALID_POINTER);
1300	PGVMM pGVMM;
1301	return gvmmR0ByVMAndEMT(pVM, idCpu, ppGVM, &pGVMM);
1302	}
1303
1304
1305	/**
1306	* Lookup a VM by its global handle.
1307	*
1308	* @returns The VM handle on success, NULL on failure.
1309	* @param hGVM The global VM handle. Asserts on bad handle.
1310	*/
1311	GVMMR0DECL(PVM) GVMMR0GetVMByHandle(uint32_t hGVM)
1312	{
1313	PGVM pGVM = GVMMR0ByHandle(hGVM);
1314	return pGVM ? pGVM->pVM : NULL;
1315	}
1316
1317
1318	/**
1319	* Looks up the VM belonging to the specified EMT thread.
1320	*
1321	* This is used by the assertion machinery in VMMR0.cpp to avoid causing
1322	* unnecessary kernel panics when the EMT thread hits an assertion. The
1323	* call may or not be an EMT thread.
1324	*
1325	* @returns The VM handle on success, NULL on failure.
1326	* @param hEMT The native thread handle of the EMT.
1327	* NIL_RTNATIVETHREAD means the current thread
1328	*/
1329	GVMMR0DECL(PVM) GVMMR0GetVMByEMT(RTNATIVETHREAD hEMT)
1330	{
1331	/*
1332	* No Assertions here as we're usually called in a AssertMsgN or
1333	* RTAssert* context.
1334	*/
1335	PGVMM pGVMM = g_pGVMM;
1336	if ( !VALID_PTR(pGVMM)
1337	\|\| pGVMM->u32Magic != GVMM_MAGIC)
1338	return NULL;
1339
1340	if (hEMT == NIL_RTNATIVETHREAD)
1341	hEMT = RTThreadNativeSelf();
1342	RTPROCESS ProcId = RTProcSelf();
1343
1344	/*
1345	* Search the handles in a linear fashion as we don't dare to take the lock (assert).
1346	*/
1347	for (unsigned i = 1; i < RT_ELEMENTS(pGVMM->aHandles); i++)
1348	{
1349	if ( pGVMM->aHandles[i].iSelf == i
1350	&& pGVMM->aHandles[i].ProcId == ProcId
1351	&& VALID_PTR(pGVMM->aHandles[i].pvObj)
1352	&& VALID_PTR(pGVMM->aHandles[i].pVM)
1353	&& VALID_PTR(pGVMM->aHandles[i].pGVM))
1354	{
1355	if (pGVMM->aHandles[i].hEMT0 == hEMT)
1356	return pGVMM->aHandles[i].pVM;
1357
1358	/* This is fearly safe with the current process per VM approach. */
1359	PGVM pGVM = pGVMM->aHandles[i].pGVM;
1360	VMCPUID const cCpus = pGVM->cCpus;
1361	if ( cCpus < 1
1362	\|\| cCpus > VMM_MAX_CPU_COUNT)
1363	continue;
1364	for (VMCPUID idCpu = 1; idCpu < cCpus; idCpu++)
1365	if (pGVM->aCpus[idCpu].hEMT == hEMT)
1366	return pGVMM->aHandles[i].pVM;
1367	}
1368	}
1369	return NULL;
1370	}
1371
1372
1373	/**
1374	* This is will wake up expired and soon-to-be expired VMs.
1375	*
1376	* @returns Number of VMs that has been woken up.
1377	* @param pGVMM Pointer to the GVMM instance data.
1378	* @param u64Now The current time.
1379	*/
1380	static unsigned gvmmR0SchedDoWakeUps(PGVMM pGVMM, uint64_t u64Now)
1381	{
1382	/** @todo Rewrite this algorithm. See performance defect XYZ. */
1383
1384	/*
1385	* The first pass will wake up VMs which have actually expired
1386	* and look for VMs that should be woken up in the 2nd and 3rd passes.
1387	*/
1388	unsigned cWoken = 0;
1389	unsigned cHalted = 0;
1390	unsigned cTodo2nd = 0;
1391	unsigned cTodo3rd = 0;
1392	for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
1393	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
1394	i = pGVMM->aHandles[i].iNext)
1395	{
1396	PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
1397	if ( VALID_PTR(pCurGVM)
1398	&& pCurGVM->u32Magic == GVM_MAGIC)
1399	{
1400	for (VMCPUID idCpu = 0; idCpu < pCurGVM->cCpus; idCpu++)
1401	{
1402	PGVMCPU pCurGVCpu = &pCurGVM->aCpus[idCpu];
1403
1404	uint64_t u64 = pCurGVCpu->gvmm.s.u64HaltExpire;
1405	if (u64)
1406	{
1407	if (u64 <= u64Now)
1408	{
1409	if (ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0))
1410	{
1411	int rc = RTSemEventMultiSignal(pCurGVCpu->gvmm.s.HaltEventMulti);
1412	AssertRC(rc);
1413	cWoken++;
1414	}
1415	}
1416	else
1417	{
1418	cHalted++;
1419	if (u64 <= u64Now + pGVMM->nsEarlyWakeUp1)
1420	cTodo2nd++;
1421	else if (u64 <= u64Now + pGVMM->nsEarlyWakeUp2)
1422	cTodo3rd++;
1423	}
1424	}
1425	}
1426	}
1427	AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
1428	}
1429
1430	if (cTodo2nd)
1431	{
1432	for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
1433	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
1434	i = pGVMM->aHandles[i].iNext)
1435	{
1436	PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
1437	if ( VALID_PTR(pCurGVM)
1438	&& pCurGVM->u32Magic == GVM_MAGIC)
1439	{
1440	for (VMCPUID idCpu = 0; idCpu < pCurGVM->cCpus; idCpu++)
1441	{
1442	PGVMCPU pCurGVCpu = &pCurGVM->aCpus[idCpu];
1443
1444	if ( pCurGVCpu->gvmm.s.u64HaltExpire
1445	&& pCurGVCpu->gvmm.s.u64HaltExpire <= u64Now + pGVMM->nsEarlyWakeUp1)
1446	{
1447	if (ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0))
1448	{
1449	int rc = RTSemEventMultiSignal(pCurGVCpu->gvmm.s.HaltEventMulti);
1450	AssertRC(rc);
1451	cWoken++;
1452	}
1453	}
1454	}
1455	}
1456	AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
1457	}
1458	}
1459
1460	if (cTodo3rd)
1461	{
1462	for (unsigned i = pGVMM->iUsedHead, cGuard = 0;
1463	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
1464	i = pGVMM->aHandles[i].iNext)
1465	{
1466	PGVM pCurGVM = pGVMM->aHandles[i].pGVM;
1467	if ( VALID_PTR(pCurGVM)
1468	&& pCurGVM->u32Magic == GVM_MAGIC)
1469	{
1470	for (VMCPUID idCpu = 0; idCpu < pCurGVM->cCpus; idCpu++)
1471	{
1472	PGVMCPU pCurGVCpu = &pCurGVM->aCpus[idCpu];
1473
1474	if ( pCurGVCpu->gvmm.s.u64HaltExpire
1475	&& pCurGVCpu->gvmm.s.u64HaltExpire <= u64Now + pGVMM->nsEarlyWakeUp2)
1476	{
1477	if (ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0))
1478	{
1479	int rc = RTSemEventMultiSignal(pCurGVCpu->gvmm.s.HaltEventMulti);
1480	AssertRC(rc);
1481	cWoken++;
1482	}
1483	}
1484	}
1485	}
1486	AssertLogRelBreak(cGuard++ < RT_ELEMENTS(pGVMM->aHandles));
1487	}
1488	}
1489
1490	return cWoken;
1491	}
1492
1493
1494	/**
1495	* Halt the EMT thread.
1496	*
1497	* @returns VINF_SUCCESS normal wakeup (timeout or kicked by other thread).
1498	* VERR_INTERRUPTED if a signal was scheduled for the thread.
1499	* @param pVM Pointer to the shared VM structure.
1500	* @param idCpu The Virtual CPU ID of the calling EMT.
1501	* @param u64ExpireGipTime The time for the sleep to expire expressed as GIP time.
1502	* @thread EMT(idCpu).
1503	*/
1504	GVMMR0DECL(int) GVMMR0SchedHalt(PVM pVM, VMCPUID idCpu, uint64_t u64ExpireGipTime)
1505	{
1506	LogFlow(("GVMMR0SchedHalt: pVM=%p\n", pVM));
1507
1508	/*
1509	* Validate the VM structure, state and handle.
1510	*/
1511	PGVM pGVM;
1512	PGVMM pGVMM;
1513	int rc = gvmmR0ByVMAndEMT(pVM, idCpu, &pGVM, &pGVMM);
1514	if (RT_FAILURE(rc))
1515	return rc;
1516	pGVM->gvmm.s.StatsSched.cHaltCalls++;
1517
1518	PGVMCPU pCurGVCpu = &pGVM->aCpus[idCpu];
1519	Assert(!pCurGVCpu->gvmm.s.u64HaltExpire);
1520
1521	/*
1522	* Take the UsedList semaphore, get the current time
1523	* and check if anyone needs waking up.
1524	* Interrupts must NOT be disabled at this point because we ask for GIP time!
1525	*/
1526	rc = gvmmR0UsedLock(pGVMM);
1527	AssertRC(rc);
1528
1529	pCurGVCpu->gvmm.s.iCpuEmt = ASMGetApicId();
1530
1531	Assert(ASMGetFlags() & X86_EFL_IF);
1532	const uint64_t u64Now = RTTimeNanoTS(); /* (GIP time) */
1533	pGVM->gvmm.s.StatsSched.cHaltWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64Now);
1534
1535	/*
1536	* Go to sleep if we must...
1537	*/
1538	if ( u64Now < u64ExpireGipTime
1539	&& u64ExpireGipTime - u64Now > (pGVMM->cVMs > pGVMM->cVMsMeansCompany
1540	? pGVMM->nsMinSleepCompany
1541	: pGVMM->nsMinSleepAlone))
1542	{
1543	pGVM->gvmm.s.StatsSched.cHaltBlocking++;
1544	ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, u64ExpireGipTime);
1545	gvmmR0UsedUnlock(pGVMM);
1546
1547	uint32_t cMillies = (u64ExpireGipTime - u64Now) / 1000000;
1548	rc = RTSemEventMultiWaitNoResume(pCurGVCpu->gvmm.s.HaltEventMulti, cMillies ? cMillies : 1);
1549	ASMAtomicXchgU64(&pCurGVCpu->gvmm.s.u64HaltExpire, 0);
1550	if (rc == VERR_TIMEOUT)
1551	{
1552	pGVM->gvmm.s.StatsSched.cHaltTimeouts++;
1553	rc = VINF_SUCCESS;
1554	}
1555	}
1556	else
1557	{
1558	pGVM->gvmm.s.StatsSched.cHaltNotBlocking++;
1559	gvmmR0UsedUnlock(pGVMM);
1560	}
1561
1562	/* Make sure false wake up calls (gvmmR0SchedDoWakeUps) cause us to spin. */
1563	RTSemEventMultiReset(pCurGVCpu->gvmm.s.HaltEventMulti);
1564
1565	return rc;
1566	}
1567
1568
1569	/**
1570	* Worker for GVMMR0SchedWakeUp and GVMMR0SchedWakeUpAndPokeCpus that wakes up
1571	* the a sleeping EMT.
1572	*
1573	* @retval VINF_SUCCESS if successfully woken up.
1574	* @retval VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
1575	*
1576	* @param pGVM The global (ring-0) VM structure.
1577	* @param pGVCpu The global (ring-0) VCPU structure.
1578	*/
1579	DECLINLINE(int) gvmmR0SchedWakeUpOne(PGVM pGVM, PGVMCPU pGVCpu)
1580	{
1581	pGVM->gvmm.s.StatsSched.cWakeUpCalls++;
1582
1583	/*
1584	* Signal the semaphore regardless of whether it's current blocked on it.
1585	*
1586	* The reason for this is that there is absolutely no way we can be 100%
1587	* certain that it isn't about go to go to sleep on it and just got
1588	* delayed a bit en route. So, we will always signal the semaphore when
1589	* the it is flagged as halted in the VMM.
1590	*/
1591	/** @todo we can optimize some of that by means of the pVCpu->enmState now. */
1592	int rc;
1593	if (pGVCpu->gvmm.s.u64HaltExpire)
1594	{
1595	rc = VINF_SUCCESS;
1596	ASMAtomicXchgU64(&pGVCpu->gvmm.s.u64HaltExpire, 0);
1597	}
1598	else
1599	{
1600	rc = VINF_GVM_NOT_BLOCKED;
1601	pGVM->gvmm.s.StatsSched.cWakeUpNotHalted++;
1602	}
1603
1604	int rc2 = RTSemEventMultiSignal(pGVCpu->gvmm.s.HaltEventMulti);
1605	AssertRC(rc2);
1606
1607	return rc;
1608	}
1609
1610
1611	/**
1612	* Wakes up the halted EMT thread so it can service a pending request.
1613	*
1614	* @returns VBox status code.
1615	* @retval VINF_SUCCESS if successfully woken up.
1616	* @retval VINF_GVM_NOT_BLOCKED if the EMT wasn't blocked.
1617	*
1618	* @param pVM Pointer to the shared VM structure.
1619	* @param idCpu The Virtual CPU ID of the EMT to wake up.
1620	* @thread Any but EMT.
1621	*/
1622	GVMMR0DECL(int) GVMMR0SchedWakeUp(PVM pVM, VMCPUID idCpu)
1623	{
1624	/*
1625	* Validate input and take the UsedLock.
1626	*/
1627	PGVM pGVM;
1628	PGVMM pGVMM;
1629	int rc = gvmmR0ByVM(pVM, &pGVM, &pGVMM, true /* fTakeUsedLock */);
1630	if (RT_SUCCESS(rc))
1631	{
1632	if (idCpu < pGVM->cCpus)
1633	{
1634	/*
1635	* Do the actual job.
1636	*/
1637	rc = gvmmR0SchedWakeUpOne(pGVM, &pGVM->aCpus[idCpu]);
1638
1639	/*
1640	* While we're here, do a round of scheduling.
1641	*/
1642	Assert(ASMGetFlags() & X86_EFL_IF);
1643	const uint64_t u64Now = RTTimeNanoTS(); /* (GIP time) */
1644	pGVM->gvmm.s.StatsSched.cWakeUpWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64Now);
1645	}
1646	else
1647	rc = VERR_INVALID_CPU_ID;
1648
1649	int rc2 = gvmmR0UsedUnlock(pGVMM);
1650	AssertRC(rc2);
1651	}
1652
1653	LogFlow(("GVMMR0SchedWakeUp: returns %Rrc\n", rc));
1654	return rc;
1655	}
1656
1657
1658	/**
1659	* Worker common to GVMMR0SchedPoke and GVMMR0SchedWakeUpAndPokeCpus that pokes
1660	* the Virtual CPU if it's still busy executing guest code.
1661	*
1662	* @returns VBox status code.
1663	* @retval VINF_SUCCESS if poked successfully.
1664	* @retval VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
1665	*
1666	* @param pGVM The global (ring-0) VM structure.
1667	* @param pVCpu The Virtual CPU handle.
1668	*/
1669	DECLINLINE(int) gvmmR0SchedPokeOne(PGVM pGVM, PVMCPU pVCpu)
1670	{
1671	pGVM->gvmm.s.StatsSched.cPokeCalls++;
1672
1673	RTCPUID idHostCpu = pVCpu->idHostCpu;
1674	if ( idHostCpu == NIL_RTCPUID
1675	\|\| VMCPU_GET_STATE(pVCpu) != VMCPUSTATE_STARTED_EXEC)
1676	{
1677	pGVM->gvmm.s.StatsSched.cPokeNotBusy++;
1678	return VINF_GVM_NOT_BUSY_IN_GC;
1679	}
1680
1681	RTMpPokeCpu(idHostCpu);
1682	return VINF_SUCCESS;
1683	}
1684
1685
1686	/**
1687	* Pokes an EMT if it's still busy running guest code.
1688	*
1689	* @returns VBox status code.
1690	* @retval VINF_SUCCESS if poked successfully.
1691	* @retval VINF_GVM_NOT_BUSY_IN_GC if the EMT wasn't busy in GC.
1692	*
1693	* @param pVM Pointer to the shared VM structure.
1694	* @param idCpu The ID of the virtual CPU to poke.
1695	*/
1696	GVMMR0DECL(int) GVMMR0SchedPoke(PVM pVM, VMCPUID idCpu)
1697	{
1698	/*
1699	* Validate input and take the UsedLock.
1700	*/
1701	PGVM pGVM;
1702	PGVMM pGVMM;
1703	int rc = gvmmR0ByVM(pVM, &pGVM, &pGVMM, true /* fTakeUsedLock */);
1704	if (RT_SUCCESS(rc))
1705	{
1706	if (idCpu < pGVM->cCpus)
1707	rc = gvmmR0SchedPokeOne(pGVM, &pVM->aCpus[idCpu]);
1708	else
1709	rc = VERR_INVALID_CPU_ID;
1710
1711	int rc2 = gvmmR0UsedUnlock(pGVMM);
1712	AssertRC(rc2);
1713	}
1714
1715	LogFlow(("GVMMR0SchedWakeUpAndPokeCpus: returns %Rrc\n", rc));
1716	return rc;
1717
1718	}
1719
1720
1721	/**
1722	* Wakes up a set of halted EMT threads so they can service pending request.
1723	*
1724	* @returns VBox status code, no informational stuff.
1725	*
1726	* @param pVM Pointer to the shared VM structure.
1727	* @param pSleepSet The set of sleepers to wake up.
1728	* @param pPokeSet The set of CPUs to poke.
1729	*/
1730	GVMMR0DECL(int) GVMMR0SchedWakeUpAndPokeCpus(PVM pVM, PCVMCPUSET pSleepSet, PCVMCPUSET pPokeSet)
1731	{
1732	AssertPtrReturn(pSleepSet, VERR_INVALID_POINTER);
1733	AssertPtrReturn(pPokeSet, VERR_INVALID_POINTER);
1734	RTNATIVETHREAD hSelf = RTThreadNativeSelf();
1735
1736	/*
1737	* Validate input and take the UsedLock.
1738	*/
1739	PGVM pGVM;
1740	PGVMM pGVMM;
1741	int rc = gvmmR0ByVM(pVM, &pGVM, &pGVMM, true /* fTakeUsedLock */);
1742	if (RT_SUCCESS(rc))
1743	{
1744	rc = VINF_SUCCESS;
1745	VMCPUID idCpu = pGVM->cCpus;
1746	while (idCpu-- > 0)
1747	{
1748	/* Don't try poke or wake up ourselves. */
1749	if (pGVM->aCpus[idCpu].hEMT == hSelf)
1750	continue;
1751
1752	/* just ignore errors for now. */
1753	if (VMCPUSET_IS_PRESENT(pSleepSet, idCpu))
1754	gvmmR0SchedWakeUpOne(pGVM, &pGVM->aCpus[idCpu]);
1755	else if (VMCPUSET_IS_PRESENT(pPokeSet, idCpu))
1756	gvmmR0SchedPokeOne(pGVM, &pVM->aCpus[idCpu]);
1757	}
1758
1759	int rc2 = gvmmR0UsedUnlock(pGVMM);
1760	AssertRC(rc2);
1761	}
1762
1763	LogFlow(("GVMMR0SchedWakeUpAndPokeCpus: returns %Rrc\n", rc));
1764	return rc;
1765	}
1766
1767
1768	/**
1769	* VMMR0 request wrapper for GVMMR0SchedWakeUpAndPokeCpus.
1770	*
1771	* @returns see GVMMR0SchedWakeUpAndPokeCpus.
1772	* @param pVM Pointer to the shared VM structure.
1773	* @param pReq The request packet.
1774	*/
1775	GVMMR0DECL(int) GVMMR0SchedWakeUpAndPokeCpusReq(PVM pVM, PGVMMSCHEDWAKEUPANDPOKECPUSREQ pReq)
1776	{
1777	/*
1778	* Validate input and pass it on.
1779	*/
1780	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
1781	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
1782
1783	return GVMMR0SchedWakeUpAndPokeCpus(pVM, &pReq->SleepSet, &pReq->PokeSet);
1784	}
1785
1786
1787
1788	/**
1789	* Poll the schedule to see if someone else should get a chance to run.
1790	*
1791	* This is a bit hackish and will not work too well if the machine is
1792	* under heavy load from non-VM processes.
1793	*
1794	* @returns VINF_SUCCESS if not yielded.
1795	* VINF_GVM_YIELDED if an attempt to switch to a different VM task was made.
1796	* @param pVM Pointer to the shared VM structure.
1797	* @param idCpu The Virtual CPU ID of the calling EMT.
1798	* @param u64ExpireGipTime The time for the sleep to expire expressed as GIP time.
1799	* @param fYield Whether to yield or not.
1800	* This is for when we're spinning in the halt loop.
1801	* @thread EMT(idCpu).
1802	*/
1803	GVMMR0DECL(int) GVMMR0SchedPoll(PVM pVM, VMCPUID idCpu, bool fYield)
1804	{
1805	/*
1806	* Validate input.
1807	*/
1808	PGVM pGVM;
1809	PGVMM pGVMM;
1810	int rc = gvmmR0ByVMAndEMT(pVM, idCpu, &pGVM, &pGVMM);
1811	if (RT_SUCCESS(rc))
1812	{
1813	rc = gvmmR0UsedLock(pGVMM);
1814	AssertRC(rc);
1815	pGVM->gvmm.s.StatsSched.cPollCalls++;
1816
1817	Assert(ASMGetFlags() & X86_EFL_IF);
1818	const uint64_t u64Now = RTTimeNanoTS(); /* (GIP time) */
1819
1820	if (!fYield)
1821	pGVM->gvmm.s.StatsSched.cPollWakeUps += gvmmR0SchedDoWakeUps(pGVMM, u64Now);
1822	else
1823	{
1824	/** @todo implement this... */
1825	rc = VERR_NOT_IMPLEMENTED;
1826	}
1827
1828	gvmmR0UsedUnlock(pGVMM);
1829	}
1830
1831	LogFlow(("GVMMR0SchedWakeUp: returns %Rrc\n", rc));
1832	return rc;
1833	}
1834
1835
1836
1837	/**
1838	* Retrieves the GVMM statistics visible to the caller.
1839	*
1840	* @returns VBox status code.
1841	*
1842	* @param pStats Where to put the statistics.
1843	* @param pSession The current session.
1844	* @param pVM The VM to obtain statistics for. Optional.
1845	*/
1846	GVMMR0DECL(int) GVMMR0QueryStatistics(PGVMMSTATS pStats, PSUPDRVSESSION pSession, PVM pVM)
1847	{
1848	LogFlow(("GVMMR0QueryStatistics: pStats=%p pSession=%p pVM=%p\n", pStats, pSession, pVM));
1849
1850	/*
1851	* Validate input.
1852	*/
1853	AssertPtrReturn(pSession, VERR_INVALID_POINTER);
1854	AssertPtrReturn(pStats, VERR_INVALID_POINTER);
1855	pStats->cVMs = 0; /* (crash before taking the sem...) */
1856
1857	/*
1858	* Take the lock and get the VM statistics.
1859	*/
1860	PGVMM pGVMM;
1861	if (pVM)
1862	{
1863	PGVM pGVM;
1864	int rc = gvmmR0ByVM(pVM, &pGVM, &pGVMM, true /fTakeUsedLock/);
1865	if (RT_FAILURE(rc))
1866	return rc;
1867	pStats->SchedVM = pGVM->gvmm.s.StatsSched;
1868	}
1869	else
1870	{
1871	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_INTERNAL_ERROR);
1872	memset(&pStats->SchedVM, 0, sizeof(pStats->SchedVM));
1873
1874	int rc = gvmmR0UsedLock(pGVMM);
1875	AssertRCReturn(rc, rc);
1876	}
1877
1878	/*
1879	* Enumerate the VMs and add the ones visibile to the statistics.
1880	*/
1881	pStats->cVMs = 0;
1882	memset(&pStats->SchedSum, 0, sizeof(pStats->SchedSum));
1883
1884	for (unsigned i = pGVMM->iUsedHead;
1885	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
1886	i = pGVMM->aHandles[i].iNext)
1887	{
1888	PGVM pGVM = pGVMM->aHandles[i].pGVM;
1889	void *pvObj = pGVMM->aHandles[i].pvObj;
1890	if ( VALID_PTR(pvObj)
1891	&& VALID_PTR(pGVM)
1892	&& pGVM->u32Magic == GVM_MAGIC
1893	&& RT_SUCCESS(SUPR0ObjVerifyAccess(pvObj, pSession, NULL)))
1894	{
1895	pStats->cVMs++;
1896
1897	pStats->SchedSum.cHaltCalls += pGVM->gvmm.s.StatsSched.cHaltCalls;
1898	pStats->SchedSum.cHaltBlocking += pGVM->gvmm.s.StatsSched.cHaltBlocking;
1899	pStats->SchedSum.cHaltTimeouts += pGVM->gvmm.s.StatsSched.cHaltTimeouts;
1900	pStats->SchedSum.cHaltNotBlocking += pGVM->gvmm.s.StatsSched.cHaltNotBlocking;
1901	pStats->SchedSum.cHaltWakeUps += pGVM->gvmm.s.StatsSched.cHaltWakeUps;
1902
1903	pStats->SchedSum.cWakeUpCalls += pGVM->gvmm.s.StatsSched.cWakeUpCalls;
1904	pStats->SchedSum.cWakeUpNotHalted += pGVM->gvmm.s.StatsSched.cWakeUpNotHalted;
1905	pStats->SchedSum.cWakeUpWakeUps += pGVM->gvmm.s.StatsSched.cWakeUpWakeUps;
1906
1907	pStats->SchedSum.cPokeCalls += pGVM->gvmm.s.StatsSched.cPokeCalls;
1908	pStats->SchedSum.cPokeNotBusy += pGVM->gvmm.s.StatsSched.cPokeNotBusy;
1909
1910	pStats->SchedSum.cPollCalls += pGVM->gvmm.s.StatsSched.cPollCalls;
1911	pStats->SchedSum.cPollHalts += pGVM->gvmm.s.StatsSched.cPollHalts;
1912	pStats->SchedSum.cPollWakeUps += pGVM->gvmm.s.StatsSched.cPollWakeUps;
1913	}
1914	}
1915
1916	gvmmR0UsedUnlock(pGVMM);
1917
1918	return VINF_SUCCESS;
1919	}
1920
1921
1922	/**
1923	* VMMR0 request wrapper for GVMMR0QueryStatistics.
1924	*
1925	* @returns see GVMMR0QueryStatistics.
1926	* @param pVM Pointer to the shared VM structure. Optional.
1927	* @param pReq The request packet.
1928	*/
1929	GVMMR0DECL(int) GVMMR0QueryStatisticsReq(PVM pVM, PGVMMQUERYSTATISTICSSREQ pReq)
1930	{
1931	/*
1932	* Validate input and pass it on.
1933	*/
1934	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
1935	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
1936
1937	return GVMMR0QueryStatistics(&pReq->Stats, pReq->pSession, pVM);
1938	}
1939
1940
1941	/**
1942	* Resets the specified GVMM statistics.
1943	*
1944	* @returns VBox status code.
1945	*
1946	* @param pStats Which statistics to reset, that is, non-zero fields indicates which to reset.
1947	* @param pSession The current session.
1948	* @param pVM The VM to reset statistics for. Optional.
1949	*/
1950	GVMMR0DECL(int) GVMMR0ResetStatistics(PCGVMMSTATS pStats, PSUPDRVSESSION pSession, PVM pVM)
1951	{
1952	LogFlow(("GVMMR0ResetStatistics: pStats=%p pSession=%p pVM=%p\n", pStats, pSession, pVM));
1953
1954	/*
1955	* Validate input.
1956	*/
1957	AssertPtrReturn(pSession, VERR_INVALID_POINTER);
1958	AssertPtrReturn(pStats, VERR_INVALID_POINTER);
1959
1960	/*
1961	* Take the lock and get the VM statistics.
1962	*/
1963	PGVMM pGVMM;
1964	if (pVM)
1965	{
1966	PGVM pGVM;
1967	int rc = gvmmR0ByVM(pVM, &pGVM, &pGVMM, true /fTakeUsedLock/);
1968	if (RT_FAILURE(rc))
1969	return rc;
1970	# define MAYBE_RESET_FIELD(field) \
1971	do { if (pStats->SchedVM. field ) { pGVM->gvmm.s.StatsSched. field = 0; } } while (0)
1972	MAYBE_RESET_FIELD(cHaltCalls);
1973	MAYBE_RESET_FIELD(cHaltBlocking);
1974	MAYBE_RESET_FIELD(cHaltTimeouts);
1975	MAYBE_RESET_FIELD(cHaltNotBlocking);
1976	MAYBE_RESET_FIELD(cHaltWakeUps);
1977	MAYBE_RESET_FIELD(cWakeUpCalls);
1978	MAYBE_RESET_FIELD(cWakeUpNotHalted);
1979	MAYBE_RESET_FIELD(cWakeUpWakeUps);
1980	MAYBE_RESET_FIELD(cPokeCalls);
1981	MAYBE_RESET_FIELD(cPokeNotBusy);
1982	MAYBE_RESET_FIELD(cPollCalls);
1983	MAYBE_RESET_FIELD(cPollHalts);
1984	MAYBE_RESET_FIELD(cPollWakeUps);
1985	# undef MAYBE_RESET_FIELD
1986	}
1987	else
1988	{
1989	GVMM_GET_VALID_INSTANCE(pGVMM, VERR_INTERNAL_ERROR);
1990
1991	int rc = gvmmR0UsedLock(pGVMM);
1992	AssertRCReturn(rc, rc);
1993	}
1994
1995	/*
1996	* Enumerate the VMs and add the ones visibile to the statistics.
1997	*/
1998	if (ASMMemIsAll8(&pStats->SchedSum, sizeof(pStats->SchedSum), 0))
1999	{
2000	for (unsigned i = pGVMM->iUsedHead;
2001	i != NIL_GVM_HANDLE && i < RT_ELEMENTS(pGVMM->aHandles);
2002	i = pGVMM->aHandles[i].iNext)
2003	{
2004	PGVM pGVM = pGVMM->aHandles[i].pGVM;
2005	void *pvObj = pGVMM->aHandles[i].pvObj;
2006	if ( VALID_PTR(pvObj)
2007	&& VALID_PTR(pGVM)
2008	&& pGVM->u32Magic == GVM_MAGIC
2009	&& RT_SUCCESS(SUPR0ObjVerifyAccess(pvObj, pSession, NULL)))
2010	{
2011	# define MAYBE_RESET_FIELD(field) \
2012	do { if (pStats->SchedSum. field ) { pGVM->gvmm.s.StatsSched. field = 0; } } while (0)
2013	MAYBE_RESET_FIELD(cHaltCalls);
2014	MAYBE_RESET_FIELD(cHaltBlocking);
2015	MAYBE_RESET_FIELD(cHaltTimeouts);
2016	MAYBE_RESET_FIELD(cHaltNotBlocking);
2017	MAYBE_RESET_FIELD(cHaltWakeUps);
2018	MAYBE_RESET_FIELD(cWakeUpCalls);
2019	MAYBE_RESET_FIELD(cWakeUpNotHalted);
2020	MAYBE_RESET_FIELD(cWakeUpWakeUps);
2021	MAYBE_RESET_FIELD(cPokeCalls);
2022	MAYBE_RESET_FIELD(cPokeNotBusy);
2023	MAYBE_RESET_FIELD(cPollCalls);
2024	MAYBE_RESET_FIELD(cPollHalts);
2025	MAYBE_RESET_FIELD(cPollWakeUps);
2026	# undef MAYBE_RESET_FIELD
2027	}
2028	}
2029	}
2030
2031	gvmmR0UsedUnlock(pGVMM);
2032
2033	return VINF_SUCCESS;
2034	}
2035
2036
2037	/**
2038	* VMMR0 request wrapper for GVMMR0ResetStatistics.
2039	*
2040	* @returns see GVMMR0ResetStatistics.
2041	* @param pVM Pointer to the shared VM structure. Optional.
2042	* @param pReq The request packet.
2043	*/
2044	GVMMR0DECL(int) GVMMR0ResetStatisticsReq(PVM pVM, PGVMMRESETSTATISTICSSREQ pReq)
2045	{
2046	/*
2047	* Validate input and pass it on.
2048	*/
2049	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
2050	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x != %#x\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
2051
2052	return GVMMR0ResetStatistics(&pReq->Stats, pReq->pSession, pVM);
2053	}
2054

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/GVMMR0.cpp@ 19606

Download in other formats: