SUPDRVShared.c@ 5031

Last change on this file since 5031 was 5031, checked in by vboxsync, 17 years ago
Use GVMMR3CreateVM. the new GVM structure is a ring-0 only VM structure. the old VM structure is the shared ring-0, ring-3 and GC VM structure.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 139.0 KB

Line
1	/* $Revision: 5031 $ */
2	/** @file
3	* VirtualBox Support Driver - Shared code.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 innotek GmbH
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License as published by the Free Software Foundation,
13	* in version 2 as it comes in the "COPYING" file of the VirtualBox OSE
14	* distribution. VirtualBox OSE is distributed in the hope that it will
15	* be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#include "SUPDRV.h"
23	#ifndef PAGE_SHIFT
24	# include <iprt/param.h>
25	#endif
26	#include <iprt/alloc.h>
27	#include <iprt/semaphore.h>
28	#include <iprt/spinlock.h>
29	#include <iprt/thread.h>
30	#include <iprt/process.h>
31	#include <iprt/log.h>
32
33
34	/*******************************************************************************
35	* Defined Constants And Macros *
36	*******************************************************************************/
37	/* from x86.h - clashes with linux thus this duplication */
38	#undef X86_CR0_PG
39	#define X86_CR0_PG BIT(31)
40	#undef X86_CR0_PE
41	#define X86_CR0_PE BIT(0)
42	#undef X86_CPUID_AMD_FEATURE_EDX_NX
43	#define X86_CPUID_AMD_FEATURE_EDX_NX BIT(20)
44	#undef MSR_K6_EFER
45	#define MSR_K6_EFER 0xc0000080
46	#undef MSR_K6_EFER_NXE
47	#define MSR_K6_EFER_NXE BIT(11)
48	#undef MSR_K6_EFER_LMA
49	#define MSR_K6_EFER_LMA BIT(10)
50	#undef X86_CR4_PGE
51	#define X86_CR4_PGE BIT(7)
52	#undef X86_CR4_PAE
53	#define X86_CR4_PAE BIT(5)
54	#undef X86_CPUID_AMD_FEATURE_EDX_LONG_MODE
55	#define X86_CPUID_AMD_FEATURE_EDX_LONG_MODE BIT(29)
56
57
58	/** The frequency by which we recalculate the u32UpdateHz and
59	* u32UpdateIntervalNS GIP members. The value must be a power of 2. */
60	#define GIP_UPDATEHZ_RECALC_FREQ 0x800
61
62	/**
63	* Validates a session pointer.
64	*
65	* @returns true/false accordingly.
66	* @param pSession The session.
67	*/
68	#define SUP_IS_SESSION_VALID(pSession) \
69	( VALID_PTR(pSession) \
70	&& pSession->u32Cookie == BIRD_INV)
71
72
73	/*******************************************************************************
74	* Global Variables *
75	*******************************************************************************/
76	/**
77	* Array of the R0 SUP API.
78	*/
79	static SUPFUNC g_aFunctions[] =
80	{
81	/* name function */
82	{ "SUPR0ObjRegister", (void *)SUPR0ObjRegister },
83	{ "SUPR0ObjAddRef", (void *)SUPR0ObjAddRef },
84	{ "SUPR0ObjRelease", (void *)SUPR0ObjRelease },
85	{ "SUPR0ObjVerifyAccess", (void *)SUPR0ObjVerifyAccess },
86	{ "SUPR0LockMem", (void *)SUPR0LockMem },
87	{ "SUPR0UnlockMem", (void *)SUPR0UnlockMem },
88	{ "SUPR0ContAlloc", (void *)SUPR0ContAlloc },
89	{ "SUPR0ContFree", (void *)SUPR0ContFree },
90	{ "SUPR0LowAlloc", (void *)SUPR0LowAlloc },
91	{ "SUPR0LowFree", (void *)SUPR0LowFree },
92	{ "SUPR0MemAlloc", (void *)SUPR0MemAlloc },
93	{ "SUPR0MemGetPhys", (void *)SUPR0MemGetPhys },
94	{ "SUPR0MemFree", (void *)SUPR0MemFree },
95	{ "SUPR0PageAlloc", (void *)SUPR0PageAlloc },
96	{ "SUPR0PageFree", (void *)SUPR0PageFree },
97	{ "SUPR0Printf", (void *)SUPR0Printf },
98	{ "RTMemAlloc", (void *)RTMemAlloc },
99	{ "RTMemAllocZ", (void *)RTMemAllocZ },
100	{ "RTMemFree", (void *)RTMemFree },
101	/{ "RTMemDup", (void )RTMemDup },*/
102	{ "RTMemRealloc", (void *)RTMemRealloc },
103	{ "RTR0MemObjAllocLow", (void *)RTR0MemObjAllocLow },
104	{ "RTR0MemObjAllocPage", (void *)RTR0MemObjAllocPage },
105	{ "RTR0MemObjAllocPhys", (void *)RTR0MemObjAllocPhys },
106	{ "RTR0MemObjAllocPhysNC", (void *)RTR0MemObjAllocPhysNC },
107	{ "RTR0MemObjLockUser", (void *)RTR0MemObjLockUser },
108	{ "RTR0MemObjMapKernel", (void *)RTR0MemObjMapKernel },
109	{ "RTR0MemObjMapUser", (void *)RTR0MemObjMapUser },
110	{ "RTR0MemObjAddress", (void *)RTR0MemObjAddress },
111	{ "RTR0MemObjAddressR3", (void *)RTR0MemObjAddressR3 },
112	{ "RTR0MemObjSize", (void *)RTR0MemObjSize },
113	{ "RTR0MemObjIsMapping", (void *)RTR0MemObjIsMapping },
114	{ "RTR0MemObjGetPagePhysAddr", (void *)RTR0MemObjGetPagePhysAddr },
115	{ "RTR0MemObjFree", (void *)RTR0MemObjFree },
116	/* These doesn't work yet on linux - use fast mutexes!
117	{ "RTSemMutexCreate", (void *)RTSemMutexCreate },
118	{ "RTSemMutexRequest", (void *)RTSemMutexRequest },
119	{ "RTSemMutexRelease", (void *)RTSemMutexRelease },
120	{ "RTSemMutexDestroy", (void *)RTSemMutexDestroy },
121	*/
122	{ "RTSemFastMutexCreate", (void *)RTSemFastMutexCreate },
123	{ "RTSemFastMutexDestroy", (void *)RTSemFastMutexDestroy },
124	{ "RTSemFastMutexRequest", (void *)RTSemFastMutexRequest },
125	{ "RTSemFastMutexRelease", (void *)RTSemFastMutexRelease },
126	{ "RTSemEventCreate", (void *)RTSemEventCreate },
127	{ "RTSemEventSignal", (void *)RTSemEventSignal },
128	{ "RTSemEventWait", (void *)RTSemEventWait },
129	{ "RTSemEventDestroy", (void *)RTSemEventDestroy },
130	{ "RTSpinlockCreate", (void *)RTSpinlockCreate },
131	{ "RTSpinlockDestroy", (void *)RTSpinlockDestroy },
132	{ "RTSpinlockAcquire", (void *)RTSpinlockAcquire },
133	{ "RTSpinlockRelease", (void *)RTSpinlockRelease },
134	{ "RTSpinlockAcquireNoInts", (void *)RTSpinlockAcquireNoInts },
135	{ "RTSpinlockReleaseNoInts", (void *)RTSpinlockReleaseNoInts },
136	{ "RTThreadNativeSelf", (void *)RTThreadNativeSelf },
137	{ "RTThreadSleep", (void *)RTThreadSleep },
138	{ "RTThreadYield", (void *)RTThreadYield },
139	#if 0 /* Thread APIs, Part 2. */
140	{ "RTThreadSelf", (void *)RTThreadSelf },
141	{ "RTThreadCreate", (void *)RTThreadCreate },
142	{ "RTThreadGetNative", (void *)RTThreadGetNative },
143	{ "RTThreadWait", (void *)RTThreadWait },
144	{ "RTThreadWaitNoResume", (void *)RTThreadWaitNoResume },
145	{ "RTThreadGetName", (void *)RTThreadGetName },
146	{ "RTThreadSelfName", (void *)RTThreadSelfName },
147	{ "RTThreadGetType", (void *)RTThreadGetType },
148	{ "RTThreadUserSignal", (void *)RTThreadUserSignal },
149	{ "RTThreadUserReset", (void *)RTThreadUserReset },
150	{ "RTThreadUserWait", (void *)RTThreadUserWait },
151	{ "RTThreadUserWaitNoResume", (void *)RTThreadUserWaitNoResume },
152	#endif
153	{ "RTLogDefaultInstance", (void *)RTLogDefaultInstance },
154	{ "RTLogRelDefaultInstance", (void *)RTLogRelDefaultInstance },
155	{ "RTLogSetDefaultInstanceThread", (void *)RTLogSetDefaultInstanceThread },
156	{ "RTLogLogger", (void *)RTLogLogger },
157	{ "RTLogLoggerEx", (void *)RTLogLoggerEx },
158	{ "RTLogLoggerExV", (void *)RTLogLoggerExV },
159	{ "RTLogPrintf", (void *)RTLogPrintf },
160	{ "RTLogPrintfV", (void *)RTLogPrintfV },
161	{ "AssertMsg1", (void *)AssertMsg1 },
162	{ "AssertMsg2", (void *)AssertMsg2 },
163	};
164
165
166	/*******************************************************************************
167	* Internal Functions *
168	*******************************************************************************/
169	static int supdrvMemAdd(PSUPDRVMEMREF pMem, PSUPDRVSESSION pSession);
170	static int supdrvMemRelease(PSUPDRVSESSION pSession, RTHCUINTPTR uPtr, SUPDRVMEMREFTYPE eType);
171	#ifdef VBOX_WITH_IDT_PATCHING
172	static int supdrvIOCtl_IdtInstall(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPIDTINSTALL pReq);
173	static PSUPDRVPATCH supdrvIdtPatchOne(PSUPDRVDEVEXT pDevExt, PSUPDRVPATCH pPatch);
174	static int supdrvIOCtl_IdtRemoveAll(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession);
175	static void supdrvIdtRemoveOne(PSUPDRVDEVEXT pDevExt, PSUPDRVPATCH pPatch);
176	static void supdrvIdtWrite(volatile void pvIdtEntry, const SUPDRVIDTE pNewIDTEntry);
177	#endif /* VBOX_WITH_IDT_PATCHING */
178	static int supdrvIOCtl_LdrOpen(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPLDROPEN pReq);
179	static int supdrvIOCtl_LdrLoad(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPLDRLOAD pReq);
180	static int supdrvIOCtl_LdrFree(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPLDRFREE pReq);
181	static int supdrvIOCtl_LdrGetSymbol(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPLDRGETSYMBOL pReq);
182	static int supdrvLdrSetR0EP(PSUPDRVDEVEXT pDevExt, void pvVMMR0, void pvVMMR0EntryInt, void pvVMMR0EntryFast, void pvVMMR0EntryEx);
183	static void supdrvLdrUnsetR0EP(PSUPDRVDEVEXT pDevExt);
184	static void supdrvLdrAddUsage(PSUPDRVSESSION pSession, PSUPDRVLDRIMAGE pImage);
185	static void supdrvLdrFree(PSUPDRVDEVEXT pDevExt, PSUPDRVLDRIMAGE pImage);
186	static SUPPAGINGMODE supdrvIOCtl_GetPagingMode(void);
187	static SUPGIPMODE supdrvGipDeterminTscMode(void);
188	#ifdef RT_OS_WINDOWS
189	static int supdrvPageGetPhys(PSUPDRVSESSION pSession, RTR3PTR pvR3, uint32_t cPages, PRTHCPHYS paPages);
190	static bool supdrvPageWasLockedByPageAlloc(PSUPDRVSESSION pSession, RTR3PTR pvR3);
191	#endif
192	#ifdef USE_NEW_OS_INTERFACE_FOR_GIP
193	static int supdrvGipCreate(PSUPDRVDEVEXT pDevExt);
194	static void supdrvGipDestroy(PSUPDRVDEVEXT pDevExt);
195	static DECLCALLBACK(void) supdrvGipTimer(PRTTIMER pTimer, void *pvUser);
196	#endif
197
198
199	/**
200	* Initializes the device extentsion structure.
201	*
202	* @returns IPRT status code.
203	* @param pDevExt The device extension to initialize.
204	*/
205	int VBOXCALL supdrvInitDevExt(PSUPDRVDEVEXT pDevExt)
206	{
207	/*
208	* Initialize it.
209	*/
210	int rc;
211	memset(pDevExt, 0, sizeof(*pDevExt));
212	rc = RTSpinlockCreate(&pDevExt->Spinlock);
213	if (!rc)
214	{
215	rc = RTSemFastMutexCreate(&pDevExt->mtxLdr);
216	if (!rc)
217	{
218	rc = RTSemFastMutexCreate(&pDevExt->mtxGip);
219	if (!rc)
220	{
221	#ifdef USE_NEW_OS_INTERFACE_FOR_GIP
222	rc = supdrvGipCreate(pDevExt);
223	if (RT_SUCCESS(rc))
224	{
225	pDevExt->u32Cookie = BIRD; /** @todo make this random? */
226	return VINF_SUCCESS;
227	}
228	#else
229	pDevExt->u32Cookie = BIRD;
230	return VINF_SUCCESS;
231	#endif
232	}
233	RTSemFastMutexDestroy(pDevExt->mtxLdr);
234	pDevExt->mtxLdr = NIL_RTSEMFASTMUTEX;
235	}
236	RTSpinlockDestroy(pDevExt->Spinlock);
237	pDevExt->Spinlock = NIL_RTSPINLOCK;
238	}
239	return rc;
240	}
241
242
243	/**
244	* Delete the device extension (e.g. cleanup members).
245	*
246	* @param pDevExt The device extension to delete.
247	*/
248	void VBOXCALL supdrvDeleteDevExt(PSUPDRVDEVEXT pDevExt)
249	{
250	#ifdef VBOX_WITH_IDT_PATCHING
251	PSUPDRVPATCH pPatch;
252	#endif
253	PSUPDRVOBJ pObj;
254	PSUPDRVUSAGE pUsage;
255
256	/*
257	* Kill mutexes and spinlocks.
258	*/
259	RTSemFastMutexDestroy(pDevExt->mtxGip);
260	pDevExt->mtxGip = NIL_RTSEMFASTMUTEX;
261	RTSemFastMutexDestroy(pDevExt->mtxLdr);
262	pDevExt->mtxLdr = NIL_RTSEMFASTMUTEX;
263	RTSpinlockDestroy(pDevExt->Spinlock);
264	pDevExt->Spinlock = NIL_RTSPINLOCK;
265
266	/*
267	* Free lists.
268	*/
269	#ifdef VBOX_WITH_IDT_PATCHING
270	/* patches */
271	/** @todo make sure we don't uninstall patches which has been patched by someone else. */
272	pPatch = pDevExt->pIdtPatchesFree;
273	pDevExt->pIdtPatchesFree = NULL;
274	while (pPatch)
275	{
276	void *pvFree = pPatch;
277	pPatch = pPatch->pNext;
278	RTMemExecFree(pvFree);
279	}
280	#endif /* VBOX_WITH_IDT_PATCHING */
281
282	/* objects. */
283	pObj = pDevExt->pObjs;
284	#if !defined(DEBUG_bird) \|\| !defined(RT_OS_LINUX) /* breaks unloading, temporary, remove me! */
285	Assert(!pObj); /* (can trigger on forced unloads) */
286	#endif
287	pDevExt->pObjs = NULL;
288	while (pObj)
289	{
290	void *pvFree = pObj;
291	pObj = pObj->pNext;
292	RTMemFree(pvFree);
293	}
294
295	/* usage records. */
296	pUsage = pDevExt->pUsageFree;
297	pDevExt->pUsageFree = NULL;
298	while (pUsage)
299	{
300	void *pvFree = pUsage;
301	pUsage = pUsage->pNext;
302	RTMemFree(pvFree);
303	}
304
305	#ifdef USE_NEW_OS_INTERFACE_FOR_GIP
306	/* kill the GIP */
307	supdrvGipDestroy(pDevExt);
308	#endif
309	}
310
311
312	/**
313	* Create session.
314	*
315	* @returns IPRT status code.
316	* @param pDevExt Device extension.
317	* @param ppSession Where to store the pointer to the session data.
318	*/
319	int VBOXCALL supdrvCreateSession(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION *ppSession)
320	{
321	/*
322	* Allocate memory for the session data.
323	*/
324	int rc = VERR_NO_MEMORY;
325	PSUPDRVSESSION pSession = ppSession = (PSUPDRVSESSION)RTMemAllocZ(sizeof(pSession));
326	if (pSession)
327	{
328	/* Initialize session data. */
329	rc = RTSpinlockCreate(&pSession->Spinlock);
330	if (!rc)
331	{
332	Assert(pSession->Spinlock != NIL_RTSPINLOCK);
333	pSession->pDevExt = pDevExt;
334	pSession->u32Cookie = BIRD_INV;
335	/*pSession->pLdrUsage = NULL;
336	pSession->pPatchUsage = NULL;
337	pSession->pUsage = NULL;
338	pSession->pGip = NULL;
339	pSession->fGipReferenced = false;
340	pSession->Bundle.cUsed = 0 */
341
342	dprintf(("Created session %p initial cookie=%#x\n", pSession, pSession->u32Cookie));
343	return VINF_SUCCESS;
344	}
345
346	RTMemFree(pSession);
347	*ppSession = NULL;
348	}
349
350	dprintf(("Failed to create spinlock, rc=%d!\n", rc));
351	return rc;
352	}
353
354
355	/**
356	* Shared code for cleaning up a session.
357	*
358	* @param pDevExt Device extension.
359	* @param pSession Session data.
360	* This data will be freed by this routine.
361	*/
362	void VBOXCALL supdrvCloseSession(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession)
363	{
364	/*
365	* Cleanup the session first.
366	*/
367	supdrvCleanupSession(pDevExt, pSession);
368
369	/*
370	* Free the rest of the session stuff.
371	*/
372	RTSpinlockDestroy(pSession->Spinlock);
373	pSession->Spinlock = NIL_RTSPINLOCK;
374	pSession->pDevExt = NULL;
375	RTMemFree(pSession);
376	dprintf2(("supdrvCloseSession: returns\n"));
377	}
378
379
380	/**
381	* Shared code for cleaning up a session (but not quite freeing it).
382	*
383	* This is primarily intended for MAC OS X where we have to clean up the memory
384	* stuff before the file handle is closed.
385	*
386	* @param pDevExt Device extension.
387	* @param pSession Session data.
388	* This data will be freed by this routine.
389	*/
390	void VBOXCALL supdrvCleanupSession(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession)
391	{
392	PSUPDRVBUNDLE pBundle;
393	dprintf(("supdrvCleanupSession: pSession=%p\n", pSession));
394
395	/*
396	* Remove logger instances related to this session.
397	* (This assumes the dprintf and dprintf2 macros doesn't use the normal logging.)
398	*/
399	RTLogSetDefaultInstanceThread(NULL, (uintptr_t)pSession);
400
401	#ifdef VBOX_WITH_IDT_PATCHING
402	/*
403	* Uninstall any IDT patches installed for this session.
404	*/
405	supdrvIOCtl_IdtRemoveAll(pDevExt, pSession);
406	#endif
407
408	/*
409	* Release object references made in this session.
410	* In theory there should be noone racing us in this session.
411	*/
412	dprintf2(("release objects - start\n"));
413	if (pSession->pUsage)
414	{
415	RTSPINLOCKTMP SpinlockTmp = RTSPINLOCKTMP_INITIALIZER;
416	PSUPDRVUSAGE pUsage;
417	RTSpinlockAcquire(pDevExt->Spinlock, &SpinlockTmp);
418
419	while ((pUsage = pSession->pUsage) != NULL)
420	{
421	PSUPDRVOBJ pObj = pUsage->pObj;
422	pSession->pUsage = pUsage->pNext;
423
424	AssertMsg(pUsage->cUsage >= 1 && pObj->cUsage >= pUsage->cUsage, ("glob %d; sess %d\n", pObj->cUsage, pUsage->cUsage));
425	if (pUsage->cUsage < pObj->cUsage)
426	{
427	pObj->cUsage -= pUsage->cUsage;
428	RTSpinlockRelease(pDevExt->Spinlock, &SpinlockTmp);
429	}
430	else
431	{
432	/* Destroy the object and free the record. */
433	if (pDevExt->pObjs == pObj)
434	pDevExt->pObjs = pObj->pNext;
435	else
436	{
437	PSUPDRVOBJ pObjPrev;
438	for (pObjPrev = pDevExt->pObjs; pObjPrev; pObjPrev = pObjPrev->pNext)
439	if (pObjPrev->pNext == pObj)
440	{
441	pObjPrev->pNext = pObj->pNext;
442	break;
443	}
444	Assert(pObjPrev);
445	}
446	RTSpinlockRelease(pDevExt->Spinlock, &SpinlockTmp);
447
448	if (pObj->pfnDestructor)
449	pObj->pfnDestructor(pObj, pObj->pvUser1, pObj->pvUser2);
450	RTMemFree(pObj);
451	}
452
453	/* free it and continue. */
454	RTMemFree(pUsage);
455
456	RTSpinlockAcquire(pDevExt->Spinlock, &SpinlockTmp);
457	}
458
459	RTSpinlockRelease(pDevExt->Spinlock, &SpinlockTmp);
460	AssertMsg(!pSession->pUsage, ("Some buster reregistered an object during desturction!\n"));
461	}
462	dprintf2(("release objects - done\n"));
463
464	/*
465	* Release memory allocated in the session.
466	*
467	* We do not serialize this as we assume that the application will
468	* not allocated memory while closing the file handle object.
469	*/
470	dprintf2(("freeing memory:\n"));
471	pBundle = &pSession->Bundle;
472	while (pBundle)
473	{
474	PSUPDRVBUNDLE pToFree;
475	unsigned i;
476
477	/*
478	* Check and unlock all entries in the bundle.
479	*/
480	for (i = 0; i < RT_ELEMENTS(pBundle->aMem); i++)
481	{
482	if (pBundle->aMem[i].MemObj != NIL_RTR0MEMOBJ)
483	{
484	int rc;
485	dprintf2(("eType=%d pvR0=%p pvR3=%p cb=%ld\n", pBundle->aMem[i].eType, RTR0MemObjAddress(pBundle->aMem[i].MemObj),
486	(void *)RTR0MemObjAddressR3(pBundle->aMem[i].MapObjR3), (long)RTR0MemObjSize(pBundle->aMem[i].MemObj)));
487	if (pBundle->aMem[i].MapObjR3 != NIL_RTR0MEMOBJ)
488	{
489	rc = RTR0MemObjFree(pBundle->aMem[i].MapObjR3, false);
490	AssertRC(rc); /** @todo figure out how to handle this. */
491	pBundle->aMem[i].MapObjR3 = NIL_RTR0MEMOBJ;
492	}
493	rc = RTR0MemObjFree(pBundle->aMem[i].MemObj, false);
494	AssertRC(rc); /** @todo figure out how to handle this. */
495	pBundle->aMem[i].MemObj = NIL_RTR0MEMOBJ;
496	pBundle->aMem[i].eType = MEMREF_TYPE_UNUSED;
497	}
498	}
499
500	/*
501	* Advance and free previous bundle.
502	*/
503	pToFree = pBundle;
504	pBundle = pBundle->pNext;
505
506	pToFree->pNext = NULL;
507	pToFree->cUsed = 0;
508	if (pToFree != &pSession->Bundle)
509	RTMemFree(pToFree);
510	}
511	dprintf2(("freeing memory - done\n"));
512
513	/*
514	* Loaded images needs to be dereferenced and possibly freed up.
515	*/
516	RTSemFastMutexRequest(pDevExt->mtxLdr);
517	dprintf2(("freeing images:\n"));
518	if (pSession->pLdrUsage)
519	{
520	PSUPDRVLDRUSAGE pUsage = pSession->pLdrUsage;
521	pSession->pLdrUsage = NULL;
522	while (pUsage)
523	{
524	void *pvFree = pUsage;
525	PSUPDRVLDRIMAGE pImage = pUsage->pImage;
526	if (pImage->cUsage > pUsage->cUsage)
527	pImage->cUsage -= pUsage->cUsage;
528	else
529	supdrvLdrFree(pDevExt, pImage);
530	pUsage->pImage = NULL;
531	pUsage = pUsage->pNext;
532	RTMemFree(pvFree);
533	}
534	}
535	RTSemFastMutexRelease(pDevExt->mtxLdr);
536	dprintf2(("freeing images - done\n"));
537
538	/*
539	* Unmap the GIP.
540	*/
541	dprintf2(("umapping GIP:\n"));
542	#ifdef USE_NEW_OS_INTERFACE_FOR_GIP
543	if (pSession->GipMapObjR3 != NIL_RTR0MEMOBJ)
544	#else
545	if (pSession->pGip)
546	#endif
547	{
548	SUPR0GipUnmap(pSession);
549	#ifndef USE_NEW_OS_INTERFACE_FOR_GIP
550	pSession->pGip = NULL;
551	#endif
552	pSession->fGipReferenced = 0;
553	}
554	dprintf2(("umapping GIP - done\n"));
555	}
556
557
558	/**
559	* Fast path I/O Control worker.
560	*
561	* @returns VBox status code that should be passed down to ring-3 unchanged.
562	* @param uIOCtl Function number.
563	* @param pDevExt Device extention.
564	* @param pSession Session data.
565	*/
566	int VBOXCALL supdrvIOCtlFast(uintptr_t uIOCtl, PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession)
567	{
568	int rc;
569
570	/*
571	* We check the two prereqs after doing this only to allow the compiler to optimize things better.
572	*/
573	if (RT_LIKELY(pSession->pVM && pDevExt->pfnVMMR0EntryFast))
574	{
575	switch (uIOCtl)
576	{
577	case SUP_IOCTL_FAST_DO_RAW_RUN:
578	rc = pDevExt->pfnVMMR0EntryFast(pSession->pVM, SUP_VMMR0_DO_RAW_RUN);
579	break;
580	case SUP_IOCTL_FAST_DO_HWACC_RUN:
581	rc = pDevExt->pfnVMMR0EntryFast(pSession->pVM, SUP_VMMR0_DO_HWACC_RUN);
582	break;
583	case SUP_IOCTL_FAST_DO_NOP:
584	rc = pDevExt->pfnVMMR0EntryFast(pSession->pVM, SUP_VMMR0_DO_NOP);
585	break;
586	default:
587	rc = VERR_INTERNAL_ERROR;
588	break;
589	}
590	}
591	else
592	rc = VERR_INTERNAL_ERROR;
593
594	return rc;
595	}
596
597
598	/**
599	* I/O Control worker.
600	*
601	* @returns 0 on success.
602	* @returns VERR_INVALID_PARAMETER if the request is invalid.
603	*
604	* @param uIOCtl Function number.
605	* @param pDevExt Device extention.
606	* @param pSession Session data.
607	* @param pReqHdr The request header.
608	*/
609	int VBOXCALL supdrvIOCtl(uintptr_t uIOCtl, PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPREQHDR pReqHdr)
610	{
611	/*
612	* Validate the request.
613	*/
614	/* this first check could probably be omitted as its also done by the OS specific code... */
615	if (RT_UNLIKELY( (pReqHdr->fFlags & SUPREQHDR_FLAGS_MAGIC_MASK) != SUPREQHDR_FLAGS_MAGIC
616	\|\| pReqHdr->cbIn < sizeof(*pReqHdr)
617	\|\| pReqHdr->cbOut < sizeof(*pReqHdr)))
618	{
619	OSDBGPRINT(("vboxdrv: Bad ioctl request header; cbIn=%#lx cbOut=%#lx fFlags=%#lx\n",
620	(long)pReqHdr->cbIn, (long)pReqHdr->cbOut, (long)pReqHdr->fFlags));
621	return VERR_INVALID_PARAMETER;
622	}
623	if (RT_UNLIKELY(uIOCtl == SUP_IOCTL_COOKIE))
624	{
625	if (pReqHdr->u32Cookie != SUPCOOKIE_INITIAL_COOKIE)
626	{
627	OSDBGPRINT(("SUP_IOCTL_COOKIE: bad cookie %#lx\n", (long)pReqHdr->u32Cookie));
628	return VERR_INVALID_PARAMETER;
629	}
630	}
631	else if (RT_UNLIKELY( pReqHdr->u32Cookie != pDevExt->u32Cookie
632	\|\| pReqHdr->u32SessionCookie != pSession->u32Cookie))
633	{
634	OSDBGPRINT(("vboxdrv: bad cookie %#lx / %#lx.\n", (long)pReqHdr->u32Cookie, (long)pReqHdr->u32SessionCookie));
635	return VERR_INVALID_PARAMETER;
636	}
637
638	/*
639	* Validation macros
640	*/
641	#define REQ_CHECK_SIZES_EX(Name, cbInExpect, cbOutExpect) \
642	do { \
643	if (RT_UNLIKELY(pReqHdr->cbIn != (cbInExpect) \|\| pReqHdr->cbOut != (cbOutExpect))) \
644	{ \
645	OSDBGPRINT(( #Name ": Invalid input/output sizes. cbIn=%ld expected %ld. cbOut=%ld expected %ld.\n", \
646	(long)pReq->Hdr.cbIn, (long)(cbInExpect), (long)pReq->Hdr.cbOut, (long)(cbOutExpect))); \
647	return pReq->Hdr.rc = VERR_INVALID_PARAMETER; \
648	} \
649	} while (0)
650
651	#define REQ_CHECK_SIZES(Name) REQ_CHECK_SIZES_EX(Name, Name ## _SIZE_IN, Name ## _SIZE_OUT)
652
653	#define REQ_CHECK_SIZE_IN(Name, cbInExpect) \
654	do { \
655	if (RT_UNLIKELY(pReqHdr->cbIn != (cbInExpect))) \
656	{ \
657	OSDBGPRINT(( #Name ": Invalid input/output sizes. cbIn=%ld expected %ld.\n", \
658	(long)pReq->Hdr.cbIn, (long)(cbInExpect))); \
659	return pReq->Hdr.rc = VERR_INVALID_PARAMETER; \
660	} \
661	} while (0)
662
663	#define REQ_CHECK_SIZE_OUT(Name, cbOutExpect) \
664	do { \
665	if (RT_UNLIKELY(pReqHdr->cbOut != (cbOutExpect))) \
666	{ \
667	OSDBGPRINT(( #Name ": Invalid input/output sizes. cbOut=%ld expected %ld.\n", \
668	(long)pReq->Hdr.cbOut, (long)(cbOutExpect))); \
669	return pReq->Hdr.rc = VERR_INVALID_PARAMETER; \
670	} \
671	} while (0)
672
673	#define REQ_CHECK_EXPR(Name, expr) \
674	do { \
675	if (RT_UNLIKELY(!(expr))) \
676	{ \
677	OSDBGPRINT(( #Name ": %s\n", #expr)); \
678	return pReq->Hdr.rc = VERR_INVALID_PARAMETER; \
679	} \
680	} while (0)
681
682	#define REQ_CHECK_EXPR_FMT(expr, fmt) \
683	do { \
684	if (RT_UNLIKELY(!(expr))) \
685	{ \
686	OSDBGPRINT( fmt ); \
687	return pReq->Hdr.rc = VERR_INVALID_PARAMETER; \
688	} \
689	} while (0)
690
691
692	/*
693	* The switch.
694	*/
695	switch (SUP_CTL_CODE_NO_SIZE(uIOCtl))
696	{
697	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_COOKIE):
698	{
699	PSUPCOOKIE pReq = (PSUPCOOKIE)pReqHdr;
700	REQ_CHECK_SIZES(SUP_IOCTL_COOKIE);
701	if (strncmp(pReq->u.In.szMagic, SUPCOOKIE_MAGIC, sizeof(pReq->u.In.szMagic)))
702	{
703	OSDBGPRINT(("SUP_IOCTL_COOKIE: invalid magic %.16s\n", pReq->u.In.szMagic));
704	pReq->Hdr.rc = VERR_INVALID_MAGIC;
705	return 0;
706	}
707
708	#if 0
709	/*
710	* Call out to the OS specific code and let it do permission checks on the
711	* client process.
712	*/
713	if (!supdrvOSValidateClientProcess(pDevExt, pSession))
714	{
715	pReq->u.Out.u32Cookie = 0xffffffff;
716	pReq->u.Out.u32SessionCookie = 0xffffffff;
717	pReq->u.Out.u32SessionVersion = 0xffffffff;
718	pReq->u.Out.u32DriverVersion = SUPDRVIOC_VERSION;
719	pReq->u.Out.pSession = NULL;
720	pReq->u.Out.cFunctions = 0;
721	pReq->Hdr.rc = VERR_PERMISSION_DENIED;
722	return 0;
723	}
724	#endif
725
726	/*
727	* Match the version.
728	* The current logic is very simple, match the major interface version.
729	*/
730	if ( pReq->u.In.u32MinVersion > SUPDRVIOC_VERSION
731	\|\| (pReq->u.In.u32MinVersion & 0xffff0000) != (SUPDRVIOC_VERSION & 0xffff0000))
732	{
733	OSDBGPRINT(("SUP_IOCTL_COOKIE: Version mismatch. Requested: %#x Min: %#x Current: %#x\n",
734	pReq->u.In.u32ReqVersion, pReq->u.In.u32MinVersion, SUPDRVIOC_VERSION));
735	pReq->u.Out.u32Cookie = 0xffffffff;
736	pReq->u.Out.u32SessionCookie = 0xffffffff;
737	pReq->u.Out.u32SessionVersion = 0xffffffff;
738	pReq->u.Out.u32DriverVersion = SUPDRVIOC_VERSION;
739	pReq->u.Out.pSession = NULL;
740	pReq->u.Out.cFunctions = 0;
741	pReq->Hdr.rc = VERR_VERSION_MISMATCH;
742	return 0;
743	}
744
745	/*
746	* Fill in return data and be gone.
747	* N.B. The first one to change SUPDRVIOC_VERSION shall makes sure that
748	* u32SessionVersion <= u32ReqVersion!
749	*/
750	/** @todo Somehow validate the client and negotiate a secure cookie... */
751	pReq->u.Out.u32Cookie = pDevExt->u32Cookie;
752	pReq->u.Out.u32SessionCookie = pSession->u32Cookie;
753	pReq->u.Out.u32SessionVersion = SUPDRVIOC_VERSION;
754	pReq->u.Out.u32DriverVersion = SUPDRVIOC_VERSION;
755	pReq->u.Out.pSession = pSession;
756	pReq->u.Out.cFunctions = sizeof(g_aFunctions) / sizeof(g_aFunctions[0]);
757	pReq->Hdr.rc = VINF_SUCCESS;
758	return 0;
759	}
760
761	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_QUERY_FUNCS(0)):
762	{
763	/* validate */
764	PSUPQUERYFUNCS pReq = (PSUPQUERYFUNCS)pReqHdr;
765	REQ_CHECK_SIZES_EX(SUP_IOCTL_QUERY_FUNCS, SUP_IOCTL_QUERY_FUNCS_SIZE_IN, SUP_IOCTL_QUERY_FUNCS_SIZE_OUT(RT_ELEMENTS(g_aFunctions)));
766
767	/* execute */
768	pReq->u.Out.cFunctions = RT_ELEMENTS(g_aFunctions);
769	memcpy(&pReq->u.Out.aFunctions[0], g_aFunctions, sizeof(g_aFunctions));
770	pReq->Hdr.rc = VINF_SUCCESS;
771	return 0;
772	}
773
774	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_IDT_INSTALL):
775	{
776	/* validate */
777	PSUPIDTINSTALL pReq = (PSUPIDTINSTALL)pReqHdr;
778	REQ_CHECK_SIZES(SUP_IOCTL_IDT_INSTALL);
779
780	/* execute */
781	#ifdef VBOX_WITH_IDT_PATCHING
782	pReq->Hdr.rc = supdrvIOCtl_IdtInstall(pDevExt, pSession, pReq);
783	#else
784	pReq->u.Out.u8Idt = 3;
785	pReq->Hdr.rc = VERR_NOT_SUPPORTED;
786	#endif
787	return 0;
788	}
789
790	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_IDT_REMOVE):
791	{
792	/* validate */
793	PSUPIDTREMOVE pReq = (PSUPIDTREMOVE)pReqHdr;
794	REQ_CHECK_SIZES(SUP_IOCTL_IDT_REMOVE);
795
796	/* execute */
797	#ifdef VBOX_WITH_IDT_PATCHING
798	pReq->Hdr.rc = supdrvIOCtl_IdtRemoveAll(pDevExt, pSession);
799	#else
800	pReq->Hdr.rc = VERR_NOT_SUPPORTED;
801	#endif
802	return 0;
803	}
804
805	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_PAGE_LOCK):
806	{
807	/* validate */
808	PSUPPAGELOCK pReq = (PSUPPAGELOCK)pReqHdr;
809	REQ_CHECK_SIZE_IN(SUP_IOCTL_PAGE_LOCK, SUP_IOCTL_PAGE_LOCK_SIZE_IN);
810	REQ_CHECK_SIZE_OUT(SUP_IOCTL_PAGE_LOCK, SUP_IOCTL_PAGE_LOCK_SIZE_OUT(pReq->u.In.cPages));
811	REQ_CHECK_EXPR(SUP_IOCTL_PAGE_LOCK, pReq->u.In.cPages > 0);
812	REQ_CHECK_EXPR(SUP_IOCTL_PAGE_LOCK, pReq->u.In.pvR3 >= PAGE_SIZE);
813
814	/* execute */
815	pReq->Hdr.rc = SUPR0LockMem(pSession, pReq->u.In.pvR3, pReq->u.In.cPages, &pReq->u.Out.aPages[0]);
816	if (RT_FAILURE(pReq->Hdr.rc))
817	pReq->Hdr.cbOut = sizeof(pReq->Hdr);
818	return 0;
819	}
820
821	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_PAGE_UNLOCK):
822	{
823	/* validate */
824	PSUPPAGEUNLOCK pReq = (PSUPPAGEUNLOCK)pReqHdr;
825	REQ_CHECK_SIZES(SUP_IOCTL_PAGE_UNLOCK);
826
827	/* execute */
828	pReq->Hdr.rc = SUPR0UnlockMem(pSession, pReq->u.In.pvR3);
829	return 0;
830	}
831
832	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_CONT_ALLOC):
833	{
834	/* validate */
835	PSUPCONTALLOC pReq = (PSUPCONTALLOC)pReqHdr;
836	REQ_CHECK_SIZES(SUP_IOCTL_CONT_ALLOC);
837
838	/* execute */
839	pReq->Hdr.rc = SUPR0ContAlloc(pSession, pReq->u.In.cPages, &pReq->u.Out.pvR0, &pReq->u.Out.pvR3, &pReq->u.Out.HCPhys);
840	if (RT_FAILURE(pReq->Hdr.rc))
841	pReq->Hdr.cbOut = sizeof(pReq->Hdr);
842	return 0;
843	}
844
845	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_CONT_FREE):
846	{
847	/* validate */
848	PSUPCONTFREE pReq = (PSUPCONTFREE)pReqHdr;
849	REQ_CHECK_SIZES(SUP_IOCTL_CONT_FREE);
850
851	/* execute */
852	pReq->Hdr.rc = SUPR0ContFree(pSession, (RTHCUINTPTR)pReq->u.In.pvR3);
853	return 0;
854	}
855
856	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_LDR_OPEN):
857	{
858	/* validate */
859	PSUPLDROPEN pReq = (PSUPLDROPEN)pReqHdr;
860	REQ_CHECK_SIZES(SUP_IOCTL_LDR_OPEN);
861	REQ_CHECK_EXPR(SUP_IOCTL_LDR_OPEN, pReq->u.In.cbImage > 0);
862	REQ_CHECK_EXPR(SUP_IOCTL_LDR_OPEN, pReq->u.In.cbImage < _1M*16);
863	REQ_CHECK_EXPR(SUP_IOCTL_LDR_OPEN, pReq->u.In.szName[0]);
864	REQ_CHECK_EXPR(SUP_IOCTL_LDR_OPEN, memchr(pReq->u.In.szName, '\0', sizeof(pReq->u.In.szName)));
865	REQ_CHECK_EXPR(SUP_IOCTL_LDR_OPEN, !strpbrk(pReq->u.In.szName, ";:()[]{}/\\\|&*%#@!~`\"'"));
866
867	/* execute */
868	pReq->Hdr.rc = supdrvIOCtl_LdrOpen(pDevExt, pSession, pReq);
869	return 0;
870	}
871
872	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_LDR_LOAD):
873	{
874	/* validate */
875	PSUPLDRLOAD pReq = (PSUPLDRLOAD)pReqHdr;
876	REQ_CHECK_EXPR(Name, pReq->Hdr.cbIn >= sizeof(*pReq));
877	REQ_CHECK_SIZES_EX(SUP_IOCTL_LDR_LOAD, SUP_IOCTL_LDR_LOAD_SIZE_IN(pReq->u.In.cbImage), SUP_IOCTL_LDR_LOAD_SIZE_OUT);
878	REQ_CHECK_EXPR(SUP_IOCTL_LDR_LOAD, pReq->u.In.cSymbols <= 16384);
879	REQ_CHECK_EXPR_FMT( !pReq->u.In.cSymbols
880	\|\| ( pReq->u.In.offSymbols < pReq->u.In.cbImage
881	&& pReq->u.In.offSymbols + pReq->u.In.cSymbols * sizeof(SUPLDRSYM) <= pReq->u.In.cbImage),
882	("SUP_IOCTL_LDR_LOAD: offSymbols=%#lx cSymbols=%#lx cbImage=%#lx\n", (long)pReq->u.In.offSymbols,
883	(long)pReq->u.In.cSymbols, (long)pReq->u.In.cbImage));
884	REQ_CHECK_EXPR_FMT( !pReq->u.In.cbStrTab
885	\|\| ( pReq->u.In.offStrTab < pReq->u.In.cbImage
886	&& pReq->u.In.offStrTab + pReq->u.In.cbStrTab <= pReq->u.In.cbImage
887	&& pReq->u.In.cbStrTab <= pReq->u.In.cbImage),
888	("SUP_IOCTL_LDR_LOAD: offStrTab=%#lx cbStrTab=%#lx cbImage=%#lx\n", (long)pReq->u.In.offStrTab,
889	(long)pReq->u.In.cbStrTab, (long)pReq->u.In.cbImage));
890
891	if (pReq->u.In.cSymbols)
892	{
893	uint32_t i;
894	PSUPLDRSYM paSyms = (PSUPLDRSYM)&pReq->u.In.achImage[pReq->u.In.offSymbols];
895	for (i = 0; i < pReq->u.In.cSymbols; i++)
896	{
897	REQ_CHECK_EXPR_FMT(paSyms[i].offSymbol < pReq->u.In.cbImage,
898	("SUP_IOCTL_LDR_LOAD: sym #%ld: symb off %#lx (max=%#lx)\n", (long)i, (long)paSyms[i].offSymbol, (long)pReq->u.In.cbImage));
899	REQ_CHECK_EXPR_FMT(paSyms[i].offName < pReq->u.In.cbStrTab,
900	("SUP_IOCTL_LDR_LOAD: sym #%ld: name off %#lx (max=%#lx)\n", (long)i, (long)paSyms[i].offName, (long)pReq->u.In.cbImage));
901	REQ_CHECK_EXPR_FMT(memchr(&pReq->u.In.achImage[pReq->u.In.offStrTab + paSyms[i].offName], '\0', pReq->u.In.cbStrTab - paSyms[i].offName),
902	("SUP_IOCTL_LDR_LOAD: sym #%ld: unterminated name! (%#lx / %#lx)\n", (long)i, (long)paSyms[i].offName, (long)pReq->u.In.cbImage));
903	}
904	}
905
906	/* execute */
907	pReq->Hdr.rc = supdrvIOCtl_LdrLoad(pDevExt, pSession, pReq);
908	return 0;
909	}
910
911	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_LDR_FREE):
912	{
913	/* validate */
914	PSUPLDRFREE pReq = (PSUPLDRFREE)pReqHdr;
915	REQ_CHECK_SIZES(SUP_IOCTL_LDR_FREE);
916
917	/* execute */
918	pReq->Hdr.rc = supdrvIOCtl_LdrFree(pDevExt, pSession, pReq);
919	return 0;
920	}
921
922	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_LDR_GET_SYMBOL):
923	{
924	/* validate */
925	PSUPLDRGETSYMBOL pReq = (PSUPLDRGETSYMBOL)pReqHdr;
926	REQ_CHECK_SIZES(SUP_IOCTL_LDR_GET_SYMBOL);
927	REQ_CHECK_EXPR(SUP_IOCTL_LDR_GET_SYMBOL, memchr(pReq->u.In.szSymbol, '\0', sizeof(pReq->u.In.szSymbol)));
928
929	/* execute */
930	pReq->Hdr.rc = supdrvIOCtl_LdrGetSymbol(pDevExt, pSession, pReq);
931	return 0;
932	}
933
934	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_CALL_VMMR0(0)):
935	{
936	/* validate */
937	PSUPCALLVMMR0 pReq = (PSUPCALLVMMR0)pReqHdr;
938	if (pReq->Hdr.cbIn == SUP_IOCTL_CALL_VMMR0_SIZE(0))
939	{
940	REQ_CHECK_SIZES_EX(SUP_IOCTL_CALL_VMMR0, SUP_IOCTL_CALL_VMMR0_SIZE_IN(0), SUP_IOCTL_CALL_VMMR0_SIZE_OUT(0));
941
942	/* execute */
943	if (RT_LIKELY(pDevExt->pfnVMMR0EntryEx))
944	pReq->Hdr.rc = pDevExt->pfnVMMR0EntryEx(pReq->u.In.pVMR0, pReq->u.In.uOperation, NULL, pReq->u.In.u64Arg);
945	else
946	pReq->Hdr.rc = VERR_WRONG_ORDER;
947	}
948	else
949	{
950	PSUPVMMR0REQHDR pVMMReq = (PSUPVMMR0REQHDR)&pReq->abReqPkt[0];
951	REQ_CHECK_EXPR_FMT(pReq->Hdr.cbIn >= SUP_IOCTL_CALL_VMMR0_SIZE(sizeof(SUPVMMR0REQHDR)),
952	("SUP_IOCTL_CALL_VMMR0: cbIn=%#x < %#x\n", pReq->Hdr.cbIn, SUP_IOCTL_CALL_VMMR0_SIZE(sizeof(SUPVMMR0REQHDR))));
953	REQ_CHECK_EXPR(SUP_IOCTL_CALL_VMMR0, pVMMReq->u32Magic == SUPVMMR0REQHDR_MAGIC);
954	REQ_CHECK_SIZES_EX(SUP_IOCTL_CALL_VMMR0, SUP_IOCTL_CALL_VMMR0_SIZE_IN(pVMMReq->cbReq), SUP_IOCTL_CALL_VMMR0_SIZE_OUT(pVMMReq->cbReq));
955
956	/* execute */
957	if (RT_LIKELY(pDevExt->pfnVMMR0EntryEx))
958	pReq->Hdr.rc = pDevExt->pfnVMMR0EntryEx(pReq->u.In.pVMR0, pReq->u.In.uOperation, pVMMReq, pReq->u.In.u64Arg);
959	else
960	pReq->Hdr.rc = VERR_WRONG_ORDER;
961	}
962	return 0;
963	}
964
965	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_GET_PAGING_MODE):
966	{
967	/* validate */
968	PSUPGETPAGINGMODE pReq = (PSUPGETPAGINGMODE)pReqHdr;
969	REQ_CHECK_SIZES(SUP_IOCTL_GET_PAGING_MODE);
970
971	/* execute */
972	pReq->Hdr.rc = VINF_SUCCESS;
973	pReq->u.Out.enmMode = supdrvIOCtl_GetPagingMode();
974	return 0;
975	}
976
977	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_LOW_ALLOC):
978	{
979	/* validate */
980	PSUPLOWALLOC pReq = (PSUPLOWALLOC)pReqHdr;
981	REQ_CHECK_EXPR(SUP_IOCTL_LOW_ALLOC, pReq->Hdr.cbIn <= SUP_IOCTL_LOW_ALLOC_SIZE_IN);
982	REQ_CHECK_SIZES_EX(SUP_IOCTL_LOW_ALLOC, SUP_IOCTL_LOW_ALLOC_SIZE_IN, SUP_IOCTL_LOW_ALLOC_SIZE_OUT(pReq->u.In.cPages));
983
984	/* execute */
985	pReq->Hdr.rc = SUPR0LowAlloc(pSession, pReq->u.In.cPages, &pReq->u.Out.pvR0, &pReq->u.Out.pvR3, &pReq->u.Out.aPages[0]);
986	if (RT_FAILURE(pReq->Hdr.rc))
987	pReq->Hdr.cbOut = sizeof(pReq->Hdr);
988	return 0;
989	}
990
991	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_LOW_FREE):
992	{
993	/* validate */
994	PSUPLOWFREE pReq = (PSUPLOWFREE)pReqHdr;
995	REQ_CHECK_SIZES(SUP_IOCTL_LOW_FREE);
996
997	/* execute */
998	pReq->Hdr.rc = SUPR0LowFree(pSession, (RTHCUINTPTR)pReq->u.In.pvR3);
999	return 0;
1000	}
1001
1002	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_GIP_MAP):
1003	{
1004	/* validate */
1005	PSUPGIPMAP pReq = (PSUPGIPMAP)pReqHdr;
1006	REQ_CHECK_SIZES(SUP_IOCTL_GIP_MAP);
1007
1008	/* execute */
1009	pReq->Hdr.rc = SUPR0GipMap(pSession, &pReq->u.Out.pGipR3, &pReq->u.Out.HCPhysGip);
1010	if (RT_SUCCESS(pReq->Hdr.rc))
1011	pReq->u.Out.pGipR0 = pDevExt->pGip;
1012	return 0;
1013	}
1014
1015	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_GIP_UNMAP):
1016	{
1017	/* validate */
1018	PSUPGIPUNMAP pReq = (PSUPGIPUNMAP)pReqHdr;
1019	REQ_CHECK_SIZES(SUP_IOCTL_GIP_UNMAP);
1020
1021	/* execute */
1022	pReq->Hdr.rc = SUPR0GipUnmap(pSession);
1023	return 0;
1024	}
1025
1026	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_SET_VM_FOR_FAST):
1027	{
1028	/* validate */
1029	PSUPSETVMFORFAST pReq = (PSUPSETVMFORFAST)pReqHdr;
1030	REQ_CHECK_SIZES(SUP_IOCTL_SET_VM_FOR_FAST);
1031	REQ_CHECK_EXPR_FMT( !pReq->u.In.pVMR0
1032	\|\| ( VALID_PTR(pReq->u.In.pVMR0)
1033	&& !((uintptr_t)pReq->u.In.pVMR0 & (PAGE_SIZE - 1))),
1034	("SUP_IOCTL_SET_VM_FOR_FAST: pVMR0=%p!\n", pReq->u.In.pVMR0));
1035	/* execute */
1036	pSession->pVM = pReq->u.In.pVMR0;
1037	pReq->Hdr.rc = VINF_SUCCESS;
1038	return 0;
1039	}
1040
1041	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_PAGE_ALLOC):
1042	{
1043	/* validate */
1044	PSUPPAGEALLOC pReq = (PSUPPAGEALLOC)pReqHdr;
1045	REQ_CHECK_EXPR(SUP_IOCTL_PAGE_ALLOC, pReq->Hdr.cbIn <= SUP_IOCTL_PAGE_ALLOC_SIZE_IN);
1046	REQ_CHECK_SIZES_EX(SUP_IOCTL_PAGE_ALLOC, SUP_IOCTL_PAGE_ALLOC_SIZE_IN, SUP_IOCTL_PAGE_ALLOC_SIZE_OUT(pReq->u.In.cPages));
1047
1048	/* execute */
1049	pReq->Hdr.rc = SUPR0PageAlloc(pSession, pReq->u.In.cPages, &pReq->u.Out.pvR3, &pReq->u.Out.aPages[0]);
1050	if (RT_FAILURE(pReq->Hdr.rc))
1051	pReq->Hdr.cbOut = sizeof(pReq->Hdr);
1052	return 0;
1053	}
1054
1055	case SUP_CTL_CODE_NO_SIZE(SUP_IOCTL_PAGE_FREE):
1056	{
1057	/* validate */
1058	PSUPPAGEFREE pReq = (PSUPPAGEFREE)pReqHdr;
1059	REQ_CHECK_SIZES(SUP_IOCTL_PAGE_FREE);
1060
1061	/* execute */
1062	pReq->Hdr.rc = SUPR0PageFree(pSession, pReq->u.In.pvR3);
1063	return 0;
1064	}
1065
1066	default:
1067	dprintf(("Unknown IOCTL %#lx\n", (long)uIOCtl));
1068	break;
1069	}
1070	return SUPDRV_ERR_GENERAL_FAILURE;
1071	}
1072
1073
1074	/**
1075	* Register a object for reference counting.
1076	* The object is registered with one reference in the specified session.
1077	*
1078	* @returns Unique identifier on success (pointer).
1079	* All future reference must use this identifier.
1080	* @returns NULL on failure.
1081	* @param pfnDestructor The destructore function which will be called when the reference count reaches 0.
1082	* @param pvUser1 The first user argument.
1083	* @param pvUser2 The second user argument.
1084	*/
1085	SUPR0DECL(void ) SUPR0ObjRegister(PSUPDRVSESSION pSession, SUPDRVOBJTYPE enmType, PFNSUPDRVDESTRUCTOR pfnDestructor, void pvUser1, void *pvUser2)
1086	{
1087	RTSPINLOCKTMP SpinlockTmp = RTSPINLOCKTMP_INITIALIZER;
1088	PSUPDRVDEVEXT pDevExt = pSession->pDevExt;
1089	PSUPDRVOBJ pObj;
1090	PSUPDRVUSAGE pUsage;
1091
1092	/*
1093	* Validate the input.
1094	*/
1095	AssertReturn(SUP_IS_SESSION_VALID(pSession), NULL);
1096	AssertReturn(enmType > SUPDRVOBJTYPE_INVALID && enmType < SUPDRVOBJTYPE_END, NULL);
1097	AssertPtrReturn(pfnDestructor, NULL);
1098
1099	/*
1100	* Allocate and initialize the object.
1101	*/
1102	pObj = (PSUPDRVOBJ)RTMemAlloc(sizeof(*pObj));
1103	if (!pObj)
1104	return NULL;
1105	pObj->u32Magic = SUPDRVOBJ_MAGIC;
1106	pObj->enmType = enmType;
1107	pObj->pNext = NULL;
1108	pObj->cUsage = 1;
1109	pObj->pfnDestructor = pfnDestructor;
1110	pObj->pvUser1 = pvUser1;
1111	pObj->pvUser2 = pvUser2;
1112	pObj->CreatorUid = pSession->Uid;
1113	pObj->CreatorGid = pSession->Gid;
1114	pObj->CreatorProcess= pSession->Process;
1115	supdrvOSObjInitCreator(pObj, pSession);
1116
1117	/*
1118	* Allocate the usage record.
1119	* (We keep freed usage records around to simplity SUPR0ObjAddRef().)
1120	*/
1121	RTSpinlockAcquire(pDevExt->Spinlock, &SpinlockTmp);
1122
1123	pUsage = pDevExt->pUsageFree;
1124	if (pUsage)
1125	pDevExt->pUsageFree = pUsage->pNext;
1126	else
1127	{
1128	RTSpinlockRelease(pDevExt->Spinlock, &SpinlockTmp);
1129	pUsage = (PSUPDRVUSAGE)RTMemAlloc(sizeof(*pUsage));
1130	if (!pUsage)
1131	{
1132	RTMemFree(pObj);
1133	return NULL;
1134	}
1135	RTSpinlockAcquire(pDevExt->Spinlock, &SpinlockTmp);
1136	}
1137
1138	/*
1139	* Insert the object and create the session usage record.
1140	*/
1141	/* The object. */
1142	pObj->pNext = pDevExt->pObjs;
1143	pDevExt->pObjs = pObj;
1144
1145	/* The session record. */
1146	pUsage->cUsage = 1;
1147	pUsage->pObj = pObj;
1148	pUsage->pNext = pSession->pUsage;
1149	dprintf(("SUPR0ObjRegister: pUsage=%p:{.pObj=%p, .pNext=%p}\n", pUsage, pUsage->pObj, pUsage->pNext));
1150	pSession->pUsage = pUsage;
1151
1152	RTSpinlockRelease(pDevExt->Spinlock, &SpinlockTmp);
1153
1154	dprintf(("SUPR0ObjRegister: returns %p (pvUser1=%p, pvUser=%p)\n", pObj, pvUser1, pvUser2));
1155	return pObj;
1156	}
1157
1158
1159	/**
1160	* Increment the reference counter for the object associating the reference
1161	* with the specified session.
1162	*
1163	* @returns IPRT status code.
1164	* @param pvObj The identifier returned by SUPR0ObjRegister().
1165	* @param pSession The session which is referencing the object.
1166	*/
1167	SUPR0DECL(int) SUPR0ObjAddRef(void *pvObj, PSUPDRVSESSION pSession)
1168	{
1169	RTSPINLOCKTMP SpinlockTmp = RTSPINLOCKTMP_INITIALIZER;
1170	PSUPDRVDEVEXT pDevExt = pSession->pDevExt;
1171	PSUPDRVOBJ pObj = (PSUPDRVOBJ)pvObj;
1172	PSUPDRVUSAGE pUsagePre;
1173	PSUPDRVUSAGE pUsage;
1174
1175	/*
1176	* Validate the input.
1177	*/
1178	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
1179	AssertMsgReturn(VALID_PTR(pObj) && pObj->u32Magic == SUPDRVOBJ_MAGIC,
1180	("Invalid pvObj=%p magic=%#x (exepcted %#x)\n", pvObj, pObj ? pObj->u32Magic : 0, SUPDRVOBJ_MAGIC),
1181	VERR_INVALID_PARAMETER);
1182
1183	/*
1184	* Preallocate the usage record.
1185	*/
1186	RTSpinlockAcquire(pDevExt->Spinlock, &SpinlockTmp);
1187
1188	pUsagePre = pDevExt->pUsageFree;
1189	if (pUsagePre)
1190	pDevExt->pUsageFree = pUsagePre->pNext;
1191	else
1192	{
1193	RTSpinlockRelease(pDevExt->Spinlock, &SpinlockTmp);
1194	pUsagePre = (PSUPDRVUSAGE)RTMemAlloc(sizeof(*pUsagePre));
1195	if (!pUsagePre)
1196	return VERR_NO_MEMORY;
1197	RTSpinlockAcquire(pDevExt->Spinlock, &SpinlockTmp);
1198	}
1199
1200	/*
1201	* Reference the object.
1202	*/
1203	pObj->cUsage++;
1204
1205	/*
1206	* Look for the session record.
1207	*/
1208	for (pUsage = pSession->pUsage; pUsage; pUsage = pUsage->pNext)
1209	{
1210	dprintf(("SUPR0AddRef: pUsage=%p:{.pObj=%p, .pNext=%p}\n", pUsage, pUsage->pObj, pUsage->pNext));
1211	if (pUsage->pObj == pObj)
1212	break;
1213	}
1214	if (pUsage)
1215	pUsage->cUsage++;
1216	else
1217	{
1218	/* create a new session record. */
1219	pUsagePre->cUsage = 1;
1220	pUsagePre->pObj = pObj;
1221	pUsagePre->pNext = pSession->pUsage;
1222	pSession->pUsage = pUsagePre;
1223	dprintf(("SUPR0AddRef: pUsagePre=%p:{.pObj=%p, .pNext=%p}\n", pUsagePre, pUsagePre->pObj, pUsagePre->pNext));
1224
1225	pUsagePre = NULL;
1226	}
1227
1228	/*
1229	* Put any unused usage record into the free list..
1230	*/
1231	if (pUsagePre)
1232	{
1233	pUsagePre->pNext = pDevExt->pUsageFree;
1234	pDevExt->pUsageFree = pUsagePre;
1235	}
1236
1237	RTSpinlockRelease(pDevExt->Spinlock, &SpinlockTmp);
1238
1239	return VINF_SUCCESS;
1240	}
1241
1242
1243	/**
1244	* Decrement / destroy a reference counter record for an object.
1245	*
1246	* The object is uniquely identified by pfnDestructor+pvUser1+pvUser2.
1247	*
1248	* @returns IPRT status code.
1249	* @param pvObj The identifier returned by SUPR0ObjRegister().
1250	* @param pSession The session which is referencing the object.
1251	*/
1252	SUPR0DECL(int) SUPR0ObjRelease(void *pvObj, PSUPDRVSESSION pSession)
1253	{
1254	RTSPINLOCKTMP SpinlockTmp = RTSPINLOCKTMP_INITIALIZER;
1255	PSUPDRVDEVEXT pDevExt = pSession->pDevExt;
1256	PSUPDRVOBJ pObj = (PSUPDRVOBJ)pvObj;
1257	bool fDestroy = false;
1258	PSUPDRVUSAGE pUsage;
1259	PSUPDRVUSAGE pUsagePrev;
1260
1261	/*
1262	* Validate the input.
1263	*/
1264	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
1265	AssertMsgReturn(VALID_PTR(pObj) && pObj->u32Magic == SUPDRVOBJ_MAGIC,
1266	("Invalid pvObj=%p magic=%#x (exepcted %#x)\n", pvObj, pObj ? pObj->u32Magic : 0, SUPDRVOBJ_MAGIC),
1267	VERR_INVALID_PARAMETER);
1268
1269	/*
1270	* Acquire the spinlock and look for the usage record.
1271	*/
1272	RTSpinlockAcquire(pDevExt->Spinlock, &SpinlockTmp);
1273
1274	for (pUsagePrev = NULL, pUsage = pSession->pUsage;
1275	pUsage;
1276	pUsagePrev = pUsage, pUsage = pUsage->pNext)
1277	{
1278	dprintf(("SUPR0ObjRelease: pUsage=%p:{.pObj=%p, .pNext=%p}\n", pUsage, pUsage->pObj, pUsage->pNext));
1279	if (pUsage->pObj == pObj)
1280	{
1281	AssertMsg(pUsage->cUsage >= 1 && pObj->cUsage >= pUsage->cUsage, ("glob %d; sess %d\n", pObj->cUsage, pUsage->cUsage));
1282	if (pUsage->cUsage > 1)
1283	{
1284	pObj->cUsage--;
1285	pUsage->cUsage--;
1286	}
1287	else
1288	{
1289	/*
1290	* Free the session record.
1291	*/
1292	if (pUsagePrev)
1293	pUsagePrev->pNext = pUsage->pNext;
1294	else
1295	pSession->pUsage = pUsage->pNext;
1296	pUsage->pNext = pDevExt->pUsageFree;
1297	pDevExt->pUsageFree = pUsage;
1298
1299	/* What about the object? */
1300	if (pObj->cUsage > 1)
1301	pObj->cUsage--;
1302	else
1303	{
1304	/*
1305	* Object is to be destroyed, unlink it.
1306	*/
1307	fDestroy = true;
1308	if (pDevExt->pObjs == pObj)
1309	pDevExt->pObjs = pObj->pNext;
1310	else
1311	{
1312	PSUPDRVOBJ pObjPrev;
1313	for (pObjPrev = pDevExt->pObjs; pObjPrev; pObjPrev = pObjPrev->pNext)
1314	if (pObjPrev->pNext == pObj)
1315	{
1316	pObjPrev->pNext = pObj->pNext;
1317	break;
1318	}
1319	Assert(pObjPrev);
1320	}
1321	}
1322	}
1323	break;
1324	}
1325	}
1326
1327	RTSpinlockRelease(pDevExt->Spinlock, &SpinlockTmp);
1328
1329	/*
1330	* Call the destructor and free the object if required.
1331	*/
1332	if (fDestroy)
1333	{
1334	pObj->u32Magic++;
1335	if (pObj->pfnDestructor)
1336	pObj->pfnDestructor(pObj, pObj->pvUser1, pObj->pvUser2);
1337	RTMemFree(pObj);
1338	}
1339
1340	AssertMsg(pUsage, ("pvObj=%p\n", pvObj));
1341	return pUsage ? VINF_SUCCESS : VERR_INVALID_PARAMETER;
1342	}
1343
1344	/**
1345	* Verifies that the current process can access the specified object.
1346	*
1347	* @returns The following IPRT status code:
1348	* @retval VINF_SUCCESS if access was granted.
1349	* @retval VERR_PERMISSION_DENIED if denied access.
1350	* @retval VERR_INVALID_PARAMETER if invalid parameter.
1351	*
1352	* @param pvObj The identifier returned by SUPR0ObjRegister().
1353	* @param pSession The session which wishes to access the object.
1354	* @param pszObjName Object string name. This is optional and depends on the object type.
1355	*
1356	* @remark The caller is responsible for making sure the object isn't removed while
1357	* we're inside this function. If uncertain about this, just call AddRef before calling us.
1358	*/
1359	SUPR0DECL(int) SUPR0ObjVerifyAccess(void pvObj, PSUPDRVSESSION pSession, const char pszObjName)
1360	{
1361	PSUPDRVOBJ pObj = (PSUPDRVOBJ)pvObj;
1362	int rc;
1363
1364	/*
1365	* Validate the input.
1366	*/
1367	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
1368	AssertMsgReturn(VALID_PTR(pObj) && pObj->u32Magic == SUPDRVOBJ_MAGIC,
1369	("Invalid pvObj=%p magic=%#x (exepcted %#x)\n", pvObj, pObj ? pObj->u32Magic : 0, SUPDRVOBJ_MAGIC),
1370	VERR_INVALID_PARAMETER);
1371
1372	/*
1373	* Check access. (returns true if a decision has been made.)
1374	*/
1375	rc = VERR_INTERNAL_ERROR;
1376	if (supdrvOSObjCanAccess(pObj, pSession, pszObjName, &rc))
1377	return rc;
1378
1379	/*
1380	* Default policy is to allow the user to access his own
1381	* stuff but nothing else.
1382	*/
1383	if (pObj->CreatorUid == pSession->Uid)
1384	return VINF_SUCCESS;
1385	return VERR_PERMISSION_DENIED;
1386	}
1387
1388
1389	/**
1390	* Lock pages.
1391	*
1392	* @returns IPRT status code.
1393	* @param pSession Session to which the locked memory should be associated.
1394	* @param pvR3 Start of the memory range to lock.
1395	* This must be page aligned.
1396	* @param cb Size of the memory range to lock.
1397	* This must be page aligned.
1398	*/
1399	SUPR0DECL(int) SUPR0LockMem(PSUPDRVSESSION pSession, RTR3PTR pvR3, uint32_t cPages, PRTHCPHYS paPages)
1400	{
1401	int rc;
1402	SUPDRVMEMREF Mem = {0};
1403	const size_t cb = (size_t)cPages << PAGE_SHIFT;
1404	dprintf(("SUPR0LockMem: pSession=%p pvR3=%p cPages=%d paPages=%p\n", pSession, (void *)pvR3, cPages, paPages));
1405
1406	/*
1407	* Verify input.
1408	*/
1409	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
1410	AssertPtrReturn(paPages, VERR_INVALID_PARAMETER);
1411	if ( RT_ALIGN_R3PT(pvR3, PAGE_SIZE, RTR3PTR) != pvR3
1412	\|\| !pvR3)
1413	{
1414	dprintf(("pvR3 (%p) must be page aligned and not NULL!\n", (void *)pvR3));
1415	return VERR_INVALID_PARAMETER;
1416	}
1417
1418	#ifdef RT_OS_WINDOWS /* A temporary hack for windows, will be removed once all ring-3 code has been cleaned up. */
1419	/* First check if we allocated it using SUPPageAlloc; if so then we don't need to lock it again */
1420	rc = supdrvPageGetPhys(pSession, pvR3, cPages, paPages);
1421	if (RT_SUCCESS(rc))
1422	return rc;
1423	#endif
1424
1425	/*
1426	* Let IPRT do the job.
1427	*/
1428	Mem.eType = MEMREF_TYPE_LOCKED;
1429	rc = RTR0MemObjLockUser(&Mem.MemObj, pvR3, cb, RTR0ProcHandleSelf());
1430	if (RT_SUCCESS(rc))
1431	{
1432	uint32_t iPage = cPages;
1433	AssertMsg(RTR0MemObjAddressR3(Mem.MemObj) == pvR3, ("%p == %p\n", RTR0MemObjAddressR3(Mem.MemObj), pvR3));
1434	AssertMsg(RTR0MemObjSize(Mem.MemObj) == cb, ("%x == %x\n", RTR0MemObjSize(Mem.MemObj), cb));
1435
1436	while (iPage-- > 0)
1437	{
1438	paPages[iPage] = RTR0MemObjGetPagePhysAddr(Mem.MemObj, iPage);
1439	if (RT_UNLIKELY(paPages[iPage] == NIL_RTCCPHYS))
1440	{
1441	AssertMsgFailed(("iPage=%d\n", iPage));
1442	rc = VERR_INTERNAL_ERROR;
1443	break;
1444	}
1445	}
1446	if (RT_SUCCESS(rc))
1447	rc = supdrvMemAdd(&Mem, pSession);
1448	if (RT_FAILURE(rc))
1449	{
1450	int rc2 = RTR0MemObjFree(Mem.MemObj, false);
1451	AssertRC(rc2);
1452	}
1453	}
1454
1455	return rc;
1456	}
1457
1458
1459	/**
1460	* Unlocks the memory pointed to by pv.
1461	*
1462	* @returns IPRT status code.
1463	* @param pSession Session to which the memory was locked.
1464	* @param pvR3 Memory to unlock.
1465	*/
1466	SUPR0DECL(int) SUPR0UnlockMem(PSUPDRVSESSION pSession, RTR3PTR pvR3)
1467	{
1468	dprintf(("SUPR0UnlockMem: pSession=%p pvR3=%p\n", pSession, (void *)pvR3));
1469	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
1470	#ifdef RT_OS_WINDOWS
1471	/*
1472	* Temporary hack for windows - SUPR0PageFree will unlock SUPR0PageAlloc
1473	* allocations; ignore this call.
1474	*/
1475	if (supdrvPageWasLockedByPageAlloc(pSession, pvR3))
1476	{
1477	dprintf(("Page will be unlocked in SUPR0PageFree -> ignore\n"));
1478	return VINF_SUCCESS;
1479	}
1480	#endif
1481	return supdrvMemRelease(pSession, (RTHCUINTPTR)pvR3, MEMREF_TYPE_LOCKED);
1482	}
1483
1484
1485	/**
1486	* Allocates a chunk of page aligned memory with contiguous and fixed physical
1487	* backing.
1488	*
1489	* @returns IPRT status code.
1490	* @param pSession Session data.
1491	* @param cb Number of bytes to allocate.
1492	* @param ppvR0 Where to put the address of Ring-0 mapping the allocated memory.
1493	* @param ppvR3 Where to put the address of Ring-3 mapping the allocated memory.
1494	* @param pHCPhys Where to put the physical address of allocated memory.
1495	*/
1496	SUPR0DECL(int) SUPR0ContAlloc(PSUPDRVSESSION pSession, uint32_t cPages, PRTR0PTR ppvR0, PRTR3PTR ppvR3, PRTHCPHYS pHCPhys)
1497	{
1498	int rc;
1499	SUPDRVMEMREF Mem = {0};
1500	dprintf(("SUPR0ContAlloc: pSession=%p cPages=%d ppvR0=%p ppvR3=%p pHCPhys=%p\n", pSession, cPages, ppvR0, ppvR3, pHCPhys));
1501
1502	/*
1503	* Validate input.
1504	*/
1505	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
1506	if (!ppvR3 \|\| !ppvR0 \|\| !pHCPhys)
1507	{
1508	dprintf(("Null pointer. All of these should be set: pSession=%p ppvR0=%p ppvR3=%p pHCPhys=%p\n",
1509	pSession, ppvR0, ppvR3, pHCPhys));
1510	return VERR_INVALID_PARAMETER;
1511
1512	}
1513	if (cPages < 1 \|\| cPages >= 256)
1514	{
1515	dprintf(("Illegal request cPages=%d, must be greater than 0 and smaller than 256\n", cPages));
1516	return VERR_INVALID_PARAMETER;
1517	}
1518
1519	/*
1520	* Let IPRT do the job.
1521	*/
1522	rc = RTR0MemObjAllocCont(&Mem.MemObj, cPages << PAGE_SHIFT, true /* executable R0 mapping */);
1523	if (RT_SUCCESS(rc))
1524	{
1525	int rc2;
1526	rc = RTR0MemObjMapUser(&Mem.MapObjR3, Mem.MemObj, (RTR3PTR)-1, 0,
1527	RTMEM_PROT_EXEC \| RTMEM_PROT_WRITE \| RTMEM_PROT_READ, RTR0ProcHandleSelf());
1528	if (RT_SUCCESS(rc))
1529	{
1530	Mem.eType = MEMREF_TYPE_CONT;
1531	rc = supdrvMemAdd(&Mem, pSession);
1532	if (!rc)
1533	{
1534	*ppvR0 = RTR0MemObjAddress(Mem.MemObj);
1535	*ppvR3 = RTR0MemObjAddressR3(Mem.MapObjR3);
1536	*pHCPhys = RTR0MemObjGetPagePhysAddr(Mem.MemObj, 0);
1537	return 0;
1538	}
1539
1540	rc2 = RTR0MemObjFree(Mem.MapObjR3, false);
1541	AssertRC(rc2);
1542	}
1543	rc2 = RTR0MemObjFree(Mem.MemObj, false);
1544	AssertRC(rc2);
1545	}
1546
1547	return rc;
1548	}
1549
1550
1551	/**
1552	* Frees memory allocated using SUPR0ContAlloc().
1553	*
1554	* @returns IPRT status code.
1555	* @param pSession The session to which the memory was allocated.
1556	* @param uPtr Pointer to the memory (ring-3 or ring-0).
1557	*/
1558	SUPR0DECL(int) SUPR0ContFree(PSUPDRVSESSION pSession, RTHCUINTPTR uPtr)
1559	{
1560	dprintf(("SUPR0ContFree: pSession=%p uPtr=%p\n", pSession, (void *)uPtr));
1561	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
1562	return supdrvMemRelease(pSession, uPtr, MEMREF_TYPE_CONT);
1563	}
1564
1565
1566	/**
1567	* Allocates a chunk of page aligned memory with fixed physical backing below 4GB.
1568	*
1569	* @returns IPRT status code.
1570	* @param pSession Session data.
1571	* @param cPages Number of pages to allocate.
1572	* @param ppvR0 Where to put the address of Ring-0 mapping of the allocated memory.
1573	* @param ppvR3 Where to put the address of Ring-3 mapping of the allocated memory.
1574	* @param paPages Where to put the physical addresses of allocated memory.
1575	*/
1576	SUPR0DECL(int) SUPR0LowAlloc(PSUPDRVSESSION pSession, uint32_t cPages, PRTR0PTR ppvR0, PRTR3PTR ppvR3, PRTHCPHYS paPages)
1577	{
1578	unsigned iPage;
1579	int rc;
1580	SUPDRVMEMREF Mem = {0};
1581	dprintf(("SUPR0LowAlloc: pSession=%p cPages=%d ppvR3=%p ppvR0=%p paPages=%p\n", pSession, cPages, ppvR3, ppvR0, paPages));
1582
1583	/*
1584	* Validate input.
1585	*/
1586	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
1587	if (!ppvR3 \|\| !ppvR0 \|\| !paPages)
1588	{
1589	dprintf(("Null pointer. All of these should be set: pSession=%p ppvR3=%p ppvR0=%p paPages=%p\n",
1590	pSession, ppvR3, ppvR0, paPages));
1591	return VERR_INVALID_PARAMETER;
1592
1593	}
1594	if (cPages < 1 \|\| cPages > 256)
1595	{
1596	dprintf(("Illegal request cPages=%d, must be greater than 0 and smaller than 256.\n", cPages));
1597	return VERR_INVALID_PARAMETER;
1598	}
1599
1600	/*
1601	* Let IPRT do the work.
1602	*/
1603	rc = RTR0MemObjAllocLow(&Mem.MemObj, cPages << PAGE_SHIFT, true /* executable ring-0 mapping */);
1604	if (RT_SUCCESS(rc))
1605	{
1606	int rc2;
1607	rc = RTR0MemObjMapUser(&Mem.MapObjR3, Mem.MemObj, (RTR3PTR)-1, 0,
1608	RTMEM_PROT_EXEC \| RTMEM_PROT_WRITE \| RTMEM_PROT_READ, RTR0ProcHandleSelf());
1609	if (RT_SUCCESS(rc))
1610	{
1611	Mem.eType = MEMREF_TYPE_LOW;
1612	rc = supdrvMemAdd(&Mem, pSession);
1613	if (!rc)
1614	{
1615	for (iPage = 0; iPage < cPages; iPage++)
1616	{
1617	paPages[iPage] = RTR0MemObjGetPagePhysAddr(Mem.MemObj, iPage);
1618	AssertMsg(!(paPages[iPage] & (PAGE_SIZE - 1)), ("iPage=%d Phys=%VHp\n", paPages[iPage]));
1619	}
1620	*ppvR0 = RTR0MemObjAddress(Mem.MemObj);
1621	*ppvR3 = RTR0MemObjAddressR3(Mem.MapObjR3);
1622	return 0;
1623	}
1624
1625	rc2 = RTR0MemObjFree(Mem.MapObjR3, false);
1626	AssertRC(rc2);
1627	}
1628
1629	rc2 = RTR0MemObjFree(Mem.MemObj, false);
1630	AssertRC(rc2);
1631	}
1632
1633	return rc;
1634	}
1635
1636
1637	/**
1638	* Frees memory allocated using SUPR0LowAlloc().
1639	*
1640	* @returns IPRT status code.
1641	* @param pSession The session to which the memory was allocated.
1642	* @param uPtr Pointer to the memory (ring-3 or ring-0).
1643	*/
1644	SUPR0DECL(int) SUPR0LowFree(PSUPDRVSESSION pSession, RTHCUINTPTR uPtr)
1645	{
1646	dprintf(("SUPR0LowFree: pSession=%p uPtr=%p\n", pSession, (void *)uPtr));
1647	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
1648	return supdrvMemRelease(pSession, uPtr, MEMREF_TYPE_LOW);
1649	}
1650
1651
1652
1653	/**
1654	* Allocates a chunk of memory with both R0 and R3 mappings.
1655	* The memory is fixed and it's possible to query the physical addresses using SUPR0MemGetPhys().
1656	*
1657	* @returns IPRT status code.
1658	* @param pSession The session to associated the allocation with.
1659	* @param cb Number of bytes to allocate.
1660	* @param ppvR0 Where to store the address of the Ring-0 mapping.
1661	* @param ppvR3 Where to store the address of the Ring-3 mapping.
1662	*/
1663	SUPR0DECL(int) SUPR0MemAlloc(PSUPDRVSESSION pSession, uint32_t cb, PRTR0PTR ppvR0, PRTR3PTR ppvR3)
1664	{
1665	int rc;
1666	SUPDRVMEMREF Mem = {0};
1667	dprintf(("SUPR0MemAlloc: pSession=%p cb=%d ppvR0=%p ppvR3=%p\n", pSession, cb, ppvR0, ppvR3));
1668
1669	/*
1670	* Validate input.
1671	*/
1672	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
1673	AssertPtrReturn(ppvR0, VERR_INVALID_POINTER);
1674	AssertPtrReturn(ppvR3, VERR_INVALID_POINTER);
1675	if (cb < 1 \|\| cb >= _4M)
1676	{
1677	dprintf(("Illegal request cb=%u; must be greater than 0 and smaller than 4MB.\n", cb));
1678	return VERR_INVALID_PARAMETER;
1679	}
1680
1681	/*
1682	* Let IPRT do the work.
1683	*/
1684	rc = RTR0MemObjAllocPage(&Mem.MemObj, cb, true /* executable ring-0 mapping */);
1685	if (RT_SUCCESS(rc))
1686	{
1687	int rc2;
1688	rc = RTR0MemObjMapUser(&Mem.MapObjR3, Mem.MemObj, (RTR3PTR)-1, 0,
1689	RTMEM_PROT_EXEC \| RTMEM_PROT_WRITE \| RTMEM_PROT_READ, RTR0ProcHandleSelf());
1690	if (RT_SUCCESS(rc))
1691	{
1692	Mem.eType = MEMREF_TYPE_MEM;
1693	rc = supdrvMemAdd(&Mem, pSession);
1694	if (!rc)
1695	{
1696	*ppvR0 = RTR0MemObjAddress(Mem.MemObj);
1697	*ppvR3 = RTR0MemObjAddressR3(Mem.MapObjR3);
1698	return VINF_SUCCESS;
1699	}
1700	rc2 = RTR0MemObjFree(Mem.MapObjR3, false);
1701	AssertRC(rc2);
1702	}
1703
1704	rc2 = RTR0MemObjFree(Mem.MemObj, false);
1705	AssertRC(rc2);
1706	}
1707
1708	return rc;
1709	}
1710
1711
1712	/**
1713	* Get the physical addresses of memory allocated using SUPR0MemAlloc().
1714	*
1715	* @returns IPRT status code.
1716	* @param pSession The session to which the memory was allocated.
1717	* @param uPtr The Ring-0 or Ring-3 address returned by SUPR0MemAlloc().
1718	* @param paPages Where to store the physical addresses.
1719	*/
1720	SUPR0DECL(int) SUPR0MemGetPhys(PSUPDRVSESSION pSession, RTHCUINTPTR uPtr, PSUPPAGE paPages) /** @todo switch this bugger to RTHCPHYS */
1721	{
1722	PSUPDRVBUNDLE pBundle;
1723	RTSPINLOCKTMP SpinlockTmp = RTSPINLOCKTMP_INITIALIZER;
1724	dprintf(("SUPR0MemGetPhys: pSession=%p uPtr=%p paPages=%p\n", pSession, (void *)uPtr, paPages));
1725
1726	/*
1727	* Validate input.
1728	*/
1729	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
1730	AssertPtrReturn(paPages, VERR_INVALID_POINTER);
1731	AssertReturn(uPtr, VERR_INVALID_PARAMETER);
1732
1733	/*
1734	* Search for the address.
1735	*/
1736	RTSpinlockAcquire(pSession->Spinlock, &SpinlockTmp);
1737	for (pBundle = &pSession->Bundle; pBundle; pBundle = pBundle->pNext)
1738	{
1739	if (pBundle->cUsed > 0)
1740	{
1741	unsigned i;
1742	for (i = 0; i < RT_ELEMENTS(pBundle->aMem); i++)
1743	{
1744	if ( pBundle->aMem[i].eType == MEMREF_TYPE_MEM
1745	&& pBundle->aMem[i].MemObj != NIL_RTR0MEMOBJ
1746	&& ( (RTHCUINTPTR)RTR0MemObjAddress(pBundle->aMem[i].MemObj) == uPtr
1747	\|\| ( pBundle->aMem[i].MapObjR3 != NIL_RTR0MEMOBJ
1748	&& RTR0MemObjAddressR3(pBundle->aMem[i].MapObjR3) == uPtr)
1749	)
1750	)
1751	{
1752	const unsigned cPages = RTR0MemObjSize(pBundle->aMem[i].MemObj) >> PAGE_SHIFT;
1753	unsigned iPage;
1754	for (iPage = 0; iPage < cPages; iPage++)
1755	{
1756	paPages[iPage].Phys = RTR0MemObjGetPagePhysAddr(pBundle->aMem[i].MemObj, iPage);
1757	paPages[iPage].uReserved = 0;
1758	}
1759	RTSpinlockRelease(pSession->Spinlock, &SpinlockTmp);
1760	return VINF_SUCCESS;
1761	}
1762	}
1763	}
1764	}
1765	RTSpinlockRelease(pSession->Spinlock, &SpinlockTmp);
1766	dprintf(("Failed to find %p!!!\n", (void *)uPtr));
1767	return VERR_INVALID_PARAMETER;
1768	}
1769
1770
1771	/**
1772	* Free memory allocated by SUPR0MemAlloc().
1773	*
1774	* @returns IPRT status code.
1775	* @param pSession The session owning the allocation.
1776	* @param uPtr The Ring-0 or Ring-3 address returned by SUPR0MemAlloc().
1777	*/
1778	SUPR0DECL(int) SUPR0MemFree(PSUPDRVSESSION pSession, RTHCUINTPTR uPtr)
1779	{
1780	dprintf(("SUPR0MemFree: pSession=%p uPtr=%p\n", pSession, (void *)uPtr));
1781	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
1782	return supdrvMemRelease(pSession, uPtr, MEMREF_TYPE_MEM);
1783	}
1784
1785
1786	/**
1787	* Allocates a chunk of memory with only a R3 mappings.
1788	* The memory is fixed and it's possible to query the physical addresses using SUPR0MemGetPhys().
1789	*
1790	* @returns IPRT status code.
1791	* @param pSession The session to associated the allocation with.
1792	* @param cPages The number of pages to allocate.
1793	* @param ppvR3 Where to store the address of the Ring-3 mapping.
1794	* @param paPages Where to store the addresses of the pages. Optional.
1795	*/
1796	SUPR0DECL(int) SUPR0PageAlloc(PSUPDRVSESSION pSession, uint32_t cPages, PRTR3PTR ppvR3, PRTHCPHYS paPages)
1797	{
1798	int rc;
1799	SUPDRVMEMREF Mem = {0};
1800	dprintf(("SUPR0PageAlloc: pSession=%p cb=%d ppvR3=%p\n", pSession, cPages, ppvR3));
1801
1802	/*
1803	* Validate input. The allowed allocation size must be at least equal to the maximum guest VRAM size.
1804	*/
1805	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
1806	AssertPtrReturn(ppvR3, VERR_INVALID_POINTER);
1807	if (cPages < 1 \|\| cPages > (128 * _1M)/PAGE_SIZE)
1808	{
1809	dprintf(("SUPR0PageAlloc: Illegal request cb=%u; must be greater than 0 and smaller than 128MB.\n", cPages));
1810	return VERR_INVALID_PARAMETER;
1811	}
1812
1813	/*
1814	* Let IPRT do the work.
1815	*/
1816	rc = RTR0MemObjAllocPhysNC(&Mem.MemObj, (size_t)cPages * PAGE_SIZE, NIL_RTHCPHYS);
1817	if (RT_SUCCESS(rc))
1818	{
1819	int rc2;
1820	rc = RTR0MemObjMapUser(&Mem.MapObjR3, Mem.MemObj, (RTR3PTR)-1, 0,
1821	RTMEM_PROT_EXEC \| RTMEM_PROT_WRITE \| RTMEM_PROT_READ, RTR0ProcHandleSelf());
1822	if (RT_SUCCESS(rc))
1823	{
1824	Mem.eType = MEMREF_TYPE_LOCKED_SUP;
1825	rc = supdrvMemAdd(&Mem, pSession);
1826	if (!rc)
1827	{
1828	*ppvR3 = RTR0MemObjAddressR3(Mem.MapObjR3);
1829	if (paPages)
1830	{
1831	uint32_t iPage = cPages;
1832	while (iPage-- > 0)
1833	{
1834	paPages[iPage] = RTR0MemObjGetPagePhysAddr(Mem.MapObjR3, iPage);
1835	Assert(paPages[iPage] != NIL_RTHCPHYS);
1836	}
1837	}
1838	return VINF_SUCCESS;
1839	}
1840	rc2 = RTR0MemObjFree(Mem.MapObjR3, false);
1841	AssertRC(rc2);
1842	}
1843
1844	rc2 = RTR0MemObjFree(Mem.MemObj, false);
1845	AssertRC(rc2);
1846	}
1847	return rc;
1848	}
1849
1850
1851	#ifdef RT_OS_WINDOWS
1852	/**
1853	* Check if the pages were locked by SUPR0PageAlloc
1854	*
1855	* This function will be removed along with the lock/unlock hacks when
1856	* we've cleaned up the ring-3 code properly.
1857	*
1858	* @returns boolean
1859	* @param pSession The session to which the memory was allocated.
1860	* @param pvR3 The Ring-3 address returned by SUPR0PageAlloc().
1861	*/
1862	static bool supdrvPageWasLockedByPageAlloc(PSUPDRVSESSION pSession, RTR3PTR pvR3)
1863	{
1864	PSUPDRVBUNDLE pBundle;
1865	RTSPINLOCKTMP SpinlockTmp = RTSPINLOCKTMP_INITIALIZER;
1866	dprintf(("SUPR0PageIsLockedByPageAlloc: pSession=%p pvR3=%p\n", pSession, (void *)pvR3));
1867
1868	/*
1869	* Search for the address.
1870	*/
1871	RTSpinlockAcquire(pSession->Spinlock, &SpinlockTmp);
1872	for (pBundle = &pSession->Bundle; pBundle; pBundle = pBundle->pNext)
1873	{
1874	if (pBundle->cUsed > 0)
1875	{
1876	unsigned i;
1877	for (i = 0; i < RT_ELEMENTS(pBundle->aMem); i++)
1878	{
1879	if ( pBundle->aMem[i].eType == MEMREF_TYPE_LOCKED_SUP
1880	&& pBundle->aMem[i].MemObj != NIL_RTR0MEMOBJ
1881	&& pBundle->aMem[i].MapObjR3 != NIL_RTR0MEMOBJ
1882	&& RTR0MemObjAddressR3(pBundle->aMem[i].MapObjR3) == pvR3)
1883	{
1884	RTSpinlockRelease(pSession->Spinlock, &SpinlockTmp);
1885	return true;
1886	}
1887	}
1888	}
1889	}
1890	RTSpinlockRelease(pSession->Spinlock, &SpinlockTmp);
1891	return false;
1892	}
1893
1894
1895	/**
1896	* Get the physical addresses of memory allocated using SUPR0PageAlloc().
1897	*
1898	* This function will be removed along with the lock/unlock hacks when
1899	* we've cleaned up the ring-3 code properly.
1900	*
1901	* @returns IPRT status code.
1902	* @param pSession The session to which the memory was allocated.
1903	* @param pvR3 The Ring-3 address returned by SUPR0PageAlloc().
1904	* @param cPages Number of pages in paPages
1905	* @param paPages Where to store the physical addresses.
1906	*/
1907	static int supdrvPageGetPhys(PSUPDRVSESSION pSession, RTR3PTR pvR3, uint32_t cPages, PRTHCPHYS paPages)
1908	{
1909	PSUPDRVBUNDLE pBundle;
1910	RTSPINLOCKTMP SpinlockTmp = RTSPINLOCKTMP_INITIALIZER;
1911	dprintf(("supdrvPageGetPhys: pSession=%p pvR3=%p cPages=%#lx paPages=%p\n", pSession, (void *)pvR3, (long)cPages, paPages));
1912
1913	/*
1914	* Search for the address.
1915	*/
1916	RTSpinlockAcquire(pSession->Spinlock, &SpinlockTmp);
1917	for (pBundle = &pSession->Bundle; pBundle; pBundle = pBundle->pNext)
1918	{
1919	if (pBundle->cUsed > 0)
1920	{
1921	unsigned i;
1922	for (i = 0; i < RT_ELEMENTS(pBundle->aMem); i++)
1923	{
1924	if ( pBundle->aMem[i].eType == MEMREF_TYPE_LOCKED_SUP
1925	&& pBundle->aMem[i].MemObj != NIL_RTR0MEMOBJ
1926	&& pBundle->aMem[i].MapObjR3 != NIL_RTR0MEMOBJ
1927	&& RTR0MemObjAddressR3(pBundle->aMem[i].MapObjR3) == pvR3)
1928	{
1929	uint32_t iPage = RTR0MemObjSize(pBundle->aMem[i].MemObj) >> PAGE_SHIFT;
1930	cPages = RT_MIN(iPage, cPages);
1931	for (iPage = 0; iPage < cPages; iPage++)
1932	paPages[iPage] = RTR0MemObjGetPagePhysAddr(pBundle->aMem[i].MemObj, iPage);
1933	RTSpinlockRelease(pSession->Spinlock, &SpinlockTmp);
1934	return VINF_SUCCESS;
1935	}
1936	}
1937	}
1938	}
1939	RTSpinlockRelease(pSession->Spinlock, &SpinlockTmp);
1940	return VERR_INVALID_PARAMETER;
1941	}
1942	#endif /* RT_OS_WINDOWS */
1943
1944
1945	/**
1946	* Free memory allocated by SUPR0PageAlloc().
1947	*
1948	* @returns IPRT status code.
1949	* @param pSession The session owning the allocation.
1950	* @param pvR3 The Ring-3 address returned by SUPR0PageAlloc().
1951	*/
1952	SUPR0DECL(int) SUPR0PageFree(PSUPDRVSESSION pSession, RTR3PTR pvR3)
1953	{
1954	dprintf(("SUPR0PageFree: pSession=%p pvR3=%p\n", pSession, (void *)pvR3));
1955	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
1956	return supdrvMemRelease(pSession, (RTHCUINTPTR)pvR3, MEMREF_TYPE_LOCKED_SUP);
1957	}
1958
1959
1960	/**
1961	* Maps the GIP into userspace and/or get the physical address of the GIP.
1962	*
1963	* @returns IPRT status code.
1964	* @param pSession Session to which the GIP mapping should belong.
1965	* @param ppGipR3 Where to store the address of the ring-3 mapping. (optional)
1966	* @param pHCPhysGip Where to store the physical address. (optional)
1967	*
1968	* @remark There is no reference counting on the mapping, so one call to this function
1969	* count globally as one reference. One call to SUPR0GipUnmap() is will unmap GIP
1970	* and remove the session as a GIP user.
1971	*/
1972	SUPR0DECL(int) SUPR0GipMap(PSUPDRVSESSION pSession, PRTR3PTR ppGipR3, PRTHCPHYS pHCPhysGip)
1973	{
1974	int rc = 0;
1975	PSUPDRVDEVEXT pDevExt = pSession->pDevExt;
1976	RTR3PTR pGip = NIL_RTR3PTR;
1977	RTHCPHYS HCPhys = NIL_RTHCPHYS;
1978	dprintf(("SUPR0GipMap: pSession=%p ppGipR3=%p pHCPhysGip=%p\n", pSession, ppGipR3, pHCPhysGip));
1979
1980	/*
1981	* Validate
1982	*/
1983	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
1984	AssertPtrNullReturn(ppGipR3, VERR_INVALID_POINTER);
1985	AssertPtrNullReturn(pHCPhysGip, VERR_INVALID_POINTER);
1986
1987	RTSemFastMutexRequest(pDevExt->mtxGip);
1988	if (pDevExt->pGip)
1989	{
1990	/*
1991	* Map it?
1992	*/
1993	if (ppGipR3)
1994	{
1995	#ifdef USE_NEW_OS_INTERFACE_FOR_GIP
1996	if (pSession->GipMapObjR3 == NIL_RTR0MEMOBJ)
1997	rc = RTR0MemObjMapUser(&pSession->GipMapObjR3, pDevExt->GipMemObj, (RTR3PTR)-1, 0,
1998	RTMEM_PROT_READ, RTR0ProcHandleSelf());
1999	if (RT_SUCCESS(rc))
2000	{
2001	pGip = RTR0MemObjAddressR3(pSession->GipMapObjR3);
2002	rc = VINF_SUCCESS; /** @todo remove this and replace the !rc below with RT_SUCCESS(rc). */
2003	}
2004	#else /* !USE_NEW_OS_INTERFACE_FOR_GIP */
2005	if (!pSession->pGip)
2006	rc = supdrvOSGipMap(pSession->pDevExt, &pSession->pGip);
2007	if (!rc)
2008	pGip = (RTR3PTR)pSession->pGip;
2009	#endif /* !USE_NEW_OS_INTERFACE_FOR_GIP */
2010	}
2011
2012	/*
2013	* Get physical address.
2014	*/
2015	if (pHCPhysGip && !rc)
2016	HCPhys = pDevExt->HCPhysGip;
2017
2018	/*
2019	* Reference globally.
2020	*/
2021	if (!pSession->fGipReferenced && !rc)
2022	{
2023	pSession->fGipReferenced = 1;
2024	pDevExt->cGipUsers++;
2025	if (pDevExt->cGipUsers == 1)
2026	{
2027	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
2028	unsigned i;
2029
2030	dprintf(("SUPR0GipMap: Resumes GIP updating\n"));
2031
2032	for (i = 0; i < RT_ELEMENTS(pGip->aCPUs); i++)
2033	ASMAtomicXchgU32(&pGip->aCPUs[i].u32TransactionId, pGip->aCPUs[i].u32TransactionId & ~(GIP_UPDATEHZ_RECALC_FREQ * 2 - 1));
2034	ASMAtomicXchgU64(&pGip->u64NanoTSLastUpdateHz, 0);
2035
2036	#ifdef USE_NEW_OS_INTERFACE_FOR_GIP
2037	rc = RTTimerStart(pDevExt->pGipTimer, 0);
2038	AssertRC(rc); rc = VINF_SUCCESS;
2039	#else
2040	supdrvOSGipResume(pDevExt);
2041	#endif
2042	}
2043	}
2044	}
2045	else
2046	{
2047	rc = SUPDRV_ERR_GENERAL_FAILURE;
2048	dprintf(("SUPR0GipMap: GIP is not available!\n"));
2049	}
2050	RTSemFastMutexRelease(pDevExt->mtxGip);
2051
2052	/*
2053	* Write returns.
2054	*/
2055	if (pHCPhysGip)
2056	*pHCPhysGip = HCPhys;
2057	if (ppGipR3)
2058	*ppGipR3 = pGip;
2059
2060	#ifdef DEBUG_DARWIN_GIP
2061	OSDBGPRINT(("SUPR0GipMap: returns %d pHCPhysGip=%lx ppGip=%p GipMapObjR3\n", rc, (unsigned long)HCPhys, pGip, pSession->GipMapObjR3));
2062	#else
2063	dprintf(("SUPR0GipMap: returns %d pHCPhysGip=%lx ppGipR3=%p\n", rc, (unsigned long)HCPhys, (void *)(uintptr_t)pGip));
2064	#endif
2065	return rc;
2066	}
2067
2068
2069	/**
2070	* Unmaps any user mapping of the GIP and terminates all GIP access
2071	* from this session.
2072	*
2073	* @returns IPRT status code.
2074	* @param pSession Session to which the GIP mapping should belong.
2075	*/
2076	SUPR0DECL(int) SUPR0GipUnmap(PSUPDRVSESSION pSession)
2077	{
2078	int rc = VINF_SUCCESS;
2079	PSUPDRVDEVEXT pDevExt = pSession->pDevExt;
2080	#ifdef DEBUG_DARWIN_GIP
2081	OSDBGPRINT(("SUPR0GipUnmap: pSession=%p pGip=%p GipMapObjR3=%p\n",
2082	pSession,
2083	pSession->GipMapObjR3 != NIL_RTR0MEMOBJ ? RTR0MemObjAddress(pSession->GipMapObjR3) : NULL,
2084	pSession->GipMapObjR3));
2085	#else
2086	dprintf(("SUPR0GipUnmap: pSession=%p\n", pSession));
2087	#endif
2088	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
2089
2090	RTSemFastMutexRequest(pDevExt->mtxGip);
2091
2092	/*
2093	* Unmap anything?
2094	*/
2095	#ifdef USE_NEW_OS_INTERFACE_FOR_GIP
2096	if (pSession->GipMapObjR3 != NIL_RTR0MEMOBJ)
2097	{
2098	rc = RTR0MemObjFree(pSession->GipMapObjR3, false);
2099	AssertRC(rc);
2100	if (RT_SUCCESS(rc))
2101	pSession->GipMapObjR3 = NIL_RTR0MEMOBJ;
2102	}
2103	#else
2104	if (pSession->pGip)
2105	{
2106	rc = supdrvOSGipUnmap(pDevExt, pSession->pGip);
2107	if (!rc)
2108	pSession->pGip = NULL;
2109	}
2110	#endif
2111
2112	/*
2113	* Dereference global GIP.
2114	*/
2115	if (pSession->fGipReferenced && !rc)
2116	{
2117	pSession->fGipReferenced = 0;
2118	if ( pDevExt->cGipUsers > 0
2119	&& !--pDevExt->cGipUsers)
2120	{
2121	dprintf(("SUPR0GipUnmap: Suspends GIP updating\n"));
2122	#ifdef USE_NEW_OS_INTERFACE_FOR_GIP
2123	rc = RTTimerStop(pDevExt->pGipTimer); AssertRC(rc); rc = 0;
2124	#else
2125	supdrvOSGipSuspend(pDevExt);
2126	#endif
2127	}
2128	}
2129
2130	RTSemFastMutexRelease(pDevExt->mtxGip);
2131
2132	return rc;
2133	}
2134
2135
2136	/**
2137	* Adds a memory object to the session.
2138	*
2139	* @returns IPRT status code.
2140	* @param pMem Memory tracking structure containing the
2141	* information to track.
2142	* @param pSession The session.
2143	*/
2144	static int supdrvMemAdd(PSUPDRVMEMREF pMem, PSUPDRVSESSION pSession)
2145	{
2146	PSUPDRVBUNDLE pBundle;
2147	RTSPINLOCKTMP SpinlockTmp = RTSPINLOCKTMP_INITIALIZER;
2148
2149	/*
2150	* Find free entry and record the allocation.
2151	*/
2152	RTSpinlockAcquire(pSession->Spinlock, &SpinlockTmp);
2153	for (pBundle = &pSession->Bundle; pBundle; pBundle = pBundle->pNext)
2154	{
2155	if (pBundle->cUsed < RT_ELEMENTS(pBundle->aMem))
2156	{
2157	unsigned i;
2158	for (i = 0; i < RT_ELEMENTS(pBundle->aMem); i++)
2159	{
2160	if (pBundle->aMem[i].MemObj == NIL_RTR0MEMOBJ)
2161	{
2162	pBundle->cUsed++;
2163	pBundle->aMem[i] = *pMem;
2164	RTSpinlockRelease(pSession->Spinlock, &SpinlockTmp);
2165	return VINF_SUCCESS;
2166	}
2167	}
2168	AssertFailed(); /* !!this can't be happening!!! */
2169	}
2170	}
2171	RTSpinlockRelease(pSession->Spinlock, &SpinlockTmp);
2172
2173	/*
2174	* Need to allocate a new bundle.
2175	* Insert into the last entry in the bundle.
2176	*/
2177	pBundle = (PSUPDRVBUNDLE)RTMemAllocZ(sizeof(*pBundle));
2178	if (!pBundle)
2179	return VERR_NO_MEMORY;
2180
2181	/* take last entry. */
2182	pBundle->cUsed++;
2183	pBundle->aMem[RT_ELEMENTS(pBundle->aMem) - 1] = *pMem;
2184
2185	/* insert into list. */
2186	RTSpinlockAcquire(pSession->Spinlock, &SpinlockTmp);
2187	pBundle->pNext = pSession->Bundle.pNext;
2188	pSession->Bundle.pNext = pBundle;
2189	RTSpinlockRelease(pSession->Spinlock, &SpinlockTmp);
2190
2191	return VINF_SUCCESS;
2192	}
2193
2194
2195	/**
2196	* Releases a memory object referenced by pointer and type.
2197	*
2198	* @returns IPRT status code.
2199	* @param pSession Session data.
2200	* @param uPtr Pointer to memory. This is matched against both the R0 and R3 addresses.
2201	* @param eType Memory type.
2202	*/
2203	static int supdrvMemRelease(PSUPDRVSESSION pSession, RTHCUINTPTR uPtr, SUPDRVMEMREFTYPE eType)
2204	{
2205	PSUPDRVBUNDLE pBundle;
2206	RTSPINLOCKTMP SpinlockTmp = RTSPINLOCKTMP_INITIALIZER;
2207
2208	/*
2209	* Validate input.
2210	*/
2211	if (!uPtr)
2212	{
2213	dprintf(("Illegal address %p\n", (void *)uPtr));
2214	return VERR_INVALID_PARAMETER;
2215	}
2216
2217	/*
2218	* Search for the address.
2219	*/
2220	RTSpinlockAcquire(pSession->Spinlock, &SpinlockTmp);
2221	for (pBundle = &pSession->Bundle; pBundle; pBundle = pBundle->pNext)
2222	{
2223	if (pBundle->cUsed > 0)
2224	{
2225	unsigned i;
2226	for (i = 0; i < RT_ELEMENTS(pBundle->aMem); i++)
2227	{
2228	if ( pBundle->aMem[i].eType == eType
2229	&& pBundle->aMem[i].MemObj != NIL_RTR0MEMOBJ
2230	&& ( (RTHCUINTPTR)RTR0MemObjAddress(pBundle->aMem[i].MemObj) == uPtr
2231	\|\| ( pBundle->aMem[i].MapObjR3 != NIL_RTR0MEMOBJ
2232	&& RTR0MemObjAddressR3(pBundle->aMem[i].MapObjR3) == uPtr))
2233	)
2234	{
2235	/* Make a copy of it and release it outside the spinlock. */
2236	SUPDRVMEMREF Mem = pBundle->aMem[i];
2237	pBundle->aMem[i].eType = MEMREF_TYPE_UNUSED;
2238	pBundle->aMem[i].MemObj = NIL_RTR0MEMOBJ;
2239	pBundle->aMem[i].MapObjR3 = NIL_RTR0MEMOBJ;
2240	RTSpinlockRelease(pSession->Spinlock, &SpinlockTmp);
2241
2242	if (Mem.MapObjR3)
2243	{
2244	int rc = RTR0MemObjFree(Mem.MapObjR3, false);
2245	AssertRC(rc); /** @todo figure out how to handle this. */
2246	}
2247	if (Mem.MemObj)
2248	{
2249	int rc = RTR0MemObjFree(Mem.MemObj, false);
2250	AssertRC(rc); /** @todo figure out how to handle this. */
2251	}
2252	return VINF_SUCCESS;
2253	}
2254	}
2255	}
2256	}
2257	RTSpinlockRelease(pSession->Spinlock, &SpinlockTmp);
2258	dprintf(("Failed to find %p!!! (eType=%d)\n", (void *)uPtr, eType));
2259	return VERR_INVALID_PARAMETER;
2260	}
2261
2262
2263	#ifdef VBOX_WITH_IDT_PATCHING
2264	/**
2265	* Install IDT for the current CPU.
2266	*
2267	* @returns One of the following IPRT status codes:
2268	* @retval VINF_SUCCESS on success.
2269	* @retval VERR_IDT_FAILED.
2270	* @retval VERR_NO_MEMORY.
2271	* @param pDevExt The device extension.
2272	* @param pSession The session data.
2273	* @param pReq The request.
2274	*/
2275	static int supdrvIOCtl_IdtInstall(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPIDTINSTALL pReq)
2276	{
2277	PSUPDRVPATCHUSAGE pUsagePre;
2278	PSUPDRVPATCH pPatchPre;
2279	RTIDTR Idtr;
2280	PSUPDRVPATCH pPatch;
2281	RTSPINLOCKTMP SpinlockTmp = RTSPINLOCKTMP_INITIALIZER;
2282	dprintf(("supdrvIOCtl_IdtInstall\n"));
2283
2284	/*
2285	* Preallocate entry for this CPU cause we don't wanna do
2286	* that inside the spinlock!
2287	*/
2288	pUsagePre = (PSUPDRVPATCHUSAGE)RTMemAlloc(sizeof(*pUsagePre));
2289	if (!pUsagePre)
2290	return VERR_NO_MEMORY;
2291
2292	/*
2293	* Take the spinlock and see what we need to do.
2294	*/
2295	RTSpinlockAcquireNoInts(pDevExt->Spinlock, &SpinlockTmp);
2296
2297	/* check if we already got a free patch. */
2298	if (!pDevExt->pIdtPatchesFree)
2299	{
2300	/*
2301	* Allocate a patch - outside the spinlock of course.
2302	*/
2303	RTSpinlockReleaseNoInts(pDevExt->Spinlock, &SpinlockTmp);
2304
2305	pPatchPre = (PSUPDRVPATCH)RTMemExecAlloc(sizeof(*pPatchPre));
2306	if (!pPatchPre)
2307	return VERR_NO_MEMORY;
2308
2309	RTSpinlockAcquireNoInts(pDevExt->Spinlock, &SpinlockTmp);
2310	}
2311	else
2312	{
2313	pPatchPre = pDevExt->pIdtPatchesFree;
2314	pDevExt->pIdtPatchesFree = pPatchPre->pNext;
2315	}
2316
2317	/* look for matching patch entry */
2318	ASMGetIDTR(&Idtr);
2319	pPatch = pDevExt->pIdtPatches;
2320	while (pPatch && pPatch->pvIdt != (void *)Idtr.pIdt)
2321	pPatch = pPatch->pNext;
2322
2323	if (!pPatch)
2324	{
2325	/*
2326	* Create patch.
2327	*/
2328	pPatch = supdrvIdtPatchOne(pDevExt, pPatchPre);
2329	if (pPatch)
2330	pPatchPre = NULL; /* mark as used. */
2331	}
2332	else
2333	{
2334	/*
2335	* Simply increment patch usage.
2336	*/
2337	pPatch->cUsage++;
2338	}
2339
2340	if (pPatch)
2341	{
2342	/*
2343	* Increment and add if need be the session usage record for this patch.
2344	*/
2345	PSUPDRVPATCHUSAGE pUsage = pSession->pPatchUsage;
2346	while (pUsage && pUsage->pPatch != pPatch)
2347	pUsage = pUsage->pNext;
2348
2349	if (!pUsage)
2350	{
2351	/*
2352	* Add usage record.
2353	*/
2354	pUsagePre->cUsage = 1;
2355	pUsagePre->pPatch = pPatch;
2356	pUsagePre->pNext = pSession->pPatchUsage;
2357	pSession->pPatchUsage = pUsagePre;
2358	pUsagePre = NULL; /* mark as used. */
2359	}
2360	else
2361	{
2362	/*
2363	* Increment usage count.
2364	*/
2365	pUsage->cUsage++;
2366	}
2367	}
2368
2369	/* free patch - we accumulate them for paranoid saftly reasons. */
2370	if (pPatchPre)
2371	{
2372	pPatchPre->pNext = pDevExt->pIdtPatchesFree;
2373	pDevExt->pIdtPatchesFree = pPatchPre;
2374	}
2375
2376	RTSpinlockReleaseNoInts(pDevExt->Spinlock, &SpinlockTmp);
2377
2378	/*
2379	* Free unused preallocated buffers.
2380	*/
2381	if (pUsagePre)
2382	RTMemFree(pUsagePre);
2383
2384	pReq->u.Out.u8Idt = pDevExt->u8Idt;
2385
2386	return pPatch ? VINF_SUCCESS : VERR_IDT_FAILED;
2387	}
2388
2389
2390	/**
2391	* This creates a IDT patch entry.
2392	* If the first patch being installed it'll also determin the IDT entry
2393	* to use.
2394	*
2395	* @returns pPatch on success.
2396	* @returns NULL on failure.
2397	* @param pDevExt Pointer to globals.
2398	* @param pPatch Patch entry to use.
2399	* This will be linked into SUPDRVDEVEXT::pIdtPatches on
2400	* successful return.
2401	* @remark Call must be owning the SUPDRVDEVEXT::Spinlock!
2402	*/
2403	static PSUPDRVPATCH supdrvIdtPatchOne(PSUPDRVDEVEXT pDevExt, PSUPDRVPATCH pPatch)
2404	{
2405	RTIDTR Idtr;
2406	PSUPDRVIDTE paIdt;
2407	dprintf(("supdrvIOCtl_IdtPatchOne: pPatch=%p\n", pPatch));
2408
2409	/*
2410	* Get IDT.
2411	*/
2412	ASMGetIDTR(&Idtr);
2413	paIdt = (PSUPDRVIDTE)Idtr.pIdt;
2414	/*
2415	* Recent Linux kernels can be configured to 1G user /3G kernel.
2416	*/
2417	if ((uintptr_t)paIdt < 0x40000000)
2418	{
2419	AssertMsgFailed(("bad paIdt=%p\n", paIdt));
2420	return NULL;
2421	}
2422
2423	if (!pDevExt->u8Idt)
2424	{
2425	/*
2426	* Test out the alternatives.
2427	*
2428	* At the moment we do not support chaining thus we ASSUME that one of
2429	* these 48 entries is unused (which is not a problem on Win32 and
2430	* Linux to my knowledge).
2431	*/
2432	/** @todo we MUST change this detection to try grab an entry which is NOT in use. This can be
2433	* combined with gathering info about which guest system call gates we can hook up directly. */
2434	unsigned i;
2435	uint8_t u8Idt = 0;
2436	static uint8_t au8Ints[] =
2437	{
2438	#ifdef RT_OS_WINDOWS /* We don't use 0xef and above because they are system stuff on linux (ef is IPI,
2439	* local apic timer, or some other frequently fireing thing). */
2440	0xef, 0xee, 0xed, 0xec,
2441	#endif
2442	0xeb, 0xea, 0xe9, 0xe8,
2443	0xdf, 0xde, 0xdd, 0xdc,
2444	0x7b, 0x7a, 0x79, 0x78,
2445	0xbf, 0xbe, 0xbd, 0xbc,
2446	};
2447	#if defined(RT_ARCH_AMD64) && defined(DEBUG)
2448	static int s_iWobble = 0;
2449	unsigned iMax = !(s_iWobble++ % 2) ? 0x80 : 0x100;
2450	dprintf(("IDT: Idtr=%p:%#x\n", (void *)Idtr.pIdt, (unsigned)Idtr.cbIdt));
2451	for (i = iMax - 0x80; i*16+15 < Idtr.cbIdt && i < iMax; i++)
2452	{
2453	dprintf(("%#x: %04x:%08x%04x%04x P=%d DPL=%d IST=%d Type1=%#x u32Reserved=%#x u5Reserved=%#x\n",
2454	i, paIdt[i].u16SegSel, paIdt[i].u32OffsetTop, paIdt[i].u16OffsetHigh, paIdt[i].u16OffsetLow,
2455	paIdt[i].u1Present, paIdt[i].u2DPL, paIdt[i].u3IST, paIdt[i].u5Type2,
2456	paIdt[i].u32Reserved, paIdt[i].u5Reserved));
2457	}
2458	#endif
2459	/* look for entries which are not present or otherwise unused. */
2460	for (i = 0; i < sizeof(au8Ints) / sizeof(au8Ints[0]); i++)
2461	{
2462	u8Idt = au8Ints[i];
2463	if ( u8Idt * sizeof(SUPDRVIDTE) < Idtr.cbIdt
2464	&& ( !paIdt[u8Idt].u1Present
2465	\|\| paIdt[u8Idt].u5Type2 == 0))
2466	break;
2467	u8Idt = 0;
2468	}
2469	if (!u8Idt)
2470	{
2471	/* try again, look for a compatible entry .*/
2472	for (i = 0; i < sizeof(au8Ints) / sizeof(au8Ints[0]); i++)
2473	{
2474	u8Idt = au8Ints[i];
2475	if ( u8Idt * sizeof(SUPDRVIDTE) < Idtr.cbIdt
2476	&& paIdt[u8Idt].u1Present
2477	&& paIdt[u8Idt].u5Type2 == SUPDRV_IDTE_TYPE2_INTERRUPT_GATE
2478	&& !(paIdt[u8Idt].u16SegSel & 3))
2479	break;
2480	u8Idt = 0;
2481	}
2482	if (!u8Idt)
2483	{
2484	dprintf(("Failed to find appropirate IDT entry!!\n"));
2485	return NULL;
2486	}
2487	}
2488	pDevExt->u8Idt = u8Idt;
2489	dprintf(("supdrvIOCtl_IdtPatchOne: u8Idt=%x\n", u8Idt));
2490	}
2491
2492	/*
2493	* Prepare the patch
2494	*/
2495	memset(pPatch, 0, sizeof(*pPatch));
2496	pPatch->pvIdt = paIdt;
2497	pPatch->cUsage = 1;
2498	pPatch->pIdtEntry = &paIdt[pDevExt->u8Idt];
2499	pPatch->SavedIdt = paIdt[pDevExt->u8Idt];
2500	pPatch->ChangedIdt.u16OffsetLow = (uint32_t)((uintptr_t)&pPatch->auCode[0] & 0xffff);
2501	pPatch->ChangedIdt.u16OffsetHigh = (uint32_t)((uintptr_t)&pPatch->auCode[0] >> 16);
2502	#ifdef RT_ARCH_AMD64
2503	pPatch->ChangedIdt.u32OffsetTop = (uint32_t)((uintptr_t)&pPatch->auCode[0] >> 32);
2504	#endif
2505	pPatch->ChangedIdt.u16SegSel = ASMGetCS();
2506	#ifdef RT_ARCH_AMD64
2507	pPatch->ChangedIdt.u3IST = 0;
2508	pPatch->ChangedIdt.u5Reserved = 0;
2509	#else /* x86 */
2510	pPatch->ChangedIdt.u5Reserved = 0;
2511	pPatch->ChangedIdt.u3Type1 = 0;
2512	#endif /* x86 */
2513	pPatch->ChangedIdt.u5Type2 = SUPDRV_IDTE_TYPE2_INTERRUPT_GATE;
2514	pPatch->ChangedIdt.u2DPL = 3;
2515	pPatch->ChangedIdt.u1Present = 1;
2516
2517	/*
2518	* Generate the patch code.
2519	*/
2520	{
2521	#ifdef RT_ARCH_AMD64
2522	union
2523	{
2524	uint8_t *pb;
2525	uint32_t *pu32;
2526	uint64_t *pu64;
2527	} u, uFixJmp, uFixCall, uNotNested;
2528	u.pb = &pPatch->auCode[0];
2529
2530	/* check the cookie */
2531	*u.pb++ = 0x3d; // cmp eax, GLOBALCOOKIE
2532	*u.pu32++ = pDevExt->u32Cookie;
2533
2534	*u.pb++ = 0x74; // jz @VBoxCall
2535	*u.pb++ = 2;
2536
2537	/* jump to forwarder code. */
2538	*u.pb++ = 0xeb;
2539	uFixJmp = u;
2540	*u.pb++ = 0xfe;
2541
2542	// @VBoxCall:
2543	*u.pb++ = 0x0f; // swapgs
2544	*u.pb++ = 0x01;
2545	*u.pb++ = 0xf8;
2546
2547	/*
2548	* Call VMMR0Entry
2549	* We don't have to push the arguments here, but we have top
2550	* reserve some stack space for the interrupt forwarding.
2551	*/
2552	# ifdef RT_OS_WINDOWS
2553	*u.pb++ = 0x50; // push rax ; alignment filler.
2554	*u.pb++ = 0x41; // push r8 ; uArg
2555	*u.pb++ = 0x50;
2556	*u.pb++ = 0x52; // push rdx ; uOperation
2557	*u.pb++ = 0x51; // push rcx ; pVM
2558	# else
2559	*u.pb++ = 0x51; // push rcx ; alignment filler.
2560	*u.pb++ = 0x52; // push rdx ; uArg
2561	*u.pb++ = 0x56; // push rsi ; uOperation
2562	*u.pb++ = 0x57; // push rdi ; pVM
2563	# endif
2564
2565	*u.pb++ = 0xff; // call qword [pfnVMMR0EntryInt wrt rip]
2566	*u.pb++ = 0x15;
2567	uFixCall = u;
2568	*u.pu32++ = 0;
2569
2570	*u.pb++ = 0x48; // add rsp, 20h ; remove call frame.
2571	*u.pb++ = 0x81;
2572	*u.pb++ = 0xc4;
2573	*u.pu32++ = 0x20;
2574
2575	*u.pb++ = 0x0f; // swapgs
2576	*u.pb++ = 0x01;
2577	*u.pb++ = 0xf8;
2578
2579	/* Return to R3. */
2580	uNotNested = u;
2581	*u.pb++ = 0x48; // iretq
2582	*u.pb++ = 0xcf;
2583
2584	while ((uintptr_t)u.pb & 0x7) // align 8
2585	*u.pb++ = 0xcc;
2586
2587	/* Pointer to the VMMR0Entry. */ // pfnVMMR0EntryInt dq StubVMMR0Entry
2588	*uFixCall.pu32 = (uint32_t)(u.pb - uFixCall.pb - 4); uFixCall.pb = NULL;
2589	pPatch->offVMMR0EntryFixup = (uint16_t)(u.pb - &pPatch->auCode[0]);
2590	*u.pu64++ = pDevExt->pvVMMR0 ? (uint64_t)pDevExt->pfnVMMR0EntryInt : (uint64_t)u.pb + 8;
2591
2592	/* stub entry. */ // StubVMMR0Entry:
2593	pPatch->offStub = (uint16_t)(u.pb - &pPatch->auCode[0]);
2594	*u.pb++ = 0x33; // xor eax, eax
2595	*u.pb++ = 0xc0;
2596
2597	*u.pb++ = 0x48; // dec rax
2598	*u.pb++ = 0xff;
2599	*u.pb++ = 0xc8;
2600
2601	*u.pb++ = 0xc3; // ret
2602
2603	/* forward to the original handler using a retf. */
2604	*uFixJmp.pb = (uint8_t)(u.pb - uFixJmp.pb - 1); uFixJmp.pb = NULL;
2605
2606	*u.pb++ = 0x68; // push <target cs>
2607	*u.pu32++ = !pPatch->SavedIdt.u5Type2 ? ASMGetCS() : pPatch->SavedIdt.u16SegSel;
2608
2609	*u.pb++ = 0x68; // push <low target rip>
2610	*u.pu32++ = !pPatch->SavedIdt.u5Type2
2611	? (uint32_t)(uintptr_t)uNotNested.pb
2612	: (uint32_t)pPatch->SavedIdt.u16OffsetLow
2613	\| (uint32_t)pPatch->SavedIdt.u16OffsetHigh << 16;
2614
2615	*u.pb++ = 0xc7; // mov dword [rsp + 4], <high target rip>
2616	*u.pb++ = 0x44;
2617	*u.pb++ = 0x24;
2618	*u.pb++ = 0x04;
2619	*u.pu32++ = !pPatch->SavedIdt.u5Type2
2620	? (uint32_t)((uint64_t)uNotNested.pb >> 32)
2621	: pPatch->SavedIdt.u32OffsetTop;
2622
2623	*u.pb++ = 0x48; // retf ; does this require prefix?
2624	*u.pb++ = 0xcb;
2625
2626	#else /* RT_ARCH_X86 */
2627
2628	union
2629	{
2630	uint8_t *pb;
2631	uint16_t *pu16;
2632	uint32_t *pu32;
2633	} u, uFixJmpNotNested, uFixJmp, uFixCall, uNotNested;
2634	u.pb = &pPatch->auCode[0];
2635
2636	/* check the cookie */
2637	*u.pb++ = 0x81; // cmp esi, GLOBALCOOKIE
2638	*u.pb++ = 0xfe;
2639	*u.pu32++ = pDevExt->u32Cookie;
2640
2641	*u.pb++ = 0x74; // jz VBoxCall
2642	uFixJmp = u;
2643	*u.pb++ = 0;
2644
2645	/* jump (far) to the original handler / not-nested-stub. */
2646	*u.pb++ = 0xea; // jmp far NotNested
2647	uFixJmpNotNested = u;
2648	*u.pu32++ = 0;
2649	*u.pu16++ = 0;
2650
2651	/* save selector registers. */ // VBoxCall:
2652	*uFixJmp.pb = (uint8_t)(u.pb - uFixJmp.pb - 1);
2653	*u.pb++ = 0x0f; // push fs
2654	*u.pb++ = 0xa0;
2655
2656	*u.pb++ = 0x1e; // push ds
2657
2658	*u.pb++ = 0x06; // push es
2659
2660	/* call frame */
2661	*u.pb++ = 0x51; // push ecx
2662
2663	*u.pb++ = 0x52; // push edx
2664
2665	*u.pb++ = 0x50; // push eax
2666
2667	/* load ds, es and perhaps fs before call. */
2668	*u.pb++ = 0xb8; // mov eax, KernelDS
2669	*u.pu32++ = ASMGetDS();
2670
2671	*u.pb++ = 0x8e; // mov ds, eax
2672	*u.pb++ = 0xd8;
2673
2674	*u.pb++ = 0x8e; // mov es, eax
2675	*u.pb++ = 0xc0;
2676
2677	#ifdef RT_OS_WINDOWS
2678	*u.pb++ = 0xb8; // mov eax, KernelFS
2679	*u.pu32++ = ASMGetFS();
2680
2681	*u.pb++ = 0x8e; // mov fs, eax
2682	*u.pb++ = 0xe0;
2683	#endif
2684
2685	/* do the call. */
2686	*u.pb++ = 0xe8; // call _VMMR0Entry / StubVMMR0Entry
2687	uFixCall = u;
2688	pPatch->offVMMR0EntryFixup = (uint16_t)(u.pb - &pPatch->auCode[0]);
2689	*u.pu32++ = 0xfffffffb;
2690
2691	*u.pb++ = 0x83; // add esp, 0ch ; cdecl
2692	*u.pb++ = 0xc4;
2693	*u.pb++ = 0x0c;
2694
2695	/* restore selector registers. */
2696	*u.pb++ = 0x07; // pop es
2697	//
2698	*u.pb++ = 0x1f; // pop ds
2699
2700	*u.pb++ = 0x0f; // pop fs
2701	*u.pb++ = 0xa1;
2702
2703	uNotNested = u; // NotNested:
2704	*u.pb++ = 0xcf; // iretd
2705
2706	/* the stub VMMR0Entry. */ // StubVMMR0Entry:
2707	pPatch->offStub = (uint16_t)(u.pb - &pPatch->auCode[0]);
2708	*u.pb++ = 0x33; // xor eax, eax
2709	*u.pb++ = 0xc0;
2710
2711	*u.pb++ = 0x48; // dec eax
2712
2713	*u.pb++ = 0xc3; // ret
2714
2715	/* Fixup the VMMR0Entry call. */
2716	if (pDevExt->pvVMMR0)
2717	*uFixCall.pu32 = (uint32_t)pDevExt->pfnVMMR0EntryInt - (uint32_t)(uFixCall.pu32 + 1);
2718	else
2719	*uFixCall.pu32 = (uint32_t)&pPatch->auCode[pPatch->offStub] - (uint32_t)(uFixCall.pu32 + 1);
2720
2721	/* Fixup the forward / nested far jump. */
2722	if (!pPatch->SavedIdt.u5Type2)
2723	{
2724	*uFixJmpNotNested.pu32++ = (uint32_t)uNotNested.pb;
2725	*uFixJmpNotNested.pu16++ = ASMGetCS();
2726	}
2727	else
2728	{
2729	*uFixJmpNotNested.pu32++ = ((uint32_t)pPatch->SavedIdt.u16OffsetHigh << 16) \| pPatch->SavedIdt.u16OffsetLow;
2730	*uFixJmpNotNested.pu16++ = pPatch->SavedIdt.u16SegSel;
2731	}
2732	#endif /* RT_ARCH_X86 */
2733	Assert(u.pb <= &pPatch->auCode[sizeof(pPatch->auCode)]);
2734	#if 0
2735	/* dump the patch code */
2736	dprintf(("patch code: %p\n", &pPatch->auCode[0]));
2737	for (uFixCall.pb = &pPatch->auCode[0]; uFixCall.pb < u.pb; uFixCall.pb++)
2738	dprintf(("0x%02x,\n", *uFixCall.pb));
2739	#endif
2740	}
2741
2742	/*
2743	* Install the patch.
2744	*/
2745	supdrvIdtWrite(pPatch->pIdtEntry, &pPatch->ChangedIdt);
2746	AssertMsg(!memcmp((void *)pPatch->pIdtEntry, &pPatch->ChangedIdt, sizeof(pPatch->ChangedIdt)), ("The stupid change code didn't work!!!!!\n"));
2747
2748	/*
2749	* Link in the patch.
2750	*/
2751	pPatch->pNext = pDevExt->pIdtPatches;
2752	pDevExt->pIdtPatches = pPatch;
2753
2754	return pPatch;
2755	}
2756
2757
2758	/**
2759	* Removes the sessions IDT references.
2760	* This will uninstall our IDT patch if we left unreferenced.
2761	*
2762	* @returns VINF_SUCCESS.
2763	* @param pDevExt Device globals.
2764	* @param pSession Session data.
2765	*/
2766	static int supdrvIOCtl_IdtRemoveAll(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession)
2767	{
2768	PSUPDRVPATCHUSAGE pUsage;
2769	RTSPINLOCKTMP SpinlockTmp = RTSPINLOCKTMP_INITIALIZER;
2770	dprintf(("supdrvIOCtl_IdtRemoveAll: pSession=%p\n", pSession));
2771
2772	/*
2773	* Take the spinlock.
2774	*/
2775	RTSpinlockAcquireNoInts(pDevExt->Spinlock, &SpinlockTmp);
2776
2777	/*
2778	* Walk usage list, removing patches as their usage count reaches zero.
2779	*/
2780	pUsage = pSession->pPatchUsage;
2781	while (pUsage)
2782	{
2783	if (pUsage->pPatch->cUsage <= pUsage->cUsage)
2784	supdrvIdtRemoveOne(pDevExt, pUsage->pPatch);
2785	else
2786	pUsage->pPatch->cUsage -= pUsage->cUsage;
2787
2788	/* next */
2789	pUsage = pUsage->pNext;
2790	}
2791
2792	/*
2793	* Empty the usage chain and we're done inside the spinlock.
2794	*/
2795	pUsage = pSession->pPatchUsage;
2796	pSession->pPatchUsage = NULL;
2797
2798	RTSpinlockReleaseNoInts(pDevExt->Spinlock, &SpinlockTmp);
2799
2800	/*
2801	* Free usage entries.
2802	*/
2803	while (pUsage)
2804	{
2805	void *pvToFree = pUsage;
2806	pUsage->cUsage = 0;
2807	pUsage->pPatch = NULL;
2808	pUsage = pUsage->pNext;
2809	RTMemFree(pvToFree);
2810	}
2811
2812	return VINF_SUCCESS;
2813	}
2814
2815
2816	/**
2817	* Remove one patch.
2818	*
2819	* Worker for supdrvIOCtl_IdtRemoveAll.
2820	*
2821	* @param pDevExt Device globals.
2822	* @param pPatch Patch entry to remove.
2823	* @remark Caller must own SUPDRVDEVEXT::Spinlock!
2824	*/
2825	static void supdrvIdtRemoveOne(PSUPDRVDEVEXT pDevExt, PSUPDRVPATCH pPatch)
2826	{
2827	dprintf(("supdrvIdtRemoveOne: pPatch=%p\n", pPatch));
2828
2829	pPatch->cUsage = 0;
2830
2831	/*
2832	* If the IDT entry was changed it have to kick around for ever!
2833	* This will be attempted freed again, perhaps next time we'll succeed :-)
2834	*/
2835	if (memcmp((void *)pPatch->pIdtEntry, &pPatch->ChangedIdt, sizeof(pPatch->ChangedIdt)))
2836	{
2837	AssertMsgFailed(("The hijacked IDT entry has CHANGED!!!\n"));
2838	return;
2839	}
2840
2841	/*
2842	* Unlink it.
2843	*/
2844	if (pDevExt->pIdtPatches != pPatch)
2845	{
2846	PSUPDRVPATCH pPatchPrev = pDevExt->pIdtPatches;
2847	while (pPatchPrev)
2848	{
2849	if (pPatchPrev->pNext == pPatch)
2850	{
2851	pPatchPrev->pNext = pPatch->pNext;
2852	break;
2853	}
2854	pPatchPrev = pPatchPrev->pNext;
2855	}
2856	Assert(!pPatchPrev);
2857	}
2858	else
2859	pDevExt->pIdtPatches = pPatch->pNext;
2860	pPatch->pNext = NULL;
2861
2862
2863	/*
2864	* Verify and restore the IDT.
2865	*/
2866	AssertMsg(!memcmp((void *)pPatch->pIdtEntry, &pPatch->ChangedIdt, sizeof(pPatch->ChangedIdt)), ("The hijacked IDT entry has CHANGED!!!\n"));
2867	supdrvIdtWrite(pPatch->pIdtEntry, &pPatch->SavedIdt);
2868	AssertMsg(!memcmp((void *)pPatch->pIdtEntry, &pPatch->SavedIdt, sizeof(pPatch->SavedIdt)), ("The hijacked IDT entry has CHANGED!!!\n"));
2869
2870	/*
2871	* Put it in the free list.
2872	* (This free list stuff is to calm my paranoia.)
2873	*/
2874	pPatch->pvIdt = NULL;
2875	pPatch->pIdtEntry = NULL;
2876
2877	pPatch->pNext = pDevExt->pIdtPatchesFree;
2878	pDevExt->pIdtPatchesFree = pPatch;
2879	}
2880
2881
2882	/**
2883	* Write to an IDT entry.
2884	*
2885	* @param pvIdtEntry Where to write.
2886	* @param pNewIDTEntry What to write.
2887	*/
2888	static void supdrvIdtWrite(volatile void pvIdtEntry, const SUPDRVIDTE pNewIDTEntry)
2889	{
2890	RTUINTREG uCR0;
2891	RTUINTREG uFlags;
2892
2893	/*
2894	* On SMP machines (P4 hyperthreading included) we must preform a
2895	* 64-bit locked write when updating the IDT entry.
2896	*
2897	* The F00F bugfix for linux (and probably other OSes) causes
2898	* the IDT to be pointing to an readonly mapping. We get around that
2899	* by temporarily turning of WP. Since we're inside a spinlock at this
2900	* point, interrupts are disabled and there isn't any way the WP bit
2901	* flipping can cause any trouble.
2902	*/
2903
2904	/* Save & Clear interrupt flag; Save & clear WP. */
2905	uFlags = ASMGetFlags();
2906	ASMSetFlags(uFlags & ~(RTUINTREG)(1 << 9)); /X86_EFL_IF/
2907	Assert(!(ASMGetFlags() & (1 << 9)));
2908	uCR0 = ASMGetCR0();
2909	ASMSetCR0(uCR0 & ~(RTUINTREG)(1 << 16)); /X86_CR0_WP/
2910
2911	/* Update IDT Entry */
2912	#ifdef RT_ARCH_AMD64
2913	ASMAtomicXchgU128((volatile uint128_t )pvIdtEntry, (uint128_t *)(uintptr_t)pNewIDTEntry);
2914	#else
2915	ASMAtomicXchgU64((volatile uint64_t )pvIdtEntry, (uint64_t *)(uintptr_t)pNewIDTEntry);
2916	#endif
2917
2918	/* Restore CR0 & Flags */
2919	ASMSetCR0(uCR0);
2920	ASMSetFlags(uFlags);
2921	}
2922	#endif /* VBOX_WITH_IDT_PATCHING */
2923
2924
2925	/**
2926	* Opens an image. If it's the first time it's opened the call must upload
2927	* the bits using the supdrvIOCtl_LdrLoad() / SUPDRV_IOCTL_LDR_LOAD function.
2928	*
2929	* This is the 1st step of the loading.
2930	*
2931	* @returns IPRT status code.
2932	* @param pDevExt Device globals.
2933	* @param pSession Session data.
2934	* @param pReq The open request.
2935	*/
2936	static int supdrvIOCtl_LdrOpen(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPLDROPEN pReq)
2937	{
2938	PSUPDRVLDRIMAGE pImage;
2939	unsigned cb;
2940	void *pv;
2941	dprintf(("supdrvIOCtl_LdrOpen: szName=%s cbImage=%d\n", pReq->u.In.szName, pReq->u.In.cbImage));
2942
2943	/*
2944	* Check if we got an instance of the image already.
2945	*/
2946	RTSemFastMutexRequest(pDevExt->mtxLdr);
2947	for (pImage = pDevExt->pLdrImages; pImage; pImage = pImage->pNext)
2948	{
2949	if (!strcmp(pImage->szName, pReq->u.In.szName))
2950	{
2951	pImage->cUsage++;
2952	pReq->u.Out.pvImageBase = pImage->pvImage;
2953	pReq->u.Out.fNeedsLoading = pImage->uState == SUP_IOCTL_LDR_OPEN;
2954	supdrvLdrAddUsage(pSession, pImage);
2955	RTSemFastMutexRelease(pDevExt->mtxLdr);
2956	return VINF_SUCCESS;
2957	}
2958	}
2959	/* (not found - add it!) */
2960
2961	/*
2962	* Allocate memory.
2963	*/
2964	cb = pReq->u.In.cbImage + sizeof(SUPDRVLDRIMAGE) + 31;
2965	pv = RTMemExecAlloc(cb);
2966	if (!pv)
2967	{
2968	RTSemFastMutexRelease(pDevExt->mtxLdr);
2969	return VERR_NO_MEMORY;
2970	}
2971
2972	/*
2973	* Setup and link in the LDR stuff.
2974	*/
2975	pImage = (PSUPDRVLDRIMAGE)pv;
2976	pImage->pvImage = ALIGNP(pImage + 1, 32);
2977	pImage->cbImage = pReq->u.In.cbImage;
2978	pImage->pfnModuleInit = NULL;
2979	pImage->pfnModuleTerm = NULL;
2980	pImage->uState = SUP_IOCTL_LDR_OPEN;
2981	pImage->cUsage = 1;
2982	strcpy(pImage->szName, pReq->u.In.szName);
2983
2984	pImage->pNext = pDevExt->pLdrImages;
2985	pDevExt->pLdrImages = pImage;
2986
2987	supdrvLdrAddUsage(pSession, pImage);
2988
2989	pReq->u.Out.pvImageBase = pImage->pvImage;
2990	pReq->u.Out.fNeedsLoading = true;
2991	RTSemFastMutexRelease(pDevExt->mtxLdr);
2992	return VINF_SUCCESS;
2993	}
2994
2995
2996	/**
2997	* Loads the image bits.
2998	*
2999	* This is the 2nd step of the loading.
3000	*
3001	* @returns IPRT status code.
3002	* @param pDevExt Device globals.
3003	* @param pSession Session data.
3004	* @param pReq The request.
3005	*/
3006	static int supdrvIOCtl_LdrLoad(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPLDRLOAD pReq)
3007	{
3008	PSUPDRVLDRUSAGE pUsage;
3009	PSUPDRVLDRIMAGE pImage;
3010	int rc;
3011	dprintf(("supdrvIOCtl_LdrLoad: pvImageBase=%p cbImage=%d\n", pReq->u.In.pvImageBase, pReq->u.In.cbImage));
3012
3013	/*
3014	* Find the ldr image.
3015	*/
3016	RTSemFastMutexRequest(pDevExt->mtxLdr);
3017	pUsage = pSession->pLdrUsage;
3018	while (pUsage && pUsage->pImage->pvImage != pReq->u.In.pvImageBase)
3019	pUsage = pUsage->pNext;
3020	if (!pUsage)
3021	{
3022	RTSemFastMutexRelease(pDevExt->mtxLdr);
3023	dprintf(("SUP_IOCTL_LDR_LOAD: couldn't find image!\n"));
3024	return VERR_INVALID_HANDLE;
3025	}
3026	pImage = pUsage->pImage;
3027	if (pImage->cbImage != pReq->u.In.cbImage)
3028	{
3029	RTSemFastMutexRelease(pDevExt->mtxLdr);
3030	dprintf(("SUP_IOCTL_LDR_LOAD: image size mismatch!! %d(prep) != %d(load)\n", pImage->cbImage, pReq->u.In.cbImage));
3031	return VERR_INVALID_HANDLE;
3032	}
3033	if (pImage->uState != SUP_IOCTL_LDR_OPEN)
3034	{
3035	unsigned uState = pImage->uState;
3036	RTSemFastMutexRelease(pDevExt->mtxLdr);
3037	if (uState != SUP_IOCTL_LDR_LOAD)
3038	AssertMsgFailed(("SUP_IOCTL_LDR_LOAD: invalid image state %d (%#x)!\n", uState, uState));
3039	return SUPDRV_ERR_ALREADY_LOADED;
3040	}
3041	switch (pReq->u.In.eEPType)
3042	{
3043	case SUPLDRLOADEP_NOTHING:
3044	break;
3045	case SUPLDRLOADEP_VMMR0:
3046	if ( !pReq->u.In.EP.VMMR0.pvVMMR0
3047	\|\| !pReq->u.In.EP.VMMR0.pvVMMR0EntryInt
3048	\|\| !pReq->u.In.EP.VMMR0.pvVMMR0EntryFast
3049	\|\| !pReq->u.In.EP.VMMR0.pvVMMR0EntryEx)
3050	{
3051	RTSemFastMutexRelease(pDevExt->mtxLdr);
3052	dprintf(("NULL pointer: pvVMMR0=%p pvVMMR0EntryInt=%p pvVMMR0EntryFast=%p pvVMMR0EntryEx=%p!\n",
3053	pReq->u.In.EP.VMMR0.pvVMMR0, pReq->u.In.EP.VMMR0.pvVMMR0EntryInt,
3054	pReq->u.In.EP.VMMR0.pvVMMR0EntryFast, pReq->u.In.EP.VMMR0.pvVMMR0EntryEx));
3055	return VERR_INVALID_PARAMETER;
3056	}
3057	if ( (uintptr_t)pReq->u.In.EP.VMMR0.pvVMMR0EntryInt - (uintptr_t)pImage->pvImage >= pReq->u.In.cbImage
3058	\|\| (uintptr_t)pReq->u.In.EP.VMMR0.pvVMMR0EntryFast - (uintptr_t)pImage->pvImage >= pReq->u.In.cbImage
3059	\|\| (uintptr_t)pReq->u.In.EP.VMMR0.pvVMMR0EntryEx - (uintptr_t)pImage->pvImage >= pReq->u.In.cbImage)
3060	{
3061	RTSemFastMutexRelease(pDevExt->mtxLdr);
3062	dprintf(("Out of range (%p LB %#x): pvVMMR0EntryInt=%p, pvVMMR0EntryFast=%p or pvVMMR0EntryEx=%p is NULL!\n",
3063	pImage->pvImage, pReq->u.In.cbImage, pReq->u.In.EP.VMMR0.pvVMMR0EntryInt,
3064	pReq->u.In.EP.VMMR0.pvVMMR0EntryFast, pReq->u.In.EP.VMMR0.pvVMMR0EntryEx));
3065	return VERR_INVALID_PARAMETER;
3066	}
3067	break;
3068	default:
3069	RTSemFastMutexRelease(pDevExt->mtxLdr);
3070	dprintf(("Invalid eEPType=%d\n", pReq->u.In.eEPType));
3071	return VERR_INVALID_PARAMETER;
3072	}
3073	if ( pReq->u.In.pfnModuleInit
3074	&& (uintptr_t)pReq->u.In.pfnModuleInit - (uintptr_t)pImage->pvImage >= pReq->u.In.cbImage)
3075	{
3076	RTSemFastMutexRelease(pDevExt->mtxLdr);
3077	dprintf(("SUP_IOCTL_LDR_LOAD: pfnModuleInit=%p is outside the image (%p %d bytes)\n",
3078	pReq->u.In.pfnModuleInit, pImage->pvImage, pReq->u.In.cbImage));
3079	return VERR_INVALID_PARAMETER;
3080	}
3081	if ( pReq->u.In.pfnModuleTerm
3082	&& (uintptr_t)pReq->u.In.pfnModuleTerm - (uintptr_t)pImage->pvImage >= pReq->u.In.cbImage)
3083	{
3084	RTSemFastMutexRelease(pDevExt->mtxLdr);
3085	dprintf(("SUP_IOCTL_LDR_LOAD: pfnModuleTerm=%p is outside the image (%p %d bytes)\n",
3086	pReq->u.In.pfnModuleTerm, pImage->pvImage, pReq->u.In.cbImage));
3087	return VERR_INVALID_PARAMETER;
3088	}
3089
3090	/*
3091	* Copy the memory.
3092	*/
3093	/* no need to do try/except as this is a buffered request. */
3094	memcpy(pImage->pvImage, &pReq->u.In.achImage[0], pImage->cbImage);
3095	pImage->uState = SUP_IOCTL_LDR_LOAD;
3096	pImage->pfnModuleInit = pReq->u.In.pfnModuleInit;
3097	pImage->pfnModuleTerm = pReq->u.In.pfnModuleTerm;
3098	pImage->offSymbols = pReq->u.In.offSymbols;
3099	pImage->cSymbols = pReq->u.In.cSymbols;
3100	pImage->offStrTab = pReq->u.In.offStrTab;
3101	pImage->cbStrTab = pReq->u.In.cbStrTab;
3102
3103	/*
3104	* Update any entry points.
3105	*/
3106	switch (pReq->u.In.eEPType)
3107	{
3108	default:
3109	case SUPLDRLOADEP_NOTHING:
3110	rc = VINF_SUCCESS;
3111	break;
3112	case SUPLDRLOADEP_VMMR0:
3113	rc = supdrvLdrSetR0EP(pDevExt, pReq->u.In.EP.VMMR0.pvVMMR0, pReq->u.In.EP.VMMR0.pvVMMR0EntryInt,
3114	pReq->u.In.EP.VMMR0.pvVMMR0EntryFast, pReq->u.In.EP.VMMR0.pvVMMR0EntryEx);
3115	break;
3116	}
3117
3118	/*
3119	* On success call the module initialization.
3120	*/
3121	dprintf(("supdrvIOCtl_LdrLoad: pfnModuleInit=%p\n", pImage->pfnModuleInit));
3122	if (RT_SUCCESS(rc) && pImage->pfnModuleInit)
3123	{
3124	dprintf(("supdrvIOCtl_LdrLoad: calling pfnModuleInit=%p\n", pImage->pfnModuleInit));
3125	rc = pImage->pfnModuleInit();
3126	if (rc && pDevExt->pvVMMR0 == pImage->pvImage)
3127	supdrvLdrUnsetR0EP(pDevExt);
3128	}
3129
3130	if (rc)
3131	pImage->uState = SUP_IOCTL_LDR_OPEN;
3132
3133	RTSemFastMutexRelease(pDevExt->mtxLdr);
3134	return rc;
3135	}
3136
3137
3138	/**
3139	* Frees a previously loaded (prep'ed) image.
3140	*
3141	* @returns IPRT status code.
3142	* @param pDevExt Device globals.
3143	* @param pSession Session data.
3144	* @param pReq The request.
3145	*/
3146	static int supdrvIOCtl_LdrFree(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPLDRFREE pReq)
3147	{
3148	int rc;
3149	PSUPDRVLDRUSAGE pUsagePrev;
3150	PSUPDRVLDRUSAGE pUsage;
3151	PSUPDRVLDRIMAGE pImage;
3152	dprintf(("supdrvIOCtl_LdrFree: pvImageBase=%p\n", pReq->u.In.pvImageBase));
3153
3154	/*
3155	* Find the ldr image.
3156	*/
3157	RTSemFastMutexRequest(pDevExt->mtxLdr);
3158	pUsagePrev = NULL;
3159	pUsage = pSession->pLdrUsage;
3160	while (pUsage && pUsage->pImage->pvImage != pReq->u.In.pvImageBase)
3161	{
3162	pUsagePrev = pUsage;
3163	pUsage = pUsage->pNext;
3164	}
3165	if (!pUsage)
3166	{
3167	RTSemFastMutexRelease(pDevExt->mtxLdr);
3168	dprintf(("SUP_IOCTL_LDR_FREE: couldn't find image!\n"));
3169	return VERR_INVALID_HANDLE;
3170	}
3171
3172	/*
3173	* Check if we can remove anything.
3174	*/
3175	rc = VINF_SUCCESS;
3176	pImage = pUsage->pImage;
3177	if (pImage->cUsage <= 1 \|\| pUsage->cUsage <= 1)
3178	{
3179	/*
3180	* Check if there are any objects with destructors in the image, if
3181	* so leave it for the session cleanup routine so we get a chance to
3182	* clean things up in the right order and not leave them all dangling.
3183	*/
3184	RTSPINLOCKTMP SpinlockTmp = RTSPINLOCKTMP_INITIALIZER;
3185	RTSpinlockAcquire(pDevExt->Spinlock, &SpinlockTmp);
3186	if (pImage->cUsage <= 1)
3187	{
3188	PSUPDRVOBJ pObj;
3189	for (pObj = pDevExt->pObjs; pObj; pObj = pObj->pNext)
3190	if (RT_UNLIKELY((uintptr_t)pObj->pfnDestructor - (uintptr_t)pImage->pvImage < pImage->cbImage))
3191	{
3192	rc = VERR_SHARING_VIOLATION; /** @todo VERR_DANGLING_OBJECTS */
3193	break;
3194	}
3195	}
3196	else
3197	{
3198	PSUPDRVUSAGE pGenUsage;
3199	for (pGenUsage = pSession->pUsage; pGenUsage; pGenUsage = pGenUsage->pNext)
3200	if (RT_UNLIKELY((uintptr_t)pGenUsage->pObj->pfnDestructor - (uintptr_t)pImage->pvImage < pImage->cbImage))
3201	{
3202	rc = VERR_SHARING_VIOLATION; /** @todo VERR_DANGLING_OBJECTS */
3203	break;
3204	}
3205	}
3206	RTSpinlockRelease(pDevExt->Spinlock, &SpinlockTmp);
3207	if (rc == VINF_SUCCESS)
3208	{
3209	/* unlink it */
3210	if (pUsagePrev)
3211	pUsagePrev->pNext = pUsage->pNext;
3212	else
3213	pSession->pLdrUsage = pUsage->pNext;
3214
3215	/* free it */
3216	pUsage->pImage = NULL;
3217	pUsage->pNext = NULL;
3218	RTMemFree(pUsage);
3219
3220	/*
3221	* Derefrence the image.
3222	*/
3223	if (pImage->cUsage <= 1)
3224	supdrvLdrFree(pDevExt, pImage);
3225	else
3226	pImage->cUsage--;
3227	}
3228	}
3229	else
3230	{
3231	/*
3232	* Dereference both image and usage.
3233	*/
3234	pImage->cUsage--;
3235	pUsage->cUsage--;
3236	}
3237
3238	RTSemFastMutexRelease(pDevExt->mtxLdr);
3239	return VINF_SUCCESS;
3240	}
3241
3242
3243	/**
3244	* Gets the address of a symbol in an open image.
3245	*
3246	* @returns 0 on success.
3247	* @returns SUPDRV_ERR_* on failure.
3248	* @param pDevExt Device globals.
3249	* @param pSession Session data.
3250	* @param pReq The request buffer.
3251	*/
3252	static int supdrvIOCtl_LdrGetSymbol(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPLDRGETSYMBOL pReq)
3253	{
3254	PSUPDRVLDRIMAGE pImage;
3255	PSUPDRVLDRUSAGE pUsage;
3256	uint32_t i;
3257	PSUPLDRSYM paSyms;
3258	const char *pchStrings;
3259	const size_t cbSymbol = strlen(pReq->u.In.szSymbol) + 1;
3260	void *pvSymbol = NULL;
3261	int rc = VERR_GENERAL_FAILURE;
3262	dprintf2(("supdrvIOCtl_LdrGetSymbol: pvImageBase=%p szSymbol=\"%s\"\n", pReq->u.In.pvImageBase, pReq->u.In.szSymbol));
3263
3264	/*
3265	* Find the ldr image.
3266	*/
3267	RTSemFastMutexRequest(pDevExt->mtxLdr);
3268	pUsage = pSession->pLdrUsage;
3269	while (pUsage && pUsage->pImage->pvImage != pReq->u.In.pvImageBase)
3270	pUsage = pUsage->pNext;
3271	if (!pUsage)
3272	{
3273	RTSemFastMutexRelease(pDevExt->mtxLdr);
3274	dprintf(("SUP_IOCTL_LDR_GET_SYMBOL: couldn't find image!\n"));
3275	return VERR_INVALID_HANDLE;
3276	}
3277	pImage = pUsage->pImage;
3278	if (pImage->uState != SUP_IOCTL_LDR_LOAD)
3279	{
3280	unsigned uState = pImage->uState;
3281	RTSemFastMutexRelease(pDevExt->mtxLdr);
3282	dprintf(("SUP_IOCTL_LDR_GET_SYMBOL: invalid image state %d (%#x)!\n", uState, uState)); NOREF(uState);
3283	return VERR_ALREADY_LOADED;
3284	}
3285
3286	/*
3287	* Search the symbol string.
3288	*/
3289	pchStrings = (const char )((uint8_t )pImage->pvImage + pImage->offStrTab);
3290	paSyms = (PSUPLDRSYM)((uint8_t *)pImage->pvImage + pImage->offSymbols);
3291	for (i = 0; i < pImage->cSymbols; i++)
3292	{
3293	if ( paSyms[i].offSymbol < pImage->cbImage /* paranoia */
3294	&& paSyms[i].offName + cbSymbol <= pImage->cbStrTab
3295	&& !memcmp(pchStrings + paSyms[i].offName, pReq->u.In.szSymbol, cbSymbol))
3296	{
3297	pvSymbol = (uint8_t *)pImage->pvImage + paSyms[i].offSymbol;
3298	rc = VINF_SUCCESS;
3299	break;
3300	}
3301	}
3302	RTSemFastMutexRelease(pDevExt->mtxLdr);
3303	pReq->u.Out.pvSymbol = pvSymbol;
3304	return rc;
3305	}
3306
3307
3308	/**
3309	* Updates the IDT patches to point to the specified VMM R0 entry
3310	* point (i.e. VMMR0Enter()).
3311	*
3312	* @returns IPRT status code.
3313	* @param pDevExt Device globals.
3314	* @param pSession Session data.
3315	* @param pVMMR0 VMMR0 image handle.
3316	* @param pvVMMR0EntryInt VMMR0EntryInt address.
3317	* @param pvVMMR0EntryFast VMMR0EntryFast address.
3318	* @param pvVMMR0EntryEx VMMR0EntryEx address.
3319	* @remark Caller must own the loader mutex.
3320	*/
3321	static int supdrvLdrSetR0EP(PSUPDRVDEVEXT pDevExt, void pvVMMR0, void pvVMMR0EntryInt, void pvVMMR0EntryFast, void pvVMMR0EntryEx)
3322	{
3323	int rc = VINF_SUCCESS;
3324	dprintf(("supdrvLdrSetR0EP pvVMMR0=%p pvVMMR0EntryInt=%p\n", pvVMMR0, pvVMMR0EntryInt));
3325
3326
3327	/*
3328	* Check if not yet set.
3329	*/
3330	if (!pDevExt->pvVMMR0)
3331	{
3332	#ifdef VBOX_WITH_IDT_PATCHING
3333	PSUPDRVPATCH pPatch;
3334	#endif
3335
3336	/*
3337	* Set it and update IDT patch code.
3338	*/
3339	pDevExt->pvVMMR0 = pvVMMR0;
3340	pDevExt->pfnVMMR0EntryInt = pvVMMR0EntryInt;
3341	pDevExt->pfnVMMR0EntryFast = pvVMMR0EntryFast;
3342	pDevExt->pfnVMMR0EntryEx = pvVMMR0EntryEx;
3343	#ifdef VBOX_WITH_IDT_PATCHING
3344	for (pPatch = pDevExt->pIdtPatches; pPatch; pPatch = pPatch->pNext)
3345	{
3346	# ifdef RT_ARCH_AMD64
3347	ASMAtomicXchgU64((volatile uint64_t *)&pPatch->auCode[pPatch->offVMMR0EntryFixup], (uint64_t)pvVMMR0);
3348	# else /* RT_ARCH_X86 */
3349	ASMAtomicXchgU32((volatile uint32_t *)&pPatch->auCode[pPatch->offVMMR0EntryFixup],
3350	(uint32_t)pvVMMR0 - (uint32_t)&pPatch->auCode[pPatch->offVMMR0EntryFixup + 4]);
3351	# endif
3352	}
3353	#endif /* VBOX_WITH_IDT_PATCHING */
3354	}
3355	else
3356	{
3357	/*
3358	* Return failure or success depending on whether the values match or not.
3359	*/
3360	if ( pDevExt->pvVMMR0 != pvVMMR0
3361	\|\| (void *)pDevExt->pfnVMMR0EntryInt != pvVMMR0EntryInt
3362	\|\| (void *)pDevExt->pfnVMMR0EntryFast != pvVMMR0EntryFast
3363	\|\| (void *)pDevExt->pfnVMMR0EntryEx != pvVMMR0EntryEx)
3364	{
3365	AssertMsgFailed(("SUP_IOCTL_LDR_SETR0EP: Already set pointing to a different module!\n"));
3366	rc = VERR_INVALID_PARAMETER;
3367	}
3368	}
3369	return rc;
3370	}
3371
3372
3373	/**
3374	* Unsets the R0 entry point installed by supdrvLdrSetR0EP.
3375	*
3376	* @param pDevExt Device globals.
3377	*/
3378	static void supdrvLdrUnsetR0EP(PSUPDRVDEVEXT pDevExt)
3379	{
3380	#ifdef VBOX_WITH_IDT_PATCHING
3381	PSUPDRVPATCH pPatch;
3382	#endif
3383
3384	pDevExt->pvVMMR0 = NULL;
3385	pDevExt->pfnVMMR0EntryInt = NULL;
3386	pDevExt->pfnVMMR0EntryFast = NULL;
3387	pDevExt->pfnVMMR0EntryEx = NULL;
3388
3389	#ifdef VBOX_WITH_IDT_PATCHING
3390	for (pPatch = pDevExt->pIdtPatches; pPatch; pPatch = pPatch->pNext)
3391	{
3392	# ifdef RT_ARCH_AMD64
3393	ASMAtomicXchgU64((volatile uint64_t *)&pPatch->auCode[pPatch->offVMMR0EntryFixup],
3394	(uint64_t)&pPatch->auCode[pPatch->offStub]);
3395	# else /* RT_ARCH_X86 */
3396	ASMAtomicXchgU32((volatile uint32_t *)&pPatch->auCode[pPatch->offVMMR0EntryFixup],
3397	(uint32_t)&pPatch->auCode[pPatch->offStub] - (uint32_t)&pPatch->auCode[pPatch->offVMMR0EntryFixup + 4]);
3398	# endif
3399	}
3400	#endif /* VBOX_WITH_IDT_PATCHING */
3401	}
3402
3403
3404	/**
3405	* Adds a usage reference in the specified session of an image.
3406	*
3407	* @param pSession Session in question.
3408	* @param pImage Image which the session is using.
3409	*/
3410	static void supdrvLdrAddUsage(PSUPDRVSESSION pSession, PSUPDRVLDRIMAGE pImage)
3411	{
3412	PSUPDRVLDRUSAGE pUsage;
3413	dprintf(("supdrvLdrAddUsage: pImage=%p\n", pImage));
3414
3415	/*
3416	* Referenced it already?
3417	*/
3418	pUsage = pSession->pLdrUsage;
3419	while (pUsage)
3420	{
3421	if (pUsage->pImage == pImage)
3422	{
3423	pUsage->cUsage++;
3424	return;
3425	}
3426	pUsage = pUsage->pNext;
3427	}
3428
3429	/*
3430	* Allocate new usage record.
3431	*/
3432	pUsage = (PSUPDRVLDRUSAGE)RTMemAlloc(sizeof(*pUsage));
3433	Assert(pUsage);
3434	if (pUsage)
3435	{
3436	pUsage->cUsage = 1;
3437	pUsage->pImage = pImage;
3438	pUsage->pNext = pSession->pLdrUsage;
3439	pSession->pLdrUsage = pUsage;
3440	}
3441	/* ignore errors... */
3442	}
3443
3444
3445	/**
3446	* Frees a load image.
3447	*
3448	* @param pDevExt Pointer to device extension.
3449	* @param pImage Pointer to the image we're gonna free.
3450	* This image must exit!
3451	* @remark The caller MUST own SUPDRVDEVEXT::mtxLdr!
3452	*/
3453	static void supdrvLdrFree(PSUPDRVDEVEXT pDevExt, PSUPDRVLDRIMAGE pImage)
3454	{
3455	PSUPDRVLDRIMAGE pImagePrev;
3456	dprintf(("supdrvLdrFree: pImage=%p\n", pImage));
3457
3458	/* find it - arg. should've used doubly linked list. */
3459	Assert(pDevExt->pLdrImages);
3460	pImagePrev = NULL;
3461	if (pDevExt->pLdrImages != pImage)
3462	{
3463	pImagePrev = pDevExt->pLdrImages;
3464	while (pImagePrev->pNext != pImage)
3465	pImagePrev = pImagePrev->pNext;
3466	Assert(pImagePrev->pNext == pImage);
3467	}
3468
3469	/* unlink */
3470	if (pImagePrev)
3471	pImagePrev->pNext = pImage->pNext;
3472	else
3473	pDevExt->pLdrImages = pImage->pNext;
3474
3475	/* check if this is VMMR0.r0 and fix the Idt patches if it is. */
3476	if (pDevExt->pvVMMR0 == pImage->pvImage)
3477	supdrvLdrUnsetR0EP(pDevExt);
3478
3479	/* check for objects with destructors in this image. (Shouldn't happen.) */
3480	if (pDevExt->pObjs)
3481	{
3482	unsigned cObjs = 0;
3483	PSUPDRVOBJ pObj;
3484	RTSPINLOCKTMP SpinlockTmp = RTSPINLOCKTMP_INITIALIZER;
3485	RTSpinlockAcquire(pDevExt->Spinlock, &SpinlockTmp);
3486	for (pObj = pDevExt->pObjs; pObj; pObj = pObj->pNext)
3487	if (RT_UNLIKELY((uintptr_t)pObj->pfnDestructor - (uintptr_t)pImage->pvImage < pImage->cbImage))
3488	{
3489	pObj->pfnDestructor = NULL;
3490	cObjs++;
3491	}
3492	RTSpinlockRelease(pDevExt->Spinlock, &SpinlockTmp);
3493	if (cObjs)
3494	OSDBGPRINT(("supdrvLdrFree: Image '%s' has %d dangling objects!\n", pImage->szName, cObjs));
3495	}
3496
3497	/* call termination function if fully loaded. */
3498	if ( pImage->pfnModuleTerm
3499	&& pImage->uState == SUP_IOCTL_LDR_LOAD)
3500	{
3501	dprintf(("supdrvIOCtl_LdrLoad: calling pfnModuleTerm=%p\n", pImage->pfnModuleTerm));
3502	pImage->pfnModuleTerm();
3503	}
3504
3505	/* free the image */
3506	pImage->cUsage = 0;
3507	pImage->pNext = 0;
3508	pImage->uState = SUP_IOCTL_LDR_FREE;
3509	RTMemExecFree(pImage);
3510	}
3511
3512
3513	/**
3514	* Gets the current paging mode of the CPU and stores in in pOut.
3515	*/
3516	static SUPPAGINGMODE supdrvIOCtl_GetPagingMode(void)
3517	{
3518	SUPPAGINGMODE enmMode;
3519
3520	RTUINTREG cr0 = ASMGetCR0();
3521	if ((cr0 & (X86_CR0_PG \| X86_CR0_PE)) != (X86_CR0_PG \| X86_CR0_PE))
3522	enmMode = SUPPAGINGMODE_INVALID;
3523	else
3524	{
3525	RTUINTREG cr4 = ASMGetCR4();
3526	uint32_t fNXEPlusLMA = 0;
3527	if (cr4 & X86_CR4_PAE)
3528	{
3529	uint32_t fAmdFeatures = ASMCpuId_EDX(0x80000001);
3530	if (fAmdFeatures & (X86_CPUID_AMD_FEATURE_EDX_NX \| X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
3531	{
3532	uint64_t efer = ASMRdMsr(MSR_K6_EFER);
3533	if ((fAmdFeatures & X86_CPUID_AMD_FEATURE_EDX_NX) && (efer & MSR_K6_EFER_NXE))
3534	fNXEPlusLMA \|= BIT(0);
3535	if ((fAmdFeatures & X86_CPUID_AMD_FEATURE_EDX_LONG_MODE) && (efer & MSR_K6_EFER_LMA))
3536	fNXEPlusLMA \|= BIT(1);
3537	}
3538	}
3539
3540	switch ((cr4 & (X86_CR4_PAE \| X86_CR4_PGE)) \| fNXEPlusLMA)
3541	{
3542	case 0:
3543	enmMode = SUPPAGINGMODE_32_BIT;
3544	break;
3545
3546	case X86_CR4_PGE:
3547	enmMode = SUPPAGINGMODE_32_BIT_GLOBAL;
3548	break;
3549
3550	case X86_CR4_PAE:
3551	enmMode = SUPPAGINGMODE_PAE;
3552	break;
3553
3554	case X86_CR4_PAE \| BIT(0):
3555	enmMode = SUPPAGINGMODE_PAE_NX;
3556	break;
3557
3558	case X86_CR4_PAE \| X86_CR4_PGE:
3559	enmMode = SUPPAGINGMODE_PAE_GLOBAL;
3560	break;
3561
3562	case X86_CR4_PAE \| X86_CR4_PGE \| BIT(0):
3563	enmMode = SUPPAGINGMODE_PAE_GLOBAL;
3564	break;
3565
3566	case BIT(1) \| X86_CR4_PAE:
3567	enmMode = SUPPAGINGMODE_AMD64;
3568	break;
3569
3570	case BIT(1) \| X86_CR4_PAE \| BIT(0):
3571	enmMode = SUPPAGINGMODE_AMD64_NX;
3572	break;
3573
3574	case BIT(1) \| X86_CR4_PAE \| X86_CR4_PGE:
3575	enmMode = SUPPAGINGMODE_AMD64_GLOBAL;
3576	break;
3577
3578	case BIT(1) \| X86_CR4_PAE \| X86_CR4_PGE \| BIT(0):
3579	enmMode = SUPPAGINGMODE_AMD64_GLOBAL_NX;
3580	break;
3581
3582	default:
3583	AssertMsgFailed(("Cannot happen! cr4=%#x fNXEPlusLMA=%d\n", cr4, fNXEPlusLMA));
3584	enmMode = SUPPAGINGMODE_INVALID;
3585	break;
3586	}
3587	}
3588	return enmMode;
3589	}
3590
3591
3592	#ifdef USE_NEW_OS_INTERFACE_FOR_GIP
3593	/**
3594	* Creates the GIP.
3595	*
3596	* @returns negative errno.
3597	* @param pDevExt Instance data. GIP stuff may be updated.
3598	*/
3599	static int supdrvGipCreate(PSUPDRVDEVEXT pDevExt)
3600	{
3601	PSUPGLOBALINFOPAGE pGip;
3602	RTHCPHYS HCPhysGip;
3603	uint32_t u32SystemResolution;
3604	uint32_t u32Interval;
3605	int rc;
3606
3607	dprintf(("supdrvGipCreate:\n"));
3608
3609	/* assert order */
3610	Assert(pDevExt->u32SystemTimerGranularityGrant == 0);
3611	Assert(pDevExt->GipMemObj == NIL_RTR0MEMOBJ);
3612	Assert(!pDevExt->pGipTimer);
3613
3614	/*
3615	* Allocate a suitable page with a default kernel mapping.
3616	*/
3617	rc = RTR0MemObjAllocLow(&pDevExt->GipMemObj, PAGE_SIZE, false);
3618	if (RT_FAILURE(rc))
3619	{
3620	OSDBGPRINT(("supdrvGipCreate: failed to allocate the GIP page. rc=%d\n", rc));
3621	return rc;
3622	}
3623	pGip = (PSUPGLOBALINFOPAGE)RTR0MemObjAddress(pDevExt->GipMemObj); AssertPtr(pGip);
3624	HCPhysGip = RTR0MemObjGetPagePhysAddr(pDevExt->GipMemObj, 0); Assert(HCPhysGip != NIL_RTHCPHYS);
3625
3626	/*
3627	* Try bump up the system timer resolution.
3628	* The more interrupts the better...
3629	*/
3630	if ( RT_SUCCESS(RTTimerRequestSystemGranularity( 976563 /* 1024 HZ */, &u32SystemResolution))
3631	\|\| RT_SUCCESS(RTTimerRequestSystemGranularity( 1000000 /* 1000 HZ */, &u32SystemResolution))
3632	\|\| RT_SUCCESS(RTTimerRequestSystemGranularity( 3906250 /* 256 HZ */, &u32SystemResolution))
3633	\|\| RT_SUCCESS(RTTimerRequestSystemGranularity( 4000000 /* 250 HZ */, &u32SystemResolution))
3634	\|\| RT_SUCCESS(RTTimerRequestSystemGranularity( 7812500 /* 128 HZ */, &u32SystemResolution))
3635	\|\| RT_SUCCESS(RTTimerRequestSystemGranularity(10000000 /* 100 HZ */, &u32SystemResolution))
3636	\|\| RT_SUCCESS(RTTimerRequestSystemGranularity(15625000 /* 64 HZ */, &u32SystemResolution))
3637	\|\| RT_SUCCESS(RTTimerRequestSystemGranularity(31250000 /* 32 HZ */, &u32SystemResolution))
3638	)
3639	{
3640	Assert(RTTimerGetSystemGranularity() <= u32SystemResolution);
3641	pDevExt->u32SystemTimerGranularityGrant = u32SystemResolution;
3642	}
3643
3644	/*
3645	* Find a reasonable update interval, something close to 10ms would be nice,
3646	* and create a recurring timer.
3647	*/
3648	u32Interval = u32SystemResolution = RTTimerGetSystemGranularity();
3649	while (u32Interval < 10000000 /* 10 ms */)
3650	u32Interval += u32SystemResolution;
3651
3652	rc = RTTimerCreateEx(&pDevExt->pGipTimer, u32Interval, 0, supdrvGipTimer, pDevExt);
3653	if (RT_FAILURE(rc))
3654	{
3655	OSDBGPRINT(("supdrvGipCreate: failed create GIP timer at %RU32 ns interval. rc=%d\n", u32Interval, rc));
3656	Assert(!pDevExt->pGipTimer);
3657	supdrvGipDestroy(pDevExt);
3658	return rc;
3659	}
3660
3661	/*
3662	* We're good.
3663	*/
3664	supdrvGipInit(pDevExt, pGip, HCPhysGip, RTTimeSystemNanoTS(), 1000000000 / u32Interval /=Hz/);
3665	return VINF_SUCCESS;
3666	}
3667
3668
3669	/**
3670	* Terminates the GIP.
3671	*
3672	* @param pDevExt Instance data. GIP stuff may be updated.
3673	*/
3674	static void supdrvGipDestroy(PSUPDRVDEVEXT pDevExt)
3675	{
3676	int rc;
3677	#ifdef DEBUG_DARWIN_GIP
3678	OSDBGPRINT(("supdrvGipDestroy: pDevExt=%p pGip=%p pGipTimer=%p GipMemObj=%p\n", pDevExt,
3679	pDevExt->GipMemObj != NIL_RTR0MEMOBJ ? RTR0MemObjAddress(pDevExt->GipMemObj) : NULL,
3680	pDevExt->pGipTimer, pDevExt->GipMemObj));
3681	#endif
3682
3683	/*
3684	* Invalid the GIP data.
3685	*/
3686	if (pDevExt->pGip)
3687	{
3688	supdrvGipTerm(pDevExt->pGip);
3689	pDevExt->pGip = NULL;
3690	}
3691
3692	/*
3693	* Destroy the timer and free the GIP memory object.
3694	*/
3695	if (pDevExt->pGipTimer)
3696	{
3697	rc = RTTimerDestroy(pDevExt->pGipTimer); AssertRC(rc);
3698	pDevExt->pGipTimer = NULL;
3699	}
3700
3701	if (pDevExt->GipMemObj != NIL_RTR0MEMOBJ)
3702	{
3703	rc = RTR0MemObjFree(pDevExt->GipMemObj, true /* free mappings */); AssertRC(rc);
3704	pDevExt->GipMemObj = NIL_RTR0MEMOBJ;
3705	}
3706
3707	/*
3708	* Finally, release the system timer resolution request if one succeeded.
3709	*/
3710	if (pDevExt->u32SystemTimerGranularityGrant)
3711	{
3712	rc = RTTimerReleaseSystemGranularity(pDevExt->u32SystemTimerGranularityGrant); AssertRC(rc);
3713	pDevExt->u32SystemTimerGranularityGrant = 0;
3714	}
3715	}
3716
3717
3718	/**
3719	* Timer callback function.
3720	* @param pTimer The timer.
3721	* @param pvUser The device extension.
3722	*/
3723	static DECLCALLBACK(void) supdrvGipTimer(PRTTIMER pTimer, void *pvUser)
3724	{
3725	PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser;
3726	supdrvGipUpdate(pDevExt->pGip, RTTimeSystemNanoTS());
3727	}
3728	#endif /* USE_NEW_OS_INTERFACE_FOR_GIP */
3729
3730
3731	/**
3732	* Initializes the GIP data.
3733	*
3734	* @returns IPRT status code.
3735	* @param pDevExt Pointer to the device instance data.
3736	* @param pGip Pointer to the read-write kernel mapping of the GIP.
3737	* @param HCPhys The physical address of the GIP.
3738	* @param u64NanoTS The current nanosecond timestamp.
3739	* @param uUpdateHz The update freqence.
3740	*/
3741	int VBOXCALL supdrvGipInit(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, RTHCPHYS HCPhys, uint64_t u64NanoTS, unsigned uUpdateHz)
3742	{
3743	unsigned i;
3744	#ifdef DEBUG_DARWIN_GIP
3745	OSDBGPRINT(("supdrvGipInit: pGip=%p HCPhys=%lx u64NanoTS=%llu uUpdateHz=%d\n", pGip, (long)HCPhys, u64NanoTS, uUpdateHz));
3746	#else
3747	dprintf(("supdrvGipInit: pGip=%p HCPhys=%lx u64NanoTS=%llu uUpdateHz=%d\n", pGip, (long)HCPhys, u64NanoTS, uUpdateHz));
3748	#endif
3749
3750	/*
3751	* Initialize the structure.
3752	*/
3753	memset(pGip, 0, PAGE_SIZE);
3754	pGip->u32Magic = SUPGLOBALINFOPAGE_MAGIC;
3755	pGip->u32Version = SUPGLOBALINFOPAGE_VERSION;
3756	pGip->u32Mode = supdrvGipDeterminTscMode();
3757	pGip->u32UpdateHz = uUpdateHz;
3758	pGip->u32UpdateIntervalNS = 1000000000 / uUpdateHz;
3759	pGip->u64NanoTSLastUpdateHz = u64NanoTS;
3760
3761	for (i = 0; i < RT_ELEMENTS(pGip->aCPUs); i++)
3762	{
3763	pGip->aCPUs[i].u32TransactionId = 2;
3764	pGip->aCPUs[i].u64NanoTS = u64NanoTS;
3765	pGip->aCPUs[i].u64TSC = ASMReadTSC();
3766
3767	/*
3768	* We don't know the following values until we've executed updates.
3769	* So, we'll just insert very high values.
3770	*/
3771	pGip->aCPUs[i].u64CpuHz = _4G + 1;
3772	pGip->aCPUs[i].u32UpdateIntervalTSC = _2G / 4;
3773	pGip->aCPUs[i].au32TSCHistory[0] = _2G / 4;
3774	pGip->aCPUs[i].au32TSCHistory[1] = _2G / 4;
3775	pGip->aCPUs[i].au32TSCHistory[2] = _2G / 4;
3776	pGip->aCPUs[i].au32TSCHistory[3] = _2G / 4;
3777	pGip->aCPUs[i].au32TSCHistory[4] = _2G / 4;
3778	pGip->aCPUs[i].au32TSCHistory[5] = _2G / 4;
3779	pGip->aCPUs[i].au32TSCHistory[6] = _2G / 4;
3780	pGip->aCPUs[i].au32TSCHistory[7] = _2G / 4;
3781	}
3782
3783	/*
3784	* Link it to the device extension.
3785	*/
3786	pDevExt->pGip = pGip;
3787	pDevExt->HCPhysGip = HCPhys;
3788	pDevExt->cGipUsers = 0;
3789
3790	return VINF_SUCCESS;
3791	}
3792
3793
3794	/**
3795	* Determin the GIP TSC mode.
3796	*
3797	* @returns The most suitable TSC mode.
3798	*/
3799	static SUPGIPMODE supdrvGipDeterminTscMode(void)
3800	{
3801	#ifndef USE_NEW_OS_INTERFACE_FOR_GIP
3802	/*
3803	* The problem here is that AMD processors with power management features
3804	* may easily end up with different TSCs because the CPUs or even cores
3805	* on the same physical chip run at different frequencies to save power.
3806	*
3807	* It is rumoured that this will be corrected with Barcelona and it's
3808	* expected that this will be indicated by the TscInvariant bit in
3809	* cpuid(0x80000007). So, the "difficult" bit here is to correctly
3810	* identify the older CPUs which don't do different frequency and
3811	* can be relied upon to have somewhat uniform TSC between the cpus.
3812	*/
3813	if (supdrvOSGetCPUCount() > 1)
3814	{
3815	uint32_t uEAX, uEBX, uECX, uEDX;
3816
3817	/* Permit user users override. */
3818	if (supdrvOSGetForcedAsyncTscMode())
3819	return SUPGIPMODE_ASYNC_TSC;
3820
3821	/* Check for "AuthenticAMD" */
3822	ASMCpuId(0, &uEAX, &uEBX, &uECX, &uEDX);
3823	if (uEAX >= 1 && uEBX == 0x68747541 && uECX == 0x444d4163 && uEDX == 0x69746e65)
3824	{
3825	/* Check for APM support and that TscInvariant is cleared. */
3826	ASMCpuId(0x80000000, &uEAX, &uEBX, &uECX, &uEDX);
3827	if (uEAX >= 0x80000007)
3828	{
3829	ASMCpuId(0x80000007, &uEAX, &uEBX, &uECX, &uEDX);
3830	if ( !(uEDX & BIT(8))/* TscInvariant */
3831	&& (uEDX & 0x3e)) /* STC\|TM\|THERMTRIP\|VID\|FID. Ignore TS. */
3832	return SUPGIPMODE_ASYNC_TSC;
3833	}
3834	}
3835	}
3836	#endif
3837	return SUPGIPMODE_SYNC_TSC;
3838	}
3839
3840
3841	/**
3842	* Invalidates the GIP data upon termination.
3843	*
3844	* @param pGip Pointer to the read-write kernel mapping of the GIP.
3845	*/
3846	void VBOXCALL supdrvGipTerm(PSUPGLOBALINFOPAGE pGip)
3847	{
3848	unsigned i;
3849	pGip->u32Magic = 0;
3850	for (i = 0; i < RT_ELEMENTS(pGip->aCPUs); i++)
3851	{
3852	pGip->aCPUs[i].u64NanoTS = 0;
3853	pGip->aCPUs[i].u64TSC = 0;
3854	pGip->aCPUs[i].iTSCHistoryHead = 0;
3855	}
3856	}
3857
3858
3859	/**
3860	* Worker routine for supdrvGipUpdate and supdrvGipUpdatePerCpu that
3861	* updates all the per cpu data except the transaction id.
3862	*
3863	* @param pGip The GIP.
3864	* @param pGipCpu Pointer to the per cpu data.
3865	* @param u64NanoTS The current time stamp.
3866	*/
3867	static void supdrvGipDoUpdateCpu(PSUPGLOBALINFOPAGE pGip, PSUPGIPCPU pGipCpu, uint64_t u64NanoTS)
3868	{
3869	uint64_t u64TSC;
3870	uint64_t u64TSCDelta;
3871	uint32_t u32UpdateIntervalTSC;
3872	uint32_t u32UpdateIntervalTSCSlack;
3873	unsigned iTSCHistoryHead;
3874	uint64_t u64CpuHz;
3875
3876	/*
3877	* Update the NanoTS.
3878	*/
3879	ASMAtomicXchgU64(&pGipCpu->u64NanoTS, u64NanoTS);
3880
3881	/*
3882	* Calc TSC delta.
3883	*/
3884	/** @todo validate the NanoTS delta, don't trust the OS to call us when it should... */
3885	u64TSC = ASMReadTSC();
3886	u64TSCDelta = u64TSC - pGipCpu->u64TSC;
3887	ASMAtomicXchgU64(&pGipCpu->u64TSC, u64TSC);
3888
3889	if (u64TSCDelta >> 32)
3890	{
3891	u64TSCDelta = pGipCpu->u32UpdateIntervalTSC;
3892	pGipCpu->cErrors++;
3893	}
3894
3895	/*
3896	* TSC History.
3897	*/
3898	Assert(ELEMENTS(pGipCpu->au32TSCHistory) == 8);
3899
3900	iTSCHistoryHead = (pGipCpu->iTSCHistoryHead + 1) & 7;
3901	ASMAtomicXchgU32(&pGipCpu->iTSCHistoryHead, iTSCHistoryHead);
3902	ASMAtomicXchgU32(&pGipCpu->au32TSCHistory[iTSCHistoryHead], (uint32_t)u64TSCDelta);
3903
3904	/*
3905	* UpdateIntervalTSC = average of last 8,2,1 intervals depending on update HZ.
3906	*/
3907	if (pGip->u32UpdateHz >= 1000)
3908	{
3909	uint32_t u32;
3910	u32 = pGipCpu->au32TSCHistory[0];
3911	u32 += pGipCpu->au32TSCHistory[1];
3912	u32 += pGipCpu->au32TSCHistory[2];
3913	u32 += pGipCpu->au32TSCHistory[3];
3914	u32 >>= 2;
3915	u32UpdateIntervalTSC = pGipCpu->au32TSCHistory[4];
3916	u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[5];
3917	u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[6];
3918	u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[7];
3919	u32UpdateIntervalTSC >>= 2;
3920	u32UpdateIntervalTSC += u32;
3921	u32UpdateIntervalTSC >>= 1;
3922
3923	/* Value choosen for a 2GHz Athlon64 running linux 2.6.10/11, . */
3924	u32UpdateIntervalTSCSlack = u32UpdateIntervalTSC >> 14;
3925	}
3926	else if (pGip->u32UpdateHz >= 90)
3927	{
3928	u32UpdateIntervalTSC = (uint32_t)u64TSCDelta;
3929	u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[(iTSCHistoryHead - 1) & 7];
3930	u32UpdateIntervalTSC >>= 1;
3931
3932	/* value choosen on a 2GHz thinkpad running windows */
3933	u32UpdateIntervalTSCSlack = u32UpdateIntervalTSC >> 7;
3934	}
3935	else
3936	{
3937	u32UpdateIntervalTSC = (uint32_t)u64TSCDelta;
3938
3939	/* This value hasn't be checked yet.. waiting for OS/2 and 33Hz timers.. :-) */
3940	u32UpdateIntervalTSCSlack = u32UpdateIntervalTSC >> 6;
3941	}
3942	ASMAtomicXchgU32(&pGipCpu->u32UpdateIntervalTSC, u32UpdateIntervalTSC + u32UpdateIntervalTSCSlack);
3943
3944	/*
3945	* CpuHz.
3946	*/
3947	u64CpuHz = ASMMult2xU32RetU64(u32UpdateIntervalTSC, pGip->u32UpdateHz);
3948	ASMAtomicXchgU64(&pGipCpu->u64CpuHz, u64CpuHz);
3949	}
3950
3951
3952	/**
3953	* Updates the GIP.
3954	*
3955	* @param pGip Pointer to the GIP.
3956	* @param u64NanoTS The current nanosecond timesamp.
3957	*/
3958	void VBOXCALL supdrvGipUpdate(PSUPGLOBALINFOPAGE pGip, uint64_t u64NanoTS)
3959	{
3960	/*
3961	* Determin the relevant CPU data.
3962	*/
3963	PSUPGIPCPU pGipCpu;
3964	if (pGip->u32Mode != SUPGIPMODE_ASYNC_TSC)
3965	pGipCpu = &pGip->aCPUs[0];
3966	else
3967	{
3968	unsigned iCpu = ASMGetApicId();
3969	if (RT_LIKELY(iCpu >= RT_ELEMENTS(pGip->aCPUs)))
3970	return;
3971	pGipCpu = &pGip->aCPUs[iCpu];
3972	}
3973
3974	/*
3975	* Start update transaction.
3976	*/
3977	if (!(ASMAtomicIncU32(&pGipCpu->u32TransactionId) & 1))
3978	{
3979	/* this can happen on win32 if we're taking to long and there are more CPUs around. shouldn't happen though. */
3980	AssertMsgFailed(("Invalid transaction id, %#x, not odd!\n", pGipCpu->u32TransactionId));
3981	ASMAtomicIncU32(&pGipCpu->u32TransactionId);
3982	pGipCpu->cErrors++;
3983	return;
3984	}
3985
3986	/*
3987	* Recalc the update frequency every 0x800th time.
3988	*/
3989	if (!(pGipCpu->u32TransactionId & (GIP_UPDATEHZ_RECALC_FREQ * 2 - 2)))
3990	{
3991	if (pGip->u64NanoTSLastUpdateHz)
3992	{
3993	#ifdef RT_ARCH_AMD64 /** @todo fix 64-bit div here to work on x86 linux. */
3994	uint64_t u64Delta = u64NanoTS - pGip->u64NanoTSLastUpdateHz;
3995	uint32_t u32UpdateHz = (uint32_t)((UINT64_C(1000000000) * GIP_UPDATEHZ_RECALC_FREQ) / u64Delta);
3996	if (u32UpdateHz <= 2000 && u32UpdateHz >= 30)
3997	{
3998	ASMAtomicXchgU32(&pGip->u32UpdateHz, u32UpdateHz);
3999	ASMAtomicXchgU32(&pGip->u32UpdateIntervalNS, 1000000000 / u32UpdateHz);
4000	}
4001	#endif
4002	}
4003	ASMAtomicXchgU64(&pGip->u64NanoTSLastUpdateHz, u64NanoTS);
4004	}
4005
4006	/*
4007	* Update the data.
4008	*/
4009	supdrvGipDoUpdateCpu(pGip, pGipCpu, u64NanoTS);
4010
4011	/*
4012	* Complete transaction.
4013	*/
4014	ASMAtomicIncU32(&pGipCpu->u32TransactionId);
4015	}
4016
4017
4018	/**
4019	* Updates the per cpu GIP data for the calling cpu.
4020	*
4021	* @param pGip Pointer to the GIP.
4022	* @param u64NanoTS The current nanosecond timesamp.
4023	* @param iCpu The CPU index.
4024	*/
4025	void VBOXCALL supdrvGipUpdatePerCpu(PSUPGLOBALINFOPAGE pGip, uint64_t u64NanoTS, unsigned iCpu)
4026	{
4027	PSUPGIPCPU pGipCpu;
4028
4029	if (RT_LIKELY(iCpu <= RT_ELEMENTS(pGip->aCPUs)))
4030	{
4031	pGipCpu = &pGip->aCPUs[iCpu];
4032
4033	/*
4034	* Start update transaction.
4035	*/
4036	if (!(ASMAtomicIncU32(&pGipCpu->u32TransactionId) & 1))
4037	{
4038	AssertMsgFailed(("Invalid transaction id, %#x, not odd!\n", pGipCpu->u32TransactionId));
4039	ASMAtomicIncU32(&pGipCpu->u32TransactionId);
4040	pGipCpu->cErrors++;
4041	return;
4042	}
4043
4044	/*
4045	* Update the data.
4046	*/
4047	supdrvGipDoUpdateCpu(pGip, pGipCpu, u64NanoTS);
4048
4049	/*
4050	* Complete transaction.
4051	*/
4052	ASMAtomicIncU32(&pGipCpu->u32TransactionId);
4053	}
4054	}
4055
4056
4057	#ifndef DEBUG /** @todo change #ifndef DEBUG -> #ifdef LOG_ENABLED */
4058	/**
4059	* Stub function for non-debug builds.
4060	*/
4061	RTDECL(PRTLOGGER) RTLogDefaultInstance(void)
4062	{
4063	return NULL;
4064	}
4065
4066	RTDECL(PRTLOGGER) RTLogRelDefaultInstance(void)
4067	{
4068	return NULL;
4069	}
4070
4071	/**
4072	* Stub function for non-debug builds.
4073	*/
4074	RTDECL(int) RTLogSetDefaultInstanceThread(PRTLOGGER pLogger, uintptr_t uKey)
4075	{
4076	return 0;
4077	}
4078
4079	/**
4080	* Stub function for non-debug builds.
4081	*/
4082	RTDECL(void) RTLogLogger(PRTLOGGER pLogger, void pvCallerRet, const char pszFormat, ...)
4083	{
4084	}
4085
4086	/**
4087	* Stub function for non-debug builds.
4088	*/
4089	RTDECL(void) RTLogLoggerEx(PRTLOGGER pLogger, unsigned fFlags, unsigned iGroup, const char *pszFormat, ...)
4090	{
4091	}
4092
4093	/**
4094	* Stub function for non-debug builds.
4095	*/
4096	RTDECL(void) RTLogLoggerExV(PRTLOGGER pLogger, unsigned fFlags, unsigned iGroup, const char *pszFormat, va_list args)
4097	{
4098	}
4099
4100	/**
4101	* Stub function for non-debug builds.
4102	*/
4103	RTDECL(void) RTLogPrintf(const char *pszFormat, ...)
4104	{
4105	}
4106
4107	/**
4108	* Stub function for non-debug builds.
4109	*/
4110	RTDECL(void) RTLogPrintfV(const char *pszFormat, va_list args)
4111	{
4112	}
4113	#endif /* !DEBUG */
4114

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/HostDrivers/Support/SUPDRVShared.c@ 5031

Download in other formats: