PGMR0DynMap.cpp@ 15344

Last change on this file since 15344 was 15344, checked in by vboxsync, 16 years ago
#3202: Optimizations of the dynamic page mapping code (ring-0). Do lots of the stuff inline, using the set as a 2st level cache and not releasing it for each inner VT-x iteration.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 72.2 KB

Line
1	/* $Id: PGMR0DynMap.cpp 15344 2008-12-12 00:13:56Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, ring-0 dynamic mapping cache.
4	*/
5
6	/*
7	* Copyright (C) 2008 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22	/*******************************************************************************
23	* Internal Functions *
24	*******************************************************************************/
25	#define LOG_GROUP LOG_GROUP_PGM
26	#include <VBox/pgm.h>
27	#include "../PGMInternal.h"
28	#include <VBox/vm.h>
29	#include <VBox/sup.h>
30	#include <VBox/err.h>
31	#include <iprt/asm.h>
32	#include <iprt/alloc.h>
33	#include <iprt/assert.h>
34	#include <iprt/cpuset.h>
35	#include <iprt/memobj.h>
36	#include <iprt/mp.h>
37	#include <iprt/semaphore.h>
38	#include <iprt/spinlock.h>
39	#include <iprt/string.h>
40
41
42	/*******************************************************************************
43	* Defined Constants And Macros *
44	*******************************************************************************/
45	/** The max size of the mapping cache (in pages). */
46	#define PGMR0DYNMAP_MAX_PAGES ((16*_1M) >> PAGE_SHIFT)
47	/** The small segment size that is adopted on out-of-memory conditions with a
48	* single big segment. */
49	#define PGMR0DYNMAP_SMALL_SEG_PAGES 128
50	/** The number of pages we reserve per CPU. */
51	#define PGMR0DYNMAP_PAGES_PER_CPU 256
52	/** The minimum number of pages we reserve per CPU.
53	* This must be equal or larger than the autoset size. */
54	#define PGMR0DYNMAP_PAGES_PER_CPU_MIN 32
55	/** The number of guard pages.
56	* @remarks Never do tuning of the hashing or whatnot with a strict build! */
57	#if defined(VBOX_STRICT)
58	# define PGMR0DYNMAP_GUARD_PAGES 1
59	#else
60	# define PGMR0DYNMAP_GUARD_PAGES 0
61	#endif
62	/** The dummy physical address of guard pages. */
63	#define PGMR0DYNMAP_GUARD_PAGE_HCPHYS UINT32_C(0x7777feed)
64	/** The dummy reference count of guard pages. (Must be non-zero.) */
65	#define PGMR0DYNMAP_GUARD_PAGE_REF_COUNT INT32_C(0x7777feed)
66	#if 0
67	/** Define this to just clear the present bit on guard pages.
68	* The alternative is to replace the entire PTE with an bad not-present
69	* PTE. Either way, XNU will screw us. :-/ */
70	#define PGMR0DYNMAP_GUARD_NP
71	#endif
72	/** The dummy PTE value for a page. */
73	#define PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE X86_PTE_PG_MASK
74	/** The dummy PTE value for a page. */
75	#define PGMR0DYNMAP_GUARD_PAGE_PAE_PTE UINT64_MAX /X86_PTE_PAE_PG_MASK/
76	/** Calcs the overload threshold. Current set at 50%. */
77	#define PGMR0DYNMAP_CALC_OVERLOAD(cPages) ((cPages) / 2)
78
79	#if 0
80	/* Assertions causes panics if preemption is disabled, this can be used to work aroudn that. */
81	//#define RTSpinlockAcquire(a,b) do {} while (0)
82	//#define RTSpinlockRelease(a,b) do {} while (0)
83	#endif
84
85
86	/*******************************************************************************
87	* Structures and Typedefs *
88	*******************************************************************************/
89	/**
90	* Ring-0 dynamic mapping cache segment.
91	*
92	* The dynamic mapping cache can be extended with additional segments if the
93	* load is found to be too high. This done the next time a VM is created, under
94	* the protection of the init mutex. The arrays is reallocated and the new
95	* segment is added to the end of these. Nothing is rehashed of course, as the
96	* indexes / addresses must remain unchanged.
97	*
98	* This structure is only modified while owning the init mutex or during module
99	* init / term.
100	*/
101	typedef struct PGMR0DYNMAPSEG
102	{
103	/** Pointer to the next segment. */
104	struct PGMR0DYNMAPSEG *pNext;
105	/** The memory object for the virtual address range that we're abusing. */
106	RTR0MEMOBJ hMemObj;
107	/** The start page in the cache. (I.e. index into the arrays.) */
108	uint16_t iPage;
109	/** The number of pages this segment contributes. */
110	uint16_t cPages;
111	/** The number of page tables. */
112	uint16_t cPTs;
113	/** The memory objects for the page tables. */
114	RTR0MEMOBJ ahMemObjPTs[1];
115	} PGMR0DYNMAPSEG;
116	/** Pointer to a ring-0 dynamic mapping cache segment. */
117	typedef PGMR0DYNMAPSEG *PPGMR0DYNMAPSEG;
118
119
120	/**
121	* Ring-0 dynamic mapping cache entry.
122	*
123	* This structure tracks
124	*/
125	typedef struct PGMR0DYNMAPENTRY
126	{
127	/** The physical address of the currently mapped page.
128	* This is duplicate for three reasons: cache locality, cache policy of the PT
129	* mappings and sanity checks. */
130	RTHCPHYS HCPhys;
131	/** Pointer to the page. */
132	void *pvPage;
133	/** The number of references. */
134	int32_t volatile cRefs;
135	/** PTE pointer union. */
136	union PGMR0DYNMAPENTRY_PPTE
137	{
138	/** PTE pointer, 32-bit legacy version. */
139	PX86PTE pLegacy;
140	/** PTE pointer, PAE version. */
141	PX86PTEPAE pPae;
142	/** PTE pointer, the void version. */
143	void *pv;
144	} uPte;
145	/** CPUs that haven't invalidated this entry after it's last update. */
146	RTCPUSET PendingSet;
147	} PGMR0DYNMAPENTRY;
148	/** Pointer to a ring-0 dynamic mapping cache entry. */
149	typedef PGMR0DYNMAPENTRY *PPGMR0DYNMAPENTRY;
150
151
152	/**
153	* Ring-0 dynamic mapping cache.
154	*
155	* This is initialized during VMMR0 module init but no segments are allocated at
156	* that time. Segments will be added when the first VM is started and removed
157	* again when the last VM shuts down, thus avoid consuming memory while dormant.
158	* At module termination, the remaining bits will be freed up.
159	*/
160	typedef struct PGMR0DYNMAP
161	{
162	/** The usual magic number / eye catcher (PGMR0DYNMAP_MAGIC). */
163	uint32_t u32Magic;
164	/** Spinlock serializing the normal operation of the cache. */
165	RTSPINLOCK hSpinlock;
166	/** Array for tracking and managing the pages. */
167	PPGMR0DYNMAPENTRY paPages;
168	/** The cache size given as a number of pages. */
169	uint32_t cPages;
170	/** Whether it's 32-bit legacy or PAE/AMD64 paging mode. */
171	bool fLegacyMode;
172	/** The current load.
173	* This does not include guard pages. */
174	uint32_t cLoad;
175	/** The max load ever.
176	* This is maintained to get trigger adding of more mapping space. */
177	uint32_t cMaxLoad;
178	/** Initialization / termination lock. */
179	RTSEMFASTMUTEX hInitLock;
180	/** The number of guard pages. */
181	uint32_t cGuardPages;
182	/** The number of users (protected by hInitLock). */
183	uint32_t cUsers;
184	/** Array containing a copy of the original page tables.
185	* The entries are either X86PTE or X86PTEPAE according to fLegacyMode. */
186	void *pvSavedPTEs;
187	/** List of segments. */
188	PPGMR0DYNMAPSEG pSegHead;
189	/** The paging mode. */
190	SUPPAGINGMODE enmPgMode;
191	} PGMR0DYNMAP;
192	/** Pointer to the ring-0 dynamic mapping cache */
193	typedef PGMR0DYNMAP *PPGMR0DYNMAP;
194
195	/** PGMR0DYNMAP::u32Magic. (Jens Christian Bugge Wesseltoft) */
196	#define PGMR0DYNMAP_MAGIC 0x19640201
197
198
199	/**
200	* Paging level data.
201	*/
202	typedef struct PGMR0DYNMAPPGLVL
203	{
204	uint32_t cLevels; /*< The number of levels. /
205	struct
206	{
207	RTHCPHYS HCPhys; /**< The address of the page for the current level,
208	* i.e. what hMemObj/hMapObj is currently mapping. */
209	RTHCPHYS fPhysMask; /*< Mask for extracting HCPhys from uEntry. /
210	RTR0MEMOBJ hMemObj; /*< Memory object for HCPhys, PAGE_SIZE. /
211	RTR0MEMOBJ hMapObj; /*< Mapping object for hMemObj. /
212	uint32_t fPtrShift; /*< The pointer shift count. /
213	uint64_t fPtrMask; /*< The mask to apply to the shifted pointer to get the table index. /
214	uint64_t fAndMask; /*< And mask to check entry flags. /
215	uint64_t fResMask; /*< The result from applying fAndMask. /
216	union
217	{
218	void pv; /< hMapObj address. /
219	PX86PGUINT paLegacy; /*< Legacy table view. /
220	PX86PGPAEUINT paPae; /*< PAE/AMD64 table view. /
221	} u;
222	} a[4];
223	} PGMR0DYNMAPPGLVL;
224	/** Pointer to paging level data. */
225	typedef PGMR0DYNMAPPGLVL *PPGMR0DYNMAPPGLVL;
226
227
228	/*******************************************************************************
229	* Global Variables *
230	*******************************************************************************/
231	/** Pointer to the ring-0 dynamic mapping cache. */
232	static PPGMR0DYNMAP g_pPGMR0DynMap;
233
234
235	/*******************************************************************************
236	* Internal Functions *
237	*******************************************************************************/
238	static void pgmR0DynMapReleasePage(PPGMR0DYNMAP pThis, uint32_t iPage, uint32_t cRefs);
239	static int pgmR0DynMapSetup(PPGMR0DYNMAP pThis);
240	static int pgmR0DynMapExpand(PPGMR0DYNMAP pThis);
241	static void pgmR0DynMapTearDown(PPGMR0DYNMAP pThis);
242	#ifdef DEBUG
243	static int pgmR0DynMapTest(PVM pVM);
244	#endif
245
246
247	/**
248	* Initializes the ring-0 dynamic mapping cache.
249	*
250	* @returns VBox status code.
251	*/
252	VMMR0DECL(int) PGMR0DynMapInit(void)
253	{
254	Assert(!g_pPGMR0DynMap);
255
256	/*
257	* Create and initialize the cache instance.
258	*/
259	PPGMR0DYNMAP pThis = (PPGMR0DYNMAP)RTMemAllocZ(sizeof(*pThis));
260	AssertLogRelReturn(pThis, VERR_NO_MEMORY);
261	int rc = VINF_SUCCESS;
262	pThis->enmPgMode = SUPR0GetPagingMode();
263	switch (pThis->enmPgMode)
264	{
265	case SUPPAGINGMODE_32_BIT:
266	case SUPPAGINGMODE_32_BIT_GLOBAL:
267	pThis->fLegacyMode = false;
268	break;
269	case SUPPAGINGMODE_PAE:
270	case SUPPAGINGMODE_PAE_GLOBAL:
271	case SUPPAGINGMODE_PAE_NX:
272	case SUPPAGINGMODE_PAE_GLOBAL_NX:
273	case SUPPAGINGMODE_AMD64:
274	case SUPPAGINGMODE_AMD64_GLOBAL:
275	case SUPPAGINGMODE_AMD64_NX:
276	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
277	pThis->fLegacyMode = false;
278	break;
279	default:
280	rc = VERR_INTERNAL_ERROR;
281	break;
282	}
283	if (RT_SUCCESS(rc))
284	{
285	rc = RTSemFastMutexCreate(&pThis->hInitLock);
286	if (RT_SUCCESS(rc))
287	{
288	rc = RTSpinlockCreate(&pThis->hSpinlock);
289	if (RT_SUCCESS(rc))
290	{
291	pThis->u32Magic = PGMR0DYNMAP_MAGIC;
292	g_pPGMR0DynMap = pThis;
293	return VINF_SUCCESS;
294	}
295	RTSemFastMutexDestroy(pThis->hInitLock);
296	}
297	}
298	RTMemFree(pThis);
299	return rc;
300	}
301
302
303	/**
304	* Terminates the ring-0 dynamic mapping cache.
305	*/
306	VMMR0DECL(void) PGMR0DynMapTerm(void)
307	{
308	/*
309	* Destroy the cache.
310	*
311	* There is not supposed to be any races here, the loader should
312	* make sure about that. So, don't bother locking anything.
313	*
314	* The VM objects should all be destroyed by now, so there is no
315	* dangling users or anything like that to clean up. This routine
316	* is just a mirror image of PGMR0DynMapInit.
317	*/
318	PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
319	if (pThis)
320	{
321	AssertPtr(pThis);
322	g_pPGMR0DynMap = NULL;
323
324	/* This should never happen, but in case it does try not to leak memory. */
325	AssertLogRelMsg(!pThis->cUsers && !pThis->paPages && !pThis->pvSavedPTEs && !pThis->cPages,
326	("cUsers=%d paPages=%p pvSavedPTEs=%p cPages=%#x\n",
327	pThis->cUsers, pThis->paPages, pThis->pvSavedPTEs, pThis->cPages));
328	if (pThis->paPages)
329	pgmR0DynMapTearDown(pThis);
330
331	/* Free the associated resources. */
332	RTSemFastMutexDestroy(pThis->hInitLock);
333	pThis->hInitLock = NIL_RTSEMFASTMUTEX;
334	RTSpinlockDestroy(pThis->hSpinlock);
335	pThis->hSpinlock = NIL_RTSPINLOCK;
336	pThis->u32Magic = UINT32_MAX;
337	RTMemFree(pThis);
338	}
339	}
340
341
342	/**
343	* Initializes the dynamic mapping cache for a new VM.
344	*
345	* @returns VBox status code.
346	* @param pVM Pointer to the shared VM structure.
347	*/
348	VMMR0DECL(int) PGMR0DynMapInitVM(PVM pVM)
349	{
350	AssertMsgReturn(!pVM->pgm.s.pvR0DynMapUsed, ("%p (pThis=%p)\n", pVM->pgm.s.pvR0DynMapUsed, g_pPGMR0DynMap), VERR_WRONG_ORDER);
351
352	/*
353	* Initialize the auto sets.
354	*/
355	VMCPUID idCpu = pVM->cCPUs;
356	AssertReturn(idCpu > 0 && idCpu <= VMCPU_MAX_CPU_COUNT, VERR_INTERNAL_ERROR);
357	while (idCpu-- > 0)
358	{
359	PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
360	uint32_t j = RT_ELEMENTS(pSet->aEntries);
361	while (j-- > 0)
362	{
363	pSet->aEntries[j].iPage = UINT16_MAX;
364	pSet->aEntries[j].cRefs = 0;
365	pSet->aEntries[j].pvPage = NULL;
366	pSet->aEntries[j].HCPhys = NIL_RTHCPHYS;
367	}
368	pSet->cEntries = PGMMAPSET_CLOSED;
369	memset(&pSet->aiHashTable[0], 0xff, sizeof(pSet->aiHashTable));
370	}
371
372	/*
373	* Do we need the cache? Skip the last bit if we don't.
374	*/
375	if (!VMMIsHwVirtExtForced(pVM))
376	return VINF_SUCCESS;
377
378	/*
379	* Reference and if necessary setup or expand the cache.
380	*/
381	PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
382	AssertPtrReturn(pThis, VERR_INTERNAL_ERROR);
383	int rc = RTSemFastMutexRequest(pThis->hInitLock);
384	AssertLogRelRCReturn(rc, rc);
385
386	pThis->cUsers++;
387	if (pThis->cUsers == 1)
388	{
389	rc = pgmR0DynMapSetup(pThis);
390	#ifdef DEBUG
391	if (RT_SUCCESS(rc))
392	{
393	rc = pgmR0DynMapTest(pVM);
394	if (RT_FAILURE(rc))
395	pgmR0DynMapTearDown(pThis);
396	}
397	#endif
398	}
399	else if (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(pThis->cPages - pThis->cGuardPages))
400	rc = pgmR0DynMapExpand(pThis);
401	if (RT_SUCCESS(rc))
402	pVM->pgm.s.pvR0DynMapUsed = pThis;
403	else
404	pThis->cUsers--;
405
406	RTSemFastMutexRelease(pThis->hInitLock);
407	return rc;
408	}
409
410
411	/**
412	* Terminates the dynamic mapping cache usage for a VM.
413	*
414	* @param pVM Pointer to the shared VM structure.
415	*/
416	VMMR0DECL(void) PGMR0DynMapTermVM(PVM pVM)
417	{
418	/*
419	* Return immediately if we're not using the cache.
420	*/
421	if (!pVM->pgm.s.pvR0DynMapUsed)
422	return;
423
424	PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
425	AssertPtrReturnVoid(pThis);
426
427	int rc = RTSemFastMutexRequest(pThis->hInitLock);
428	AssertLogRelRCReturnVoid(rc);
429
430	if (pVM->pgm.s.pvR0DynMapUsed == pThis)
431	{
432	pVM->pgm.s.pvR0DynMapUsed = NULL;
433
434	#ifdef VBOX_STRICT
435	PGMR0DynMapAssertIntegrity();
436	#endif
437
438	/*
439	* Clean up and check the auto sets.
440	*/
441	VMCPUID idCpu = pVM->cCPUs;
442	while (idCpu-- > 0)
443	{
444	PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
445	uint32_t j = pSet->cEntries;
446	if (j <= RT_ELEMENTS(pSet->aEntries))
447	{
448	/*
449	* The set is open, close it.
450	*/
451	while (j-- > 0)
452	{
453	int32_t cRefs = pSet->aEntries[j].cRefs;
454	uint32_t iPage = pSet->aEntries[j].iPage;
455	LogRel(("PGMR0DynMapTermVM: %d dangling refs to %#x\n", cRefs, iPage));
456	if (iPage < pThis->cPages && cRefs > 0)
457	pgmR0DynMapReleasePage(pThis, iPage, cRefs);
458	else
459	AssertLogRelMsgFailed(("cRefs=%d iPage=%#x cPages=%u\n", cRefs, iPage, pThis->cPages));
460
461	pSet->aEntries[j].iPage = UINT16_MAX;
462	pSet->aEntries[j].cRefs = 0;
463	pSet->aEntries[j].pvPage = NULL;
464	pSet->aEntries[j].HCPhys = NIL_RTHCPHYS;
465	}
466	pSet->cEntries = PGMMAPSET_CLOSED;
467	}
468	else
469	AssertMsg(j == PGMMAPSET_CLOSED, ("cEntries=%#x\n", j));
470
471	j = RT_ELEMENTS(pSet->aEntries);
472	while (j-- > 0)
473	{
474	Assert(pSet->aEntries[j].iPage == UINT16_MAX);
475	Assert(!pSet->aEntries[j].cRefs);
476	}
477	}
478
479	/*
480	* Release our reference to the mapping cache.
481	*/
482	Assert(pThis->cUsers > 0);
483	pThis->cUsers--;
484	if (!pThis->cUsers)
485	pgmR0DynMapTearDown(pThis);
486	}
487	else
488	AssertLogRelMsgFailed(("pvR0DynMapUsed=%p pThis=%p\n", pVM->pgm.s.pvR0DynMapUsed, pThis));
489
490	RTSemFastMutexRelease(pThis->hInitLock);
491	}
492
493
494	/**
495	* Shoots down the TLBs for all the cache pages, pgmR0DynMapTearDown helper.
496	*
497	* @param idCpu The current CPU.
498	* @param pvUser1 The dynamic mapping cache instance.
499	* @param pvUser2 Unused, NULL.
500	*/
501	static DECLCALLBACK(void) pgmR0DynMapShootDownTlbs(RTCPUID idCpu, void pvUser1, void pvUser2)
502	{
503	Assert(!pvUser2);
504	PPGMR0DYNMAP pThis = (PPGMR0DYNMAP)pvUser1;
505	Assert(pThis == g_pPGMR0DynMap);
506	PPGMR0DYNMAPENTRY paPages = pThis->paPages;
507	uint32_t iPage = pThis->cPages;
508	while (iPage-- > 0)
509	ASMInvalidatePage(paPages[iPage].pvPage);
510	}
511
512
513	/**
514	* Shoot down the TLBs for every single cache entry on all CPUs.
515	*
516	* @returns IPRT status code (RTMpOnAll).
517	* @param pThis The dynamic mapping cache instance.
518	*/
519	static int pgmR0DynMapTlbShootDown(PPGMR0DYNMAP pThis)
520	{
521	int rc = RTMpOnAll(pgmR0DynMapShootDownTlbs, pThis, NULL);
522	AssertRC(rc);
523	if (RT_FAILURE(rc))
524	{
525	uint32_t iPage = pThis->cPages;
526	while (iPage-- > 0)
527	ASMInvalidatePage(pThis->paPages[iPage].pvPage);
528	}
529	return rc;
530	}
531
532
533	/**
534	* Calculate the new cache size based on cMaxLoad statistics.
535	*
536	* @returns Number of pages.
537	* @param pThis The dynamic mapping cache instance.
538	* @param pcMinPages The minimal size in pages.
539	*/
540	static uint32_t pgmR0DynMapCalcNewSize(PPGMR0DYNMAP pThis, uint32_t *pcMinPages)
541	{
542	Assert(pThis->cPages <= PGMR0DYNMAP_MAX_PAGES);
543
544	/* cCpus * PGMR0DYNMAP_PAGES_PER_CPU(_MIN). */
545	RTCPUID cCpus = RTMpGetCount();
546	AssertReturn(cCpus > 0 && cCpus <= RTCPUSET_MAX_CPUS, 0);
547	uint32_t cPages = cCpus * PGMR0DYNMAP_PAGES_PER_CPU;
548	uint32_t cMinPages = cCpus * PGMR0DYNMAP_PAGES_PER_CPU_MIN;
549
550	/* adjust against cMaxLoad. */
551	AssertMsg(pThis->cMaxLoad <= PGMR0DYNMAP_MAX_PAGES, ("%#x\n", pThis->cMaxLoad));
552	if (pThis->cMaxLoad > PGMR0DYNMAP_MAX_PAGES)
553	pThis->cMaxLoad = 0;
554
555	while (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(cPages))
556	cPages += PGMR0DYNMAP_PAGES_PER_CPU;
557
558	if (pThis->cMaxLoad > cMinPages)
559	cMinPages = pThis->cMaxLoad;
560
561	/* adjust against max and current size. */
562	if (cPages < pThis->cPages)
563	cPages = pThis->cPages;
564	cPages *= PGMR0DYNMAP_GUARD_PAGES + 1;
565	if (cPages > PGMR0DYNMAP_MAX_PAGES)
566	cPages = PGMR0DYNMAP_MAX_PAGES;
567
568	if (cMinPages < pThis->cPages)
569	cMinPages = pThis->cPages;
570	cMinPages *= PGMR0DYNMAP_GUARD_PAGES + 1;
571	if (cMinPages > PGMR0DYNMAP_MAX_PAGES)
572	cMinPages = PGMR0DYNMAP_MAX_PAGES;
573
574	Assert(cMinPages);
575	*pcMinPages = cMinPages;
576	return cPages;
577	}
578
579
580	/**
581	* Initializes the paging level data.
582	*
583	* @param pThis The dynamic mapping cache instance.
584	* @param pPgLvl The paging level data.
585	*/
586	void pgmR0DynMapPagingArrayInit(PPGMR0DYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl)
587	{
588	RTCCUINTREG cr4 = ASMGetCR4();
589	switch (pThis->enmPgMode)
590	{
591	case SUPPAGINGMODE_32_BIT:
592	case SUPPAGINGMODE_32_BIT_GLOBAL:
593	pPgLvl->cLevels = 2;
594	pPgLvl->a[0].fPhysMask = X86_CR3_PAGE_MASK;
595	pPgLvl->a[0].fAndMask = X86_PDE_P \| X86_PDE_RW \| (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
596	pPgLvl->a[0].fResMask = X86_PDE_P \| X86_PDE_RW;
597	pPgLvl->a[0].fPtrMask = X86_PD_MASK;
598	pPgLvl->a[0].fPtrShift = X86_PD_SHIFT;
599
600	pPgLvl->a[1].fPhysMask = X86_PDE_PG_MASK;
601	pPgLvl->a[1].fAndMask = X86_PTE_P \| X86_PTE_RW;
602	pPgLvl->a[1].fResMask = X86_PTE_P \| X86_PTE_RW;
603	pPgLvl->a[1].fPtrMask = X86_PT_MASK;
604	pPgLvl->a[1].fPtrShift = X86_PT_SHIFT;
605	break;
606
607	case SUPPAGINGMODE_PAE:
608	case SUPPAGINGMODE_PAE_GLOBAL:
609	case SUPPAGINGMODE_PAE_NX:
610	case SUPPAGINGMODE_PAE_GLOBAL_NX:
611	pPgLvl->cLevels = 3;
612	pPgLvl->a[0].fPhysMask = X86_CR3_PAE_PAGE_MASK;
613	pPgLvl->a[0].fPtrMask = X86_PDPT_MASK_PAE;
614	pPgLvl->a[0].fPtrShift = X86_PDPT_SHIFT;
615	pPgLvl->a[0].fAndMask = X86_PDPE_P;
616	pPgLvl->a[0].fResMask = X86_PDPE_P;
617
618	pPgLvl->a[1].fPhysMask = X86_PDPE_PG_MASK;
619	pPgLvl->a[1].fPtrMask = X86_PD_PAE_MASK;
620	pPgLvl->a[1].fPtrShift = X86_PD_PAE_SHIFT;
621	pPgLvl->a[1].fAndMask = X86_PDE_P \| X86_PDE_RW \| (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
622	pPgLvl->a[1].fResMask = X86_PDE_P \| X86_PDE_RW;
623
624	pPgLvl->a[2].fPhysMask = X86_PDE_PAE_PG_MASK;
625	pPgLvl->a[2].fPtrMask = X86_PT_PAE_MASK;
626	pPgLvl->a[2].fPtrShift = X86_PT_PAE_SHIFT;
627	pPgLvl->a[2].fAndMask = X86_PTE_P \| X86_PTE_RW;
628	pPgLvl->a[2].fResMask = X86_PTE_P \| X86_PTE_RW;
629	break;
630
631	case SUPPAGINGMODE_AMD64:
632	case SUPPAGINGMODE_AMD64_GLOBAL:
633	case SUPPAGINGMODE_AMD64_NX:
634	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
635	pPgLvl->cLevels = 4;
636	pPgLvl->a[0].fPhysMask = X86_CR3_AMD64_PAGE_MASK;
637	pPgLvl->a[0].fPtrShift = X86_PML4_SHIFT;
638	pPgLvl->a[0].fPtrMask = X86_PML4_MASK;
639	pPgLvl->a[0].fAndMask = X86_PML4E_P \| X86_PML4E_RW;
640	pPgLvl->a[0].fResMask = X86_PML4E_P \| X86_PML4E_RW;
641
642	pPgLvl->a[1].fPhysMask = X86_PML4E_PG_MASK;
643	pPgLvl->a[1].fPtrShift = X86_PDPT_SHIFT;
644	pPgLvl->a[1].fPtrMask = X86_PDPT_MASK_AMD64;
645	pPgLvl->a[1].fAndMask = X86_PDPE_P \| X86_PDPE_RW /** @todo check for X86_PDPT_PS support. */;
646	pPgLvl->a[1].fResMask = X86_PDPE_P \| X86_PDPE_RW;
647
648	pPgLvl->a[2].fPhysMask = X86_PDPE_PG_MASK;
649	pPgLvl->a[2].fPtrShift = X86_PD_PAE_SHIFT;
650	pPgLvl->a[2].fPtrMask = X86_PD_PAE_MASK;
651	pPgLvl->a[2].fAndMask = X86_PDE_P \| X86_PDE_RW \| (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
652	pPgLvl->a[2].fResMask = X86_PDE_P \| X86_PDE_RW;
653
654	pPgLvl->a[3].fPhysMask = X86_PDE_PAE_PG_MASK;
655	pPgLvl->a[3].fPtrShift = X86_PT_PAE_SHIFT;
656	pPgLvl->a[3].fPtrMask = X86_PT_PAE_MASK;
657	pPgLvl->a[3].fAndMask = X86_PTE_P \| X86_PTE_RW;
658	pPgLvl->a[3].fResMask = X86_PTE_P \| X86_PTE_RW;
659	break;
660
661	default:
662	AssertFailed();
663	pPgLvl->cLevels = 0;
664	break;
665	}
666
667	for (uint32_t i = 0; i < 4; i++) /* ASSUMING array size. */
668	{
669	pPgLvl->a[i].HCPhys = NIL_RTHCPHYS;
670	pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
671	pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ;
672	pPgLvl->a[i].u.pv = NULL;
673	}
674	}
675
676
677	/**
678	* Maps a PTE.
679	*
680	* This will update the segment structure when new PTs are mapped.
681	*
682	* It also assumes that we (for paranoid reasons) wish to establish a mapping
683	* chain from CR3 to the PT that all corresponds to the processor we're
684	* currently running on, and go about this by running with interrupts disabled
685	* and restarting from CR3 for every change.
686	*
687	* @returns VBox status code, VINF_TRY_AGAIN if we changed any mappings and had
688	* to re-enable interrupts.
689	* @param pThis The dynamic mapping cache instance.
690	* @param pPgLvl The paging level structure.
691	* @param pvPage The page.
692	* @param pSeg The segment.
693	* @param cMaxPTs The max number of PTs expected in the segment.
694	* @param ppvPTE Where to store the PTE address.
695	*/
696	static int pgmR0DynMapPagingArrayMapPte(PPGMR0DYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl, void *pvPage,
697	PPGMR0DYNMAPSEG pSeg, uint32_t cMaxPTs, void **ppvPTE)
698	{
699	Assert(!(ASMGetFlags() & X86_EFL_IF));
700	void *pvEntry = NULL;
701	X86PGPAEUINT uEntry = ASMGetCR3();
702	for (uint32_t i = 0; i < pPgLvl->cLevels; i++)
703	{
704	RTHCPHYS HCPhys = uEntry & pPgLvl->a[i].fPhysMask;
705	if (pPgLvl->a[i].HCPhys != HCPhys)
706	{
707	/*
708	* Need to remap this level.
709	* The final level, the PT, will not be freed since that is what it's all about.
710	*/
711	ASMIntEnable();
712	if (i + 1 == pPgLvl->cLevels)
713	AssertReturn(pSeg->cPTs < cMaxPTs, VERR_INTERNAL_ERROR);
714	else
715	{
716	int rc2 = RTR0MemObjFree(pPgLvl->a[i].hMemObj, true /* fFreeMappings */); AssertRC(rc2);
717	pPgLvl->a[i].hMemObj = pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
718	}
719
720	int rc = RTR0MemObjEnterPhys(&pPgLvl->a[i].hMemObj, HCPhys, PAGE_SIZE);
721	if (RT_SUCCESS(rc))
722	{
723	rc = RTR0MemObjMapKernel(&pPgLvl->a[i].hMapObj, pPgLvl->a[i].hMemObj,
724	(void )-1 / pvFixed /, 0 / cbAlignment */,
725	RTMEM_PROT_WRITE \| RTMEM_PROT_READ);
726	if (RT_SUCCESS(rc))
727	{
728	pPgLvl->a[i].u.pv = RTR0MemObjAddress(pPgLvl->a[i].hMapObj);
729	AssertMsg(((uintptr_t)pPgLvl->a[i].u.pv & ~(uintptr_t)PAGE_OFFSET_MASK), ("%p\n", pPgLvl->a[i].u.pv));
730	pPgLvl->a[i].HCPhys = HCPhys;
731	if (i + 1 == pPgLvl->cLevels)
732	pSeg->ahMemObjPTs[pSeg->cPTs++] = pPgLvl->a[i].hMemObj;
733	ASMIntDisable();
734	return VINF_TRY_AGAIN;
735	}
736
737	pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
738	}
739	else
740	pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ;
741	pPgLvl->a[i].HCPhys = NIL_RTHCPHYS;
742	return rc;
743	}
744
745	/*
746	* The next level.
747	*/
748	uint32_t iEntry = ((uint64_t)(uintptr_t)pvPage >> pPgLvl->a[i].fPtrShift) & pPgLvl->a[i].fPtrMask;
749	if (pThis->fLegacyMode)
750	{
751	pvEntry = &pPgLvl->a[i].u.paLegacy[iEntry];
752	uEntry = pPgLvl->a[i].u.paLegacy[iEntry];
753	}
754	else
755	{
756	pvEntry = &pPgLvl->a[i].u.paPae[iEntry];
757	uEntry = pPgLvl->a[i].u.paPae[iEntry];
758	}
759
760	if ((uEntry & pPgLvl->a[i].fAndMask) != pPgLvl->a[i].fResMask)
761	{
762	LogRel(("PGMR0DynMap: internal error - iPgLvl=%u cLevels=%u uEntry=%#llx fAnd=%#llx fRes=%#llx got=%#llx\n"
763	"PGMR0DynMap: pv=%p pvPage=%p iEntry=%#x fLegacyMode=%RTbool\n",
764	i, pPgLvl->cLevels, uEntry, pPgLvl->a[i].fAndMask, pPgLvl->a[i].fResMask, uEntry & pPgLvl->a[i].fAndMask,
765	pPgLvl->a[i].u.pv, pvPage, iEntry, pThis->fLegacyMode));
766	return VERR_INTERNAL_ERROR;
767	}
768	/Log(("#%d: iEntry=%4d uEntry=%#llx pvEntry=%p HCPhys=%RHp \n", i, iEntry, uEntry, pvEntry, pPgLvl->a[i].HCPhys));/
769	}
770
771	/* made it thru without needing to remap anything. */
772	*ppvPTE = pvEntry;
773	return VINF_SUCCESS;
774	}
775
776
777	/**
778	* Sets up a guard page.
779	*
780	* @param pThis The dynamic mapping cache instance.
781	* @param pPage The page.
782	*/
783	DECLINLINE(void) pgmR0DynMapSetupGuardPage(PPGMR0DYNMAP pThis, PPGMR0DYNMAPENTRY pPage)
784	{
785	memset(pPage->pvPage, 0xfd, PAGE_SIZE);
786	pPage->cRefs = PGMR0DYNMAP_GUARD_PAGE_REF_COUNT;
787	pPage->HCPhys = PGMR0DYNMAP_GUARD_PAGE_HCPHYS;
788	#ifdef PGMR0DYNMAP_GUARD_NP
789	ASMAtomicBitClear(pPage->uPte.pv, X86_PTE_BIT_P);
790	#else
791	if (pThis->fLegacyMode)
792	ASMAtomicWriteU32(&pPage->uPte.pLegacy->u, PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE);
793	else
794	ASMAtomicWriteU64(&pPage->uPte.pPae->u, PGMR0DYNMAP_GUARD_PAGE_PAE_PTE);
795	#endif
796	pThis->cGuardPages++;
797	}
798
799
800	/**
801	* Adds a new segment of the specified size.
802	*
803	* @returns VBox status code.
804	* @param pThis The dynamic mapping cache instance.
805	* @param cPages The size of the new segment, give as a page count.
806	*/
807	static int pgmR0DynMapAddSeg(PPGMR0DYNMAP pThis, uint32_t cPages)
808	{
809	int rc2;
810	AssertReturn(ASMGetFlags() & X86_EFL_IF, VERR_PREEMPT_DISABLED);
811
812	/*
813	* Do the array reallocations first.
814	* (The pages array has to be replaced behind the spinlock of course.)
815	*/
816	void pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) (pThis->cPages + cPages));
817	if (!pvSavedPTEs)
818	return VERR_NO_MEMORY;
819	pThis->pvSavedPTEs = pvSavedPTEs;
820
821	void pvPages = RTMemAllocZ(sizeof(pThis->paPages[0]) (pThis->cPages + cPages));
822	if (!pvPages)
823	{
824	pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) * pThis->cPages);
825	if (pvSavedPTEs)
826	pThis->pvSavedPTEs = pvSavedPTEs;
827	return VERR_NO_MEMORY;
828	}
829
830	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
831	RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
832
833	memcpy(pvPages, pThis->paPages, sizeof(pThis->paPages[0]) * pThis->cPages);
834	void *pvToFree = pThis->paPages;
835	pThis->paPages = (PPGMR0DYNMAPENTRY)pvPages;
836
837	RTSpinlockRelease(pThis->hSpinlock, &Tmp);
838	RTMemFree(pvToFree);
839
840	/*
841	* Allocate the segment structure and pages of memory, then touch all the pages (paranoia).
842	*/
843	uint32_t cMaxPTs = cPages / (pThis->fLegacyMode ? X86_PG_ENTRIES : X86_PG_PAE_ENTRIES) + 2;
844	PPGMR0DYNMAPSEG pSeg = (PPGMR0DYNMAPSEG)RTMemAllocZ(RT_UOFFSETOF(PGMR0DYNMAPSEG, ahMemObjPTs[cMaxPTs]));
845	if (!pSeg)
846	return VERR_NO_MEMORY;
847	pSeg->pNext = NULL;
848	pSeg->cPages = cPages;
849	pSeg->iPage = pThis->cPages;
850	pSeg->cPTs = 0;
851	int rc = RTR0MemObjAllocPage(&pSeg->hMemObj, cPages << PAGE_SHIFT, false);
852	if (RT_SUCCESS(rc))
853	{
854	uint8_t pbPage = (uint8_t )RTR0MemObjAddress(pSeg->hMemObj);
855	AssertMsg(VALID_PTR(pbPage) && !((uintptr_t)pbPage & PAGE_OFFSET_MASK), ("%p\n", pbPage));
856	memset(pbPage, 0xfe, cPages << PAGE_SHIFT);
857
858	/*
859	* Walk thru the pages and set them up with a mapping of their PTE and everything.
860	*/
861	ASMIntDisable();
862	PGMR0DYNMAPPGLVL PgLvl;
863	pgmR0DynMapPagingArrayInit(pThis, &PgLvl);
864	uint32_t const iEndPage = pSeg->iPage + cPages;
865	for (uint32_t iPage = pSeg->iPage;
866	iPage < iEndPage;
867	iPage++, pbPage += PAGE_SIZE)
868	{
869	/* Initialize the page data. */
870	pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
871	pThis->paPages[iPage].pvPage = pbPage;
872	pThis->paPages[iPage].cRefs = 0;
873	pThis->paPages[iPage].uPte.pPae = 0;
874	RTCpuSetFill(&pThis->paPages[iPage].PendingSet);
875
876	/* Map its page table, retry until we've got a clean run (paranoia). */
877	do
878	rc = pgmR0DynMapPagingArrayMapPte(pThis, &PgLvl, pbPage, pSeg, cMaxPTs,
879	&pThis->paPages[iPage].uPte.pv);
880	while (rc == VINF_TRY_AGAIN);
881	if (RT_FAILURE(rc))
882	break;
883
884	/* Save the PTE. */
885	if (pThis->fLegacyMode)
886	((PX86PGUINT)pThis->pvSavedPTEs)[iPage] = pThis->paPages[iPage].uPte.pLegacy->u;
887	else
888	((PX86PGPAEUINT)pThis->pvSavedPTEs)[iPage] = pThis->paPages[iPage].uPte.pPae->u;
889
890	#ifdef VBOX_STRICT
891	/* Check that we've got the right entry. */
892	RTHCPHYS HCPhysPage = RTR0MemObjGetPagePhysAddr(pSeg->hMemObj, iPage - pSeg->iPage);
893	RTHCPHYS HCPhysPte = pThis->fLegacyMode
894	? pThis->paPages[iPage].uPte.pLegacy->u & X86_PTE_PG_MASK
895	: pThis->paPages[iPage].uPte.pPae->u & X86_PTE_PAE_PG_MASK;
896	if (HCPhysPage != HCPhysPte)
897	{
898	LogRel(("pgmR0DynMapAddSeg: internal error - page #%u HCPhysPage=%RHp HCPhysPte=%RHp pbPage=%p pvPte=%p\n",
899	iPage - pSeg->iPage, HCPhysPage, HCPhysPte, pbPage, pThis->paPages[iPage].uPte.pv));
900	rc = VERR_INTERNAL_ERROR;
901	break;
902	}
903	#endif
904	} /* for each page */
905	ASMIntEnable();
906
907	/* cleanup non-PT mappings */
908	for (uint32_t i = 0; i < PgLvl.cLevels - 1; i++)
909	RTR0MemObjFree(PgLvl.a[i].hMemObj, true /* fFreeMappings */);
910
911	if (RT_SUCCESS(rc))
912	{
913	#if PGMR0DYNMAP_GUARD_PAGES > 0
914	/*
915	* Setup guard pages.
916	* (Note: TLBs will be shot down later on.)
917	*/
918	uint32_t iPage = pSeg->iPage;
919	while (iPage < iEndPage)
920	{
921	for (uint32_t iGPg = 0; iGPg < PGMR0DYNMAP_GUARD_PAGES && iPage < iEndPage; iGPg++, iPage++)
922	pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]);
923	iPage++; /* the guarded page */
924	}
925
926	/* Make sure the very last page is a guard page too. */
927	iPage = iEndPage - 1;
928	if (pThis->paPages[iPage].cRefs != PGMR0DYNMAP_GUARD_PAGE_REF_COUNT)
929	pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]);
930	#endif /* PGMR0DYNMAP_GUARD_PAGES > 0 */
931
932	/*
933	* Commit it by adding the segment to the list and updating the page count.
934	*/
935	pSeg->pNext = pThis->pSegHead;
936	pThis->pSegHead = pSeg;
937	pThis->cPages += cPages;
938	return VINF_SUCCESS;
939	}
940
941	/*
942	* Bail out.
943	*/
944	while (pSeg->cPTs-- > 0)
945	{
946	rc2 = RTR0MemObjFree(pSeg->ahMemObjPTs[pSeg->cPTs], true /* fFreeMappings */);
947	AssertRC(rc2);
948	pSeg->ahMemObjPTs[pSeg->cPTs] = NIL_RTR0MEMOBJ;
949	}
950
951	rc2 = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */);
952	AssertRC(rc2);
953	pSeg->hMemObj = NIL_RTR0MEMOBJ;
954	}
955	RTMemFree(pSeg);
956
957	/* Don't bother resizing the arrays, but free them if we're the only user. */
958	if (!pThis->cPages)
959	{
960	RTMemFree(pThis->paPages);
961	pThis->paPages = NULL;
962	RTMemFree(pThis->pvSavedPTEs);
963	pThis->pvSavedPTEs = NULL;
964	}
965	return rc;
966	}
967
968
969	/**
970	* Called by PGMR0DynMapInitVM under the init lock.
971	*
972	* @returns VBox status code.
973	* @param pThis The dynamic mapping cache instance.
974	*/
975	static int pgmR0DynMapSetup(PPGMR0DYNMAP pThis)
976	{
977	/*
978	* Calc the size and add a segment of that size.
979	*/
980	uint32_t cMinPages;
981	uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages);
982	AssertReturn(cPages, VERR_INTERNAL_ERROR);
983	int rc = pgmR0DynMapAddSeg(pThis, cPages);
984	if (rc == VERR_NO_MEMORY)
985	{
986	/*
987	* Try adding smaller segments.
988	*/
989	do
990	rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES);
991	while (RT_SUCCESS(rc) && pThis->cPages < cPages);
992	if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages)
993	rc = VINF_SUCCESS;
994	if (rc == VERR_NO_MEMORY)
995	{
996	if (pThis->cPages)
997	pgmR0DynMapTearDown(pThis);
998	rc = VERR_PGM_DYNMAP_SETUP_ERROR;
999	}
1000	}
1001	Assert(ASMGetFlags() & X86_EFL_IF);
1002
1003	#if PGMR0DYNMAP_GUARD_PAGES > 0
1004	/* paranoia */
1005	if (RT_SUCCESS(rc))
1006	pgmR0DynMapTlbShootDown(pThis);
1007	#endif
1008	return rc;
1009	}
1010
1011
1012	/**
1013	* Called by PGMR0DynMapInitVM under the init lock.
1014	*
1015	* @returns VBox status code.
1016	* @param pThis The dynamic mapping cache instance.
1017	*/
1018	static int pgmR0DynMapExpand(PPGMR0DYNMAP pThis)
1019	{
1020	/*
1021	* Calc the new target size and add a segment of the appropriate size.
1022	*/
1023	uint32_t cMinPages;
1024	uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages);
1025	AssertReturn(cPages, VERR_INTERNAL_ERROR);
1026	if (pThis->cPages >= cPages)
1027	return VINF_SUCCESS;
1028
1029	uint32_t cAdd = cPages - pThis->cPages;
1030	int rc = pgmR0DynMapAddSeg(pThis, cAdd);
1031	if (rc == VERR_NO_MEMORY)
1032	{
1033	/*
1034	* Try adding smaller segments.
1035	*/
1036	do
1037	rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES);
1038	while (RT_SUCCESS(rc) && pThis->cPages < cPages);
1039	if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages)
1040	rc = VINF_SUCCESS;
1041	if (rc == VERR_NO_MEMORY)
1042	rc = VERR_PGM_DYNMAP_EXPAND_ERROR;
1043	}
1044	Assert(ASMGetFlags() & X86_EFL_IF);
1045
1046	#if PGMR0DYNMAP_GUARD_PAGES > 0
1047	/* paranoia */
1048	if (RT_SUCCESS(rc))
1049	pgmR0DynMapTlbShootDown(pThis);
1050	#endif
1051	return rc;
1052	}
1053
1054
1055	/**
1056	* Called by PGMR0DynMapTermVM under the init lock.
1057	*
1058	* @returns VBox status code.
1059	* @param pThis The dynamic mapping cache instance.
1060	*/
1061	static void pgmR0DynMapTearDown(PPGMR0DYNMAP pThis)
1062	{
1063	/*
1064	* Restore the original page table entries
1065	*/
1066	PPGMR0DYNMAPENTRY paPages = pThis->paPages;
1067	uint32_t iPage = pThis->cPages;
1068	if (pThis->fLegacyMode)
1069	{
1070	X86PGUINT const paSavedPTEs = (X86PGUINT const )pThis->pvSavedPTEs;
1071	while (iPage-- > 0)
1072	{
1073	X86PGUINT uOld = paPages[iPage].uPte.pLegacy->u;
1074	X86PGUINT uOld2 = uOld; NOREF(uOld2);
1075	X86PGUINT uNew = paSavedPTEs[iPage];
1076	while (!ASMAtomicCmpXchgExU32(&paPages[iPage].uPte.pLegacy->u, uNew, uOld, &uOld))
1077	AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
1078	Assert(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage]);
1079	}
1080	}
1081	else
1082	{
1083	X86PGPAEUINT const paSavedPTEs = (X86PGPAEUINT const )pThis->pvSavedPTEs;
1084	while (iPage-- > 0)
1085	{
1086	X86PGPAEUINT uOld = paPages[iPage].uPte.pPae->u;
1087	X86PGPAEUINT uOld2 = uOld; NOREF(uOld2);
1088	X86PGPAEUINT uNew = paSavedPTEs[iPage];
1089	while (!ASMAtomicCmpXchgExU64(&paPages[iPage].uPte.pPae->u, uNew, uOld, &uOld))
1090	AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
1091	Assert(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage]);
1092	}
1093	}
1094
1095	/*
1096	* Shoot down the TLBs on all CPUs before freeing them.
1097	*/
1098	pgmR0DynMapTlbShootDown(pThis);
1099
1100	/*
1101	* Free the segments.
1102	*/
1103	while (pThis->pSegHead)
1104	{
1105	int rc;
1106	PPGMR0DYNMAPSEG pSeg = pThis->pSegHead;
1107	pThis->pSegHead = pSeg->pNext;
1108
1109	uint32_t iPT = pSeg->cPTs;
1110	while (iPT-- > 0)
1111	{
1112	rc = RTR0MemObjFree(pSeg->ahMemObjPTs[iPT], true /* fFreeMappings */); AssertRC(rc);
1113	pSeg->ahMemObjPTs[iPT] = NIL_RTR0MEMOBJ;
1114	}
1115	rc = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */); AssertRC(rc);
1116	pSeg->hMemObj = NIL_RTR0MEMOBJ;
1117	pSeg->pNext = NULL;
1118	pSeg->iPage = UINT16_MAX;
1119	pSeg->cPages = 0;
1120	pSeg->cPTs = 0;
1121	RTMemFree(pSeg);
1122	}
1123
1124	/*
1125	* Free the arrays and restore the initial state.
1126	* The cLoadMax value is left behind for the next setup.
1127	*/
1128	RTMemFree(pThis->paPages);
1129	pThis->paPages = NULL;
1130	RTMemFree(pThis->pvSavedPTEs);
1131	pThis->pvSavedPTEs = NULL;
1132	pThis->cPages = 0;
1133	pThis->cLoad = 0;
1134	pThis->cGuardPages = 0;
1135	}
1136
1137
1138	/**
1139	* Release references to a page, caller owns the spin lock.
1140	*
1141	* @param pThis The dynamic mapping cache instance.
1142	* @param iPage The page.
1143	* @param cRefs The number of references to release.
1144	*/
1145	DECLINLINE(void) pgmR0DynMapReleasePageLocked(PPGMR0DYNMAP pThis, uint32_t iPage, int32_t cRefs)
1146	{
1147	cRefs = ASMAtomicSubS32(&pThis->paPages[iPage].cRefs, cRefs) - cRefs;
1148	AssertMsg(cRefs >= 0, ("%d\n", cRefs));
1149	if (!cRefs)
1150	pThis->cLoad--;
1151	}
1152
1153
1154	/**
1155	* Release references to a page, caller does not own the spin lock.
1156	*
1157	* @param pThis The dynamic mapping cache instance.
1158	* @param iPage The page.
1159	* @param cRefs The number of references to release.
1160	*/
1161	static void pgmR0DynMapReleasePage(PPGMR0DYNMAP pThis, uint32_t iPage, uint32_t cRefs)
1162	{
1163	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1164	RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1165	pgmR0DynMapReleasePageLocked(pThis, iPage, cRefs);
1166	RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1167	}
1168
1169
1170	/**
1171	* pgmR0DynMapPage worker that deals with the tedious bits.
1172	*
1173	* @returns The page index on success, UINT32_MAX on failure.
1174	* @param pThis The dynamic mapping cache instance.
1175	* @param HCPhys The address of the page to be mapped.
1176	* @param iPage The page index pgmR0DynMapPage hashed HCPhys to.
1177	* @param pVM The shared VM structure, for statistics only.
1178	*/
1179	static uint32_t pgmR0DynMapPageSlow(PPGMR0DYNMAP pThis, RTHCPHYS HCPhys, uint32_t iPage, PVM pVM)
1180	{
1181	STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageSlow);
1182
1183	/*
1184	* Check if any of the first 3 pages are unreferenced since the caller
1185	* already has made sure they aren't matching.
1186	*/
1187	#ifdef VBOX_WITH_STATISTICS
1188	bool fLooped = false;
1189	#endif
1190	uint32_t const cPages = pThis->cPages;
1191	PPGMR0DYNMAPENTRY paPages = pThis->paPages;
1192	uint32_t iFreePage;
1193	if (!paPages[iPage].cRefs)
1194	iFreePage = iPage;
1195	else if (!paPages[(iPage + 1) % cPages].cRefs)
1196	iFreePage = (iPage + 1) % cPages;
1197	else if (!paPages[(iPage + 2) % cPages].cRefs)
1198	iFreePage = (iPage + 2) % cPages;
1199	else
1200	{
1201	/*
1202	* Search for an unused or matching entry.
1203	*/
1204	iFreePage = (iPage + 3) % cPages;
1205	for (;;)
1206	{
1207	if (paPages[iFreePage].HCPhys == HCPhys)
1208	{
1209	STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageSlowLoopHits);
1210	return iFreePage;
1211	}
1212	if (!paPages[iFreePage].cRefs)
1213	break;
1214
1215	/* advance */
1216	iFreePage = (iFreePage + 1) % cPages;
1217	if (RT_UNLIKELY(iFreePage == iPage))
1218	return UINT32_MAX;
1219	}
1220	STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageSlowLoopMisses);
1221	#ifdef VBOX_WITH_STATISTICS
1222	fLooped = true;
1223	#endif
1224	}
1225	Assert(iFreePage < cPages);
1226
1227	#if 0 //def VBOX_WITH_STATISTICS
1228	/* Check for lost hits. */
1229	if (!fLooped)
1230	for (uint32_t iPage2 = (iPage + 3) % cPages; iPage2 != iPage; iPage2 = (iPage2 + 1) % cPages)
1231	if (paPages[iPage2].HCPhys == HCPhys)
1232	STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageSlowLostHits);
1233	#endif
1234
1235	/*
1236	* Setup the new entry.
1237	*/
1238	/Log6(("pgmR0DynMapPageSlow: old - %RHp %#x %#llx\n", paPages[iFreePage].HCPhys, paPages[iFreePage].cRefs, paPages[iFreePage].uPte.pPae->u));/
1239	paPages[iFreePage].HCPhys = HCPhys;
1240	RTCpuSetFill(&paPages[iFreePage].PendingSet);
1241	if (pThis->fLegacyMode)
1242	{
1243	X86PGUINT uOld = paPages[iFreePage].uPte.pLegacy->u;
1244	X86PGUINT uOld2 = uOld; NOREF(uOld2);
1245	X86PGUINT uNew = (uOld & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1246	\| X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1247	\| (HCPhys & X86_PTE_PG_MASK);
1248	while (!ASMAtomicCmpXchgExU32(&paPages[iFreePage].uPte.pLegacy->u, uNew, uOld, &uOld))
1249	AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
1250	Assert(paPages[iFreePage].uPte.pLegacy->u == uNew);
1251	}
1252	else
1253	{
1254	X86PGPAEUINT uOld = paPages[iFreePage].uPte.pPae->u;
1255	X86PGPAEUINT uOld2 = uOld; NOREF(uOld2);
1256	X86PGPAEUINT uNew = (uOld & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1257	\| X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1258	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1259	while (!ASMAtomicCmpXchgExU64(&paPages[iFreePage].uPte.pPae->u, uNew, uOld, &uOld))
1260	AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
1261	Assert(paPages[iFreePage].uPte.pPae->u == uNew);
1262	/Log6(("pgmR0DynMapPageSlow: #%x - %RHp %p %#llx\n", iFreePage, HCPhys, paPages[iFreePage].pvPage, uNew));/
1263	}
1264	return iFreePage;
1265	}
1266
1267
1268	/**
1269	* Maps a page into the pool.
1270	*
1271	* @returns Page index on success, UINT32_MAX on failure.
1272	* @param pThis The dynamic mapping cache instance.
1273	* @param HCPhys The address of the page to be mapped.
1274	* @param pVM The shared VM structure, for statistics only.
1275	* @param ppvPage Where to the page address.
1276	*/
1277	DECLINLINE(uint32_t) pgmR0DynMapPage(PPGMR0DYNMAP pThis, RTHCPHYS HCPhys, PVM pVM, void **ppvPage)
1278	{
1279	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1280	RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1281	AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
1282	STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPage);
1283
1284	/*
1285	* Find an entry, if possible a matching one. The HCPhys address is hashed
1286	* down to a page index, collisions are handled by linear searching.
1287	* Optimized for a hit in the first 3 pages.
1288	*
1289	* To the cheap hits here and defer the tedious searching and inserting
1290	* to a helper function.
1291	*/
1292	uint32_t const cPages = pThis->cPages;
1293	uint32_t iPage = (HCPhys >> PAGE_SHIFT) % cPages;
1294	PPGMR0DYNMAPENTRY paPages = pThis->paPages;
1295	if (RT_LIKELY(paPages[iPage].HCPhys == HCPhys))
1296	STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageHits0);
1297	else
1298	{
1299	uint32_t iPage2 = (iPage + 1) % cPages;
1300	if (RT_LIKELY(paPages[iPage2].HCPhys == HCPhys))
1301	{
1302	iPage = iPage2;
1303	STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageHits1);
1304	}
1305	else
1306	{
1307	iPage2 = (iPage + 2) % cPages;
1308	if (paPages[iPage2].HCPhys == HCPhys)
1309	{
1310	iPage = iPage2;
1311	STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageHits2);
1312	}
1313	else
1314	{
1315	iPage = pgmR0DynMapPageSlow(pThis, HCPhys, iPage, pVM);
1316	if (RT_UNLIKELY(iPage == UINT32_MAX))
1317	{
1318	RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1319	return iPage;
1320	}
1321	}
1322	}
1323	}
1324
1325	/*
1326	* Reference it, update statistics and get the return address.
1327	*/
1328	int32_t cRefs = ASMAtomicIncS32(&paPages[iPage].cRefs);
1329	if (cRefs == 1)
1330	{
1331	pThis->cLoad++;
1332	if (pThis->cLoad > pThis->cMaxLoad)
1333	pThis->cMaxLoad = pThis->cLoad;
1334	AssertMsg(pThis->cLoad <= pThis->cPages - pThis->cGuardPages, ("%d/%d\n", pThis->cLoad, pThis->cPages - pThis->cGuardPages));
1335	}
1336	else if (RT_UNLIKELY(cRefs <= 0))
1337	{
1338	ASMAtomicDecS32(&paPages[iPage].cRefs);
1339	RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1340	AssertLogRelMsgFailedReturn(("cRefs=%d iPage=%p HCPhys=%RHp\n", cRefs, iPage, HCPhys), UINT32_MAX);
1341	}
1342	void *pvPage = paPages[iPage].pvPage;
1343
1344	/*
1345	* Invalidate the entry?
1346	*/
1347	RTCPUID idRealCpu = RTMpCpuId();
1348	bool fInvalidateIt = RTCpuSetIsMember(&paPages[iPage].PendingSet, idRealCpu);
1349	if (RT_UNLIKELY(fInvalidateIt))
1350	RTCpuSetDel(&paPages[iPage].PendingSet, idRealCpu);
1351
1352	RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1353
1354	/*
1355	* Do the actual invalidation outside the spinlock.
1356	*/
1357	if (RT_UNLIKELY(fInvalidateIt))
1358	{
1359	STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageInvlPg);
1360	ASMInvalidatePage(pvPage);
1361	}
1362
1363	*ppvPage = pvPage;
1364	return iPage;
1365	}
1366
1367
1368	/**
1369	* Assert the the integrity of the pool.
1370	*
1371	* @returns VBox status code.
1372	*/
1373	VMMR0DECL(int) PGMR0DynMapAssertIntegrity(void)
1374	{
1375	/*
1376	* Basic pool stuff that doesn't require any lock, just assumes we're a user.
1377	*/
1378	PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
1379	if (!pThis)
1380	return VINF_SUCCESS;
1381	AssertPtrReturn(pThis, VERR_INVALID_POINTER);
1382	AssertReturn(pThis->u32Magic == PGMR0DYNMAP_MAGIC, VERR_INVALID_MAGIC);
1383	if (!pThis->cUsers)
1384	return VERR_INVALID_PARAMETER;
1385
1386
1387	int rc = VINF_SUCCESS;
1388	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1389	RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1390
1391	#define CHECK_RET(expr, a) \
1392	do { \
1393	if (RT_UNLIKELY(!(expr))) \
1394	{ \
1395	RTSpinlockRelease(pThis->hSpinlock, &Tmp); \
1396	AssertMsg1(#expr, __LINE__, __FILE__, __PRETTY_FUNCTION__); \
1397	AssertMsg2 a; \
1398	return VERR_INTERNAL_ERROR; \
1399	} \
1400	} while (0)
1401
1402	/*
1403	* Check that the PTEs are correct.
1404	*/
1405	uint32_t cGuard = 0;
1406	uint32_t cLoad = 0;
1407	PPGMR0DYNMAPENTRY paPages = pThis->paPages;
1408	uint32_t iPage = pThis->cPages;
1409	if (pThis->fLegacyMode)
1410	{
1411	PCX86PGUINT paSavedPTEs = (PCX86PGUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
1412	while (iPage-- > 0)
1413	{
1414	CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
1415	if ( paPages[iPage].cRefs == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT
1416	&& paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS)
1417	{
1418	#ifdef PGMR0DYNMAP_GUARD_NP
1419	CHECK_RET(paPages[iPage].uPte.pLegacy->u == (paSavedPTEs[iPage] & ~(X86PGUINT)X86_PTE_P),
1420	("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage]));
1421	#else
1422	CHECK_RET(paPages[iPage].uPte.pLegacy->u == PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE,
1423	("#%u: %#x", iPage, paPages[iPage].uPte.pLegacy->u));
1424	#endif
1425	cGuard++;
1426	}
1427	else if (paPages[iPage].HCPhys != NIL_RTHCPHYS)
1428	{
1429	CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
1430	X86PGUINT uPte = (paSavedPTEs[iPage] & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1431	\| X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1432	\| (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
1433	CHECK_RET(paPages[iPage].uPte.pLegacy->u == uPte,
1434	("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, uPte));
1435	if (paPages[iPage].cRefs)
1436	cLoad++;
1437	}
1438	else
1439	CHECK_RET(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage],
1440	("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage]));
1441	}
1442	}
1443	else
1444	{
1445	PCX86PGPAEUINT paSavedPTEs = (PCX86PGPAEUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
1446	while (iPage-- > 0)
1447	{
1448	CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
1449	if ( paPages[iPage].cRefs == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT
1450	&& paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS)
1451	{
1452	#ifdef PGMR0DYNMAP_GUARD_NP
1453	CHECK_RET(paPages[iPage].uPte.pPae->u == (paSavedPTEs[iPage] & ~(X86PGPAEUINT)X86_PTE_P),
1454	("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage]));
1455	#else
1456	CHECK_RET(paPages[iPage].uPte.pPae->u == PGMR0DYNMAP_GUARD_PAGE_PAE_PTE,
1457	("#%u: %#llx", iPage, paPages[iPage].uPte.pPae->u));
1458	#endif
1459	cGuard++;
1460	}
1461	else if (paPages[iPage].HCPhys != NIL_RTHCPHYS)
1462	{
1463	CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
1464	X86PGPAEUINT uPte = (paSavedPTEs[iPage] & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1465	\| X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1466	\| (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
1467	CHECK_RET(paPages[iPage].uPte.pPae->u == uPte,
1468	("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pLegacy->u, uPte));
1469	if (paPages[iPage].cRefs)
1470	cLoad++;
1471	}
1472	else
1473	CHECK_RET(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage],
1474	("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage]));
1475	}
1476	}
1477
1478	CHECK_RET(cLoad == pThis->cLoad, ("%u %u\n", cLoad, pThis->cLoad));
1479	CHECK_RET(cGuard == pThis->cGuardPages, ("%u %u\n", cGuard, pThis->cGuardPages));
1480
1481	#undef CHECK_RET
1482	RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1483	return VINF_SUCCESS;
1484	}
1485
1486
1487	/**
1488	* Signals the start of a new set of mappings.
1489	*
1490	* Mostly for strictness. PGMDynMapHCPage won't work unless this
1491	* API is called.
1492	*
1493	* @param pVCpu The shared data for the current virtual CPU.
1494	*/
1495	VMMDECL(void) PGMDynMapStartAutoSet(PVMCPU pVCpu)
1496	{
1497	Assert(pVCpu->pgm.s.AutoSet.cEntries == PGMMAPSET_CLOSED);
1498	pVCpu->pgm.s.AutoSet.cEntries = 0;
1499	}
1500
1501
1502	/**
1503	* Worker that performs the actual flushing of the set.
1504	*
1505	* @param pSet The set to flush.
1506	* @param cEntries The number of entries.
1507	*/
1508	DECLINLINE(void) pgmDynMapFlushAutoSetWorker(PPGMMAPSET pSet, uint32_t cEntries)
1509	{
1510	/*
1511	* Release any pages it's referencing.
1512	*/
1513	if ( cEntries != 0
1514	&& RT_LIKELY(cEntries <= RT_ELEMENTS(pSet->aEntries)))
1515	{
1516	PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
1517	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1518	RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1519
1520	uint32_t i = cEntries;
1521	while (i-- > 0)
1522	{
1523	uint32_t iPage = pSet->aEntries[i].iPage;
1524	Assert(iPage < pThis->cPages);
1525	int32_t cRefs = pSet->aEntries[i].cRefs;
1526	Assert(cRefs > 0);
1527	pgmR0DynMapReleasePageLocked(pThis, iPage, cRefs);
1528
1529	pSet->aEntries[i].iPage = UINT16_MAX;
1530	pSet->aEntries[i].cRefs = 0;
1531	}
1532
1533	Assert(pThis->cLoad <= pThis->cPages - pThis->cGuardPages);
1534	RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1535	}
1536	}
1537
1538
1539	/**
1540	* Releases the dynamic memory mappings made by PGMDynMapHCPage and associates
1541	* since the PGMDynMapStartAutoSet call.
1542	*
1543	* @param pVCpu The shared data for the current virtual CPU.
1544	*/
1545	VMMDECL(void) PGMDynMapReleaseAutoSet(PVMCPU pVCpu)
1546	{
1547	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1548
1549	/*
1550	* Close and flush the set.
1551	*/
1552	uint32_t cEntries = pSet->cEntries;
1553	AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
1554	AssertMsg(cEntries <= RT_ELEMENTS(pSet->aEntries), ("%#x (%u)\n", cEntries, cEntries));
1555	pSet->cEntries = PGMMAPSET_CLOSED;
1556
1557	pgmDynMapFlushAutoSetWorker(pSet, cEntries);
1558	}
1559
1560
1561	/**
1562	* Flushes the set if it's above a certain threshold.
1563	*
1564	* @param pVCpu The shared data for the current virtual CPU.
1565	*/
1566	VMMDECL(void) PGMDynMapFlushAutoSet(PVMCPU pVCpu)
1567	{
1568	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1569
1570	/*
1571	* Only flush it if it's 50% full.
1572	*/
1573	uint32_t cEntries = pSet->cEntries;
1574	AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
1575	if (cEntries >= RT_ELEMENTS(pSet->aEntries) / 2)
1576	{
1577	AssertMsg(cEntries <= RT_ELEMENTS(pSet->aEntries), ("%#x (%u)\n", cEntries, cEntries));
1578	pSet->cEntries = 0;
1579
1580	pgmDynMapFlushAutoSetWorker(pSet, cEntries);
1581	}
1582	}
1583
1584
1585
1586	/**
1587	* Migrates the automatic mapping set of the current vCPU if it's active and
1588	* necessary.
1589	*
1590	* This is called when re-entering the hardware assisted execution mode after a
1591	* nip down to ring-3. We run the risk that the CPU might have change and we
1592	* will therefore make sure all the cache entries currently in the auto set will
1593	* be valid on the new CPU. If the cpu didn't change nothing will happen as all
1594	* the entries will have been flagged as invalidated.
1595	*
1596	* @param pVCpu The shared data for the current virtual CPU.
1597	* @thread EMT
1598	*/
1599	VMMDECL(void) PGMDynMapMigrateAutoSet(PVMCPU pVCpu)
1600	{
1601	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1602	uint32_t i = pSet->cEntries;
1603	if (i != PGMMAPSET_CLOSED)
1604	{
1605	AssertMsg(i <= RT_ELEMENTS(pSet->aEntries), ("%#x (%u)\n", i, i));
1606	if (i != 0 && RT_LIKELY(i <= RT_ELEMENTS(pSet->aEntries)))
1607	{
1608	PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
1609	RTCPUID idRealCpu = RTMpCpuId();
1610
1611	while (i-- > 0)
1612	{
1613	Assert(pSet->aEntries[i].cRefs > 0);
1614	uint32_t iPage = pSet->aEntries[i].iPage;
1615	Assert(iPage < pThis->cPages);
1616	if (RTCpuSetIsMember(&pThis->paPages[iPage].PendingSet, idRealCpu))
1617	{
1618	RTCpuSetDel(&pThis->paPages[iPage].PendingSet, idRealCpu);
1619	ASMInvalidatePage(pThis->paPages[iPage].pvPage);
1620	STAM_COUNTER_INC(&pVCpu->pVMR0->pgm.s.StatR0DynMapMigrateInvlPg);
1621	}
1622	}
1623	}
1624	}
1625	}
1626
1627
1628	/**
1629	* As a final resort for a full auto set, try merge duplicate entries.
1630	*
1631	* @param pSet The set.
1632	*/
1633	static void pgmDynMapOptimizeAutoSet(PPGMMAPSET pSet)
1634	{
1635	for (uint32_t i = 0 ; i < pSet->cEntries; i++)
1636	{
1637	uint16_t const iPage = pSet->aEntries[i].iPage;
1638	uint32_t j = i + 1;
1639	while (j < pSet->cEntries)
1640	{
1641	if (pSet->aEntries[j].iPage != iPage)
1642	j++;
1643	else if ((uint32_t)pSet->aEntries[i].cRefs + (uint32_t)pSet->aEntries[j].cRefs < UINT16_MAX)
1644	{
1645	/* merge j into i removing j. */
1646	pSet->aEntries[i].cRefs += pSet->aEntries[j].cRefs;
1647	pSet->cEntries--;
1648	if (j < pSet->cEntries)
1649	{
1650	pSet->aEntries[j] = pSet->aEntries[pSet->cEntries];
1651	pSet->aEntries[pSet->cEntries].iPage = UINT16_MAX;
1652	pSet->aEntries[pSet->cEntries].cRefs = 0;
1653	}
1654	else
1655	{
1656	pSet->aEntries[j].iPage = UINT16_MAX;
1657	pSet->aEntries[j].cRefs = 0;
1658	}
1659	}
1660	else
1661	{
1662	/* migrate the max number of refs from j into i and quit the inner loop. */
1663	uint32_t cMigrate = UINT16_MAX - 1 - pSet->aEntries[i].cRefs;
1664	Assert(pSet->aEntries[j].cRefs > cMigrate);
1665	pSet->aEntries[j].cRefs -= cMigrate;
1666	pSet->aEntries[i].cRefs = UINT16_MAX - 1;
1667	break;
1668	}
1669	}
1670	}
1671	}
1672
1673
1674	/**
1675	* Common worker code for PGMDynMapHCPhys, pgmR0DynMapHCPageInlined and
1676	* pgmR0DynMapGCPageInlined.
1677	*
1678	* @returns VBox status code.
1679	* @param pVM The shared VM structure (for statistics).
1680	* @param pSet The set.
1681	* @param HCPhys The physical address of the page.
1682	* @param ppv Where to store the address of the mapping on success.
1683	*/
1684	int pgmR0DynMapHCPageCommon(PVM pVM, PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv)
1685	{
1686	/*
1687	* Map it.
1688	*/
1689	void *pvPage;
1690	uint32_t const iPage = pgmR0DynMapPage(g_pPGMR0DynMap, HCPhys, pVM, &pvPage);
1691	if (RT_UNLIKELY(iPage == UINT32_MAX))
1692	{
1693	static uint32_t s_cBitched = 0;
1694	if (++s_cBitched < 10)
1695	LogRel(("PGMDynMapHCPage: cLoad=%u/%u cPages=%u cGuardPages=%u\n",
1696	g_pPGMR0DynMap->cLoad, g_pPGMR0DynMap->cMaxLoad, g_pPGMR0DynMap->cPages, g_pPGMR0DynMap->cGuardPages));
1697	*ppv = NULL;
1698	return VERR_PGM_DYNMAP_FAILED;
1699	}
1700
1701	/*
1702	* Add the page to the auto reference set.
1703	*
1704	* The typical usage pattern means that the same pages will be mapped
1705	* several times in the same set. We can catch most of these
1706	* remappings by looking a few pages back into the set. (The searching
1707	* and set optimizing path will hardly ever be used when doing this.)
1708	*/
1709	AssertCompile(RT_ELEMENTS(pSet->aEntries) >= 8);
1710	int32_t i = pSet->cEntries;
1711	if (i-- < 5)
1712	{
1713	unsigned iEntry = pSet->cEntries++;
1714	pSet->aEntries[iEntry].cRefs = 1;
1715	pSet->aEntries[iEntry].iPage = iPage;
1716	pSet->aEntries[iEntry].pvPage = pvPage;
1717	pSet->aEntries[iEntry].HCPhys = HCPhys;
1718	pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
1719	}
1720	/* Any of the last 5 pages? */
1721	else if ( pSet->aEntries[i - 0].iPage == iPage
1722	&& pSet->aEntries[i - 0].cRefs < UINT16_MAX - 1)
1723	pSet->aEntries[i - 0].cRefs++;
1724	else if ( pSet->aEntries[i - 1].iPage == iPage
1725	&& pSet->aEntries[i - 1].cRefs < UINT16_MAX - 1)
1726	pSet->aEntries[i - 1].cRefs++;
1727	else if ( pSet->aEntries[i - 2].iPage == iPage
1728	&& pSet->aEntries[i - 2].cRefs < UINT16_MAX - 1)
1729	pSet->aEntries[i - 2].cRefs++;
1730	else if ( pSet->aEntries[i - 3].iPage == iPage
1731	&& pSet->aEntries[i - 3].cRefs < UINT16_MAX - 1)
1732	pSet->aEntries[i - 3].cRefs++;
1733	else if ( pSet->aEntries[i - 4].iPage == iPage
1734	&& pSet->aEntries[i - 4].cRefs < UINT16_MAX - 1)
1735	pSet->aEntries[i - 4].cRefs++;
1736	/* Don't bother searching unless we're above a 75% load. */
1737	else if (RT_LIKELY(i <= (int32_t)RT_ELEMENTS(pSet->aEntries) / 4 * 3))
1738	{
1739	unsigned iEntry = pSet->cEntries++;
1740	pSet->aEntries[iEntry].cRefs = 1;
1741	pSet->aEntries[iEntry].iPage = iPage;
1742	pSet->aEntries[iEntry].pvPage = pvPage;
1743	pSet->aEntries[iEntry].HCPhys = HCPhys;
1744	pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
1745	}
1746	else
1747	{
1748	/* Search the rest of the set. */
1749	Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
1750	i -= 4;
1751	while (i-- > 0)
1752	if ( pSet->aEntries[i].iPage == iPage
1753	&& pSet->aEntries[i].cRefs < UINT16_MAX - 1)
1754	{
1755	pSet->aEntries[i].cRefs++;
1756	STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapSetSearchHits);
1757	break;
1758	}
1759	if (i < 0)
1760	{
1761	STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapSetSearchMisses);
1762	if (RT_UNLIKELY(pSet->cEntries >= RT_ELEMENTS(pSet->aEntries)))
1763	{
1764	STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapSetOptimize);
1765	pgmDynMapOptimizeAutoSet(pSet);
1766	}
1767	if (RT_LIKELY(pSet->cEntries < RT_ELEMENTS(pSet->aEntries)))
1768	{
1769	unsigned iEntry = pSet->cEntries++;
1770	pSet->aEntries[iEntry].cRefs = 1;
1771	pSet->aEntries[iEntry].iPage = iPage;
1772	pSet->aEntries[iEntry].pvPage = pvPage;
1773	pSet->aEntries[iEntry].HCPhys = HCPhys;
1774	pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
1775	}
1776	else
1777	{
1778	/* We're screwed. */
1779	pgmR0DynMapReleasePage(g_pPGMR0DynMap, iPage, 1);
1780
1781	static uint32_t s_cBitched = 0;
1782	if (++s_cBitched < 10)
1783	LogRel(("PGMDynMapHCPage: set is full!\n"));
1784	*ppv = NULL;
1785	return VERR_PGM_DYNMAP_FULL_SET;
1786	}
1787	}
1788	}
1789
1790	*ppv = pvPage;
1791	return VINF_SUCCESS;
1792	}
1793
1794
1795	/* documented elsewhere - a bit of a mess.
1796	This is a VERY hot path. */
1797	VMMDECL(int) PGMDynMapHCPage(PVM pVM, RTHCPHYS HCPhys, void **ppv)
1798	{
1799	/*
1800	* Validate state.
1801	*/
1802	STAM_PROFILE_START(&pVM->pgm.s.StatR0DynMapHCPage, a);
1803	AssertPtr(ppv);
1804	AssertMsgReturn(pVM->pgm.s.pvR0DynMapUsed == g_pPGMR0DynMap,
1805	("%p != %p\n", pVM->pgm.s.pvR0DynMapUsed, g_pPGMR0DynMap),
1806	VERR_ACCESS_DENIED);
1807	AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
1808	PVMCPU pVCpu = VMMGetCpu(pVM);
1809	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1810	AssertPtrReturn(pVCpu, VERR_INTERNAL_ERROR);
1811	AssertMsgReturn(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries),
1812	("%#x (%u)\n", pSet->cEntries, pSet->cEntries), VERR_WRONG_ORDER);
1813
1814	/*
1815	* Call common code.
1816	*/
1817	int rc = pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
1818
1819	STAM_PROFILE_STOP(&pVM->pgm.s.StatR0DynMapHCPage, a);
1820	return rc;
1821	}
1822
1823
1824	#ifdef DEBUG
1825	/** For pgmR0DynMapTest3PerCpu. */
1826	typedef struct PGMR0DYNMAPTEST
1827	{
1828	uint32_t u32Expect;
1829	uint32_t *pu32;
1830	uint32_t volatile cFailures;
1831	} PGMR0DYNMAPTEST;
1832	typedef PGMR0DYNMAPTEST *PPGMR0DYNMAPTEST;
1833
1834	/**
1835	* Checks that the content of the page is the same on all CPUs, i.e. that there
1836	* are no CPU specfic PTs or similar nasty stuff involved.
1837	*
1838	* @param idCpu The current CPU.
1839	* @param pvUser1 Pointer a PGMR0DYNMAPTEST structure.
1840	* @param pvUser2 Unused, ignored.
1841	*/
1842	static DECLCALLBACK(void) pgmR0DynMapTest3PerCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
1843	{
1844	PPGMR0DYNMAPTEST pTest = (PPGMR0DYNMAPTEST)pvUser1;
1845	ASMInvalidatePage(pTest->pu32);
1846	if (*pTest->pu32 != pTest->u32Expect)
1847	ASMAtomicIncU32(&pTest->cFailures);
1848	NOREF(pvUser2); NOREF(idCpu);
1849	}
1850
1851
1852	/**
1853	* Performs some basic tests in debug builds.
1854	*/
1855	static int pgmR0DynMapTest(PVM pVM)
1856	{
1857	LogRel(("pgmR0DynMapTest: **** START ****\n"));
1858	PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
1859	PPGMMAPSET pSet = &pVM->aCpus[0].pgm.s.AutoSet;
1860	uint32_t i;
1861
1862	/*
1863	* Assert internal integrity first.
1864	*/
1865	LogRel(("Test #0\n"));
1866	int rc = PGMR0DynMapAssertIntegrity();
1867	if (RT_FAILURE(rc))
1868	return rc;
1869
1870	void *pvR0DynMapUsedSaved = pVM->pgm.s.pvR0DynMapUsed;
1871	pVM->pgm.s.pvR0DynMapUsed = pThis;
1872
1873	/*
1874	* Simple test, map CR3 twice and check that we're getting the
1875	* same mapping address back.
1876	*/
1877	LogRel(("Test #1\n"));
1878	ASMIntDisable();
1879	PGMDynMapStartAutoSet(&pVM->aCpus[0]);
1880
1881	uint64_t cr3 = ASMGetCR3() & ~(uint64_t)PAGE_OFFSET_MASK;
1882	void pv = (void )(intptr_t)-1;
1883	void pv2 = (void )(intptr_t)-2;
1884	rc = PGMDynMapHCPage(pVM, cr3, &pv);
1885	int rc2 = PGMDynMapHCPage(pVM, cr3, &pv2);
1886	ASMIntEnable();
1887	if ( RT_SUCCESS(rc2)
1888	&& RT_SUCCESS(rc)
1889	&& pv == pv2)
1890	{
1891	LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
1892	rc = PGMR0DynMapAssertIntegrity();
1893
1894	/*
1895	* Check that the simple set overflow code works by filling it
1896	* with more CR3 mappings.
1897	*/
1898	LogRel(("Test #2\n"));
1899	ASMIntDisable();
1900	for (i = 0 ; i < UINT16_MAX*2 - 1 && RT_SUCCESS(rc) && pv2 == pv; i++)
1901	{
1902	pv2 = (void *)(intptr_t)-4;
1903	rc = PGMDynMapHCPage(pVM, cr3, &pv2);
1904	}
1905	ASMIntEnable();
1906	if (RT_FAILURE(rc) \|\| pv != pv2)
1907	{
1908	LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%p\n", __LINE__, rc, pv, pv2, i));
1909	if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
1910	}
1911	else if (pSet->cEntries != 5)
1912	{
1913	LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries) / 2));
1914	rc = VERR_INTERNAL_ERROR;
1915	}
1916	else if ( pSet->aEntries[4].cRefs != UINT16_MAX - 1
1917	\|\| pSet->aEntries[3].cRefs != UINT16_MAX - 1
1918	\|\| pSet->aEntries[2].cRefs != 1
1919	\|\| pSet->aEntries[1].cRefs != 1
1920	\|\| pSet->aEntries[0].cRefs != 1)
1921	{
1922	LogRel(("failed(%d): bad set dist: ", __LINE__));
1923	for (i = 0; i < pSet->cEntries; i++)
1924	LogRel(("[%d]=%d, ", i, pSet->aEntries[i].cRefs));
1925	LogRel(("\n"));
1926	rc = VERR_INTERNAL_ERROR;
1927	}
1928	if (RT_SUCCESS(rc))
1929	rc = PGMR0DynMapAssertIntegrity();
1930	if (RT_SUCCESS(rc))
1931	{
1932	/*
1933	* Trigger an set optimization run (exactly).
1934	*/
1935	LogRel(("Test #3\n"));
1936	ASMIntDisable();
1937	pv2 = NULL;
1938	for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) - 5 && RT_SUCCESS(rc) && pv2 != pv; i++)
1939	{
1940	pv2 = (void *)(intptr_t)(-5 - i);
1941	rc = PGMDynMapHCPage(pVM, cr3 + PAGE_SIZE * (i + 5), &pv2);
1942	}
1943	ASMIntEnable();
1944	if (RT_FAILURE(rc) \|\| pv == pv2)
1945	{
1946	LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%d\n", __LINE__, rc, pv, pv2, i));
1947	if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
1948	}
1949	else if (pSet->cEntries != RT_ELEMENTS(pSet->aEntries))
1950	{
1951	LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
1952	rc = VERR_INTERNAL_ERROR;
1953	}
1954	LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
1955	if (RT_SUCCESS(rc))
1956	rc = PGMR0DynMapAssertIntegrity();
1957	if (RT_SUCCESS(rc))
1958	{
1959	/*
1960	* Trigger an overflow error.
1961	*/
1962	LogRel(("Test #4\n"));
1963	ASMIntDisable();
1964	for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) + 2; i++)
1965	{
1966	rc = PGMDynMapHCPage(pVM, cr3 - PAGE_SIZE * (i + 5), &pv2);
1967	if (RT_SUCCESS(rc))
1968	rc = PGMR0DynMapAssertIntegrity();
1969	if (RT_FAILURE(rc))
1970	break;
1971	}
1972	ASMIntEnable();
1973	if (rc == VERR_PGM_DYNMAP_FULL_SET)
1974	{
1975	/* flush the set. */
1976	LogRel(("Test #5\n"));
1977	ASMIntDisable();
1978	PGMDynMapMigrateAutoSet(&pVM->aCpus[0]);
1979	PGMDynMapReleaseAutoSet(&pVM->aCpus[0]);
1980	PGMDynMapStartAutoSet(&pVM->aCpus[0]);
1981	ASMIntEnable();
1982
1983	rc = PGMR0DynMapAssertIntegrity();
1984	}
1985	else
1986	{
1987	LogRel(("failed(%d): rc=%Rrc, wanted %d ; pv2=%p Set=%u/%u; i=%d\n", __LINE__,
1988	rc, VERR_PGM_DYNMAP_FULL_SET, pv2, pSet->cEntries, RT_ELEMENTS(pSet->aEntries), i));
1989	if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
1990	}
1991	}
1992	}
1993	}
1994	else
1995	{
1996	LogRel(("failed(%d): rc=%Rrc rc2=%Rrc; pv=%p pv2=%p\n", __LINE__, rc, rc2, pv, pv2));
1997	if (RT_SUCCESS(rc))
1998	rc = rc2;
1999	}
2000
2001	/*
2002	* Check that everyone sees the same stuff.
2003	*/
2004	if (RT_SUCCESS(rc))
2005	{
2006	LogRel(("Test #5\n"));
2007	ASMIntDisable();
2008	RTHCPHYS HCPhysPT = RTR0MemObjGetPagePhysAddr(pThis->pSegHead->ahMemObjPTs[0], 0);
2009	rc = PGMDynMapHCPage(pVM, HCPhysPT, &pv);
2010	if (RT_SUCCESS(rc))
2011	{
2012	PGMR0DYNMAPTEST Test;
2013	uint32_t *pu32Real = &pThis->paPages[pThis->pSegHead->iPage].uPte.pLegacy->u;
2014	Test.pu32 = (uint32_t *)((uintptr_t)pv \| ((uintptr_t)pu32Real & PAGE_OFFSET_MASK));
2015	Test.u32Expect = *pu32Real;
2016	ASMAtomicWriteU32(&Test.cFailures, 0);
2017	ASMIntEnable();
2018
2019	rc = RTMpOnAll(pgmR0DynMapTest3PerCpu, &Test, NULL);
2020	if (RT_FAILURE(rc))
2021	LogRel(("failed(%d): RTMpOnAll rc=%Rrc\n", __LINE__, rc));
2022	else if (Test.cFailures)
2023	{
2024	LogRel(("failed(%d): cFailures=%d pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n", __LINE__,
2025	Test.cFailures, pu32Real, Test.pu32, Test.u32Expect, *Test.pu32));
2026	rc = VERR_INTERNAL_ERROR;
2027	}
2028	else
2029	LogRel(("pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n",
2030	pu32Real, Test.pu32, Test.u32Expect, *Test.pu32));
2031	}
2032	else
2033	{
2034	ASMIntEnable();
2035	LogRel(("failed(%d): rc=%Rrc\n", rc));
2036	}
2037	}
2038
2039	/*
2040	* Clean up.
2041	*/
2042	LogRel(("Cleanup.\n"));
2043	ASMIntDisable();
2044	PGMDynMapMigrateAutoSet(&pVM->aCpus[0]);
2045	PGMDynMapFlushAutoSet(&pVM->aCpus[0]);
2046	PGMDynMapReleaseAutoSet(&pVM->aCpus[0]);
2047	ASMIntEnable();
2048
2049	if (RT_SUCCESS(rc))
2050	rc = PGMR0DynMapAssertIntegrity();
2051	else
2052	PGMR0DynMapAssertIntegrity();
2053
2054	LogRel(("Result: rc=%Rrc Load=%u/%u/%u Set=%#x/%u\n", rc,
2055	pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2056	pVM->pgm.s.pvR0DynMapUsed = pvR0DynMapUsedSaved;
2057	LogRel(("pgmR0DynMapTest: **** END ****\n"));
2058	return rc;
2059	}
2060	#endif /* DEBUG */
2061

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/PGMR0DynMap.cpp@ 15344

Download in other formats: