PGMRZDynMap.cpp@ 36416

Last change on this file since 36416 was 36416, checked in by vboxsync, 14 years ago
PGMRZDynMap.cpp: Committed a slightly modified version of pgm_dynmap.patch. See #3862.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 97.3 KB

Line
1	/* $Id: PGMRZDynMap.cpp 36416 2011-03-24 18:25:59Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, dynamic mapping cache.
4	*/
5
6	/*
7	* Copyright (C) 2008-2011 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Internal Functions *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM_DYNMAP
23	#include <VBox/vmm/pgm.h>
24	#include "PGMInternal.h"
25	#include <VBox/vmm/vm.h>
26	#include "PGMInline.h"
27	#include <VBox/err.h>
28	#include <VBox/param.h>
29	#include <VBox/sup.h>
30	#include <iprt/asm.h>
31	#include <iprt/asm-amd64-x86.h>
32	#include <iprt/assert.h>
33	#ifndef IN_RC
34	# include <iprt/cpuset.h>
35	# include <iprt/mem.h>
36	# include <iprt/memobj.h>
37	# include <iprt/mp.h>
38	# include <iprt/semaphore.h>
39	# include <iprt/spinlock.h>
40	#endif
41	#include <iprt/string.h>
42
43
44	/*******************************************************************************
45	* Defined Constants And Macros *
46	*******************************************************************************/
47	#ifdef IN_RING0
48	/** The max size of the mapping cache (in pages). */
49	# define PGMR0DYNMAP_MAX_PAGES ((16*_1M) >> PAGE_SHIFT)
50	/** The small segment size that is adopted on out-of-memory conditions with a
51	* single big segment. */
52	# define PGMR0DYNMAP_SMALL_SEG_PAGES 128
53	/** The number of pages we reserve per CPU. */
54	# define PGMR0DYNMAP_PAGES_PER_CPU 256
55	/** The minimum number of pages we reserve per CPU.
56	* This must be equal or larger than the autoset size. */
57	# define PGMR0DYNMAP_PAGES_PER_CPU_MIN 64
58	/** Calcs the overload threshold (safety margin). Current set at 50%. */
59	# define PGMR0DYNMAP_CALC_OVERLOAD(cPages) ((cPages) / 2)
60	/** The number of guard pages.
61	* @remarks Never do tuning of the hashing or whatnot with a strict build! */
62	# if defined(VBOX_STRICT)
63	# define PGMR0DYNMAP_GUARD_PAGES 1
64	# else
65	# define PGMR0DYNMAP_GUARD_PAGES 0
66	# endif
67	#endif /* IN_RING0 */
68	/** The dummy physical address of guard pages. */
69	#define PGMR0DYNMAP_GUARD_PAGE_HCPHYS UINT32_C(0x7777feed)
70	/** The dummy reference count of guard pages. (Must be non-zero.) */
71	#define PGMR0DYNMAP_GUARD_PAGE_REF_COUNT INT32_C(0x7777feed)
72	#if 0
73	/** Define this to just clear the present bit on guard pages.
74	* The alternative is to replace the entire PTE with an bad not-present
75	* PTE. Either way, XNU will screw us. :-/ */
76	# define PGMR0DYNMAP_GUARD_NP
77	#endif
78	/** The dummy PTE value for a page. */
79	#define PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE X86_PTE_PG_MASK
80	/** The dummy PTE value for a page. */
81	#define PGMR0DYNMAP_GUARD_PAGE_PAE_PTE UINT64_MAX /X86_PTE_PAE_PG_MASK/
82
83	#ifdef IN_RING0 /* Note! Assertions causes panics if preemption is disabled,
84	* disable this to work around that. */
85	/**
86	* Acquire the spinlock.
87	* This will declare a temporary variable and expands to two statements!
88	*/
89	# define PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis) \
90	RTSPINLOCKTMP MySpinlockTmp = RTSPINLOCKTMP_INITIALIZER; \
91	RTSpinlockAcquire((pThis)->hSpinlock, &MySpinlockTmp)
92	/**
93	* Releases the spinlock.
94	*/
95	# define PGMRZDYNMAP_SPINLOCK_RELEASE(pThis) \
96	RTSpinlockRelease((pThis)->hSpinlock, &MySpinlockTmp)
97
98	/**
99	* Re-acquires the spinlock.
100	*/
101	# define PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis) \
102	RTSpinlockAcquire((pThis)->hSpinlock, &MySpinlockTmp)
103	#else
104	# define PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis) do { } while (0)
105	# define PGMRZDYNMAP_SPINLOCK_RELEASE(pThis) do { } while (0)
106	# define PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis) do { } while (0)
107	#endif
108
109
110	/** Converts a PGMCPUM::AutoSet pointer into a PVMCPU. */
111	#define PGMRZDYNMAP_SET_2_VMCPU(pSet) (RT_FROM_MEMBER(pSet, VMCPU, pgm.s.AutoSet))
112
113	/** Converts a PGMCPUM::AutoSet pointer into a PVM. */
114	#define PGMRZDYNMAP_SET_2_VM(pSet) (PGMRZDYNMAP_SET_2_VMCPU(pSet)->CTX_SUFF(pVM))
115
116	/** Converts a PGMCPUM::AutoSet pointer into a PVM. */
117	#ifdef IN_RC
118	# define PGMRZDYNMAP_SET_2_DYNMAP(pSet) (PGMRZDYNMAP_SET_2_VM(pSet)->pgm.s.pRCDynMap)
119	#else
120	# define PGMRZDYNMAP_SET_2_DYNMAP(pSet) (g_pPGMR0DynMap)
121	#endif
122
123	/**
124	* Gets the set index of the current CPU.
125	*
126	* This always returns 0 when in raw-mode context because there is only ever
127	* one EMT in that context (at least presently).
128	*/
129	#ifdef IN_RC
130	# define PGMRZDYNMAP_CUR_CPU() (0)
131	#else
132	# define PGMRZDYNMAP_CUR_CPU() RTMpCpuIdToSetIndex(RTMpCpuId())
133	#endif
134
135	/** PGMRZDYNMAP::u32Magic. (Jens Christian Bugge Wesseltoft) */
136	#define PGMRZDYNMAP_MAGIC UINT32_C(0x19640201)
137
138
139	/** Zaps an set entry. */
140	#define PGMRZDYNMAP_ZAP_ENTRY(pEntry) \
141	do \
142	{ \
143	(pEntry)->iPage = UINT16_MAX; \
144	(pEntry)->cRefs = 0; \
145	(pEntry)->cInlinedRefs = 0; \
146	(pEntry)->cUnrefs = 0; \
147	} while (0)
148
149
150	/** @def PGMRZDYNMAP_STRICT_RELEASE
151	* Define this to force pages to be released and make non-present ASAP after
152	* use. This should not normally be enabled as it is a bit expensive. */
153	#if 0 \|\| defined(DOXYGEN_RUNNING)
154	# define PGMRZDYNMAP_STRICT_RELEASE
155	#endif
156
157
158	/*******************************************************************************
159	* Structures and Typedefs *
160	*******************************************************************************/
161	#ifdef IN_RING0
162	/**
163	* Ring-0 dynamic mapping cache segment.
164	*
165	* The dynamic mapping cache can be extended with additional segments if the
166	* load is found to be too high. This done the next time a VM is created, under
167	* the protection of the init mutex. The arrays is reallocated and the new
168	* segment is added to the end of these. Nothing is rehashed of course, as the
169	* indexes / addresses must remain unchanged.
170	*
171	* This structure is only modified while owning the init mutex or during module
172	* init / term.
173	*/
174	typedef struct PGMR0DYNMAPSEG
175	{
176	/** Pointer to the next segment. */
177	struct PGMR0DYNMAPSEG *pNext;
178	/** The memory object for the virtual address range that we're abusing. */
179	RTR0MEMOBJ hMemObj;
180	/** The start page in the cache. (I.e. index into the arrays.) */
181	uint16_t iPage;
182	/** The number of pages this segment contributes. */
183	uint16_t cPages;
184	/** The number of page tables. */
185	uint16_t cPTs;
186	/** The memory objects for the page tables. */
187	RTR0MEMOBJ ahMemObjPTs[1];
188	} PGMR0DYNMAPSEG;
189	/** Pointer to a ring-0 dynamic mapping cache segment. */
190	typedef PGMR0DYNMAPSEG *PPGMR0DYNMAPSEG;
191
192
193	/**
194	* Ring-0 dynamic mapping cache entry.
195	*
196	* @sa PGMRZDYNMAPENTRY, PGMRCDYNMAPENTRY.
197	*/
198	typedef struct PGMR0DYNMAPENTRY
199	{
200	/** The physical address of the currently mapped page.
201	* This is duplicate for three reasons: cache locality, cache policy of the PT
202	* mappings and sanity checks. */
203	RTHCPHYS HCPhys;
204	/** Pointer to the page. */
205	void *pvPage;
206	/** The number of references. */
207	int32_t volatile cRefs;
208	#ifndef IN_RC
209	/** PTE pointer union. */
210	union PGMR0DYNMAPENTRY_PPTE
211	{
212	/** PTE pointer, 32-bit legacy version. */
213	PX86PTE pLegacy;
214	/** PTE pointer, PAE version. */
215	PX86PTEPAE pPae;
216	/** PTE pointer, the void version. */
217	void *pv;
218	} uPte;
219	#else
220	/** PTE pointers. */
221	struct PGMR0DYNMAPENTRY_PPTE
222	{
223	/** PTE pointer, 32-bit legacy version. */
224	PX86PTE pLegacy;
225	/** PTE pointer, PAE version. */
226	PX86PTEPAE pPae;
227	} uPte;
228	#endif
229	# ifndef IN_RC
230	/** CPUs that haven't invalidated this entry after it's last update. */
231	RTCPUSET PendingSet;
232	# endif
233	} PGMR0DYNMAPENTRY;
234	/** Pointer a mapping cache entry for the ring-0.
235	* @sa PPGMRZDYNMAPENTRY, PPGMRCDYNMAPENTRY, */
236	typedef PGMR0DYNMAPENTRY *PPGMR0DYNMAPENTRY;
237
238
239	/**
240	* Dynamic mapping cache for ring-0.
241	*
242	* This is initialized during VMMR0 module init but no segments are allocated
243	* at that time. Segments will be added when the first VM is started and
244	* removed again when the last VM shuts down, thus avoid consuming memory while
245	* dormant. At module termination, the remaining bits will be freed up.
246	*
247	* @sa PPGMRZDYNMAP, PGMRCDYNMAP.
248	*/
249	typedef struct PGMR0DYNMAP
250	{
251	/** The usual magic number / eye catcher (PGMRZDYNMAP_MAGIC). */
252	uint32_t u32Magic;
253	# ifndef IN_RC
254	/** Spinlock serializing the normal operation of the cache. */
255	RTSPINLOCK hSpinlock;
256	# endif
257	/** Array for tracking and managing the pages. */
258	PPGMR0DYNMAPENTRY paPages;
259	/** The cache size given as a number of pages. */
260	uint32_t cPages;
261	# ifndef IN_RC
262	/** Whether it's 32-bit legacy or PAE/AMD64 paging mode. */
263	bool fLegacyMode;
264	# endif
265	/** The current load.
266	* This does not include guard pages. */
267	uint32_t cLoad;
268	/** The max load ever.
269	* This is maintained to trigger the adding of more mapping space. */
270	uint32_t cMaxLoad;
271	# ifndef IN_RC
272	/** Initialization / termination lock. */
273	RTSEMFASTMUTEX hInitLock;
274	# endif
275	/** The number of guard pages. */
276	uint32_t cGuardPages;
277	/** The number of users (protected by hInitLock). */
278	uint32_t cUsers;
279	# ifndef IN_RC
280	/** Array containing a copy of the original page tables.
281	* The entries are either X86PTE or X86PTEPAE according to fLegacyMode. */
282	void *pvSavedPTEs;
283	/** List of segments. */
284	PPGMR0DYNMAPSEG pSegHead;
285	/** The paging mode. */
286	SUPPAGINGMODE enmPgMode;
287	# endif
288	} PGMR0DYNMAP;
289
290
291	/**
292	* Paging level data.
293	*/
294	typedef struct PGMR0DYNMAPPGLVL
295	{
296	uint32_t cLevels; /*< The number of levels. /
297	struct
298	{
299	RTHCPHYS HCPhys; /**< The address of the page for the current level,
300	* i.e. what hMemObj/hMapObj is currently mapping. */
301	RTHCPHYS fPhysMask; /*< Mask for extracting HCPhys from uEntry. /
302	RTR0MEMOBJ hMemObj; /*< Memory object for HCPhys, PAGE_SIZE. /
303	RTR0MEMOBJ hMapObj; /*< Mapping object for hMemObj. /
304	uint32_t fPtrShift; /*< The pointer shift count. /
305	uint64_t fPtrMask; /*< The mask to apply to the shifted pointer to get the table index. /
306	uint64_t fAndMask; /*< And mask to check entry flags. /
307	uint64_t fResMask; /*< The result from applying fAndMask. /
308	union
309	{
310	void pv; /< hMapObj address. /
311	PX86PGUINT paLegacy; /*< Legacy table view. /
312	PX86PGPAEUINT paPae; /*< PAE/AMD64 table view. /
313	} u;
314	} a[4];
315	} PGMR0DYNMAPPGLVL;
316	/** Pointer to paging level data. */
317	typedef PGMR0DYNMAPPGLVL *PPGMR0DYNMAPPGLVL;
318	#endif
319
320	/** Mapping cache entry for the current context.
321	* @sa PGMR0DYNMAPENTRY, PGMRCDYNMAPENTRY */
322	typedef CTX_MID(PGM,DYNMAPENTRY) PGMRZDYNMAPENTRY;
323	/** Pointer a mapping cache entry for the current context.
324	* @sa PGMR0DYNMAPENTRY, PGMRCDYNMAPENTRY */
325	typedef PGMRZDYNMAPENTRY *PPGMRZDYNMAPENTRY;
326
327	/** Pointer the mapping cache instance for the current context.
328	* @sa PGMR0DYNMAP, PGMRCDYNMAP */
329	typedef CTX_MID(PGM,DYNMAP) *PPGMRZDYNMAP;
330
331
332
333	/*******************************************************************************
334	* Global Variables *
335	*******************************************************************************/
336	#ifdef IN_RING0
337	/** Pointer to the ring-0 dynamic mapping cache. */
338	static PGMR0DYNMAP *g_pPGMR0DynMap;
339	#endif
340	/** For overflow testing. */
341	static bool g_fPGMR0DynMapTestRunning = false;
342
343
344	/*******************************************************************************
345	* Internal Functions *
346	*******************************************************************************/
347	static void pgmRZDynMapReleasePage(PPGMRZDYNMAP pThis, uint32_t iPage, uint32_t cRefs);
348	#ifdef IN_RING0
349	static int pgmR0DynMapSetup(PPGMRZDYNMAP pThis);
350	static int pgmR0DynMapExpand(PPGMRZDYNMAP pThis);
351	static void pgmR0DynMapTearDown(PPGMRZDYNMAP pThis);
352	#endif
353	#if 0 /def DEBUG/
354	static int pgmR0DynMapTest(PVM pVM);
355	#endif
356
357
358	/**
359	* Initializes the auto mapping sets for a VM.
360	*
361	* @returns VINF_SUCCESS on success, VERR_INTERNAL_ERROR on failure.
362	* @param pVM The VM in question.
363	*/
364	static int pgmRZDynMapInitAutoSetsForVM(PVM pVM)
365	{
366	VMCPUID idCpu = pVM->cCpus;
367	AssertReturn(idCpu > 0 && idCpu <= VMM_MAX_CPU_COUNT, VERR_INTERNAL_ERROR);
368	while (idCpu-- > 0)
369	{
370	PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
371	uint32_t j = RT_ELEMENTS(pSet->aEntries);
372	while (j-- > 0)
373	{
374	pSet->aEntries[j].pvPage = NULL;
375	pSet->aEntries[j].HCPhys = NIL_RTHCPHYS;
376	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]);
377	}
378	pSet->cEntries = PGMMAPSET_CLOSED;
379	pSet->iSubset = UINT32_MAX;
380	pSet->iCpu = -1;
381	memset(&pSet->aiHashTable[0], 0xff, sizeof(pSet->aiHashTable));
382	}
383
384	return VINF_SUCCESS;
385	}
386
387
388	#ifdef IN_RING0
389
390	/**
391	* Initializes the ring-0 dynamic mapping cache.
392	*
393	* @returns VBox status code.
394	*/
395	VMMR0DECL(int) PGMR0DynMapInit(void)
396	{
397	Assert(!g_pPGMR0DynMap);
398
399	/*
400	* Create and initialize the cache instance.
401	*/
402	PPGMRZDYNMAP pThis = (PPGMRZDYNMAP)RTMemAllocZ(sizeof(*pThis));
403	AssertLogRelReturn(pThis, VERR_NO_MEMORY);
404	int rc = VINF_SUCCESS;
405	pThis->enmPgMode = SUPR0GetPagingMode();
406	switch (pThis->enmPgMode)
407	{
408	case SUPPAGINGMODE_32_BIT:
409	case SUPPAGINGMODE_32_BIT_GLOBAL:
410	pThis->fLegacyMode = false;
411	break;
412	case SUPPAGINGMODE_PAE:
413	case SUPPAGINGMODE_PAE_GLOBAL:
414	case SUPPAGINGMODE_PAE_NX:
415	case SUPPAGINGMODE_PAE_GLOBAL_NX:
416	case SUPPAGINGMODE_AMD64:
417	case SUPPAGINGMODE_AMD64_GLOBAL:
418	case SUPPAGINGMODE_AMD64_NX:
419	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
420	pThis->fLegacyMode = false;
421	break;
422	default:
423	rc = VERR_INTERNAL_ERROR;
424	break;
425	}
426	if (RT_SUCCESS(rc))
427	{
428	rc = RTSemFastMutexCreate(&pThis->hInitLock);
429	if (RT_SUCCESS(rc))
430	{
431	rc = RTSpinlockCreate(&pThis->hSpinlock);
432	if (RT_SUCCESS(rc))
433	{
434	pThis->u32Magic = PGMRZDYNMAP_MAGIC;
435	g_pPGMR0DynMap = pThis;
436	return VINF_SUCCESS;
437	}
438	RTSemFastMutexDestroy(pThis->hInitLock);
439	}
440	}
441	RTMemFree(pThis);
442	return rc;
443	}
444
445
446	/**
447	* Terminates the ring-0 dynamic mapping cache.
448	*/
449	VMMR0DECL(void) PGMR0DynMapTerm(void)
450	{
451	/*
452	* Destroy the cache.
453	*
454	* There is not supposed to be any races here, the loader should
455	* make sure about that. So, don't bother locking anything.
456	*
457	* The VM objects should all be destroyed by now, so there is no
458	* dangling users or anything like that to clean up. This routine
459	* is just a mirror image of PGMR0DynMapInit.
460	*/
461	PPGMRZDYNMAP pThis = g_pPGMR0DynMap;
462	if (pThis)
463	{
464	AssertPtr(pThis);
465	g_pPGMR0DynMap = NULL;
466
467	/* This should never happen, but in case it does try not to leak memory. */
468	AssertLogRelMsg(!pThis->cUsers && !pThis->paPages && !pThis->pvSavedPTEs && !pThis->cPages,
469	("cUsers=%d paPages=%p pvSavedPTEs=%p cPages=%#x\n",
470	pThis->cUsers, pThis->paPages, pThis->pvSavedPTEs, pThis->cPages));
471	if (pThis->paPages)
472	pgmR0DynMapTearDown(pThis);
473
474	/* Free the associated resources. */
475	RTSemFastMutexDestroy(pThis->hInitLock);
476	pThis->hInitLock = NIL_RTSEMFASTMUTEX;
477	RTSpinlockDestroy(pThis->hSpinlock);
478	pThis->hSpinlock = NIL_RTSPINLOCK;
479	pThis->u32Magic = UINT32_MAX;
480	RTMemFree(pThis);
481	}
482	}
483
484
485	/**
486	* Initializes the dynamic mapping cache for a new VM.
487	*
488	* @returns VBox status code.
489	* @param pVM Pointer to the shared VM structure.
490	*/
491	VMMR0DECL(int) PGMR0DynMapInitVM(PVM pVM)
492	{
493	AssertMsgReturn(!pVM->pgm.s.pvR0DynMapUsed, ("%p (pThis=%p)\n", pVM->pgm.s.pvR0DynMapUsed, g_pPGMR0DynMap), VERR_WRONG_ORDER);
494
495	/*
496	* Initialize the auto sets.
497	*/
498	int rc = pgmRZDynMapInitAutoSetsForVM(pVM);
499	if (RT_FAILURE(rc))
500	return rc;
501
502	/*
503	* Do we need the cache? Skip the last bit if we don't.
504	*/
505	if (!VMMIsHwVirtExtForced(pVM))
506	return VINF_SUCCESS;
507
508	/*
509	* Reference and if necessary setup or expand the cache.
510	*/
511	PPGMRZDYNMAP pThis = g_pPGMR0DynMap;
512	AssertPtrReturn(pThis, VERR_INTERNAL_ERROR);
513	rc = RTSemFastMutexRequest(pThis->hInitLock);
514	AssertLogRelRCReturn(rc, rc);
515
516	pThis->cUsers++;
517	if (pThis->cUsers == 1)
518	{
519	rc = pgmR0DynMapSetup(pThis);
520	#if 0 /def DEBUG/
521	if (RT_SUCCESS(rc))
522	{
523	rc = pgmR0DynMapTest(pVM);
524	if (RT_FAILURE(rc))
525	pgmR0DynMapTearDown(pThis);
526	}
527	#endif
528	}
529	else if (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(pThis->cPages - pThis->cGuardPages))
530	rc = pgmR0DynMapExpand(pThis);
531	if (RT_SUCCESS(rc))
532	pVM->pgm.s.pvR0DynMapUsed = pThis;
533	else
534	pThis->cUsers--;
535
536	RTSemFastMutexRelease(pThis->hInitLock);
537	return rc;
538	}
539
540
541	/**
542	* Terminates the dynamic mapping cache usage for a VM.
543	*
544	* @param pVM Pointer to the shared VM structure.
545	*/
546	VMMR0DECL(void) PGMR0DynMapTermVM(PVM pVM)
547	{
548	/*
549	* Return immediately if we're not using the cache.
550	*/
551	if (!pVM->pgm.s.pvR0DynMapUsed)
552	return;
553
554	PPGMRZDYNMAP pThis = g_pPGMR0DynMap;
555	AssertPtrReturnVoid(pThis);
556
557	int rc = RTSemFastMutexRequest(pThis->hInitLock);
558	AssertLogRelRCReturnVoid(rc);
559
560	if (pVM->pgm.s.pvR0DynMapUsed == pThis)
561	{
562	pVM->pgm.s.pvR0DynMapUsed = NULL;
563
564	#ifdef VBOX_STRICT
565	PGMR0DynMapAssertIntegrity();
566	#endif
567
568	/*
569	* Clean up and check the auto sets.
570	*/
571	VMCPUID idCpu = pVM->cCpus;
572	while (idCpu-- > 0)
573	{
574	PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
575	uint32_t j = pSet->cEntries;
576	if (j <= RT_ELEMENTS(pSet->aEntries))
577	{
578	/*
579	* The set is open, close it.
580	*/
581	while (j-- > 0)
582	{
583	int32_t cRefs = pSet->aEntries[j].cRefs;
584	uint32_t iPage = pSet->aEntries[j].iPage;
585	LogRel(("PGMR0DynMapTermVM: %d dangling refs to %#x\n", cRefs, iPage));
586	if (iPage < pThis->cPages && cRefs > 0)
587	pgmRZDynMapReleasePage(pThis, iPage, cRefs);
588	else
589	AssertLogRelMsgFailed(("cRefs=%d iPage=%#x cPages=%u\n", cRefs, iPage, pThis->cPages));
590
591	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]);
592	}
593	pSet->cEntries = PGMMAPSET_CLOSED;
594	pSet->iSubset = UINT32_MAX;
595	pSet->iCpu = -1;
596	}
597	else
598	AssertMsg(j == PGMMAPSET_CLOSED, ("cEntries=%#x\n", j));
599
600	j = RT_ELEMENTS(pSet->aEntries);
601	while (j-- > 0)
602	{
603	Assert(pSet->aEntries[j].iPage == UINT16_MAX);
604	Assert(!pSet->aEntries[j].cRefs);
605	}
606	}
607
608	/*
609	* Release our reference to the mapping cache.
610	*/
611	Assert(pThis->cUsers > 0);
612	pThis->cUsers--;
613	if (!pThis->cUsers)
614	pgmR0DynMapTearDown(pThis);
615	}
616	else
617	AssertLogRelMsgFailed(("pvR0DynMapUsed=%p pThis=%p\n", pVM->pgm.s.pvR0DynMapUsed, pThis));
618
619	RTSemFastMutexRelease(pThis->hInitLock);
620	}
621
622
623	/**
624	* Shoots down the TLBs for all the cache pages, pgmR0DynMapTearDown helper.
625	*
626	* @param idCpu The current CPU.
627	* @param pvUser1 The dynamic mapping cache instance.
628	* @param pvUser2 Unused, NULL.
629	*/
630	static DECLCALLBACK(void) pgmR0DynMapShootDownTlbs(RTCPUID idCpu, void pvUser1, void pvUser2)
631	{
632	Assert(!pvUser2);
633	PPGMRZDYNMAP pThis = (PPGMRZDYNMAP)pvUser1;
634	Assert(pThis == g_pPGMR0DynMap);
635	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
636	uint32_t iPage = pThis->cPages;
637	while (iPage-- > 0)
638	ASMInvalidatePage(paPages[iPage].pvPage);
639	}
640
641
642	/**
643	* Shoot down the TLBs for every single cache entry on all CPUs.
644	*
645	* @returns IPRT status code (RTMpOnAll).
646	* @param pThis The dynamic mapping cache instance.
647	*/
648	static int pgmR0DynMapTlbShootDown(PPGMRZDYNMAP pThis)
649	{
650	int rc = RTMpOnAll(pgmR0DynMapShootDownTlbs, pThis, NULL);
651	AssertRC(rc);
652	if (RT_FAILURE(rc))
653	{
654	uint32_t iPage = pThis->cPages;
655	while (iPage-- > 0)
656	ASMInvalidatePage(pThis->paPages[iPage].pvPage);
657	}
658	return rc;
659	}
660
661
662	/**
663	* Calculate the new cache size based on cMaxLoad statistics.
664	*
665	* @returns Number of pages.
666	* @param pThis The dynamic mapping cache instance.
667	* @param pcMinPages The minimal size in pages.
668	*/
669	static uint32_t pgmR0DynMapCalcNewSize(PPGMRZDYNMAP pThis, uint32_t *pcMinPages)
670	{
671	Assert(pThis->cPages <= PGMR0DYNMAP_MAX_PAGES);
672
673	/* cCpus * PGMR0DYNMAP_PAGES_PER_CPU(_MIN). */
674	RTCPUID cCpus = RTMpGetCount();
675	AssertReturn(cCpus > 0 && cCpus <= RTCPUSET_MAX_CPUS, 0);
676	uint32_t cPages = cCpus * PGMR0DYNMAP_PAGES_PER_CPU;
677	uint32_t cMinPages = cCpus * PGMR0DYNMAP_PAGES_PER_CPU_MIN;
678
679	/* adjust against cMaxLoad. */
680	AssertMsg(pThis->cMaxLoad <= PGMR0DYNMAP_MAX_PAGES, ("%#x\n", pThis->cMaxLoad));
681	if (pThis->cMaxLoad > PGMR0DYNMAP_MAX_PAGES)
682	pThis->cMaxLoad = 0;
683
684	while (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(cPages))
685	cPages += PGMR0DYNMAP_PAGES_PER_CPU;
686
687	if (pThis->cMaxLoad > cMinPages)
688	cMinPages = pThis->cMaxLoad;
689
690	/* adjust against max and current size. */
691	if (cPages < pThis->cPages)
692	cPages = pThis->cPages;
693	cPages *= PGMR0DYNMAP_GUARD_PAGES + 1;
694	if (cPages > PGMR0DYNMAP_MAX_PAGES)
695	cPages = PGMR0DYNMAP_MAX_PAGES;
696
697	if (cMinPages < pThis->cPages)
698	cMinPages = pThis->cPages;
699	cMinPages *= PGMR0DYNMAP_GUARD_PAGES + 1;
700	if (cMinPages > PGMR0DYNMAP_MAX_PAGES)
701	cMinPages = PGMR0DYNMAP_MAX_PAGES;
702
703	Assert(cMinPages);
704	*pcMinPages = cMinPages;
705	return cPages;
706	}
707
708
709	/**
710	* Initializes the paging level data.
711	*
712	* @param pThis The dynamic mapping cache instance.
713	* @param pPgLvl The paging level data.
714	*/
715	void pgmR0DynMapPagingArrayInit(PPGMRZDYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl)
716	{
717	RTCCUINTREG cr4 = ASMGetCR4();
718	switch (pThis->enmPgMode)
719	{
720	case SUPPAGINGMODE_32_BIT:
721	case SUPPAGINGMODE_32_BIT_GLOBAL:
722	pPgLvl->cLevels = 2;
723	pPgLvl->a[0].fPhysMask = X86_CR3_PAGE_MASK;
724	pPgLvl->a[0].fAndMask = X86_PDE_P \| X86_PDE_RW \| (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
725	pPgLvl->a[0].fResMask = X86_PDE_P \| X86_PDE_RW;
726	pPgLvl->a[0].fPtrMask = X86_PD_MASK;
727	pPgLvl->a[0].fPtrShift = X86_PD_SHIFT;
728
729	pPgLvl->a[1].fPhysMask = X86_PDE_PG_MASK;
730	pPgLvl->a[1].fAndMask = X86_PTE_P \| X86_PTE_RW;
731	pPgLvl->a[1].fResMask = X86_PTE_P \| X86_PTE_RW;
732	pPgLvl->a[1].fPtrMask = X86_PT_MASK;
733	pPgLvl->a[1].fPtrShift = X86_PT_SHIFT;
734	break;
735
736	case SUPPAGINGMODE_PAE:
737	case SUPPAGINGMODE_PAE_GLOBAL:
738	case SUPPAGINGMODE_PAE_NX:
739	case SUPPAGINGMODE_PAE_GLOBAL_NX:
740	pPgLvl->cLevels = 3;
741	pPgLvl->a[0].fPhysMask = X86_CR3_PAE_PAGE_MASK;
742	pPgLvl->a[0].fPtrMask = X86_PDPT_MASK_PAE;
743	pPgLvl->a[0].fPtrShift = X86_PDPT_SHIFT;
744	pPgLvl->a[0].fAndMask = X86_PDPE_P;
745	pPgLvl->a[0].fResMask = X86_PDPE_P;
746
747	pPgLvl->a[1].fPhysMask = X86_PDPE_PG_MASK;
748	pPgLvl->a[1].fPtrMask = X86_PD_PAE_MASK;
749	pPgLvl->a[1].fPtrShift = X86_PD_PAE_SHIFT;
750	pPgLvl->a[1].fAndMask = X86_PDE_P \| X86_PDE_RW \| (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
751	pPgLvl->a[1].fResMask = X86_PDE_P \| X86_PDE_RW;
752
753	pPgLvl->a[2].fPhysMask = X86_PDE_PAE_PG_MASK;
754	pPgLvl->a[2].fPtrMask = X86_PT_PAE_MASK;
755	pPgLvl->a[2].fPtrShift = X86_PT_PAE_SHIFT;
756	pPgLvl->a[2].fAndMask = X86_PTE_P \| X86_PTE_RW;
757	pPgLvl->a[2].fResMask = X86_PTE_P \| X86_PTE_RW;
758	break;
759
760	case SUPPAGINGMODE_AMD64:
761	case SUPPAGINGMODE_AMD64_GLOBAL:
762	case SUPPAGINGMODE_AMD64_NX:
763	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
764	pPgLvl->cLevels = 4;
765	pPgLvl->a[0].fPhysMask = X86_CR3_AMD64_PAGE_MASK;
766	pPgLvl->a[0].fPtrShift = X86_PML4_SHIFT;
767	pPgLvl->a[0].fPtrMask = X86_PML4_MASK;
768	pPgLvl->a[0].fAndMask = X86_PML4E_P \| X86_PML4E_RW;
769	pPgLvl->a[0].fResMask = X86_PML4E_P \| X86_PML4E_RW;
770
771	pPgLvl->a[1].fPhysMask = X86_PML4E_PG_MASK;
772	pPgLvl->a[1].fPtrShift = X86_PDPT_SHIFT;
773	pPgLvl->a[1].fPtrMask = X86_PDPT_MASK_AMD64;
774	pPgLvl->a[1].fAndMask = X86_PDPE_P \| X86_PDPE_RW /** @todo check for X86_PDPT_PS support. */;
775	pPgLvl->a[1].fResMask = X86_PDPE_P \| X86_PDPE_RW;
776
777	pPgLvl->a[2].fPhysMask = X86_PDPE_PG_MASK;
778	pPgLvl->a[2].fPtrShift = X86_PD_PAE_SHIFT;
779	pPgLvl->a[2].fPtrMask = X86_PD_PAE_MASK;
780	pPgLvl->a[2].fAndMask = X86_PDE_P \| X86_PDE_RW \| (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
781	pPgLvl->a[2].fResMask = X86_PDE_P \| X86_PDE_RW;
782
783	pPgLvl->a[3].fPhysMask = X86_PDE_PAE_PG_MASK;
784	pPgLvl->a[3].fPtrShift = X86_PT_PAE_SHIFT;
785	pPgLvl->a[3].fPtrMask = X86_PT_PAE_MASK;
786	pPgLvl->a[3].fAndMask = X86_PTE_P \| X86_PTE_RW;
787	pPgLvl->a[3].fResMask = X86_PTE_P \| X86_PTE_RW;
788	break;
789
790	default:
791	AssertFailed();
792	pPgLvl->cLevels = 0;
793	break;
794	}
795
796	for (uint32_t i = 0; i < 4; i++) /* ASSUMING array size. */
797	{
798	pPgLvl->a[i].HCPhys = NIL_RTHCPHYS;
799	pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
800	pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ;
801	pPgLvl->a[i].u.pv = NULL;
802	}
803	}
804
805
806	/**
807	* Maps a PTE.
808	*
809	* This will update the segment structure when new PTs are mapped.
810	*
811	* It also assumes that we (for paranoid reasons) wish to establish a mapping
812	* chain from CR3 to the PT that all corresponds to the processor we're
813	* currently running on, and go about this by running with interrupts disabled
814	* and restarting from CR3 for every change.
815	*
816	* @returns VBox status code, VINF_TRY_AGAIN if we changed any mappings and had
817	* to re-enable interrupts.
818	* @param pThis The dynamic mapping cache instance.
819	* @param pPgLvl The paging level structure.
820	* @param pvPage The page.
821	* @param pSeg The segment.
822	* @param cMaxPTs The max number of PTs expected in the segment.
823	* @param ppvPTE Where to store the PTE address.
824	*/
825	static int pgmR0DynMapPagingArrayMapPte(PPGMRZDYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl, void *pvPage,
826	PPGMR0DYNMAPSEG pSeg, uint32_t cMaxPTs, void **ppvPTE)
827	{
828	Assert(!(ASMGetFlags() & X86_EFL_IF));
829	void *pvEntry = NULL;
830	X86PGPAEUINT uEntry = ASMGetCR3();
831	for (uint32_t i = 0; i < pPgLvl->cLevels; i++)
832	{
833	RTHCPHYS HCPhys = uEntry & pPgLvl->a[i].fPhysMask;
834	if (pPgLvl->a[i].HCPhys != HCPhys)
835	{
836	/*
837	* Need to remap this level.
838	* The final level, the PT, will not be freed since that is what it's all about.
839	*/
840	ASMIntEnable();
841	if (i + 1 == pPgLvl->cLevels)
842	AssertReturn(pSeg->cPTs < cMaxPTs, VERR_INTERNAL_ERROR);
843	else
844	{
845	int rc2 = RTR0MemObjFree(pPgLvl->a[i].hMemObj, true /* fFreeMappings */); AssertRC(rc2);
846	pPgLvl->a[i].hMemObj = pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
847	}
848
849	int rc = RTR0MemObjEnterPhys(&pPgLvl->a[i].hMemObj, HCPhys, PAGE_SIZE, RTMEM_CACHE_POLICY_DONT_CARE);
850	if (RT_SUCCESS(rc))
851	{
852	rc = RTR0MemObjMapKernel(&pPgLvl->a[i].hMapObj, pPgLvl->a[i].hMemObj,
853	(void )-1 / pvFixed /, 0 / cbAlignment */,
854	RTMEM_PROT_WRITE \| RTMEM_PROT_READ);
855	if (RT_SUCCESS(rc))
856	{
857	pPgLvl->a[i].u.pv = RTR0MemObjAddress(pPgLvl->a[i].hMapObj);
858	AssertMsg(((uintptr_t)pPgLvl->a[i].u.pv & ~(uintptr_t)PAGE_OFFSET_MASK), ("%p\n", pPgLvl->a[i].u.pv));
859	pPgLvl->a[i].HCPhys = HCPhys;
860	if (i + 1 == pPgLvl->cLevels)
861	pSeg->ahMemObjPTs[pSeg->cPTs++] = pPgLvl->a[i].hMemObj;
862	ASMIntDisable();
863	return VINF_TRY_AGAIN;
864	}
865
866	pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
867	}
868	else
869	pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ;
870	pPgLvl->a[i].HCPhys = NIL_RTHCPHYS;
871	return rc;
872	}
873
874	/*
875	* The next level.
876	*/
877	uint32_t iEntry = ((uint64_t)(uintptr_t)pvPage >> pPgLvl->a[i].fPtrShift) & pPgLvl->a[i].fPtrMask;
878	if (pThis->fLegacyMode)
879	{
880	pvEntry = &pPgLvl->a[i].u.paLegacy[iEntry];
881	uEntry = pPgLvl->a[i].u.paLegacy[iEntry];
882	}
883	else
884	{
885	pvEntry = &pPgLvl->a[i].u.paPae[iEntry];
886	uEntry = pPgLvl->a[i].u.paPae[iEntry];
887	}
888
889	if ((uEntry & pPgLvl->a[i].fAndMask) != pPgLvl->a[i].fResMask)
890	{
891	LogRel(("PGMR0DynMap: internal error - iPgLvl=%u cLevels=%u uEntry=%#llx fAnd=%#llx fRes=%#llx got=%#llx\n"
892	"PGMR0DynMap: pv=%p pvPage=%p iEntry=%#x fLegacyMode=%RTbool\n",
893	i, pPgLvl->cLevels, uEntry, pPgLvl->a[i].fAndMask, pPgLvl->a[i].fResMask, uEntry & pPgLvl->a[i].fAndMask,
894	pPgLvl->a[i].u.pv, pvPage, iEntry, pThis->fLegacyMode));
895	return VERR_INTERNAL_ERROR;
896	}
897	/Log(("#%d: iEntry=%4d uEntry=%#llx pvEntry=%p HCPhys=%RHp \n", i, iEntry, uEntry, pvEntry, pPgLvl->a[i].HCPhys));/
898	}
899
900	/* made it thru without needing to remap anything. */
901	*ppvPTE = pvEntry;
902	return VINF_SUCCESS;
903	}
904
905
906	/**
907	* Sets up a guard page.
908	*
909	* @param pThis The dynamic mapping cache instance.
910	* @param pPage The page.
911	*/
912	DECLINLINE(void) pgmR0DynMapSetupGuardPage(PPGMRZDYNMAP pThis, PPGMRZDYNMAPENTRY pPage)
913	{
914	memset(pPage->pvPage, 0xfd, PAGE_SIZE);
915	pPage->cRefs = PGMR0DYNMAP_GUARD_PAGE_REF_COUNT;
916	pPage->HCPhys = PGMR0DYNMAP_GUARD_PAGE_HCPHYS;
917	#ifdef PGMR0DYNMAP_GUARD_NP
918	ASMAtomicBitClear(pPage->uPte.pv, X86_PTE_BIT_P);
919	#else
920	if (pThis->fLegacyMode)
921	ASMAtomicWriteU32(&pPage->uPte.pLegacy->u, PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE);
922	else
923	ASMAtomicWriteU64(&pPage->uPte.pPae->u, PGMR0DYNMAP_GUARD_PAGE_PAE_PTE);
924	#endif
925	pThis->cGuardPages++;
926	}
927
928
929	/**
930	* Adds a new segment of the specified size.
931	*
932	* @returns VBox status code.
933	* @param pThis The dynamic mapping cache instance.
934	* @param cPages The size of the new segment, give as a page count.
935	*/
936	static int pgmR0DynMapAddSeg(PPGMRZDYNMAP pThis, uint32_t cPages)
937	{
938	int rc2;
939	AssertReturn(ASMGetFlags() & X86_EFL_IF, VERR_PREEMPT_DISABLED);
940
941	/*
942	* Do the array reallocations first.
943	* (The pages array has to be replaced behind the spinlock of course.)
944	*/
945	void pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) (pThis->cPages + cPages));
946	if (!pvSavedPTEs)
947	return VERR_NO_MEMORY;
948	pThis->pvSavedPTEs = pvSavedPTEs;
949
950	void pvPages = RTMemAllocZ(sizeof(pThis->paPages[0]) (pThis->cPages + cPages));
951	if (!pvPages)
952	{
953	pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) * pThis->cPages);
954	if (pvSavedPTEs)
955	pThis->pvSavedPTEs = pvSavedPTEs;
956	return VERR_NO_MEMORY;
957	}
958
959	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
960
961	memcpy(pvPages, pThis->paPages, sizeof(pThis->paPages[0]) * pThis->cPages);
962	void *pvToFree = pThis->paPages;
963	pThis->paPages = (PPGMRZDYNMAPENTRY)pvPages;
964
965	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
966	RTMemFree(pvToFree);
967
968	/*
969	* Allocate the segment structure and pages of memory, then touch all the pages (paranoia).
970	*/
971	uint32_t cMaxPTs = cPages / (pThis->fLegacyMode ? X86_PG_ENTRIES : X86_PG_PAE_ENTRIES) + 2;
972	PPGMR0DYNMAPSEG pSeg = (PPGMR0DYNMAPSEG)RTMemAllocZ(RT_UOFFSETOF(PGMR0DYNMAPSEG, ahMemObjPTs[cMaxPTs]));
973	if (!pSeg)
974	return VERR_NO_MEMORY;
975	pSeg->pNext = NULL;
976	pSeg->cPages = cPages;
977	pSeg->iPage = pThis->cPages;
978	pSeg->cPTs = 0;
979	int rc = RTR0MemObjAllocPage(&pSeg->hMemObj, cPages << PAGE_SHIFT, false);
980	if (RT_SUCCESS(rc))
981	{
982	uint8_t pbPage = (uint8_t )RTR0MemObjAddress(pSeg->hMemObj);
983	AssertMsg(VALID_PTR(pbPage) && !((uintptr_t)pbPage & PAGE_OFFSET_MASK), ("%p\n", pbPage));
984	memset(pbPage, 0xfe, cPages << PAGE_SHIFT);
985
986	/*
987	* Walk thru the pages and set them up with a mapping of their PTE and everything.
988	*/
989	ASMIntDisable();
990	PGMR0DYNMAPPGLVL PgLvl;
991	pgmR0DynMapPagingArrayInit(pThis, &PgLvl);
992	uint32_t const iEndPage = pSeg->iPage + cPages;
993	for (uint32_t iPage = pSeg->iPage;
994	iPage < iEndPage;
995	iPage++, pbPage += PAGE_SIZE)
996	{
997	/* Initialize the page data. */
998	pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
999	pThis->paPages[iPage].pvPage = pbPage;
1000	pThis->paPages[iPage].cRefs = 0;
1001	pThis->paPages[iPage].uPte.pPae = 0;
1002	#ifndef IN_RC
1003	RTCpuSetFill(&pThis->paPages[iPage].PendingSet);
1004	#endif
1005
1006	/* Map its page table, retry until we've got a clean run (paranoia). */
1007	do
1008	rc = pgmR0DynMapPagingArrayMapPte(pThis, &PgLvl, pbPage, pSeg, cMaxPTs,
1009	&pThis->paPages[iPage].uPte.pv);
1010	while (rc == VINF_TRY_AGAIN);
1011	if (RT_FAILURE(rc))
1012	break;
1013
1014	/* Save the PTE. */
1015	if (pThis->fLegacyMode)
1016	((PX86PGUINT)pThis->pvSavedPTEs)[iPage] = pThis->paPages[iPage].uPte.pLegacy->u;
1017	else
1018	((PX86PGPAEUINT)pThis->pvSavedPTEs)[iPage] = pThis->paPages[iPage].uPte.pPae->u;
1019
1020	#ifdef VBOX_STRICT
1021	/* Check that we've got the right entry. */
1022	RTHCPHYS HCPhysPage = RTR0MemObjGetPagePhysAddr(pSeg->hMemObj, iPage - pSeg->iPage);
1023	RTHCPHYS HCPhysPte = pThis->fLegacyMode
1024	? pThis->paPages[iPage].uPte.pLegacy->u & X86_PTE_PG_MASK
1025	: pThis->paPages[iPage].uPte.pPae->u & X86_PTE_PAE_PG_MASK;
1026	if (HCPhysPage != HCPhysPte)
1027	{
1028	LogRel(("pgmR0DynMapAddSeg: internal error - page #%u HCPhysPage=%RHp HCPhysPte=%RHp pbPage=%p pvPte=%p\n",
1029	iPage - pSeg->iPage, HCPhysPage, HCPhysPte, pbPage, pThis->paPages[iPage].uPte.pv));
1030	rc = VERR_INTERNAL_ERROR;
1031	break;
1032	}
1033	#endif
1034	} /* for each page */
1035	ASMIntEnable();
1036
1037	/* cleanup non-PT mappings */
1038	for (uint32_t i = 0; i < PgLvl.cLevels - 1; i++)
1039	RTR0MemObjFree(PgLvl.a[i].hMemObj, true /* fFreeMappings */);
1040
1041	if (RT_SUCCESS(rc))
1042	{
1043	#if PGMR0DYNMAP_GUARD_PAGES > 0
1044	/*
1045	* Setup guard pages.
1046	* (Note: TLBs will be shot down later on.)
1047	*/
1048	uint32_t iPage = pSeg->iPage;
1049	while (iPage < iEndPage)
1050	{
1051	for (uint32_t iGPg = 0; iGPg < PGMR0DYNMAP_GUARD_PAGES && iPage < iEndPage; iGPg++, iPage++)
1052	pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]);
1053	iPage++; /* the guarded page */
1054	}
1055
1056	/* Make sure the very last page is a guard page too. */
1057	iPage = iEndPage - 1;
1058	if (pThis->paPages[iPage].cRefs != PGMR0DYNMAP_GUARD_PAGE_REF_COUNT)
1059	pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]);
1060	#endif /* PGMR0DYNMAP_GUARD_PAGES > 0 */
1061
1062	/*
1063	* Commit it by adding the segment to the list and updating the page count.
1064	*/
1065	pSeg->pNext = pThis->pSegHead;
1066	pThis->pSegHead = pSeg;
1067	pThis->cPages += cPages;
1068	return VINF_SUCCESS;
1069	}
1070
1071	/*
1072	* Bail out.
1073	*/
1074	while (pSeg->cPTs-- > 0)
1075	{
1076	rc2 = RTR0MemObjFree(pSeg->ahMemObjPTs[pSeg->cPTs], true /* fFreeMappings */);
1077	AssertRC(rc2);
1078	pSeg->ahMemObjPTs[pSeg->cPTs] = NIL_RTR0MEMOBJ;
1079	}
1080
1081	rc2 = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */);
1082	AssertRC(rc2);
1083	pSeg->hMemObj = NIL_RTR0MEMOBJ;
1084	}
1085	RTMemFree(pSeg);
1086
1087	/* Don't bother resizing the arrays, but free them if we're the only user. */
1088	if (!pThis->cPages)
1089	{
1090	RTMemFree(pThis->paPages);
1091	pThis->paPages = NULL;
1092	RTMemFree(pThis->pvSavedPTEs);
1093	pThis->pvSavedPTEs = NULL;
1094	}
1095	return rc;
1096	}
1097
1098
1099	/**
1100	* Called by PGMR0DynMapInitVM under the init lock.
1101	*
1102	* @returns VBox status code.
1103	* @param pThis The dynamic mapping cache instance.
1104	*/
1105	static int pgmR0DynMapSetup(PPGMRZDYNMAP pThis)
1106	{
1107	/*
1108	* Calc the size and add a segment of that size.
1109	*/
1110	uint32_t cMinPages;
1111	uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages);
1112	AssertReturn(cPages, VERR_INTERNAL_ERROR);
1113	int rc = pgmR0DynMapAddSeg(pThis, cPages);
1114	if (rc == VERR_NO_MEMORY)
1115	{
1116	/*
1117	* Try adding smaller segments.
1118	*/
1119	do
1120	rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES);
1121	while (RT_SUCCESS(rc) && pThis->cPages < cPages);
1122	if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages)
1123	rc = VINF_SUCCESS;
1124	if (rc == VERR_NO_MEMORY)
1125	{
1126	if (pThis->cPages)
1127	pgmR0DynMapTearDown(pThis);
1128	rc = VERR_PGM_DYNMAP_SETUP_ERROR;
1129	}
1130	}
1131	Assert(ASMGetFlags() & X86_EFL_IF);
1132
1133	#if PGMR0DYNMAP_GUARD_PAGES > 0
1134	/* paranoia */
1135	if (RT_SUCCESS(rc))
1136	pgmR0DynMapTlbShootDown(pThis);
1137	#endif
1138	return rc;
1139	}
1140
1141
1142	/**
1143	* Called by PGMR0DynMapInitVM under the init lock.
1144	*
1145	* @returns VBox status code.
1146	* @param pThis The dynamic mapping cache instance.
1147	*/
1148	static int pgmR0DynMapExpand(PPGMRZDYNMAP pThis)
1149	{
1150	/*
1151	* Calc the new target size and add a segment of the appropriate size.
1152	*/
1153	uint32_t cMinPages;
1154	uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages);
1155	AssertReturn(cPages, VERR_INTERNAL_ERROR);
1156	if (pThis->cPages >= cPages)
1157	return VINF_SUCCESS;
1158
1159	uint32_t cAdd = cPages - pThis->cPages;
1160	int rc = pgmR0DynMapAddSeg(pThis, cAdd);
1161	if (rc == VERR_NO_MEMORY)
1162	{
1163	/*
1164	* Try adding smaller segments.
1165	*/
1166	do
1167	rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES);
1168	while (RT_SUCCESS(rc) && pThis->cPages < cPages);
1169	if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages)
1170	rc = VINF_SUCCESS;
1171	if (rc == VERR_NO_MEMORY)
1172	rc = VERR_PGM_DYNMAP_EXPAND_ERROR;
1173	}
1174	Assert(ASMGetFlags() & X86_EFL_IF);
1175
1176	#if PGMR0DYNMAP_GUARD_PAGES > 0
1177	/* paranoia */
1178	if (RT_SUCCESS(rc))
1179	pgmR0DynMapTlbShootDown(pThis);
1180	#endif
1181	return rc;
1182	}
1183
1184
1185	/**
1186	* Called by PGMR0DynMapTermVM under the init lock.
1187	*
1188	* @returns VBox status code.
1189	* @param pThis The dynamic mapping cache instance.
1190	*/
1191	static void pgmR0DynMapTearDown(PPGMRZDYNMAP pThis)
1192	{
1193	/*
1194	* Restore the original page table entries
1195	*/
1196	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
1197	uint32_t iPage = pThis->cPages;
1198	if (pThis->fLegacyMode)
1199	{
1200	X86PGUINT const paSavedPTEs = (X86PGUINT const )pThis->pvSavedPTEs;
1201	while (iPage-- > 0)
1202	{
1203	X86PGUINT uOld = paPages[iPage].uPte.pLegacy->u;
1204	X86PGUINT uOld2 = uOld; NOREF(uOld2);
1205	X86PGUINT uNew = paSavedPTEs[iPage];
1206	while (!ASMAtomicCmpXchgExU32(&paPages[iPage].uPte.pLegacy->u, uNew, uOld, &uOld))
1207	AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
1208	Assert(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage]);
1209	}
1210	}
1211	else
1212	{
1213	X86PGPAEUINT const paSavedPTEs = (X86PGPAEUINT const )pThis->pvSavedPTEs;
1214	while (iPage-- > 0)
1215	{
1216	X86PGPAEUINT uOld = paPages[iPage].uPte.pPae->u;
1217	X86PGPAEUINT uOld2 = uOld; NOREF(uOld2);
1218	X86PGPAEUINT uNew = paSavedPTEs[iPage];
1219	while (!ASMAtomicCmpXchgExU64(&paPages[iPage].uPte.pPae->u, uNew, uOld, &uOld))
1220	AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
1221	Assert(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage]);
1222	}
1223	}
1224
1225	/*
1226	* Shoot down the TLBs on all CPUs before freeing them.
1227	*/
1228	pgmR0DynMapTlbShootDown(pThis);
1229
1230	/*
1231	* Free the segments.
1232	*/
1233	while (pThis->pSegHead)
1234	{
1235	int rc;
1236	PPGMR0DYNMAPSEG pSeg = pThis->pSegHead;
1237	pThis->pSegHead = pSeg->pNext;
1238
1239	uint32_t iPT = pSeg->cPTs;
1240	while (iPT-- > 0)
1241	{
1242	rc = RTR0MemObjFree(pSeg->ahMemObjPTs[iPT], true /* fFreeMappings */); AssertRC(rc);
1243	pSeg->ahMemObjPTs[iPT] = NIL_RTR0MEMOBJ;
1244	}
1245	rc = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */); AssertRC(rc);
1246	pSeg->hMemObj = NIL_RTR0MEMOBJ;
1247	pSeg->pNext = NULL;
1248	pSeg->iPage = UINT16_MAX;
1249	pSeg->cPages = 0;
1250	pSeg->cPTs = 0;
1251	RTMemFree(pSeg);
1252	}
1253
1254	/*
1255	* Free the arrays and restore the initial state.
1256	* The cLoadMax value is left behind for the next setup.
1257	*/
1258	RTMemFree(pThis->paPages);
1259	pThis->paPages = NULL;
1260	RTMemFree(pThis->pvSavedPTEs);
1261	pThis->pvSavedPTEs = NULL;
1262	pThis->cPages = 0;
1263	pThis->cLoad = 0;
1264	pThis->cGuardPages = 0;
1265	}
1266
1267	#endif /* IN_RING0 */
1268	#ifdef IN_RC
1269
1270	/**
1271	* Initializes the dynamic mapping cache in raw-mode context.
1272	*
1273	* @returns VBox status code.
1274	* @param pVM The VM handle.
1275	*/
1276	VMMRCDECL(int) PGMRCDynMapInit(PVM pVM)
1277	{
1278	/*
1279	* Allocate and initialize the instance data and page array.
1280	*/
1281	PPGMRZDYNMAP pThis;
1282	size_t const cPages = MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE;
1283	size_t const cb = RT_ALIGN_Z(sizeof(*pThis), 32)
1284	+ sizeof(PGMRZDYNMAPENTRY) * cPages;
1285	int rc = MMHyperAlloc(pVM, cb, 32, MM_TAG_PGM, (void **)&pThis);
1286	if (RT_FAILURE(rc))
1287	return rc;
1288
1289	pThis->u32Magic = PGMRZDYNMAP_MAGIC;
1290	pThis->paPages = RT_ALIGN_PT(pThis + 1, 32, PPGMRZDYNMAPENTRY);
1291	pThis->cPages = cPages;
1292	pThis->cLoad = 0;
1293	pThis->cMaxLoad = 0;
1294	pThis->cGuardPages = 0;
1295	pThis->cUsers = 1;
1296
1297	for (size_t iPage = 0; iPage < cPages; iPage++)
1298	{
1299	pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
1300	pThis->paPages[iPage].pvPage = pVM->pgm.s.pbDynPageMapBaseGC + iPage * PAGE_SIZE;
1301	pThis->paPages[iPage].cRefs = 0;
1302	pThis->paPages[iPage].uPte.pLegacy = &pVM->pgm.s.paDynPageMap32BitPTEsGC[iPage];
1303	pThis->paPages[iPage].uPte.pPae = (PX86PTEPAE)&pVM->pgm.s.paDynPageMapPaePTEsGC[iPage];
1304	}
1305
1306	pVM->pgm.s.pRCDynMap = pThis;
1307
1308	/*
1309	* Initialize the autosets the VM.
1310	*/
1311	rc = pgmRZDynMapInitAutoSetsForVM(pVM);
1312	if (RT_FAILURE(rc))
1313	return rc;
1314
1315	return VINF_SUCCESS;
1316	}
1317
1318	#endif /* IN_RC */
1319
1320	/**
1321	* Release references to a page, caller owns the spin lock.
1322	*
1323	* @param pThis The dynamic mapping cache instance.
1324	* @param iPage The page.
1325	* @param cRefs The number of references to release.
1326	*/
1327	DECLINLINE(void) pgmRZDynMapReleasePageLocked(PPGMRZDYNMAP pThis, uint32_t iPage, int32_t cRefs)
1328	{
1329	cRefs = ASMAtomicSubS32(&pThis->paPages[iPage].cRefs, cRefs) - cRefs;
1330	AssertMsg(cRefs >= 0, ("%d\n", cRefs));
1331	if (!cRefs)
1332	{
1333	pThis->cLoad--;
1334	#ifdef PGMRZDYNMAP_STRICT_RELEASE
1335	pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
1336	ASMAtomicBitClear(pThis->paPages[iPage].uPte.pv, X86_PTE_BIT_P);
1337	ASMInvalidatePage(pThis->paPages[iPage].pvPage);
1338	#endif
1339	}
1340	}
1341
1342
1343	/**
1344	* Release references to a page, caller does not own the spin lock.
1345	*
1346	* @param pThis The dynamic mapping cache instance.
1347	* @param iPage The page.
1348	* @param cRefs The number of references to release.
1349	*/
1350	static void pgmRZDynMapReleasePage(PPGMRZDYNMAP pThis, uint32_t iPage, uint32_t cRefs)
1351	{
1352	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
1353	pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs);
1354	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1355	}
1356
1357
1358	/**
1359	* pgmR0DynMapPage worker that deals with the tedious bits.
1360	*
1361	* @returns The page index on success, UINT32_MAX on failure.
1362	* @param pThis The dynamic mapping cache instance.
1363	* @param HCPhys The address of the page to be mapped.
1364	* @param iPage The page index pgmR0DynMapPage hashed HCPhys to.
1365	* @param pVCpu The current CPU, for statistics.
1366	* @param pfNew Set to @c true if a new entry was made and @c false if
1367	* an old entry was found and reused.
1368	*/
1369	static uint32_t pgmR0DynMapPageSlow(PPGMRZDYNMAP pThis, RTHCPHYS HCPhys, uint32_t iPage, PVMCPU pVCpu, bool *pfNew)
1370	{
1371	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlow);
1372
1373	/*
1374	* Check if any of the first 3 pages are unreferenced since the caller
1375	* already has made sure they aren't matching.
1376	*/
1377	#ifdef VBOX_WITH_STATISTICS
1378	bool fLooped = false;
1379	#endif
1380	uint32_t const cPages = pThis->cPages;
1381	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
1382	uint32_t iFreePage;
1383	if (!paPages[iPage].cRefs)
1384	iFreePage = iPage;
1385	else if (!paPages[(iPage + 1) % cPages].cRefs)
1386	iFreePage = (iPage + 1) % cPages;
1387	else if (!paPages[(iPage + 2) % cPages].cRefs)
1388	iFreePage = (iPage + 2) % cPages;
1389	else
1390	{
1391	/*
1392	* Search for an unused or matching entry.
1393	*/
1394	iFreePage = (iPage + 3) % cPages;
1395	for (;;)
1396	{
1397	if (paPages[iFreePage].HCPhys == HCPhys)
1398	{
1399	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlowLoopHits);
1400	*pfNew = false;
1401	return iFreePage;
1402	}
1403	if (!paPages[iFreePage].cRefs)
1404	break;
1405
1406	/* advance */
1407	iFreePage = (iFreePage + 1) % cPages;
1408	if (RT_UNLIKELY(iFreePage == iPage))
1409	return UINT32_MAX;
1410	}
1411	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlowLoopMisses);
1412	#ifdef VBOX_WITH_STATISTICS
1413	fLooped = true;
1414	#endif
1415	}
1416	Assert(iFreePage < cPages);
1417
1418	#if 0 //def VBOX_WITH_STATISTICS
1419	/* Check for lost hits. */
1420	if (!fLooped)
1421	for (uint32_t iPage2 = (iPage + 3) % cPages; iPage2 != iPage; iPage2 = (iPage2 + 1) % cPages)
1422	if (paPages[iPage2].HCPhys == HCPhys)
1423	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZDynMapPageSlowLostHits);
1424	#endif
1425
1426	/*
1427	* Setup the new entry.
1428	*/
1429	*pfNew = true;
1430	/Log6(("pgmR0DynMapPageSlow: old - %RHp %#x %#llx\n", paPages[iFreePage].HCPhys, paPages[iFreePage].cRefs, paPages[iFreePage].uPte.pPae->u));/
1431	paPages[iFreePage].HCPhys = HCPhys;
1432	#ifndef IN_RC
1433	RTCpuSetFill(&paPages[iFreePage].PendingSet);
1434
1435	if (pThis->fLegacyMode)
1436	#endif
1437	{
1438	X86PGUINT uOld = paPages[iFreePage].uPte.pLegacy->u;
1439	X86PGUINT uOld2 = uOld; NOREF(uOld2);
1440	X86PGUINT uNew = (uOld & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1441	\| X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1442	\| (HCPhys & X86_PTE_PG_MASK);
1443	while (!ASMAtomicCmpXchgExU32(&paPages[iFreePage].uPte.pLegacy->u, uNew, uOld, &uOld))
1444	AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
1445	Assert(paPages[iFreePage].uPte.pLegacy->u == uNew);
1446	}
1447	#ifndef IN_RC
1448	else
1449	#endif
1450	{
1451	X86PGPAEUINT uOld = paPages[iFreePage].uPte.pPae->u;
1452	X86PGPAEUINT uOld2 = uOld; NOREF(uOld2);
1453	X86PGPAEUINT uNew = (uOld & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1454	\| X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1455	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1456	while (!ASMAtomicCmpXchgExU64(&paPages[iFreePage].uPte.pPae->u, uNew, uOld, &uOld))
1457	AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
1458	Assert(paPages[iFreePage].uPte.pPae->u == uNew);
1459	/Log6(("pgmR0DynMapPageSlow: #%x - %RHp %p %#llx\n", iFreePage, HCPhys, paPages[iFreePage].pvPage, uNew));/
1460	}
1461	return iFreePage;
1462	}
1463
1464
1465	/**
1466	* Maps a page into the pool.
1467	*
1468	* @returns Page index on success, UINT32_MAX on failure.
1469	* @param pThis The dynamic mapping cache instance.
1470	* @param HCPhys The address of the page to be mapped.
1471	* @param iRealCpu The real cpu set index. (optimization)
1472	* @param pVCpu The current CPU (for statistics).
1473	* @param ppvPage Where to the page address.
1474	*/
1475	DECLINLINE(uint32_t) pgmR0DynMapPage(PPGMRZDYNMAP pThis, RTHCPHYS HCPhys, int32_t iRealCpu, PVMCPU pVCpu, void **ppvPage)
1476	{
1477	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
1478	AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
1479	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPage);
1480
1481	/*
1482	* Find an entry, if possible a matching one. The HCPhys address is hashed
1483	* down to a page index, collisions are handled by linear searching.
1484	* Optimized for a hit in the first 3 pages.
1485	*
1486	* Field easy hits here and defer the tedious searching and inserting
1487	* to pgmR0DynMapPageSlow().
1488	*/
1489	bool fNew = false;
1490	uint32_t const cPages = pThis->cPages;
1491	uint32_t iPage = (HCPhys >> PAGE_SHIFT) % cPages;
1492	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
1493	if (RT_LIKELY(paPages[iPage].HCPhys == HCPhys))
1494	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits0);
1495	else
1496	{
1497	uint32_t iPage2 = (iPage + 1) % cPages;
1498	if (RT_LIKELY(paPages[iPage2].HCPhys == HCPhys))
1499	{
1500	iPage = iPage2;
1501	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits1);
1502	}
1503	else
1504	{
1505	iPage2 = (iPage + 2) % cPages;
1506	if (paPages[iPage2].HCPhys == HCPhys)
1507	{
1508	iPage = iPage2;
1509	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits2);
1510	}
1511	else
1512	{
1513	iPage = pgmR0DynMapPageSlow(pThis, HCPhys, iPage, pVCpu, &fNew);
1514	if (RT_UNLIKELY(iPage == UINT32_MAX))
1515	{
1516	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1517	*ppvPage = NULL;
1518	return iPage;
1519	}
1520	}
1521	}
1522	}
1523
1524	/*
1525	* Reference it, update statistics and get the return address.
1526	*/
1527	int32_t cRefs = ASMAtomicIncS32(&paPages[iPage].cRefs);
1528	if (cRefs == 1)
1529	{
1530	pThis->cLoad++;
1531	if (pThis->cLoad > pThis->cMaxLoad)
1532	pThis->cMaxLoad = pThis->cLoad;
1533	AssertMsg(pThis->cLoad <= pThis->cPages - pThis->cGuardPages, ("%d/%d\n", pThis->cLoad, pThis->cPages - pThis->cGuardPages));
1534	}
1535	else if (RT_UNLIKELY(cRefs <= 0))
1536	{
1537	ASMAtomicDecS32(&paPages[iPage].cRefs);
1538	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1539	*ppvPage = NULL;
1540	AssertLogRelMsgFailedReturn(("cRefs=%d iPage=%p HCPhys=%RHp\n", cRefs, iPage, HCPhys), UINT32_MAX);
1541	}
1542	void *pvPage = paPages[iPage].pvPage;
1543
1544	#ifndef IN_RC
1545	/*
1546	* Invalidate the entry?
1547	*/
1548	bool fInvalidateIt = RTCpuSetIsMemberByIndex(&paPages[iPage].PendingSet, iRealCpu);
1549	if (RT_UNLIKELY(fInvalidateIt))
1550	RTCpuSetDelByIndex(&paPages[iPage].PendingSet, iRealCpu);
1551	#endif
1552
1553	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1554
1555	/*
1556	* Do the actual invalidation outside the spinlock.
1557	*/
1558	#ifdef IN_RC
1559	if (RT_UNLIKELY(fNew))
1560	#else
1561	if (RT_UNLIKELY(fInvalidateIt))
1562	#endif
1563	{
1564	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageInvlPg);
1565	ASMInvalidatePage(pvPage);
1566	}
1567
1568	*ppvPage = pvPage;
1569	return iPage;
1570	}
1571
1572
1573	/**
1574	* Assert the the integrity of the pool.
1575	*
1576	* @returns VBox status code.
1577	*/
1578	static int pgmRZDynMapAssertIntegrity(PPGMRZDYNMAP pThis)
1579	{
1580	/*
1581	* Basic pool stuff that doesn't require any lock, just assumes we're a user.
1582	*/
1583	if (!pThis)
1584	return VINF_SUCCESS;
1585	AssertPtrReturn(pThis, VERR_INVALID_POINTER);
1586	AssertReturn(pThis->u32Magic == PGMRZDYNMAP_MAGIC, VERR_INVALID_MAGIC);
1587	if (!pThis->cUsers)
1588	return VERR_INVALID_PARAMETER;
1589
1590
1591	int rc = VINF_SUCCESS;
1592	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
1593
1594	#define CHECK_RET(expr, a) \
1595	do { \
1596	if (RT_UNLIKELY(!(expr))) \
1597	{ \
1598	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis); \
1599	RTAssertMsg1Weak(#expr, __LINE__, __FILE__, __PRETTY_FUNCTION__); \
1600	RTAssertMsg2Weak a; \
1601	return VERR_INTERNAL_ERROR; \
1602	} \
1603	} while (0)
1604
1605	/*
1606	* Check that the PTEs are correct.
1607	*/
1608	uint32_t cGuard = 0;
1609	uint32_t cLoad = 0;
1610	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
1611	uint32_t iPage = pThis->cPages;
1612	if (pThis->fLegacyMode)
1613	{
1614	#ifdef IN_RING0
1615	PCX86PGUINT paSavedPTEs = (PCX86PGUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
1616	#endif
1617	while (iPage-- > 0)
1618	{
1619	CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
1620	if ( paPages[iPage].cRefs == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT
1621	&& paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS)
1622	{
1623	#ifdef PGMR0DYNMAP_GUARD_NP
1624	CHECK_RET(paPages[iPage].uPte.pLegacy->u == (paSavedPTEs[iPage] & ~(X86PGUINT)X86_PTE_P),
1625	("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage]));
1626	#else
1627	CHECK_RET(paPages[iPage].uPte.pLegacy->u == PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE,
1628	("#%u: %#x", iPage, paPages[iPage].uPte.pLegacy->u));
1629	#endif
1630	cGuard++;
1631	}
1632	else if (paPages[iPage].HCPhys != NIL_RTHCPHYS)
1633	{
1634	CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
1635	X86PGUINT uPte = X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1636	#ifdef IN_RING0
1637	\| (paSavedPTEs[iPage] & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1638	#endif
1639	\| (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
1640	CHECK_RET(paPages[iPage].uPte.pLegacy->u == uPte,
1641	("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, uPte));
1642	if (paPages[iPage].cRefs)
1643	cLoad++;
1644	}
1645	#if defined(IN_RING0) && !defined(PGMRZDYNMAP_STRICT_RELEASE)
1646	else
1647	CHECK_RET(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage],
1648	("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage]));
1649	#endif
1650	}
1651	}
1652	else
1653	{
1654	#ifdef IN_RING0
1655	PCX86PGPAEUINT paSavedPTEs = (PCX86PGPAEUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
1656	#endif
1657	while (iPage-- > 0)
1658	{
1659	CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
1660	if ( paPages[iPage].cRefs == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT
1661	&& paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS)
1662	{
1663	#ifdef PGMR0DYNMAP_GUARD_NP
1664	CHECK_RET(paPages[iPage].uPte.pPae->u == (paSavedPTEs[iPage] & ~(X86PGPAEUINT)X86_PTE_P),
1665	("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage]));
1666	#else
1667	CHECK_RET(paPages[iPage].uPte.pPae->u == PGMR0DYNMAP_GUARD_PAGE_PAE_PTE,
1668	("#%u: %#llx", iPage, paPages[iPage].uPte.pPae->u));
1669	#endif
1670	cGuard++;
1671	}
1672	else if (paPages[iPage].HCPhys != NIL_RTHCPHYS)
1673	{
1674	CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
1675	X86PGPAEUINT uPte = X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1676	#ifdef IN_RING0
1677	\| (paSavedPTEs[iPage] & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1678	#endif
1679	\| (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
1680	CHECK_RET(paPages[iPage].uPte.pPae->u == uPte,
1681	("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pLegacy->u, uPte));
1682	if (paPages[iPage].cRefs)
1683	cLoad++;
1684	}
1685	#ifdef IN_RING0
1686	else
1687	CHECK_RET(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage],
1688	("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage]));
1689	#endif
1690	}
1691	}
1692
1693	CHECK_RET(cLoad == pThis->cLoad, ("%u %u\n", cLoad, pThis->cLoad));
1694	CHECK_RET(cGuard == pThis->cGuardPages, ("%u %u\n", cGuard, pThis->cGuardPages));
1695
1696	#undef CHECK_RET
1697	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1698	return VINF_SUCCESS;
1699	}
1700
1701	#ifdef IN_RING0
1702	/**
1703	* Assert the the integrity of the pool.
1704	*
1705	* @returns VBox status code.
1706	*/
1707	VMMR0DECL(int) PGMR0DynMapAssertIntegrity(void)
1708	{
1709	return pgmRZDynMapAssertIntegrity(g_pPGMR0DynMap);
1710	}
1711	#endif /* IN_RING0 */
1712
1713	#ifdef IN_RC
1714	/**
1715	* Assert the the integrity of the pool.
1716	*
1717	* @returns VBox status code.
1718	*/
1719	VMMRCDECL(int) PGMRCDynMapAssertIntegrity(PVM pVM)
1720	{
1721	return pgmRZDynMapAssertIntegrity((PPGMRZDYNMAP)pVM->pgm.s.pRCDynMap);
1722	}
1723	#endif /* IN_RC */
1724
1725
1726	/**
1727	* As a final resort for a (somewhat) full auto set or full cache, try merge
1728	* duplicate entries and flush the ones we can.
1729	*
1730	* @param pSet The set.
1731	*/
1732	static void pgmDynMapOptimizeAutoSet(PPGMMAPSET pSet)
1733	{
1734	LogFlow(("pgmDynMapOptimizeAutoSet\n"));
1735
1736	for (uint32_t i = 0 ; i < pSet->cEntries; i++)
1737	{
1738	/*
1739	* Try merge entries.
1740	*/
1741	uint16_t const iPage = pSet->aEntries[i].iPage;
1742	uint32_t j = i + 1;
1743	while ( j < pSet->cEntries
1744	&& ( pSet->iSubset == UINT32_MAX
1745	\|\| pSet->iSubset < pSet->cEntries) )
1746	{
1747	if (pSet->aEntries[j].iPage != iPage)
1748	j++;
1749	else
1750	{
1751	uint32_t const cHardRefs = (uint32_t)pSet->aEntries[i].cRefs
1752	+ (uint32_t)pSet->aEntries[j].cRefs;
1753	uint32_t cInlinedRefs = (uint32_t)pSet->aEntries[i].cInlinedRefs
1754	+ (uint32_t)pSet->aEntries[j].cInlinedRefs;
1755	uint32_t cUnrefs = (uint32_t)pSet->aEntries[i].cUnrefs
1756	+ (uint32_t)pSet->aEntries[j].cUnrefs;
1757	uint32_t cSub = RT_MIN(cUnrefs, cInlinedRefs);
1758	cInlinedRefs -= cSub;
1759	cUnrefs -= cSub;
1760
1761	if ( cHardRefs < UINT16_MAX
1762	&& cInlinedRefs < UINT16_MAX
1763	&& cUnrefs < UINT16_MAX)
1764	{
1765	/* merge j into i removing j. */
1766	Log2(("pgmDynMapOptimizeAutoSet: Merging #%u into #%u\n", j, i));
1767	pSet->aEntries[i].cRefs = cHardRefs;
1768	pSet->aEntries[i].cInlinedRefs = cInlinedRefs;
1769	pSet->aEntries[i].cUnrefs = cUnrefs;
1770	pSet->cEntries--;
1771	if (j < pSet->cEntries)
1772	{
1773	pSet->aEntries[j] = pSet->aEntries[pSet->cEntries];
1774	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[pSet->cEntries]);
1775	}
1776	else
1777	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]);
1778	}
1779	#if 0 /* too complicated, skip it. */
1780	else
1781	{
1782	/* migrate the max number of refs from j into i and quit the inner loop. */
1783	uint32_t cMigrate = UINT16_MAX - 1 - pSet->aEntries[i].cRefs;
1784	Assert(pSet->aEntries[j].cRefs > cMigrate);
1785	pSet->aEntries[j].cRefs -= cMigrate;
1786	pSet->aEntries[i].cRefs = UINT16_MAX - 1;
1787	break;
1788	}
1789	#endif
1790	}
1791	}
1792
1793	/*
1794	* Try make use of the unused hinting (cUnrefs) to evict entries
1795	* from both the set as well as the mapping cache.
1796	*/
1797
1798	uint32_t const cTotalRefs = (uint32_t)pSet->aEntries[i].cRefs + pSet->aEntries[i].cInlinedRefs;
1799	Log2(("pgmDynMapOptimizeAutoSet: #%u/%u/%u pvPage=%p iPage=%u cRefs=%u cInlinedRefs=%u cUnrefs=%u cTotalRefs=%u\n",
1800	i,
1801	pSet->iSubset,
1802	pSet->cEntries,
1803	pSet->aEntries[i].pvPage,
1804	pSet->aEntries[i].iPage,
1805	pSet->aEntries[i].cRefs,
1806	pSet->aEntries[i].cInlinedRefs,
1807	pSet->aEntries[i].cUnrefs,
1808	cTotalRefs));
1809	Assert(cTotalRefs >= pSet->aEntries[i].cUnrefs);
1810
1811	if ( cTotalRefs == pSet->aEntries[i].cUnrefs
1812	&& ( pSet->iSubset == UINT32_MAX
1813	\|\| pSet->iSubset < pSet->cEntries)
1814	)
1815	{
1816	Log2(("pgmDynMapOptimizeAutoSet: Releasing iPage=%d/%p\n", pSet->aEntries[i].iPage, pSet->aEntries[i].pvPage));
1817	//LogFlow(("pgmDynMapOptimizeAutoSet: Releasing iPage=%d/%p\n", pSet->aEntries[i].iPage, pSet->aEntries[i].pvPage));
1818	pgmRZDynMapReleasePage(PGMRZDYNMAP_SET_2_DYNMAP(pSet),
1819	pSet->aEntries[i].iPage,
1820	pSet->aEntries[i].cRefs);
1821	pSet->cEntries--;
1822	if (i < pSet->cEntries)
1823	{
1824	pSet->aEntries[i] = pSet->aEntries[pSet->cEntries];
1825	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[pSet->cEntries]);
1826	}
1827
1828	i--;
1829	}
1830	}
1831	}
1832
1833
1834
1835
1836	/**
1837	* Signals the start of a new set of mappings.
1838	*
1839	* Mostly for strictness. PGMDynMapHCPage won't work unless this
1840	* API is called.
1841	*
1842	* @param pVCpu The shared data for the current virtual CPU.
1843	*/
1844	VMMDECL(void) PGMRZDynMapStartAutoSet(PVMCPU pVCpu)
1845	{
1846	LogFlow(("PGMRZDynMapStartAutoSet:\n"));
1847	Assert(pVCpu->pgm.s.AutoSet.cEntries == PGMMAPSET_CLOSED);
1848	Assert(pVCpu->pgm.s.AutoSet.iSubset == UINT32_MAX);
1849	pVCpu->pgm.s.AutoSet.cEntries = 0;
1850	pVCpu->pgm.s.AutoSet.iCpu = PGMRZDYNMAP_CUR_CPU();
1851	}
1852
1853
1854	#ifdef IN_RING0
1855	/**
1856	* Starts or migrates the autoset of a virtual CPU.
1857	*
1858	* This is used by HWACCMR0Enter. When we've longjumped out of the HWACCM
1859	* execution loop with the set open, we'll migrate it when re-entering. While
1860	* under normal circumstances, we'll start it so VMXR0LoadGuestState can access
1861	* guest memory.
1862	*
1863	* @returns @c true if started, @c false if migrated.
1864	* @param pVCpu The shared data for the current virtual CPU.
1865	* @thread EMT
1866	*/
1867	VMMR0DECL(bool) PGMR0DynMapStartOrMigrateAutoSet(PVMCPU pVCpu)
1868	{
1869	bool fStartIt = pVCpu->pgm.s.AutoSet.cEntries == PGMMAPSET_CLOSED;
1870	if (fStartIt)
1871	PGMRZDynMapStartAutoSet(pVCpu);
1872	else
1873	PGMR0DynMapMigrateAutoSet(pVCpu);
1874	return fStartIt;
1875	}
1876	#endif /* IN_RING0 */
1877
1878
1879	/**
1880	* Checks if the set has high load.
1881	*
1882	* @returns true on high load, otherwise false.
1883	* @param pSet The set.
1884	*/
1885	DECLINLINE(bool) pgmRZDynMapHasHighLoad(PPGMMAPSET pSet)
1886	{
1887	#ifdef IN_RC
1888	if (pSet->cEntries < MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE / 2)
1889	return false;
1890	#endif
1891
1892	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
1893	uint32_t cUnusedPages = pThis->cPages - pThis->cLoad;
1894	#ifdef IN_RC
1895	return cUnusedPages <= MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE * 36 / 100;
1896	#else
1897	return cUnusedPages <= PGMR0DYNMAP_PAGES_PER_CPU_MIN;
1898	#endif
1899	}
1900
1901
1902	/**
1903	* Worker that performs the actual flushing of the set.
1904	*
1905	* @param pSet The set to flush.
1906	* @param cEntries The number of entries.
1907	*/
1908	DECLINLINE(void) pgmDynMapFlushAutoSetWorker(PPGMMAPSET pSet, uint32_t cEntries)
1909	{
1910	/*
1911	* Release any pages it's referencing.
1912	*/
1913	if ( cEntries != 0
1914	&& RT_LIKELY(cEntries <= RT_ELEMENTS(pSet->aEntries)))
1915	{
1916	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
1917	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
1918
1919	uint32_t i = cEntries;
1920	while (i-- > 0)
1921	{
1922	uint32_t iPage = pSet->aEntries[i].iPage;
1923	Assert(iPage < pThis->cPages);
1924	int32_t cRefs = pSet->aEntries[i].cRefs;
1925	Assert(cRefs > 0);
1926	pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs);
1927
1928	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[i]);
1929	}
1930
1931	Assert(pThis->cLoad <= pThis->cPages - pThis->cGuardPages);
1932	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1933	}
1934	}
1935
1936
1937	/**
1938	* Releases the dynamic memory mappings made by PGMDynMapHCPage and associates
1939	* since the PGMDynMapStartAutoSet call.
1940	*
1941	* @param pVCpu The shared data for the current virtual CPU.
1942	*/
1943	VMMDECL(void) PGMRZDynMapReleaseAutoSet(PVMCPU pVCpu)
1944	{
1945	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1946
1947	/*
1948	* Close and flush the set.
1949	*/
1950	uint32_t cEntries = pSet->cEntries;
1951	AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
1952	pSet->cEntries = PGMMAPSET_CLOSED;
1953	pSet->iSubset = UINT32_MAX;
1954	pSet->iCpu = -1;
1955
1956	#ifdef IN_RC
1957	if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)
1958	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
1959	else
1960	#endif
1961	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
1962	if (cEntries > RT_ELEMENTS(pSet->aEntries) * 50 / 100)
1963	Log(("PGMRZDynMapReleaseAutoSet: cEntries=%d\n", cEntries));
1964	else
1965	LogFlow(("PGMRZDynMapReleaseAutoSet: cEntries=%d\n", cEntries));
1966
1967	pgmDynMapFlushAutoSetWorker(pSet, cEntries);
1968	}
1969
1970
1971	/**
1972	* Flushes the set if it's above a certain threshold.
1973	*
1974	* @param pVCpu The shared data for the current virtual CPU.
1975	*/
1976	VMMDECL(void) PGMRZDynMapFlushAutoSet(PVMCPU pVCpu)
1977	{
1978	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1979	AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
1980
1981	/*
1982	* Only flush it if it's 45% full.
1983	*/
1984	uint32_t cEntries = pSet->cEntries;
1985	AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
1986	Assert(pSet->iSubset == UINT32_MAX);
1987	#ifdef IN_RC
1988	if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)
1989	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
1990	else
1991	#endif
1992	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
1993	if ( cEntries >= RT_ELEMENTS(pSet->aEntries) * 45 / 100
1994	\|\| pgmRZDynMapHasHighLoad(pSet))
1995	{
1996	pSet->cEntries = 0;
1997	Log(("PGMDynMapFlushAutoSet: cEntries=%d\n", pSet->cEntries));
1998
1999	pgmDynMapFlushAutoSetWorker(pSet, cEntries);
2000	AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
2001	}
2002	}
2003
2004
2005	#ifndef IN_RC
2006	/**
2007	* Migrates the automatic mapping set of the current vCPU if it's active and
2008	* necessary.
2009	*
2010	* This is called when re-entering the hardware assisted execution mode after a
2011	* nip down to ring-3. We run the risk that the CPU might have change and we
2012	* will therefore make sure all the cache entries currently in the auto set will
2013	* be valid on the new CPU. If the cpu didn't change nothing will happen as all
2014	* the entries will have been flagged as invalidated.
2015	*
2016	* @param pVCpu The shared data for the current virtual CPU.
2017	* @thread EMT
2018	*/
2019	VMMR0DECL(void) PGMR0DynMapMigrateAutoSet(PVMCPU pVCpu)
2020	{
2021	LogFlow(("PGMR0DynMapMigrateAutoSet\n"));
2022	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
2023	int32_t iRealCpu = PGMRZDYNMAP_CUR_CPU();
2024	if (pSet->iCpu != iRealCpu)
2025	{
2026	uint32_t i = pSet->cEntries;
2027	if (i != PGMMAPSET_CLOSED)
2028	{
2029	AssertMsg(i <= RT_ELEMENTS(pSet->aEntries), ("%#x (%u)\n", i, i));
2030	if (i != 0 && RT_LIKELY(i <= RT_ELEMENTS(pSet->aEntries)))
2031	{
2032	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
2033	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
2034
2035	while (i-- > 0)
2036	{
2037	Assert(pSet->aEntries[i].cRefs > 0);
2038	uint32_t iPage = pSet->aEntries[i].iPage;
2039	Assert(iPage < pThis->cPages);
2040	if (RTCpuSetIsMemberByIndex(&pThis->paPages[iPage].PendingSet, iRealCpu))
2041	{
2042	RTCpuSetDelByIndex(&pThis->paPages[iPage].PendingSet, iRealCpu);
2043	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
2044
2045	ASMInvalidatePage(pThis->paPages[iPage].pvPage);
2046	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapMigrateInvlPg);
2047
2048	PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis);
2049	}
2050	}
2051
2052	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
2053	}
2054	}
2055	pSet->iCpu = iRealCpu;
2056	}
2057	}
2058	#endif /* !IN_RC */
2059
2060
2061	/**
2062	* Worker function that flushes the current subset.
2063	*
2064	* This is called when the set is popped or when the set
2065	* hash a too high load. As also pointed out elsewhere, the
2066	* whole subset thing is a hack for working around code that
2067	* accesses too many pages. Like PGMPool.
2068	*
2069	* @param pSet The set which subset to flush.
2070	*/
2071	static void pgmDynMapFlushSubset(PPGMMAPSET pSet)
2072	{
2073	uint32_t iSubset = pSet->iSubset;
2074	uint32_t i = pSet->cEntries;
2075	Assert(i <= RT_ELEMENTS(pSet->aEntries));
2076	if ( i > iSubset
2077	&& i <= RT_ELEMENTS(pSet->aEntries))
2078	{
2079	Log(("pgmDynMapFlushSubset: cEntries=%d iSubset=%d\n", pSet->cEntries, iSubset));
2080	pSet->cEntries = iSubset;
2081
2082	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
2083	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
2084
2085	while (i-- > iSubset)
2086	{
2087	uint32_t iPage = pSet->aEntries[i].iPage;
2088	Assert(iPage < pThis->cPages);
2089	int32_t cRefs = pSet->aEntries[i].cRefs;
2090	Assert(cRefs > 0);
2091	pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs);
2092
2093	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[i]);
2094	}
2095
2096	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
2097	}
2098	}
2099
2100
2101	/**
2102	* Creates a subset.
2103	*
2104	* A subset is a hack to avoid having to rewrite code that touches a lot of
2105	* pages. It prevents the mapping set from being overflowed by automatically
2106	* flushing previous mappings when a certain threshold is reached.
2107	*
2108	* Pages mapped after calling this function are only valid until the next page
2109	* is mapped.
2110	*
2111	* @returns The index of the previous subset. Pass this to
2112	* PGMDynMapPopAutoSubset when popping it.
2113	* @param pVCpu Pointer to the virtual cpu data.
2114	*/
2115	VMMDECL(uint32_t) PGMRZDynMapPushAutoSubset(PVMCPU pVCpu)
2116	{
2117	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
2118	AssertReturn(pSet->cEntries != PGMMAPSET_CLOSED, UINT32_MAX);
2119	uint32_t iPrevSubset = pSet->iSubset;
2120	LogFlow(("PGMRZDynMapPushAutoSubset: pVCpu=%p iPrevSubset=%u\n", pVCpu, iPrevSubset));
2121
2122	/*
2123	* If it looks like we're approaching the max set size or mapping space
2124	* optimize the set to drop off unused pages.
2125	*/
2126	if ( pSet->cEntries > RT_ELEMENTS(pSet->aEntries) * 60 / 100
2127	\|\| pgmRZDynMapHasHighLoad(pSet))
2128	{
2129	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize);
2130	pgmDynMapOptimizeAutoSet(pSet);
2131	}
2132
2133	pSet->iSubset = pSet->cEntries;
2134	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSubsets);
2135
2136	AssertMsg(iPrevSubset <= pSet->iSubset \|\| iPrevSubset == UINT32_MAX, ("iPrevSubset=%#x iSubset=%#x\n", iPrevSubset, pSet->iSubset));
2137	return iPrevSubset;
2138	}
2139
2140
2141	/**
2142	* Pops a subset created by a previous call to PGMDynMapPushAutoSubset.
2143	*
2144	* @param pVCpu Pointer to the virtual cpu data.
2145	* @param iPrevSubset What PGMDynMapPushAutoSubset returned.
2146	*/
2147	VMMDECL(void) PGMRZDynMapPopAutoSubset(PVMCPU pVCpu, uint32_t iPrevSubset)
2148	{
2149	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
2150	uint32_t cEntries = pSet->cEntries;
2151	LogFlow(("PGMRZDynMapPopAutoSubset: pVCpu=%p iPrevSubset=%u iSubset=%u cEntries=%u\n", pVCpu, iPrevSubset, pSet->iSubset, cEntries));
2152	AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
2153	AssertMsgReturnVoid(pSet->iSubset >= iPrevSubset \|\| iPrevSubset == UINT32_MAX, ("iPrevSubset=%u iSubset=%u cEntries=%u\n", iPrevSubset, pSet->iSubset, cEntries));
2154	#ifdef IN_RC
2155	if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)
2156	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
2157	else
2158	#endif
2159	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
2160	if ( cEntries >= RT_ELEMENTS(pSet->aEntries) * 40 / 100
2161	&& cEntries != pSet->iSubset)
2162	{
2163	pgmDynMapFlushSubset(pSet);
2164	Assert(pSet->cEntries >= iPrevSubset \|\| iPrevSubset == UINT32_MAX);
2165	}
2166	pSet->iSubset = iPrevSubset;
2167	}
2168
2169
2170	/**
2171	* Indicates that the given page is unused and its mapping can be re-used.
2172	*
2173	* @param pVCpu The current CPU.
2174	* @param pvHint The page that is now unused. This does not have to
2175	* point at the start of the page. NULL is ignored.
2176	*/
2177	#ifdef LOG_ENABLED
2178	void pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint, RT_SRC_POS_DECL)
2179	#else
2180	void pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint)
2181	#endif
2182	{
2183	/*
2184	* Ignore NULL pointers and mask off the page offset bits.
2185	*/
2186	if (pvHint == NULL)
2187	return;
2188	pvHint = (void *)((uintptr_t)pvHint & ~(uintptr_t)PAGE_OFFSET_MASK);
2189
2190	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
2191	uint32_t iEntry = pSet->cEntries;
2192	AssertReturnVoid(iEntry > 0);
2193
2194	/*
2195	* Find the entry in the usual unrolled fashion.
2196	*/
2197	/** @todo add a hint to the set which entry was used last since it's not
2198	* always the last entry? */
2199	#define IS_MATCHING_ENTRY(pSet, iEntry, pvHint) \
2200	( (pSet)->aEntries[(iEntry)].pvPage == (pvHint) \
2201	&& (uint32_t)(pSet)->aEntries[(iEntry)].cRefs + (pSet)->aEntries[(iEntry)].cInlinedRefs \
2202	> (pSet)->aEntries[(iEntry)].cUnrefs )
2203	if ( iEntry >= 1 && IS_MATCHING_ENTRY(pSet, iEntry - 1, pvHint))
2204	iEntry = iEntry - 1;
2205	else if (iEntry >= 2 && IS_MATCHING_ENTRY(pSet, iEntry - 2, pvHint))
2206	iEntry = iEntry - 2;
2207	else if (iEntry >= 3 && IS_MATCHING_ENTRY(pSet, iEntry - 3, pvHint))
2208	iEntry = iEntry - 3;
2209	else if (iEntry >= 4 && IS_MATCHING_ENTRY(pSet, iEntry - 4, pvHint))
2210	iEntry = iEntry - 4;
2211	else if (iEntry >= 5 && IS_MATCHING_ENTRY(pSet, iEntry - 5, pvHint))
2212	iEntry = iEntry - 5;
2213	else if (iEntry >= 6 && IS_MATCHING_ENTRY(pSet, iEntry - 6, pvHint))
2214	iEntry = iEntry - 6;
2215	else if (iEntry >= 7 && IS_MATCHING_ENTRY(pSet, iEntry - 7, pvHint))
2216	iEntry = iEntry - 7;
2217	else
2218	{
2219	/*
2220	* Loop till we find it.
2221	*/
2222	bool fFound = false;
2223	if (iEntry > 7)
2224	{
2225	iEntry -= 7;
2226	while (iEntry-- > 0)
2227	if (IS_MATCHING_ENTRY(pSet, iEntry, pvHint))
2228	{
2229	fFound = true;
2230	break;
2231	}
2232	}
2233	AssertMsgReturnVoid(fFound,
2234	("pvHint=%p cEntries=%#x iSubset=%#x\n"
2235	"aEntries[0] = {%#x, %#x, %#x, %#x, %p}\n"
2236	"aEntries[1] = {%#x, %#x, %#x, %#x, %p}\n"
2237	"aEntries[2] = {%#x, %#x, %#x, %#x, %p}\n"
2238	"aEntries[3] = {%#x, %#x, %#x, %#x, %p}\n"
2239	"aEntries[4] = {%#x, %#x, %#x, %#x, %p}\n"
2240	"aEntries[5] = {%#x, %#x, %#x, %#x, %p}\n"
2241	,
2242	pvHint, pSet->cEntries, pSet->iSubset,
2243	pSet->aEntries[0].iPage, pSet->aEntries[0].cRefs, pSet->aEntries[0].cInlinedRefs, pSet->aEntries[0].cUnrefs, pSet->aEntries[0].pvPage,
2244	pSet->aEntries[1].iPage, pSet->aEntries[1].cRefs, pSet->aEntries[1].cInlinedRefs, pSet->aEntries[1].cUnrefs, pSet->aEntries[1].pvPage,
2245	pSet->aEntries[2].iPage, pSet->aEntries[2].cRefs, pSet->aEntries[2].cInlinedRefs, pSet->aEntries[2].cUnrefs, pSet->aEntries[2].pvPage,
2246	pSet->aEntries[3].iPage, pSet->aEntries[3].cRefs, pSet->aEntries[3].cInlinedRefs, pSet->aEntries[3].cUnrefs, pSet->aEntries[3].pvPage,
2247	pSet->aEntries[4].iPage, pSet->aEntries[4].cRefs, pSet->aEntries[4].cInlinedRefs, pSet->aEntries[4].cUnrefs, pSet->aEntries[4].pvPage,
2248	pSet->aEntries[5].iPage, pSet->aEntries[5].cRefs, pSet->aEntries[5].cInlinedRefs, pSet->aEntries[5].cUnrefs, pSet->aEntries[5].pvPage));
2249	}
2250	#undef IS_MATCHING_ENTRY
2251
2252	/*
2253	* Update it.
2254	*/
2255	uint32_t const cTotalRefs = (uint32_t)pSet->aEntries[iEntry].cRefs + pSet->aEntries[iEntry].cInlinedRefs;
2256	uint32_t const cUnrefs = pSet->aEntries[iEntry].cUnrefs;
2257	LogFlow(("pgmRZDynMapUnusedHint: pvHint=%p #%u cRefs=%d cInlinedRefs=%d cUnrefs=%d (+1) cTotalRefs=%d %s(%d) %s\n",
2258	pvHint, iEntry, pSet->aEntries[iEntry].cRefs, pSet->aEntries[iEntry].cInlinedRefs, cUnrefs, cTotalRefs, pszFile, iLine, pszFunction));
2259	AssertReturnVoid(cTotalRefs > cUnrefs);
2260
2261	if (RT_LIKELY(cUnrefs < UINT16_MAX - 1))
2262	pSet->aEntries[iEntry].cUnrefs++;
2263	else if (pSet->aEntries[iEntry].cInlinedRefs)
2264	{
2265	uint32_t cSub = RT_MIN(pSet->aEntries[iEntry].cInlinedRefs, pSet->aEntries[iEntry].cUnrefs);
2266	pSet->aEntries[iEntry].cInlinedRefs -= cSub;
2267	pSet->aEntries[iEntry].cUnrefs -= cSub;
2268	pSet->aEntries[iEntry].cUnrefs++;
2269	}
2270	else
2271	Log(("pgmRZDynMapUnusedHint: pvHint=%p ignored because of overflow! %s(%d) %s\n", pvHint, pszFile, iLine, pszFunction));
2272
2273	#ifdef PGMRZDYNMAP_STRICT_RELEASE
2274	/*
2275	* Optimize the set to trigger the unmapping and invalidation of the page.
2276	*/
2277	if (cUnrefs + 1 == cTotalRefs)
2278	pgmDynMapOptimizeAutoSet(pSet);
2279	#endif
2280	}
2281
2282
2283	/**
2284	* Common worker code for pgmRZDynMapHCPageInlined, pgmRZDynMapHCPageV2Inlined
2285	* and pgmR0DynMapGCPageOffInlined.
2286	*
2287	* @returns VINF_SUCCESS, bails out to ring-3 on failure.
2288	* @param pSet The set.
2289	* @param HCPhys The physical address of the page.
2290	* @param ppv Where to store the address of the mapping on success.
2291	*
2292	* @remarks This is a very hot path.
2293	*/
2294	int pgmRZDynMapHCPageCommon(PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
2295	{
2296	AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
2297	PVMCPU pVCpu = PGMRZDYNMAP_SET_2_VMCPU(pSet);
2298	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
2299
2300	/*
2301	* Map it.
2302	*/
2303	void *pvPage;
2304	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
2305	uint32_t iPage = pgmR0DynMapPage(pThis, HCPhys, pSet->iCpu, pVCpu, &pvPage);
2306	if (RT_UNLIKELY(iPage == UINT32_MAX))
2307	{
2308	/*
2309	* We're out of mapping space, optimize our set to try remedy the
2310	* situation. (Only works if there are unreference hints.)
2311	*/
2312	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize);
2313	pgmDynMapOptimizeAutoSet(pSet);
2314
2315	iPage = pgmR0DynMapPage(pThis, HCPhys, pSet->iCpu, pVCpu, &pvPage);
2316	if (RT_UNLIKELY(iPage == UINT32_MAX))
2317	{
2318	RTAssertMsg2Weak("pgmRZDynMapHCPageCommon: cLoad=%u/%u cPages=%u cGuardPages=%u\n",
2319	pThis->cLoad, pThis->cMaxLoad, pThis->cPages, pThis->cGuardPages);
2320	if (!g_fPGMR0DynMapTestRunning)
2321	VMMRZCallRing3NoCpu(PGMRZDYNMAP_SET_2_VM(pSet), VMMCALLRING3_VM_R0_ASSERTION, 0);
2322	*ppv = NULL;
2323	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
2324	return VERR_PGM_DYNMAP_FAILED;
2325	}
2326	}
2327
2328	/*
2329	* Add the page to the auto reference set.
2330	*
2331	* The typical usage pattern means that the same pages will be mapped
2332	* several times in the same set. We can catch most of these
2333	* remappings by looking a few pages back into the set. (The searching
2334	* and set optimizing path will hardly ever be used when doing this.)
2335	*/
2336	AssertCompile(RT_ELEMENTS(pSet->aEntries) >= 8);
2337	int32_t i = pSet->cEntries;
2338	if (i-- < 5)
2339	{
2340	unsigned iEntry = pSet->cEntries++;
2341	pSet->aEntries[iEntry].cRefs = 1;
2342	pSet->aEntries[iEntry].cUnrefs = 0;
2343	pSet->aEntries[iEntry].cInlinedRefs = 0;
2344	pSet->aEntries[iEntry].iPage = iPage;
2345	pSet->aEntries[iEntry].pvPage = pvPage;
2346	pSet->aEntries[iEntry].HCPhys = HCPhys;
2347	pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
2348	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/0/0 iPage=%#x [a] %s(%d) %s\n",
2349	pSet, HCPhys, iEntry, iEntry + 1, pvPage, 1, iPage, pszFile, iLine, pszFunction));
2350	}
2351	/* Any of the last 5 pages? */
2352	else if ( pSet->aEntries[i - 0].iPage == iPage
2353	&& pSet->aEntries[i - 0].cRefs < UINT16_MAX - 1)
2354	{
2355	pSet->aEntries[i - 0].cRefs++;
2356	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [0] %s(%d) %s\n", pSet, HCPhys, i - 0, pSet->cEntries, pvPage, pSet->aEntries[i - 0].cRefs, pSet->aEntries[i - 0].cInlinedRefs, pSet->aEntries[i - 0].cUnrefs, iPage, pszFile, iLine, pszFunction));
2357	}
2358	else if ( pSet->aEntries[i - 1].iPage == iPage
2359	&& pSet->aEntries[i - 1].cRefs < UINT16_MAX - 1)
2360	{
2361	pSet->aEntries[i - 1].cRefs++;
2362	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [1] %s(%d) %s\n", pSet, HCPhys, i - 1, pSet->cEntries, pvPage, pSet->aEntries[i - 1].cRefs, pSet->aEntries[i - 1].cInlinedRefs, pSet->aEntries[i - 1].cUnrefs, iPage, pszFile, iLine, pszFunction));
2363	}
2364	else if ( pSet->aEntries[i - 2].iPage == iPage
2365	&& pSet->aEntries[i - 2].cRefs < UINT16_MAX - 1)
2366	{
2367	pSet->aEntries[i - 2].cRefs++;
2368	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [2] %s(%d) %s\n", pSet, HCPhys, i - 2, pSet->cEntries, pvPage, pSet->aEntries[i - 2].cRefs, pSet->aEntries[i - 2].cInlinedRefs, pSet->aEntries[i - 2].cUnrefs, iPage, pszFile, iLine, pszFunction));
2369	}
2370	else if ( pSet->aEntries[i - 3].iPage == iPage
2371	&& pSet->aEntries[i - 3].cRefs < UINT16_MAX - 1)
2372	{
2373	pSet->aEntries[i - 3].cRefs++;
2374	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [4] %s(%d) %s\n", pSet, HCPhys, i - 3, pSet->cEntries, pvPage, pSet->aEntries[i - 3].cRefs, pSet->aEntries[i - 3].cInlinedRefs, pSet->aEntries[i - 3].cUnrefs, iPage, pszFile, iLine, pszFunction));
2375	}
2376	else if ( pSet->aEntries[i - 4].iPage == iPage
2377	&& pSet->aEntries[i - 4].cRefs < UINT16_MAX - 1)
2378	{
2379	pSet->aEntries[i - 4].cRefs++;
2380	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [4] %s(%d) %s\n", pSet, HCPhys, i - 4, pSet->cEntries, pvPage, pSet->aEntries[i - 4].cRefs, pSet->aEntries[i - 4].cInlinedRefs, pSet->aEntries[i - 4].cUnrefs, iPage, pszFile, iLine, pszFunction));
2381	}
2382	/* Don't bother searching unless we're above a 60% load. */
2383	else if (RT_LIKELY(i <= (int32_t)RT_ELEMENTS(pSet->aEntries) * 60 / 100))
2384	{
2385	unsigned iEntry = pSet->cEntries++;
2386	pSet->aEntries[iEntry].cRefs = 1;
2387	pSet->aEntries[iEntry].cUnrefs = 0;
2388	pSet->aEntries[iEntry].cInlinedRefs = 0;
2389	pSet->aEntries[iEntry].iPage = iPage;
2390	pSet->aEntries[iEntry].pvPage = pvPage;
2391	pSet->aEntries[iEntry].HCPhys = HCPhys;
2392	pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
2393	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=1/0/0 iPage=%#x [b] %s(%d) %s\n", pSet, HCPhys, iEntry, pSet->cEntries, pvPage, iPage, pszFile, iLine, pszFunction));
2394	}
2395	else
2396	{
2397	/* Search the rest of the set. */
2398	Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
2399	i -= 4;
2400	while (i-- > 0)
2401	if ( pSet->aEntries[i].iPage == iPage
2402	&& pSet->aEntries[i].cRefs < UINT16_MAX - 1)
2403	{
2404	pSet->aEntries[i].cRefs++;
2405	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchHits);
2406	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [c] %s(%d) %s\n", pSet, HCPhys, i, pSet->cEntries, pvPage, pSet->aEntries[i].cRefs, pSet->aEntries[i].cInlinedRefs, pSet->aEntries[i].cUnrefs, iPage, pszFile, iLine, pszFunction));
2407	break;
2408	}
2409	if (i < 0)
2410	{
2411	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchMisses);
2412	#if 0 /* this is very bogus */
2413	if (pSet->iSubset < pSet->cEntries)
2414	{
2415	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchFlushes);
2416	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(pSet->cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
2417	pgmDynMapFlushSubset(pSet);
2418	}
2419	#endif
2420
2421	if (RT_UNLIKELY(pSet->cEntries >= RT_ELEMENTS(pSet->aEntries)))
2422	{
2423	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize);
2424	pgmDynMapOptimizeAutoSet(pSet);
2425	}
2426
2427	if (RT_LIKELY(pSet->cEntries < RT_ELEMENTS(pSet->aEntries)))
2428	{
2429	unsigned iEntry = pSet->cEntries++;
2430	pSet->aEntries[iEntry].cRefs = 1;
2431	pSet->aEntries[iEntry].cUnrefs = 0;
2432	pSet->aEntries[iEntry].cInlinedRefs = 0;
2433	pSet->aEntries[iEntry].iPage = iPage;
2434	pSet->aEntries[iEntry].pvPage = pvPage;
2435	pSet->aEntries[iEntry].HCPhys = HCPhys;
2436	pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
2437	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=1/0/0 iPage=%#x [d] %s(%d) %s\n", pSet, HCPhys, iEntry, pSet->cEntries, pvPage, iPage, pszFile, iLine, pszFunction));
2438	}
2439	else
2440	{
2441	/* We're screwed. */
2442	pgmRZDynMapReleasePage(pThis, iPage, 1);
2443
2444	RTAssertMsg2Weak("pgmRZDynMapHCPageCommon: set is full!\n");
2445	if (!g_fPGMR0DynMapTestRunning)
2446	VMMRZCallRing3NoCpu(PGMRZDYNMAP_SET_2_VM(pSet), VMMCALLRING3_VM_R0_ASSERTION, 0);
2447	*ppv = NULL;
2448	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
2449	return VERR_PGM_DYNMAP_FULL_SET;
2450	}
2451	}
2452	}
2453
2454	*ppv = pvPage;
2455	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
2456	return VINF_SUCCESS;
2457	}
2458
2459
2460	#if 0 /def DEBUG/
2461	/** For pgmR0DynMapTest3PerCpu. */
2462	typedef struct PGMR0DYNMAPTEST
2463	{
2464	uint32_t u32Expect;
2465	uint32_t *pu32;
2466	uint32_t volatile cFailures;
2467	} PGMR0DYNMAPTEST;
2468	typedef PGMR0DYNMAPTEST *PPGMR0DYNMAPTEST;
2469
2470	/**
2471	* Checks that the content of the page is the same on all CPUs, i.e. that there
2472	* are no CPU specific PTs or similar nasty stuff involved.
2473	*
2474	* @param idCpu The current CPU.
2475	* @param pvUser1 Pointer a PGMR0DYNMAPTEST structure.
2476	* @param pvUser2 Unused, ignored.
2477	*/
2478	static DECLCALLBACK(void) pgmR0DynMapTest3PerCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
2479	{
2480	PPGMR0DYNMAPTEST pTest = (PPGMR0DYNMAPTEST)pvUser1;
2481	ASMInvalidatePage(pTest->pu32);
2482	if (*pTest->pu32 != pTest->u32Expect)
2483	ASMAtomicIncU32(&pTest->cFailures);
2484	NOREF(pvUser2); NOREF(idCpu);
2485	}
2486
2487
2488	/**
2489	* Performs some basic tests in debug builds.
2490	*/
2491	static int pgmR0DynMapTest(PVM pVM)
2492	{
2493	LogRel(("pgmR0DynMapTest: **** START ****\n"));
2494	PPGMMAPSET pSet = &pVM->aCpus[0].pgm.s.AutoSet;
2495	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
2496	uint32_t i;
2497
2498	/*
2499	* Assert internal integrity first.
2500	*/
2501	LogRel(("Test #0\n"));
2502	int rc = PGMR0DynMapAssertIntegrity();
2503	if (RT_FAILURE(rc))
2504	return rc;
2505
2506	void *pvR0DynMapUsedSaved = pVM->pgm.s.pvR0DynMapUsed;
2507	pVM->pgm.s.pvR0DynMapUsed = pThis;
2508	g_fPGMR0DynMapTestRunning = true;
2509
2510	/*
2511	* Simple test, map CR3 twice and check that we're getting the
2512	* same mapping address back.
2513	*/
2514	LogRel(("Test #1\n"));
2515	ASMIntDisable();
2516	PGMRZDynMapStartAutoSet(&pVM->aCpus[0]);
2517
2518	uint64_t cr3 = ASMGetCR3() & ~(uint64_t)PAGE_OFFSET_MASK;
2519	void pv = (void )(intptr_t)-1;
2520	void pv2 = (void )(intptr_t)-2;
2521	rc = pgmRZDynMapHCPageCommon(pVM, cr3, &pv RTLOG_COMMA_SRC_POS);
2522	int rc2 = pgmRZDynMapHCPageCommon(pVM, cr3, &pv2 RTLOG_COMMA_SRC_POS);
2523	ASMIntEnable();
2524	if ( RT_SUCCESS(rc2)
2525	&& RT_SUCCESS(rc)
2526	&& pv == pv2)
2527	{
2528	LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2529	rc = PGMR0DynMapAssertIntegrity();
2530
2531	/*
2532	* Check that the simple set overflow code works by filling it
2533	* with more CR3 mappings.
2534	*/
2535	LogRel(("Test #2\n"));
2536	ASMIntDisable();
2537	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2538	for (i = 0 ; i < UINT16_MAX*2 - 1 && RT_SUCCESS(rc) && pv2 == pv; i++)
2539	{
2540	pv2 = (void *)(intptr_t)-4;
2541	rc = pgmRZDynMapHCPageCommon(pVM, cr3, &pv2 RTLOG_COMMA_SRC_POS);
2542	}
2543	ASMIntEnable();
2544	if (RT_FAILURE(rc) \|\| pv != pv2)
2545	{
2546	LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%p\n", __LINE__, rc, pv, pv2, i));
2547	if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
2548	}
2549	else if (pSet->cEntries != 5)
2550	{
2551	LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries) / 2));
2552	rc = VERR_INTERNAL_ERROR;
2553	}
2554	else if ( pSet->aEntries[4].cRefs != UINT16_MAX - 1
2555	\|\| pSet->aEntries[3].cRefs != UINT16_MAX - 1
2556	\|\| pSet->aEntries[2].cRefs != 1
2557	\|\| pSet->aEntries[1].cRefs != 1
2558	\|\| pSet->aEntries[0].cRefs != 1)
2559	{
2560	LogRel(("failed(%d): bad set dist: ", __LINE__));
2561	for (i = 0; i < pSet->cEntries; i++)
2562	LogRel(("[%d]=%d, ", i, pSet->aEntries[i].cRefs));
2563	LogRel(("\n"));
2564	rc = VERR_INTERNAL_ERROR;
2565	}
2566	if (RT_SUCCESS(rc))
2567	rc = PGMR0DynMapAssertIntegrity();
2568	if (RT_SUCCESS(rc))
2569	{
2570	/*
2571	* Trigger an set optimization run (exactly).
2572	*/
2573	LogRel(("Test #3\n"));
2574	ASMIntDisable();
2575	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2576	pv2 = NULL;
2577	for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) - 5 && RT_SUCCESS(rc) && pv2 != pv; i++)
2578	{
2579	pv2 = (void *)(intptr_t)(-5 - i);
2580	rc = pgmRZDynMapHCPageCommon(pVM, cr3 + PAGE_SIZE * (i + 5), &pv2 RTLOG_COMMA_SRC_POS);
2581	}
2582	ASMIntEnable();
2583	if (RT_FAILURE(rc) \|\| pv == pv2)
2584	{
2585	LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%d\n", __LINE__, rc, pv, pv2, i));
2586	if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
2587	}
2588	else if (pSet->cEntries != RT_ELEMENTS(pSet->aEntries))
2589	{
2590	LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2591	rc = VERR_INTERNAL_ERROR;
2592	}
2593	LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2594	if (RT_SUCCESS(rc))
2595	rc = PGMR0DynMapAssertIntegrity();
2596	if (RT_SUCCESS(rc))
2597	{
2598	/*
2599	* Trigger an overflow error.
2600	*/
2601	LogRel(("Test #4\n"));
2602	ASMIntDisable();
2603	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2604	for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) + 2; i++)
2605	{
2606	rc = pgmRZDynMapHCPageCommon(pVM, cr3 - PAGE_SIZE * (i + 5), &pv2 RTLOG_COMMA_SRC_POS);
2607	if (RT_SUCCESS(rc))
2608	rc = PGMR0DynMapAssertIntegrity();
2609	if (RT_FAILURE(rc))
2610	break;
2611	}
2612	ASMIntEnable();
2613	if (rc == VERR_PGM_DYNMAP_FULL_SET)
2614	{
2615	/* flush the set. */
2616	LogRel(("Test #5\n"));
2617	ASMIntDisable();
2618	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2619	PGMRZDynMapReleaseAutoSet(&pVM->aCpus[0]);
2620	PGMRZDynMapStartAutoSet(&pVM->aCpus[0]);
2621	ASMIntEnable();
2622
2623	rc = PGMR0DynMapAssertIntegrity();
2624	}
2625	else
2626	{
2627	LogRel(("failed(%d): rc=%Rrc, wanted %d ; pv2=%p Set=%u/%u; i=%d\n", __LINE__,
2628	rc, VERR_PGM_DYNMAP_FULL_SET, pv2, pSet->cEntries, RT_ELEMENTS(pSet->aEntries), i));
2629	if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
2630	}
2631	}
2632	}
2633	}
2634	else
2635	{
2636	LogRel(("failed(%d): rc=%Rrc rc2=%Rrc; pv=%p pv2=%p\n", __LINE__, rc, rc2, pv, pv2));
2637	if (RT_SUCCESS(rc))
2638	rc = rc2;
2639	}
2640
2641	/*
2642	* Check that everyone sees the same stuff.
2643	*/
2644	if (RT_SUCCESS(rc))
2645	{
2646	LogRel(("Test #5\n"));
2647	ASMIntDisable();
2648	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2649	RTHCPHYS HCPhysPT = RTR0MemObjGetPagePhysAddr(pThis->pSegHead->ahMemObjPTs[0], 0);
2650	rc = pgmRZDynMapHCPageCommon(pVM, HCPhysPT, &pv RTLOG_COMMA_SRC_POS);
2651	if (RT_SUCCESS(rc))
2652	{
2653	PGMR0DYNMAPTEST Test;
2654	uint32_t *pu32Real = &pThis->paPages[pThis->pSegHead->iPage].uPte.pLegacy->u;
2655	Test.pu32 = (uint32_t *)((uintptr_t)pv \| ((uintptr_t)pu32Real & PAGE_OFFSET_MASK));
2656	Test.u32Expect = *pu32Real;
2657	ASMAtomicWriteU32(&Test.cFailures, 0);
2658	ASMIntEnable();
2659
2660	rc = RTMpOnAll(pgmR0DynMapTest3PerCpu, &Test, NULL);
2661	if (RT_FAILURE(rc))
2662	LogRel(("failed(%d): RTMpOnAll rc=%Rrc\n", __LINE__, rc));
2663	else if (Test.cFailures)
2664	{
2665	LogRel(("failed(%d): cFailures=%d pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n", __LINE__,
2666	Test.cFailures, pu32Real, Test.pu32, Test.u32Expect, *Test.pu32));
2667	rc = VERR_INTERNAL_ERROR;
2668	}
2669	else
2670	LogRel(("pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n",
2671	pu32Real, Test.pu32, Test.u32Expect, *Test.pu32));
2672	}
2673	else
2674	{
2675	ASMIntEnable();
2676	LogRel(("failed(%d): rc=%Rrc\n", rc));
2677	}
2678	}
2679
2680	/*
2681	* Clean up.
2682	*/
2683	LogRel(("Cleanup.\n"));
2684	ASMIntDisable();
2685	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2686	PGMRZDynMapFlushAutoSet(&pVM->aCpus[0]);
2687	PGMRZDynMapReleaseAutoSet(&pVM->aCpus[0]);
2688	ASMIntEnable();
2689
2690	if (RT_SUCCESS(rc))
2691	rc = PGMR0DynMapAssertIntegrity();
2692	else
2693	PGMR0DynMapAssertIntegrity();
2694
2695	g_fPGMR0DynMapTestRunning = false;
2696	LogRel(("Result: rc=%Rrc Load=%u/%u/%u Set=%#x/%u\n", rc,
2697	pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2698	pVM->pgm.s.pvR0DynMapUsed = pvR0DynMapUsedSaved;
2699	LogRel(("pgmR0DynMapTest: **** END ****\n"));
2700	return rc;
2701	}
2702	#endif /* DEBUG */
2703

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMRZ/PGMRZDynMap.cpp@ 36416

Download in other formats: