PGMRZDynMap.cpp@ 36944

Last change on this file since 36944 was 36627, checked in by vboxsync, 14 years ago
Fix regression introduced with r70774. Change the RC structure of the mapping cache instead of the R0 one
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 96.9 KB

Line
1	/* $Id: PGMRZDynMap.cpp 36627 2011-04-08 15:38:47Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, dynamic mapping cache.
4	*/
5
6	/*
7	* Copyright (C) 2008-2011 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Internal Functions *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM_DYNMAP
23	#include <VBox/vmm/pgm.h>
24	#include "PGMInternal.h"
25	#include <VBox/vmm/vm.h>
26	#include "PGMInline.h"
27	#include <VBox/err.h>
28	#include <VBox/param.h>
29	#include <VBox/sup.h>
30	#include <iprt/asm.h>
31	#include <iprt/asm-amd64-x86.h>
32	#include <iprt/assert.h>
33	#ifndef IN_RC
34	# include <iprt/cpuset.h>
35	# include <iprt/mem.h>
36	# include <iprt/memobj.h>
37	# include <iprt/mp.h>
38	# include <iprt/semaphore.h>
39	# include <iprt/spinlock.h>
40	#endif
41	#include <iprt/string.h>
42
43
44	/*******************************************************************************
45	* Defined Constants And Macros *
46	*******************************************************************************/
47	#ifdef IN_RING0
48	/** The max size of the mapping cache (in pages). */
49	# define PGMR0DYNMAP_MAX_PAGES ((16*_1M) >> PAGE_SHIFT)
50	/** The small segment size that is adopted on out-of-memory conditions with a
51	* single big segment. */
52	# define PGMR0DYNMAP_SMALL_SEG_PAGES 128
53	/** The number of pages we reserve per CPU. */
54	# define PGMR0DYNMAP_PAGES_PER_CPU 256
55	/** The minimum number of pages we reserve per CPU.
56	* This must be equal or larger than the autoset size. */
57	# define PGMR0DYNMAP_PAGES_PER_CPU_MIN 64
58	/** Calcs the overload threshold (safety margin). Current set at 50%. */
59	# define PGMR0DYNMAP_CALC_OVERLOAD(cPages) ((cPages) / 2)
60	/** The number of guard pages.
61	* @remarks Never do tuning of the hashing or whatnot with a strict build! */
62	# if defined(VBOX_STRICT)
63	# define PGMR0DYNMAP_GUARD_PAGES 1
64	# else
65	# define PGMR0DYNMAP_GUARD_PAGES 0
66	# endif
67	#endif /* IN_RING0 */
68	/** The dummy physical address of guard pages. */
69	#define PGMR0DYNMAP_GUARD_PAGE_HCPHYS UINT32_C(0x7777feed)
70	/** The dummy reference count of guard pages. (Must be non-zero.) */
71	#define PGMR0DYNMAP_GUARD_PAGE_REF_COUNT INT32_C(0x7777feed)
72	#if 0
73	/** Define this to just clear the present bit on guard pages.
74	* The alternative is to replace the entire PTE with an bad not-present
75	* PTE. Either way, XNU will screw us. :-/ */
76	# define PGMR0DYNMAP_GUARD_NP
77	#endif
78	/** The dummy PTE value for a page. */
79	#define PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE X86_PTE_PG_MASK
80	/** The dummy PTE value for a page. */
81	#define PGMR0DYNMAP_GUARD_PAGE_PAE_PTE UINT64_MAX /X86_PTE_PAE_PG_MASK/
82
83	#ifdef IN_RING0 /* Note! Assertions causes panics if preemption is disabled,
84	* disable this to work around that. */
85	/**
86	* Acquire the spinlock.
87	* This will declare a temporary variable and expands to two statements!
88	*/
89	# define PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis) \
90	RTSPINLOCKTMP MySpinlockTmp = RTSPINLOCKTMP_INITIALIZER; \
91	RTSpinlockAcquire((pThis)->hSpinlock, &MySpinlockTmp)
92	/**
93	* Releases the spinlock.
94	*/
95	# define PGMRZDYNMAP_SPINLOCK_RELEASE(pThis) \
96	RTSpinlockRelease((pThis)->hSpinlock, &MySpinlockTmp)
97
98	/**
99	* Re-acquires the spinlock.
100	*/
101	# define PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis) \
102	RTSpinlockAcquire((pThis)->hSpinlock, &MySpinlockTmp)
103	#else
104	# define PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis) do { } while (0)
105	# define PGMRZDYNMAP_SPINLOCK_RELEASE(pThis) do { } while (0)
106	# define PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis) do { } while (0)
107	#endif
108
109
110	/** Converts a PGMCPUM::AutoSet pointer into a PVMCPU. */
111	#define PGMRZDYNMAP_SET_2_VMCPU(pSet) (RT_FROM_MEMBER(pSet, VMCPU, pgm.s.AutoSet))
112
113	/** Converts a PGMCPUM::AutoSet pointer into a PVM. */
114	#define PGMRZDYNMAP_SET_2_VM(pSet) (PGMRZDYNMAP_SET_2_VMCPU(pSet)->CTX_SUFF(pVM))
115
116	/** Converts a PGMCPUM::AutoSet pointer into a PVM. */
117	#ifdef IN_RC
118	# define PGMRZDYNMAP_SET_2_DYNMAP(pSet) (PGMRZDYNMAP_SET_2_VM(pSet)->pgm.s.pRCDynMap)
119	#else
120	# define PGMRZDYNMAP_SET_2_DYNMAP(pSet) (g_pPGMR0DynMap)
121	#endif
122
123	/**
124	* Gets the set index of the current CPU.
125	*
126	* This always returns 0 when in raw-mode context because there is only ever
127	* one EMT in that context (at least presently).
128	*/
129	#ifdef IN_RC
130	# define PGMRZDYNMAP_CUR_CPU() (0)
131	#else
132	# define PGMRZDYNMAP_CUR_CPU() RTMpCpuIdToSetIndex(RTMpCpuId())
133	#endif
134
135	/** PGMRZDYNMAP::u32Magic. (Jens Christian Bugge Wesseltoft) */
136	#define PGMRZDYNMAP_MAGIC UINT32_C(0x19640201)
137
138
139	/** Zaps an set entry. */
140	#define PGMRZDYNMAP_ZAP_ENTRY(pEntry) \
141	do \
142	{ \
143	(pEntry)->iPage = UINT16_MAX; \
144	(pEntry)->cRefs = 0; \
145	(pEntry)->cInlinedRefs = 0; \
146	(pEntry)->cUnrefs = 0; \
147	} while (0)
148
149
150	/** @def PGMRZDYNMAP_STRICT_RELEASE
151	* Define this to force pages to be released and make non-present ASAP after
152	* use. This should not normally be enabled as it is a bit expensive. */
153	#if 0 \|\| defined(DOXYGEN_RUNNING)
154	# define PGMRZDYNMAP_STRICT_RELEASE
155	#endif
156
157
158	/*******************************************************************************
159	* Structures and Typedefs *
160	*******************************************************************************/
161	#ifdef IN_RING0
162	/**
163	* Ring-0 dynamic mapping cache segment.
164	*
165	* The dynamic mapping cache can be extended with additional segments if the
166	* load is found to be too high. This done the next time a VM is created, under
167	* the protection of the init mutex. The arrays is reallocated and the new
168	* segment is added to the end of these. Nothing is rehashed of course, as the
169	* indexes / addresses must remain unchanged.
170	*
171	* This structure is only modified while owning the init mutex or during module
172	* init / term.
173	*/
174	typedef struct PGMR0DYNMAPSEG
175	{
176	/** Pointer to the next segment. */
177	struct PGMR0DYNMAPSEG *pNext;
178	/** The memory object for the virtual address range that we're abusing. */
179	RTR0MEMOBJ hMemObj;
180	/** The start page in the cache. (I.e. index into the arrays.) */
181	uint16_t iPage;
182	/** The number of pages this segment contributes. */
183	uint16_t cPages;
184	/** The number of page tables. */
185	uint16_t cPTs;
186	/** The memory objects for the page tables. */
187	RTR0MEMOBJ ahMemObjPTs[1];
188	} PGMR0DYNMAPSEG;
189	/** Pointer to a ring-0 dynamic mapping cache segment. */
190	typedef PGMR0DYNMAPSEG *PPGMR0DYNMAPSEG;
191
192
193	/**
194	* Ring-0 dynamic mapping cache entry.
195	*
196	* @sa PGMRZDYNMAPENTRY, PGMRCDYNMAPENTRY.
197	*/
198	typedef struct PGMR0DYNMAPENTRY
199	{
200	/** The physical address of the currently mapped page.
201	* This is duplicate for three reasons: cache locality, cache policy of the PT
202	* mappings and sanity checks. */
203	RTHCPHYS HCPhys;
204	/** Pointer to the page. */
205	void *pvPage;
206	/** The number of references. */
207	int32_t volatile cRefs;
208	/** PTE pointer union. */
209	union PGMR0DYNMAPENTRY_PPTE
210	{
211	/** PTE pointer, 32-bit legacy version. */
212	PX86PTE pLegacy;
213	/** PTE pointer, PAE version. */
214	PX86PTEPAE pPae;
215	/** PTE pointer, the void version. */
216	void *pv;
217	} uPte;
218	/** CPUs that haven't invalidated this entry after it's last update. */
219	RTCPUSET PendingSet;
220	} PGMR0DYNMAPENTRY;
221	/** Pointer a mapping cache entry for the ring-0.
222	* @sa PPGMRZDYNMAPENTRY, PPGMRCDYNMAPENTRY, */
223	typedef PGMR0DYNMAPENTRY *PPGMR0DYNMAPENTRY;
224
225
226	/**
227	* Dynamic mapping cache for ring-0.
228	*
229	* This is initialized during VMMR0 module init but no segments are allocated
230	* at that time. Segments will be added when the first VM is started and
231	* removed again when the last VM shuts down, thus avoid consuming memory while
232	* dormant. At module termination, the remaining bits will be freed up.
233	*
234	* @sa PPGMRZDYNMAP, PGMRCDYNMAP.
235	*/
236	typedef struct PGMR0DYNMAP
237	{
238	/** The usual magic number / eye catcher (PGMRZDYNMAP_MAGIC). */
239	uint32_t u32Magic;
240	/** Spinlock serializing the normal operation of the cache. */
241	RTSPINLOCK hSpinlock;
242	/** Array for tracking and managing the pages. */
243	PPGMR0DYNMAPENTRY paPages;
244	/** The cache size given as a number of pages. */
245	uint32_t cPages;
246	/** Whether it's 32-bit legacy or PAE/AMD64 paging mode. */
247	bool fLegacyMode;
248	/** The current load.
249	* This does not include guard pages. */
250	uint32_t cLoad;
251	/** The max load ever.
252	* This is maintained to trigger the adding of more mapping space. */
253	uint32_t cMaxLoad;
254	/** Initialization / termination lock. */
255	RTSEMFASTMUTEX hInitLock;
256	/** The number of guard pages. */
257	uint32_t cGuardPages;
258	/** The number of users (protected by hInitLock). */
259	uint32_t cUsers;
260	/** Array containing a copy of the original page tables.
261	* The entries are either X86PTE or X86PTEPAE according to fLegacyMode. */
262	void *pvSavedPTEs;
263	/** List of segments. */
264	PPGMR0DYNMAPSEG pSegHead;
265	/** The paging mode. */
266	SUPPAGINGMODE enmPgMode;
267	} PGMR0DYNMAP;
268
269
270	/**
271	* Paging level data.
272	*/
273	typedef struct PGMR0DYNMAPPGLVL
274	{
275	uint32_t cLevels; /*< The number of levels. /
276	struct
277	{
278	RTHCPHYS HCPhys; /**< The address of the page for the current level,
279	* i.e. what hMemObj/hMapObj is currently mapping. */
280	RTHCPHYS fPhysMask; /*< Mask for extracting HCPhys from uEntry. /
281	RTR0MEMOBJ hMemObj; /*< Memory object for HCPhys, PAGE_SIZE. /
282	RTR0MEMOBJ hMapObj; /*< Mapping object for hMemObj. /
283	uint32_t fPtrShift; /*< The pointer shift count. /
284	uint64_t fPtrMask; /*< The mask to apply to the shifted pointer to get the table index. /
285	uint64_t fAndMask; /*< And mask to check entry flags. /
286	uint64_t fResMask; /*< The result from applying fAndMask. /
287	union
288	{
289	void pv; /< hMapObj address. /
290	PX86PGUINT paLegacy; /*< Legacy table view. /
291	PX86PGPAEUINT paPae; /*< PAE/AMD64 table view. /
292	} u;
293	} a[4];
294	} PGMR0DYNMAPPGLVL;
295	/** Pointer to paging level data. */
296	typedef PGMR0DYNMAPPGLVL *PPGMR0DYNMAPPGLVL;
297	#endif
298
299	/** Mapping cache entry for the current context.
300	* @sa PGMR0DYNMAPENTRY, PGMRCDYNMAPENTRY */
301	typedef CTX_MID(PGM,DYNMAPENTRY) PGMRZDYNMAPENTRY;
302	/** Pointer a mapping cache entry for the current context.
303	* @sa PGMR0DYNMAPENTRY, PGMRCDYNMAPENTRY */
304	typedef PGMRZDYNMAPENTRY *PPGMRZDYNMAPENTRY;
305
306	/** Pointer the mapping cache instance for the current context.
307	* @sa PGMR0DYNMAP, PGMRCDYNMAP */
308	typedef CTX_MID(PGM,DYNMAP) *PPGMRZDYNMAP;
309
310
311
312	/*******************************************************************************
313	* Global Variables *
314	*******************************************************************************/
315	#ifdef IN_RING0
316	/** Pointer to the ring-0 dynamic mapping cache. */
317	static PGMR0DYNMAP *g_pPGMR0DynMap;
318	#endif
319	/** For overflow testing. */
320	static bool g_fPGMR0DynMapTestRunning = false;
321
322
323	/*******************************************************************************
324	* Internal Functions *
325	*******************************************************************************/
326	static void pgmRZDynMapReleasePage(PPGMRZDYNMAP pThis, uint32_t iPage, uint32_t cRefs);
327	#ifdef IN_RING0
328	static int pgmR0DynMapSetup(PPGMRZDYNMAP pThis);
329	static int pgmR0DynMapExpand(PPGMRZDYNMAP pThis);
330	static void pgmR0DynMapTearDown(PPGMRZDYNMAP pThis);
331	#endif
332	#if 0 /def DEBUG/
333	static int pgmR0DynMapTest(PVM pVM);
334	#endif
335
336
337	/**
338	* Initializes the auto mapping sets for a VM.
339	*
340	* @returns VINF_SUCCESS on success, VERR_INTERNAL_ERROR on failure.
341	* @param pVM The VM in question.
342	*/
343	static int pgmRZDynMapInitAutoSetsForVM(PVM pVM)
344	{
345	VMCPUID idCpu = pVM->cCpus;
346	AssertReturn(idCpu > 0 && idCpu <= VMM_MAX_CPU_COUNT, VERR_INTERNAL_ERROR);
347	while (idCpu-- > 0)
348	{
349	PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
350	uint32_t j = RT_ELEMENTS(pSet->aEntries);
351	while (j-- > 0)
352	{
353	pSet->aEntries[j].pvPage = NULL;
354	pSet->aEntries[j].HCPhys = NIL_RTHCPHYS;
355	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]);
356	}
357	pSet->cEntries = PGMMAPSET_CLOSED;
358	pSet->iSubset = UINT32_MAX;
359	pSet->iCpu = -1;
360	memset(&pSet->aiHashTable[0], 0xff, sizeof(pSet->aiHashTable));
361	}
362
363	return VINF_SUCCESS;
364	}
365
366
367	#ifdef IN_RING0
368
369	/**
370	* Initializes the ring-0 dynamic mapping cache.
371	*
372	* @returns VBox status code.
373	*/
374	VMMR0DECL(int) PGMR0DynMapInit(void)
375	{
376	Assert(!g_pPGMR0DynMap);
377
378	/*
379	* Create and initialize the cache instance.
380	*/
381	PPGMRZDYNMAP pThis = (PPGMRZDYNMAP)RTMemAllocZ(sizeof(*pThis));
382	AssertLogRelReturn(pThis, VERR_NO_MEMORY);
383	int rc = VINF_SUCCESS;
384	pThis->enmPgMode = SUPR0GetPagingMode();
385	switch (pThis->enmPgMode)
386	{
387	case SUPPAGINGMODE_32_BIT:
388	case SUPPAGINGMODE_32_BIT_GLOBAL:
389	pThis->fLegacyMode = false;
390	break;
391	case SUPPAGINGMODE_PAE:
392	case SUPPAGINGMODE_PAE_GLOBAL:
393	case SUPPAGINGMODE_PAE_NX:
394	case SUPPAGINGMODE_PAE_GLOBAL_NX:
395	case SUPPAGINGMODE_AMD64:
396	case SUPPAGINGMODE_AMD64_GLOBAL:
397	case SUPPAGINGMODE_AMD64_NX:
398	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
399	pThis->fLegacyMode = false;
400	break;
401	default:
402	rc = VERR_INTERNAL_ERROR;
403	break;
404	}
405	if (RT_SUCCESS(rc))
406	{
407	rc = RTSemFastMutexCreate(&pThis->hInitLock);
408	if (RT_SUCCESS(rc))
409	{
410	rc = RTSpinlockCreate(&pThis->hSpinlock);
411	if (RT_SUCCESS(rc))
412	{
413	pThis->u32Magic = PGMRZDYNMAP_MAGIC;
414	g_pPGMR0DynMap = pThis;
415	return VINF_SUCCESS;
416	}
417	RTSemFastMutexDestroy(pThis->hInitLock);
418	}
419	}
420	RTMemFree(pThis);
421	return rc;
422	}
423
424
425	/**
426	* Terminates the ring-0 dynamic mapping cache.
427	*/
428	VMMR0DECL(void) PGMR0DynMapTerm(void)
429	{
430	/*
431	* Destroy the cache.
432	*
433	* There is not supposed to be any races here, the loader should
434	* make sure about that. So, don't bother locking anything.
435	*
436	* The VM objects should all be destroyed by now, so there is no
437	* dangling users or anything like that to clean up. This routine
438	* is just a mirror image of PGMR0DynMapInit.
439	*/
440	PPGMRZDYNMAP pThis = g_pPGMR0DynMap;
441	if (pThis)
442	{
443	AssertPtr(pThis);
444	g_pPGMR0DynMap = NULL;
445
446	/* This should never happen, but in case it does try not to leak memory. */
447	AssertLogRelMsg(!pThis->cUsers && !pThis->paPages && !pThis->pvSavedPTEs && !pThis->cPages,
448	("cUsers=%d paPages=%p pvSavedPTEs=%p cPages=%#x\n",
449	pThis->cUsers, pThis->paPages, pThis->pvSavedPTEs, pThis->cPages));
450	if (pThis->paPages)
451	pgmR0DynMapTearDown(pThis);
452
453	/* Free the associated resources. */
454	RTSemFastMutexDestroy(pThis->hInitLock);
455	pThis->hInitLock = NIL_RTSEMFASTMUTEX;
456	RTSpinlockDestroy(pThis->hSpinlock);
457	pThis->hSpinlock = NIL_RTSPINLOCK;
458	pThis->u32Magic = UINT32_MAX;
459	RTMemFree(pThis);
460	}
461	}
462
463
464	/**
465	* Initializes the dynamic mapping cache for a new VM.
466	*
467	* @returns VBox status code.
468	* @param pVM Pointer to the shared VM structure.
469	*/
470	VMMR0DECL(int) PGMR0DynMapInitVM(PVM pVM)
471	{
472	AssertMsgReturn(!pVM->pgm.s.pvR0DynMapUsed, ("%p (pThis=%p)\n", pVM->pgm.s.pvR0DynMapUsed, g_pPGMR0DynMap), VERR_WRONG_ORDER);
473
474	/*
475	* Initialize the auto sets.
476	*/
477	int rc = pgmRZDynMapInitAutoSetsForVM(pVM);
478	if (RT_FAILURE(rc))
479	return rc;
480
481	/*
482	* Do we need the cache? Skip the last bit if we don't.
483	*/
484	if (!VMMIsHwVirtExtForced(pVM))
485	return VINF_SUCCESS;
486
487	/*
488	* Reference and if necessary setup or expand the cache.
489	*/
490	PPGMRZDYNMAP pThis = g_pPGMR0DynMap;
491	AssertPtrReturn(pThis, VERR_INTERNAL_ERROR);
492	rc = RTSemFastMutexRequest(pThis->hInitLock);
493	AssertLogRelRCReturn(rc, rc);
494
495	pThis->cUsers++;
496	if (pThis->cUsers == 1)
497	{
498	rc = pgmR0DynMapSetup(pThis);
499	#if 0 /def DEBUG/
500	if (RT_SUCCESS(rc))
501	{
502	rc = pgmR0DynMapTest(pVM);
503	if (RT_FAILURE(rc))
504	pgmR0DynMapTearDown(pThis);
505	}
506	#endif
507	}
508	else if (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(pThis->cPages - pThis->cGuardPages))
509	rc = pgmR0DynMapExpand(pThis);
510	if (RT_SUCCESS(rc))
511	pVM->pgm.s.pvR0DynMapUsed = pThis;
512	else
513	pThis->cUsers--;
514
515	RTSemFastMutexRelease(pThis->hInitLock);
516	return rc;
517	}
518
519
520	/**
521	* Terminates the dynamic mapping cache usage for a VM.
522	*
523	* @param pVM Pointer to the shared VM structure.
524	*/
525	VMMR0DECL(void) PGMR0DynMapTermVM(PVM pVM)
526	{
527	/*
528	* Return immediately if we're not using the cache.
529	*/
530	if (!pVM->pgm.s.pvR0DynMapUsed)
531	return;
532
533	PPGMRZDYNMAP pThis = g_pPGMR0DynMap;
534	AssertPtrReturnVoid(pThis);
535
536	int rc = RTSemFastMutexRequest(pThis->hInitLock);
537	AssertLogRelRCReturnVoid(rc);
538
539	if (pVM->pgm.s.pvR0DynMapUsed == pThis)
540	{
541	pVM->pgm.s.pvR0DynMapUsed = NULL;
542
543	#ifdef VBOX_STRICT
544	PGMR0DynMapAssertIntegrity();
545	#endif
546
547	/*
548	* Clean up and check the auto sets.
549	*/
550	VMCPUID idCpu = pVM->cCpus;
551	while (idCpu-- > 0)
552	{
553	PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
554	uint32_t j = pSet->cEntries;
555	if (j <= RT_ELEMENTS(pSet->aEntries))
556	{
557	/*
558	* The set is open, close it.
559	*/
560	while (j-- > 0)
561	{
562	int32_t cRefs = pSet->aEntries[j].cRefs;
563	uint32_t iPage = pSet->aEntries[j].iPage;
564	LogRel(("PGMR0DynMapTermVM: %d dangling refs to %#x\n", cRefs, iPage));
565	if (iPage < pThis->cPages && cRefs > 0)
566	pgmRZDynMapReleasePage(pThis, iPage, cRefs);
567	else
568	AssertLogRelMsgFailed(("cRefs=%d iPage=%#x cPages=%u\n", cRefs, iPage, pThis->cPages));
569
570	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]);
571	}
572	pSet->cEntries = PGMMAPSET_CLOSED;
573	pSet->iSubset = UINT32_MAX;
574	pSet->iCpu = -1;
575	}
576	else
577	AssertMsg(j == PGMMAPSET_CLOSED, ("cEntries=%#x\n", j));
578
579	j = RT_ELEMENTS(pSet->aEntries);
580	while (j-- > 0)
581	{
582	Assert(pSet->aEntries[j].iPage == UINT16_MAX);
583	Assert(!pSet->aEntries[j].cRefs);
584	}
585	}
586
587	/*
588	* Release our reference to the mapping cache.
589	*/
590	Assert(pThis->cUsers > 0);
591	pThis->cUsers--;
592	if (!pThis->cUsers)
593	pgmR0DynMapTearDown(pThis);
594	}
595	else
596	AssertLogRelMsgFailed(("pvR0DynMapUsed=%p pThis=%p\n", pVM->pgm.s.pvR0DynMapUsed, pThis));
597
598	RTSemFastMutexRelease(pThis->hInitLock);
599	}
600
601
602	/**
603	* Shoots down the TLBs for all the cache pages, pgmR0DynMapTearDown helper.
604	*
605	* @param idCpu The current CPU.
606	* @param pvUser1 The dynamic mapping cache instance.
607	* @param pvUser2 Unused, NULL.
608	*/
609	static DECLCALLBACK(void) pgmR0DynMapShootDownTlbs(RTCPUID idCpu, void pvUser1, void pvUser2)
610	{
611	Assert(!pvUser2);
612	PPGMRZDYNMAP pThis = (PPGMRZDYNMAP)pvUser1;
613	Assert(pThis == g_pPGMR0DynMap);
614	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
615	uint32_t iPage = pThis->cPages;
616	while (iPage-- > 0)
617	ASMInvalidatePage(paPages[iPage].pvPage);
618	}
619
620
621	/**
622	* Shoot down the TLBs for every single cache entry on all CPUs.
623	*
624	* @returns IPRT status code (RTMpOnAll).
625	* @param pThis The dynamic mapping cache instance.
626	*/
627	static int pgmR0DynMapTlbShootDown(PPGMRZDYNMAP pThis)
628	{
629	int rc = RTMpOnAll(pgmR0DynMapShootDownTlbs, pThis, NULL);
630	AssertRC(rc);
631	if (RT_FAILURE(rc))
632	{
633	uint32_t iPage = pThis->cPages;
634	while (iPage-- > 0)
635	ASMInvalidatePage(pThis->paPages[iPage].pvPage);
636	}
637	return rc;
638	}
639
640
641	/**
642	* Calculate the new cache size based on cMaxLoad statistics.
643	*
644	* @returns Number of pages.
645	* @param pThis The dynamic mapping cache instance.
646	* @param pcMinPages The minimal size in pages.
647	*/
648	static uint32_t pgmR0DynMapCalcNewSize(PPGMRZDYNMAP pThis, uint32_t *pcMinPages)
649	{
650	Assert(pThis->cPages <= PGMR0DYNMAP_MAX_PAGES);
651
652	/* cCpus * PGMR0DYNMAP_PAGES_PER_CPU(_MIN). */
653	RTCPUID cCpus = RTMpGetCount();
654	AssertReturn(cCpus > 0 && cCpus <= RTCPUSET_MAX_CPUS, 0);
655	uint32_t cPages = cCpus * PGMR0DYNMAP_PAGES_PER_CPU;
656	uint32_t cMinPages = cCpus * PGMR0DYNMAP_PAGES_PER_CPU_MIN;
657
658	/* adjust against cMaxLoad. */
659	AssertMsg(pThis->cMaxLoad <= PGMR0DYNMAP_MAX_PAGES, ("%#x\n", pThis->cMaxLoad));
660	if (pThis->cMaxLoad > PGMR0DYNMAP_MAX_PAGES)
661	pThis->cMaxLoad = 0;
662
663	while (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(cPages))
664	cPages += PGMR0DYNMAP_PAGES_PER_CPU;
665
666	if (pThis->cMaxLoad > cMinPages)
667	cMinPages = pThis->cMaxLoad;
668
669	/* adjust against max and current size. */
670	if (cPages < pThis->cPages)
671	cPages = pThis->cPages;
672	cPages *= PGMR0DYNMAP_GUARD_PAGES + 1;
673	if (cPages > PGMR0DYNMAP_MAX_PAGES)
674	cPages = PGMR0DYNMAP_MAX_PAGES;
675
676	if (cMinPages < pThis->cPages)
677	cMinPages = pThis->cPages;
678	cMinPages *= PGMR0DYNMAP_GUARD_PAGES + 1;
679	if (cMinPages > PGMR0DYNMAP_MAX_PAGES)
680	cMinPages = PGMR0DYNMAP_MAX_PAGES;
681
682	Assert(cMinPages);
683	*pcMinPages = cMinPages;
684	return cPages;
685	}
686
687
688	/**
689	* Initializes the paging level data.
690	*
691	* @param pThis The dynamic mapping cache instance.
692	* @param pPgLvl The paging level data.
693	*/
694	void pgmR0DynMapPagingArrayInit(PPGMRZDYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl)
695	{
696	RTCCUINTREG cr4 = ASMGetCR4();
697	switch (pThis->enmPgMode)
698	{
699	case SUPPAGINGMODE_32_BIT:
700	case SUPPAGINGMODE_32_BIT_GLOBAL:
701	pPgLvl->cLevels = 2;
702	pPgLvl->a[0].fPhysMask = X86_CR3_PAGE_MASK;
703	pPgLvl->a[0].fAndMask = X86_PDE_P \| X86_PDE_RW \| (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
704	pPgLvl->a[0].fResMask = X86_PDE_P \| X86_PDE_RW;
705	pPgLvl->a[0].fPtrMask = X86_PD_MASK;
706	pPgLvl->a[0].fPtrShift = X86_PD_SHIFT;
707
708	pPgLvl->a[1].fPhysMask = X86_PDE_PG_MASK;
709	pPgLvl->a[1].fAndMask = X86_PTE_P \| X86_PTE_RW;
710	pPgLvl->a[1].fResMask = X86_PTE_P \| X86_PTE_RW;
711	pPgLvl->a[1].fPtrMask = X86_PT_MASK;
712	pPgLvl->a[1].fPtrShift = X86_PT_SHIFT;
713	break;
714
715	case SUPPAGINGMODE_PAE:
716	case SUPPAGINGMODE_PAE_GLOBAL:
717	case SUPPAGINGMODE_PAE_NX:
718	case SUPPAGINGMODE_PAE_GLOBAL_NX:
719	pPgLvl->cLevels = 3;
720	pPgLvl->a[0].fPhysMask = X86_CR3_PAE_PAGE_MASK;
721	pPgLvl->a[0].fPtrMask = X86_PDPT_MASK_PAE;
722	pPgLvl->a[0].fPtrShift = X86_PDPT_SHIFT;
723	pPgLvl->a[0].fAndMask = X86_PDPE_P;
724	pPgLvl->a[0].fResMask = X86_PDPE_P;
725
726	pPgLvl->a[1].fPhysMask = X86_PDPE_PG_MASK;
727	pPgLvl->a[1].fPtrMask = X86_PD_PAE_MASK;
728	pPgLvl->a[1].fPtrShift = X86_PD_PAE_SHIFT;
729	pPgLvl->a[1].fAndMask = X86_PDE_P \| X86_PDE_RW \| (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
730	pPgLvl->a[1].fResMask = X86_PDE_P \| X86_PDE_RW;
731
732	pPgLvl->a[2].fPhysMask = X86_PDE_PAE_PG_MASK;
733	pPgLvl->a[2].fPtrMask = X86_PT_PAE_MASK;
734	pPgLvl->a[2].fPtrShift = X86_PT_PAE_SHIFT;
735	pPgLvl->a[2].fAndMask = X86_PTE_P \| X86_PTE_RW;
736	pPgLvl->a[2].fResMask = X86_PTE_P \| X86_PTE_RW;
737	break;
738
739	case SUPPAGINGMODE_AMD64:
740	case SUPPAGINGMODE_AMD64_GLOBAL:
741	case SUPPAGINGMODE_AMD64_NX:
742	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
743	pPgLvl->cLevels = 4;
744	pPgLvl->a[0].fPhysMask = X86_CR3_AMD64_PAGE_MASK;
745	pPgLvl->a[0].fPtrShift = X86_PML4_SHIFT;
746	pPgLvl->a[0].fPtrMask = X86_PML4_MASK;
747	pPgLvl->a[0].fAndMask = X86_PML4E_P \| X86_PML4E_RW;
748	pPgLvl->a[0].fResMask = X86_PML4E_P \| X86_PML4E_RW;
749
750	pPgLvl->a[1].fPhysMask = X86_PML4E_PG_MASK;
751	pPgLvl->a[1].fPtrShift = X86_PDPT_SHIFT;
752	pPgLvl->a[1].fPtrMask = X86_PDPT_MASK_AMD64;
753	pPgLvl->a[1].fAndMask = X86_PDPE_P \| X86_PDPE_RW /** @todo check for X86_PDPT_PS support. */;
754	pPgLvl->a[1].fResMask = X86_PDPE_P \| X86_PDPE_RW;
755
756	pPgLvl->a[2].fPhysMask = X86_PDPE_PG_MASK;
757	pPgLvl->a[2].fPtrShift = X86_PD_PAE_SHIFT;
758	pPgLvl->a[2].fPtrMask = X86_PD_PAE_MASK;
759	pPgLvl->a[2].fAndMask = X86_PDE_P \| X86_PDE_RW \| (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
760	pPgLvl->a[2].fResMask = X86_PDE_P \| X86_PDE_RW;
761
762	pPgLvl->a[3].fPhysMask = X86_PDE_PAE_PG_MASK;
763	pPgLvl->a[3].fPtrShift = X86_PT_PAE_SHIFT;
764	pPgLvl->a[3].fPtrMask = X86_PT_PAE_MASK;
765	pPgLvl->a[3].fAndMask = X86_PTE_P \| X86_PTE_RW;
766	pPgLvl->a[3].fResMask = X86_PTE_P \| X86_PTE_RW;
767	break;
768
769	default:
770	AssertFailed();
771	pPgLvl->cLevels = 0;
772	break;
773	}
774
775	for (uint32_t i = 0; i < 4; i++) /* ASSUMING array size. */
776	{
777	pPgLvl->a[i].HCPhys = NIL_RTHCPHYS;
778	pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
779	pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ;
780	pPgLvl->a[i].u.pv = NULL;
781	}
782	}
783
784
785	/**
786	* Maps a PTE.
787	*
788	* This will update the segment structure when new PTs are mapped.
789	*
790	* It also assumes that we (for paranoid reasons) wish to establish a mapping
791	* chain from CR3 to the PT that all corresponds to the processor we're
792	* currently running on, and go about this by running with interrupts disabled
793	* and restarting from CR3 for every change.
794	*
795	* @returns VBox status code, VINF_TRY_AGAIN if we changed any mappings and had
796	* to re-enable interrupts.
797	* @param pThis The dynamic mapping cache instance.
798	* @param pPgLvl The paging level structure.
799	* @param pvPage The page.
800	* @param pSeg The segment.
801	* @param cMaxPTs The max number of PTs expected in the segment.
802	* @param ppvPTE Where to store the PTE address.
803	*/
804	static int pgmR0DynMapPagingArrayMapPte(PPGMRZDYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl, void *pvPage,
805	PPGMR0DYNMAPSEG pSeg, uint32_t cMaxPTs, void **ppvPTE)
806	{
807	Assert(!(ASMGetFlags() & X86_EFL_IF));
808	void *pvEntry = NULL;
809	X86PGPAEUINT uEntry = ASMGetCR3();
810	for (uint32_t i = 0; i < pPgLvl->cLevels; i++)
811	{
812	RTHCPHYS HCPhys = uEntry & pPgLvl->a[i].fPhysMask;
813	if (pPgLvl->a[i].HCPhys != HCPhys)
814	{
815	/*
816	* Need to remap this level.
817	* The final level, the PT, will not be freed since that is what it's all about.
818	*/
819	ASMIntEnable();
820	if (i + 1 == pPgLvl->cLevels)
821	AssertReturn(pSeg->cPTs < cMaxPTs, VERR_INTERNAL_ERROR);
822	else
823	{
824	int rc2 = RTR0MemObjFree(pPgLvl->a[i].hMemObj, true /* fFreeMappings */); AssertRC(rc2);
825	pPgLvl->a[i].hMemObj = pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
826	}
827
828	int rc = RTR0MemObjEnterPhys(&pPgLvl->a[i].hMemObj, HCPhys, PAGE_SIZE, RTMEM_CACHE_POLICY_DONT_CARE);
829	if (RT_SUCCESS(rc))
830	{
831	rc = RTR0MemObjMapKernel(&pPgLvl->a[i].hMapObj, pPgLvl->a[i].hMemObj,
832	(void )-1 / pvFixed /, 0 / cbAlignment */,
833	RTMEM_PROT_WRITE \| RTMEM_PROT_READ);
834	if (RT_SUCCESS(rc))
835	{
836	pPgLvl->a[i].u.pv = RTR0MemObjAddress(pPgLvl->a[i].hMapObj);
837	AssertMsg(((uintptr_t)pPgLvl->a[i].u.pv & ~(uintptr_t)PAGE_OFFSET_MASK), ("%p\n", pPgLvl->a[i].u.pv));
838	pPgLvl->a[i].HCPhys = HCPhys;
839	if (i + 1 == pPgLvl->cLevels)
840	pSeg->ahMemObjPTs[pSeg->cPTs++] = pPgLvl->a[i].hMemObj;
841	ASMIntDisable();
842	return VINF_TRY_AGAIN;
843	}
844
845	pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
846	}
847	else
848	pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ;
849	pPgLvl->a[i].HCPhys = NIL_RTHCPHYS;
850	return rc;
851	}
852
853	/*
854	* The next level.
855	*/
856	uint32_t iEntry = ((uint64_t)(uintptr_t)pvPage >> pPgLvl->a[i].fPtrShift) & pPgLvl->a[i].fPtrMask;
857	if (pThis->fLegacyMode)
858	{
859	pvEntry = &pPgLvl->a[i].u.paLegacy[iEntry];
860	uEntry = pPgLvl->a[i].u.paLegacy[iEntry];
861	}
862	else
863	{
864	pvEntry = &pPgLvl->a[i].u.paPae[iEntry];
865	uEntry = pPgLvl->a[i].u.paPae[iEntry];
866	}
867
868	if ((uEntry & pPgLvl->a[i].fAndMask) != pPgLvl->a[i].fResMask)
869	{
870	LogRel(("PGMR0DynMap: internal error - iPgLvl=%u cLevels=%u uEntry=%#llx fAnd=%#llx fRes=%#llx got=%#llx\n"
871	"PGMR0DynMap: pv=%p pvPage=%p iEntry=%#x fLegacyMode=%RTbool\n",
872	i, pPgLvl->cLevels, uEntry, pPgLvl->a[i].fAndMask, pPgLvl->a[i].fResMask, uEntry & pPgLvl->a[i].fAndMask,
873	pPgLvl->a[i].u.pv, pvPage, iEntry, pThis->fLegacyMode));
874	return VERR_INTERNAL_ERROR;
875	}
876	/Log(("#%d: iEntry=%4d uEntry=%#llx pvEntry=%p HCPhys=%RHp \n", i, iEntry, uEntry, pvEntry, pPgLvl->a[i].HCPhys));/
877	}
878
879	/* made it thru without needing to remap anything. */
880	*ppvPTE = pvEntry;
881	return VINF_SUCCESS;
882	}
883
884
885	/**
886	* Sets up a guard page.
887	*
888	* @param pThis The dynamic mapping cache instance.
889	* @param pPage The page.
890	*/
891	DECLINLINE(void) pgmR0DynMapSetupGuardPage(PPGMRZDYNMAP pThis, PPGMRZDYNMAPENTRY pPage)
892	{
893	memset(pPage->pvPage, 0xfd, PAGE_SIZE);
894	pPage->cRefs = PGMR0DYNMAP_GUARD_PAGE_REF_COUNT;
895	pPage->HCPhys = PGMR0DYNMAP_GUARD_PAGE_HCPHYS;
896	#ifdef PGMR0DYNMAP_GUARD_NP
897	ASMAtomicBitClear(pPage->uPte.pv, X86_PTE_BIT_P);
898	#else
899	if (pThis->fLegacyMode)
900	ASMAtomicWriteU32(&pPage->uPte.pLegacy->u, PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE);
901	else
902	ASMAtomicWriteU64(&pPage->uPte.pPae->u, PGMR0DYNMAP_GUARD_PAGE_PAE_PTE);
903	#endif
904	pThis->cGuardPages++;
905	}
906
907
908	/**
909	* Adds a new segment of the specified size.
910	*
911	* @returns VBox status code.
912	* @param pThis The dynamic mapping cache instance.
913	* @param cPages The size of the new segment, give as a page count.
914	*/
915	static int pgmR0DynMapAddSeg(PPGMRZDYNMAP pThis, uint32_t cPages)
916	{
917	int rc2;
918	AssertReturn(ASMGetFlags() & X86_EFL_IF, VERR_PREEMPT_DISABLED);
919
920	/*
921	* Do the array reallocations first.
922	* (The pages array has to be replaced behind the spinlock of course.)
923	*/
924	void pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) (pThis->cPages + cPages));
925	if (!pvSavedPTEs)
926	return VERR_NO_MEMORY;
927	pThis->pvSavedPTEs = pvSavedPTEs;
928
929	void pvPages = RTMemAllocZ(sizeof(pThis->paPages[0]) (pThis->cPages + cPages));
930	if (!pvPages)
931	{
932	pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) * pThis->cPages);
933	if (pvSavedPTEs)
934	pThis->pvSavedPTEs = pvSavedPTEs;
935	return VERR_NO_MEMORY;
936	}
937
938	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
939
940	memcpy(pvPages, pThis->paPages, sizeof(pThis->paPages[0]) * pThis->cPages);
941	void *pvToFree = pThis->paPages;
942	pThis->paPages = (PPGMRZDYNMAPENTRY)pvPages;
943
944	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
945	RTMemFree(pvToFree);
946
947	/*
948	* Allocate the segment structure and pages of memory, then touch all the pages (paranoia).
949	*/
950	uint32_t cMaxPTs = cPages / (pThis->fLegacyMode ? X86_PG_ENTRIES : X86_PG_PAE_ENTRIES) + 2;
951	PPGMR0DYNMAPSEG pSeg = (PPGMR0DYNMAPSEG)RTMemAllocZ(RT_UOFFSETOF(PGMR0DYNMAPSEG, ahMemObjPTs[cMaxPTs]));
952	if (!pSeg)
953	return VERR_NO_MEMORY;
954	pSeg->pNext = NULL;
955	pSeg->cPages = cPages;
956	pSeg->iPage = pThis->cPages;
957	pSeg->cPTs = 0;
958	int rc = RTR0MemObjAllocPage(&pSeg->hMemObj, cPages << PAGE_SHIFT, false);
959	if (RT_SUCCESS(rc))
960	{
961	uint8_t pbPage = (uint8_t )RTR0MemObjAddress(pSeg->hMemObj);
962	AssertMsg(VALID_PTR(pbPage) && !((uintptr_t)pbPage & PAGE_OFFSET_MASK), ("%p\n", pbPage));
963	memset(pbPage, 0xfe, cPages << PAGE_SHIFT);
964
965	/*
966	* Walk thru the pages and set them up with a mapping of their PTE and everything.
967	*/
968	ASMIntDisable();
969	PGMR0DYNMAPPGLVL PgLvl;
970	pgmR0DynMapPagingArrayInit(pThis, &PgLvl);
971	uint32_t const iEndPage = pSeg->iPage + cPages;
972	for (uint32_t iPage = pSeg->iPage;
973	iPage < iEndPage;
974	iPage++, pbPage += PAGE_SIZE)
975	{
976	/* Initialize the page data. */
977	pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
978	pThis->paPages[iPage].pvPage = pbPage;
979	pThis->paPages[iPage].cRefs = 0;
980	pThis->paPages[iPage].uPte.pPae = 0;
981	#ifndef IN_RC
982	RTCpuSetFill(&pThis->paPages[iPage].PendingSet);
983	#endif
984
985	/* Map its page table, retry until we've got a clean run (paranoia). */
986	do
987	rc = pgmR0DynMapPagingArrayMapPte(pThis, &PgLvl, pbPage, pSeg, cMaxPTs,
988	&pThis->paPages[iPage].uPte.pv);
989	while (rc == VINF_TRY_AGAIN);
990	if (RT_FAILURE(rc))
991	break;
992
993	/* Save the PTE. */
994	if (pThis->fLegacyMode)
995	((PX86PGUINT)pThis->pvSavedPTEs)[iPage] = pThis->paPages[iPage].uPte.pLegacy->u;
996	else
997	((PX86PGPAEUINT)pThis->pvSavedPTEs)[iPage] = pThis->paPages[iPage].uPte.pPae->u;
998
999	#ifdef VBOX_STRICT
1000	/* Check that we've got the right entry. */
1001	RTHCPHYS HCPhysPage = RTR0MemObjGetPagePhysAddr(pSeg->hMemObj, iPage - pSeg->iPage);
1002	RTHCPHYS HCPhysPte = pThis->fLegacyMode
1003	? pThis->paPages[iPage].uPte.pLegacy->u & X86_PTE_PG_MASK
1004	: pThis->paPages[iPage].uPte.pPae->u & X86_PTE_PAE_PG_MASK;
1005	if (HCPhysPage != HCPhysPte)
1006	{
1007	LogRel(("pgmR0DynMapAddSeg: internal error - page #%u HCPhysPage=%RHp HCPhysPte=%RHp pbPage=%p pvPte=%p\n",
1008	iPage - pSeg->iPage, HCPhysPage, HCPhysPte, pbPage, pThis->paPages[iPage].uPte.pv));
1009	rc = VERR_INTERNAL_ERROR;
1010	break;
1011	}
1012	#endif
1013	} /* for each page */
1014	ASMIntEnable();
1015
1016	/* cleanup non-PT mappings */
1017	for (uint32_t i = 0; i < PgLvl.cLevels - 1; i++)
1018	RTR0MemObjFree(PgLvl.a[i].hMemObj, true /* fFreeMappings */);
1019
1020	if (RT_SUCCESS(rc))
1021	{
1022	#if PGMR0DYNMAP_GUARD_PAGES > 0
1023	/*
1024	* Setup guard pages.
1025	* (Note: TLBs will be shot down later on.)
1026	*/
1027	uint32_t iPage = pSeg->iPage;
1028	while (iPage < iEndPage)
1029	{
1030	for (uint32_t iGPg = 0; iGPg < PGMR0DYNMAP_GUARD_PAGES && iPage < iEndPage; iGPg++, iPage++)
1031	pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]);
1032	iPage++; /* the guarded page */
1033	}
1034
1035	/* Make sure the very last page is a guard page too. */
1036	iPage = iEndPage - 1;
1037	if (pThis->paPages[iPage].cRefs != PGMR0DYNMAP_GUARD_PAGE_REF_COUNT)
1038	pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]);
1039	#endif /* PGMR0DYNMAP_GUARD_PAGES > 0 */
1040
1041	/*
1042	* Commit it by adding the segment to the list and updating the page count.
1043	*/
1044	pSeg->pNext = pThis->pSegHead;
1045	pThis->pSegHead = pSeg;
1046	pThis->cPages += cPages;
1047	return VINF_SUCCESS;
1048	}
1049
1050	/*
1051	* Bail out.
1052	*/
1053	while (pSeg->cPTs-- > 0)
1054	{
1055	rc2 = RTR0MemObjFree(pSeg->ahMemObjPTs[pSeg->cPTs], true /* fFreeMappings */);
1056	AssertRC(rc2);
1057	pSeg->ahMemObjPTs[pSeg->cPTs] = NIL_RTR0MEMOBJ;
1058	}
1059
1060	rc2 = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */);
1061	AssertRC(rc2);
1062	pSeg->hMemObj = NIL_RTR0MEMOBJ;
1063	}
1064	RTMemFree(pSeg);
1065
1066	/* Don't bother resizing the arrays, but free them if we're the only user. */
1067	if (!pThis->cPages)
1068	{
1069	RTMemFree(pThis->paPages);
1070	pThis->paPages = NULL;
1071	RTMemFree(pThis->pvSavedPTEs);
1072	pThis->pvSavedPTEs = NULL;
1073	}
1074	return rc;
1075	}
1076
1077
1078	/**
1079	* Called by PGMR0DynMapInitVM under the init lock.
1080	*
1081	* @returns VBox status code.
1082	* @param pThis The dynamic mapping cache instance.
1083	*/
1084	static int pgmR0DynMapSetup(PPGMRZDYNMAP pThis)
1085	{
1086	/*
1087	* Calc the size and add a segment of that size.
1088	*/
1089	uint32_t cMinPages;
1090	uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages);
1091	AssertReturn(cPages, VERR_INTERNAL_ERROR);
1092	int rc = pgmR0DynMapAddSeg(pThis, cPages);
1093	if (rc == VERR_NO_MEMORY)
1094	{
1095	/*
1096	* Try adding smaller segments.
1097	*/
1098	do
1099	rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES);
1100	while (RT_SUCCESS(rc) && pThis->cPages < cPages);
1101	if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages)
1102	rc = VINF_SUCCESS;
1103	if (rc == VERR_NO_MEMORY)
1104	{
1105	if (pThis->cPages)
1106	pgmR0DynMapTearDown(pThis);
1107	rc = VERR_PGM_DYNMAP_SETUP_ERROR;
1108	}
1109	}
1110	Assert(ASMGetFlags() & X86_EFL_IF);
1111
1112	#if PGMR0DYNMAP_GUARD_PAGES > 0
1113	/* paranoia */
1114	if (RT_SUCCESS(rc))
1115	pgmR0DynMapTlbShootDown(pThis);
1116	#endif
1117	return rc;
1118	}
1119
1120
1121	/**
1122	* Called by PGMR0DynMapInitVM under the init lock.
1123	*
1124	* @returns VBox status code.
1125	* @param pThis The dynamic mapping cache instance.
1126	*/
1127	static int pgmR0DynMapExpand(PPGMRZDYNMAP pThis)
1128	{
1129	/*
1130	* Calc the new target size and add a segment of the appropriate size.
1131	*/
1132	uint32_t cMinPages;
1133	uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages);
1134	AssertReturn(cPages, VERR_INTERNAL_ERROR);
1135	if (pThis->cPages >= cPages)
1136	return VINF_SUCCESS;
1137
1138	uint32_t cAdd = cPages - pThis->cPages;
1139	int rc = pgmR0DynMapAddSeg(pThis, cAdd);
1140	if (rc == VERR_NO_MEMORY)
1141	{
1142	/*
1143	* Try adding smaller segments.
1144	*/
1145	do
1146	rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES);
1147	while (RT_SUCCESS(rc) && pThis->cPages < cPages);
1148	if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages)
1149	rc = VINF_SUCCESS;
1150	if (rc == VERR_NO_MEMORY)
1151	rc = VERR_PGM_DYNMAP_EXPAND_ERROR;
1152	}
1153	Assert(ASMGetFlags() & X86_EFL_IF);
1154
1155	#if PGMR0DYNMAP_GUARD_PAGES > 0
1156	/* paranoia */
1157	if (RT_SUCCESS(rc))
1158	pgmR0DynMapTlbShootDown(pThis);
1159	#endif
1160	return rc;
1161	}
1162
1163
1164	/**
1165	* Called by PGMR0DynMapTermVM under the init lock.
1166	*
1167	* @returns VBox status code.
1168	* @param pThis The dynamic mapping cache instance.
1169	*/
1170	static void pgmR0DynMapTearDown(PPGMRZDYNMAP pThis)
1171	{
1172	/*
1173	* Restore the original page table entries
1174	*/
1175	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
1176	uint32_t iPage = pThis->cPages;
1177	if (pThis->fLegacyMode)
1178	{
1179	X86PGUINT const paSavedPTEs = (X86PGUINT const )pThis->pvSavedPTEs;
1180	while (iPage-- > 0)
1181	{
1182	X86PGUINT uOld = paPages[iPage].uPte.pLegacy->u;
1183	X86PGUINT uOld2 = uOld; NOREF(uOld2);
1184	X86PGUINT uNew = paSavedPTEs[iPage];
1185	while (!ASMAtomicCmpXchgExU32(&paPages[iPage].uPte.pLegacy->u, uNew, uOld, &uOld))
1186	AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
1187	Assert(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage]);
1188	}
1189	}
1190	else
1191	{
1192	X86PGPAEUINT const paSavedPTEs = (X86PGPAEUINT const )pThis->pvSavedPTEs;
1193	while (iPage-- > 0)
1194	{
1195	X86PGPAEUINT uOld = paPages[iPage].uPte.pPae->u;
1196	X86PGPAEUINT uOld2 = uOld; NOREF(uOld2);
1197	X86PGPAEUINT uNew = paSavedPTEs[iPage];
1198	while (!ASMAtomicCmpXchgExU64(&paPages[iPage].uPte.pPae->u, uNew, uOld, &uOld))
1199	AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
1200	Assert(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage]);
1201	}
1202	}
1203
1204	/*
1205	* Shoot down the TLBs on all CPUs before freeing them.
1206	*/
1207	pgmR0DynMapTlbShootDown(pThis);
1208
1209	/*
1210	* Free the segments.
1211	*/
1212	while (pThis->pSegHead)
1213	{
1214	int rc;
1215	PPGMR0DYNMAPSEG pSeg = pThis->pSegHead;
1216	pThis->pSegHead = pSeg->pNext;
1217
1218	uint32_t iPT = pSeg->cPTs;
1219	while (iPT-- > 0)
1220	{
1221	rc = RTR0MemObjFree(pSeg->ahMemObjPTs[iPT], true /* fFreeMappings */); AssertRC(rc);
1222	pSeg->ahMemObjPTs[iPT] = NIL_RTR0MEMOBJ;
1223	}
1224	rc = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */); AssertRC(rc);
1225	pSeg->hMemObj = NIL_RTR0MEMOBJ;
1226	pSeg->pNext = NULL;
1227	pSeg->iPage = UINT16_MAX;
1228	pSeg->cPages = 0;
1229	pSeg->cPTs = 0;
1230	RTMemFree(pSeg);
1231	}
1232
1233	/*
1234	* Free the arrays and restore the initial state.
1235	* The cLoadMax value is left behind for the next setup.
1236	*/
1237	RTMemFree(pThis->paPages);
1238	pThis->paPages = NULL;
1239	RTMemFree(pThis->pvSavedPTEs);
1240	pThis->pvSavedPTEs = NULL;
1241	pThis->cPages = 0;
1242	pThis->cLoad = 0;
1243	pThis->cGuardPages = 0;
1244	}
1245
1246	#endif /* IN_RING0 */
1247	#ifdef IN_RC
1248
1249	/**
1250	* Initializes the dynamic mapping cache in raw-mode context.
1251	*
1252	* @returns VBox status code.
1253	* @param pVM The VM handle.
1254	*/
1255	VMMRCDECL(int) PGMRCDynMapInit(PVM pVM)
1256	{
1257	/*
1258	* Allocate and initialize the instance data and page array.
1259	*/
1260	PPGMRZDYNMAP pThis;
1261	size_t const cPages = MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE;
1262	size_t const cb = RT_ALIGN_Z(sizeof(*pThis), 32)
1263	+ sizeof(PGMRZDYNMAPENTRY) * cPages;
1264	int rc = MMHyperAlloc(pVM, cb, 32, MM_TAG_PGM, (void **)&pThis);
1265	if (RT_FAILURE(rc))
1266	return rc;
1267
1268	pThis->u32Magic = PGMRZDYNMAP_MAGIC;
1269	pThis->paPages = RT_ALIGN_PT(pThis + 1, 32, PPGMRZDYNMAPENTRY);
1270	pThis->cPages = cPages;
1271	pThis->cLoad = 0;
1272	pThis->cMaxLoad = 0;
1273	pThis->cGuardPages = 0;
1274	pThis->cUsers = 1;
1275
1276	for (size_t iPage = 0; iPage < cPages; iPage++)
1277	{
1278	pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
1279	pThis->paPages[iPage].pvPage = pVM->pgm.s.pbDynPageMapBaseGC + iPage * PAGE_SIZE;
1280	pThis->paPages[iPage].cRefs = 0;
1281	pThis->paPages[iPage].uPte.pLegacy = &pVM->pgm.s.paDynPageMap32BitPTEsGC[iPage];
1282	pThis->paPages[iPage].uPte.pPae = (PX86PTEPAE)&pVM->pgm.s.paDynPageMapPaePTEsGC[iPage];
1283	}
1284
1285	pVM->pgm.s.pRCDynMap = pThis;
1286
1287	/*
1288	* Initialize the autosets the VM.
1289	*/
1290	rc = pgmRZDynMapInitAutoSetsForVM(pVM);
1291	if (RT_FAILURE(rc))
1292	return rc;
1293
1294	return VINF_SUCCESS;
1295	}
1296
1297	#endif /* IN_RC */
1298
1299	/**
1300	* Release references to a page, caller owns the spin lock.
1301	*
1302	* @param pThis The dynamic mapping cache instance.
1303	* @param iPage The page.
1304	* @param cRefs The number of references to release.
1305	*/
1306	DECLINLINE(void) pgmRZDynMapReleasePageLocked(PPGMRZDYNMAP pThis, uint32_t iPage, int32_t cRefs)
1307	{
1308	cRefs = ASMAtomicSubS32(&pThis->paPages[iPage].cRefs, cRefs) - cRefs;
1309	AssertMsg(cRefs >= 0, ("%d\n", cRefs));
1310	if (!cRefs)
1311	{
1312	pThis->cLoad--;
1313	#ifdef PGMRZDYNMAP_STRICT_RELEASE
1314	pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
1315	ASMAtomicBitClear(pThis->paPages[iPage].uPte.pv, X86_PTE_BIT_P);
1316	ASMInvalidatePage(pThis->paPages[iPage].pvPage);
1317	#endif
1318	}
1319	}
1320
1321
1322	/**
1323	* Release references to a page, caller does not own the spin lock.
1324	*
1325	* @param pThis The dynamic mapping cache instance.
1326	* @param iPage The page.
1327	* @param cRefs The number of references to release.
1328	*/
1329	static void pgmRZDynMapReleasePage(PPGMRZDYNMAP pThis, uint32_t iPage, uint32_t cRefs)
1330	{
1331	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
1332	pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs);
1333	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1334	}
1335
1336
1337	/**
1338	* pgmR0DynMapPage worker that deals with the tedious bits.
1339	*
1340	* @returns The page index on success, UINT32_MAX on failure.
1341	* @param pThis The dynamic mapping cache instance.
1342	* @param HCPhys The address of the page to be mapped.
1343	* @param iPage The page index pgmR0DynMapPage hashed HCPhys to.
1344	* @param pVCpu The current CPU, for statistics.
1345	* @param pfNew Set to @c true if a new entry was made and @c false if
1346	* an old entry was found and reused.
1347	*/
1348	static uint32_t pgmR0DynMapPageSlow(PPGMRZDYNMAP pThis, RTHCPHYS HCPhys, uint32_t iPage, PVMCPU pVCpu, bool *pfNew)
1349	{
1350	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlow);
1351
1352	/*
1353	* Check if any of the first 3 pages are unreferenced since the caller
1354	* already has made sure they aren't matching.
1355	*/
1356	#ifdef VBOX_WITH_STATISTICS
1357	bool fLooped = false;
1358	#endif
1359	uint32_t const cPages = pThis->cPages;
1360	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
1361	uint32_t iFreePage;
1362	if (!paPages[iPage].cRefs)
1363	iFreePage = iPage;
1364	else if (!paPages[(iPage + 1) % cPages].cRefs)
1365	iFreePage = (iPage + 1) % cPages;
1366	else if (!paPages[(iPage + 2) % cPages].cRefs)
1367	iFreePage = (iPage + 2) % cPages;
1368	else
1369	{
1370	/*
1371	* Search for an unused or matching entry.
1372	*/
1373	iFreePage = (iPage + 3) % cPages;
1374	for (;;)
1375	{
1376	if (paPages[iFreePage].HCPhys == HCPhys)
1377	{
1378	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlowLoopHits);
1379	*pfNew = false;
1380	return iFreePage;
1381	}
1382	if (!paPages[iFreePage].cRefs)
1383	break;
1384
1385	/* advance */
1386	iFreePage = (iFreePage + 1) % cPages;
1387	if (RT_UNLIKELY(iFreePage == iPage))
1388	return UINT32_MAX;
1389	}
1390	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlowLoopMisses);
1391	#ifdef VBOX_WITH_STATISTICS
1392	fLooped = true;
1393	#endif
1394	}
1395	Assert(iFreePage < cPages);
1396
1397	#if 0 //def VBOX_WITH_STATISTICS
1398	/* Check for lost hits. */
1399	if (!fLooped)
1400	for (uint32_t iPage2 = (iPage + 3) % cPages; iPage2 != iPage; iPage2 = (iPage2 + 1) % cPages)
1401	if (paPages[iPage2].HCPhys == HCPhys)
1402	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZDynMapPageSlowLostHits);
1403	#endif
1404
1405	/*
1406	* Setup the new entry.
1407	*/
1408	*pfNew = true;
1409	/Log6(("pgmR0DynMapPageSlow: old - %RHp %#x %#llx\n", paPages[iFreePage].HCPhys, paPages[iFreePage].cRefs, paPages[iFreePage].uPte.pPae->u));/
1410	paPages[iFreePage].HCPhys = HCPhys;
1411	#ifndef IN_RC
1412	RTCpuSetFill(&paPages[iFreePage].PendingSet);
1413
1414	if (pThis->fLegacyMode)
1415	#endif
1416	{
1417	X86PGUINT uOld = paPages[iFreePage].uPte.pLegacy->u;
1418	X86PGUINT uOld2 = uOld; NOREF(uOld2);
1419	X86PGUINT uNew = (uOld & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1420	\| X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1421	\| (HCPhys & X86_PTE_PG_MASK);
1422	while (!ASMAtomicCmpXchgExU32(&paPages[iFreePage].uPte.pLegacy->u, uNew, uOld, &uOld))
1423	AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
1424	Assert(paPages[iFreePage].uPte.pLegacy->u == uNew);
1425	}
1426	#ifndef IN_RC
1427	else
1428	#endif
1429	{
1430	X86PGPAEUINT uOld = paPages[iFreePage].uPte.pPae->u;
1431	X86PGPAEUINT uOld2 = uOld; NOREF(uOld2);
1432	X86PGPAEUINT uNew = (uOld & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1433	\| X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1434	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1435	while (!ASMAtomicCmpXchgExU64(&paPages[iFreePage].uPte.pPae->u, uNew, uOld, &uOld))
1436	AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
1437	Assert(paPages[iFreePage].uPte.pPae->u == uNew);
1438	/Log6(("pgmR0DynMapPageSlow: #%x - %RHp %p %#llx\n", iFreePage, HCPhys, paPages[iFreePage].pvPage, uNew));/
1439	}
1440	return iFreePage;
1441	}
1442
1443
1444	/**
1445	* Maps a page into the pool.
1446	*
1447	* @returns Page index on success, UINT32_MAX on failure.
1448	* @param pThis The dynamic mapping cache instance.
1449	* @param HCPhys The address of the page to be mapped.
1450	* @param iRealCpu The real cpu set index. (optimization)
1451	* @param pVCpu The current CPU (for statistics).
1452	* @param ppvPage Where to the page address.
1453	*/
1454	DECLINLINE(uint32_t) pgmR0DynMapPage(PPGMRZDYNMAP pThis, RTHCPHYS HCPhys, int32_t iRealCpu, PVMCPU pVCpu, void **ppvPage)
1455	{
1456	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
1457	AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
1458	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPage);
1459
1460	/*
1461	* Find an entry, if possible a matching one. The HCPhys address is hashed
1462	* down to a page index, collisions are handled by linear searching.
1463	* Optimized for a hit in the first 3 pages.
1464	*
1465	* Field easy hits here and defer the tedious searching and inserting
1466	* to pgmR0DynMapPageSlow().
1467	*/
1468	bool fNew = false;
1469	uint32_t const cPages = pThis->cPages;
1470	uint32_t iPage = (HCPhys >> PAGE_SHIFT) % cPages;
1471	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
1472	if (RT_LIKELY(paPages[iPage].HCPhys == HCPhys))
1473	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits0);
1474	else
1475	{
1476	uint32_t iPage2 = (iPage + 1) % cPages;
1477	if (RT_LIKELY(paPages[iPage2].HCPhys == HCPhys))
1478	{
1479	iPage = iPage2;
1480	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits1);
1481	}
1482	else
1483	{
1484	iPage2 = (iPage + 2) % cPages;
1485	if (paPages[iPage2].HCPhys == HCPhys)
1486	{
1487	iPage = iPage2;
1488	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits2);
1489	}
1490	else
1491	{
1492	iPage = pgmR0DynMapPageSlow(pThis, HCPhys, iPage, pVCpu, &fNew);
1493	if (RT_UNLIKELY(iPage == UINT32_MAX))
1494	{
1495	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1496	*ppvPage = NULL;
1497	return iPage;
1498	}
1499	}
1500	}
1501	}
1502
1503	/*
1504	* Reference it, update statistics and get the return address.
1505	*/
1506	int32_t cRefs = ASMAtomicIncS32(&paPages[iPage].cRefs);
1507	if (cRefs == 1)
1508	{
1509	pThis->cLoad++;
1510	if (pThis->cLoad > pThis->cMaxLoad)
1511	pThis->cMaxLoad = pThis->cLoad;
1512	AssertMsg(pThis->cLoad <= pThis->cPages - pThis->cGuardPages, ("%d/%d\n", pThis->cLoad, pThis->cPages - pThis->cGuardPages));
1513	}
1514	else if (RT_UNLIKELY(cRefs <= 0))
1515	{
1516	ASMAtomicDecS32(&paPages[iPage].cRefs);
1517	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1518	*ppvPage = NULL;
1519	AssertLogRelMsgFailedReturn(("cRefs=%d iPage=%p HCPhys=%RHp\n", cRefs, iPage, HCPhys), UINT32_MAX);
1520	}
1521	void *pvPage = paPages[iPage].pvPage;
1522
1523	#ifndef IN_RC
1524	/*
1525	* Invalidate the entry?
1526	*/
1527	bool fInvalidateIt = RTCpuSetIsMemberByIndex(&paPages[iPage].PendingSet, iRealCpu);
1528	if (RT_UNLIKELY(fInvalidateIt))
1529	RTCpuSetDelByIndex(&paPages[iPage].PendingSet, iRealCpu);
1530	#endif
1531
1532	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1533
1534	/*
1535	* Do the actual invalidation outside the spinlock.
1536	*/
1537	#ifdef IN_RC
1538	if (RT_UNLIKELY(fNew))
1539	#else
1540	if (RT_UNLIKELY(fInvalidateIt))
1541	#endif
1542	{
1543	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageInvlPg);
1544	ASMInvalidatePage(pvPage);
1545	}
1546
1547	*ppvPage = pvPage;
1548	return iPage;
1549	}
1550
1551
1552	/**
1553	* Assert the the integrity of the pool.
1554	*
1555	* @returns VBox status code.
1556	*/
1557	static int pgmRZDynMapAssertIntegrity(PPGMRZDYNMAP pThis)
1558	{
1559	/*
1560	* Basic pool stuff that doesn't require any lock, just assumes we're a user.
1561	*/
1562	if (!pThis)
1563	return VINF_SUCCESS;
1564	AssertPtrReturn(pThis, VERR_INVALID_POINTER);
1565	AssertReturn(pThis->u32Magic == PGMRZDYNMAP_MAGIC, VERR_INVALID_MAGIC);
1566	if (!pThis->cUsers)
1567	return VERR_INVALID_PARAMETER;
1568
1569
1570	int rc = VINF_SUCCESS;
1571	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
1572
1573	#define CHECK_RET(expr, a) \
1574	do { \
1575	if (RT_UNLIKELY(!(expr))) \
1576	{ \
1577	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis); \
1578	RTAssertMsg1Weak(#expr, __LINE__, __FILE__, __PRETTY_FUNCTION__); \
1579	RTAssertMsg2Weak a; \
1580	return VERR_INTERNAL_ERROR; \
1581	} \
1582	} while (0)
1583
1584	/*
1585	* Check that the PTEs are correct.
1586	*/
1587	uint32_t cGuard = 0;
1588	uint32_t cLoad = 0;
1589	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
1590	uint32_t iPage = pThis->cPages;
1591
1592	#ifndef IN_RC
1593	if (pThis->fLegacyMode)
1594	#endif
1595	{
1596	#ifdef IN_RING0
1597	PCX86PGUINT paSavedPTEs = (PCX86PGUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
1598	#endif
1599	while (iPage-- > 0)
1600	{
1601	CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
1602	if ( paPages[iPage].cRefs == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT
1603	&& paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS)
1604	{
1605	#ifdef PGMR0DYNMAP_GUARD_NP
1606	CHECK_RET(paPages[iPage].uPte.pLegacy->u == (paSavedPTEs[iPage] & ~(X86PGUINT)X86_PTE_P),
1607	("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage]));
1608	#else
1609	CHECK_RET(paPages[iPage].uPte.pLegacy->u == PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE,
1610	("#%u: %#x", iPage, paPages[iPage].uPte.pLegacy->u));
1611	#endif
1612	cGuard++;
1613	}
1614	else if (paPages[iPage].HCPhys != NIL_RTHCPHYS)
1615	{
1616	CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
1617	X86PGUINT uPte = X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1618	#ifdef IN_RING0
1619	\| (paSavedPTEs[iPage] & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1620	#endif
1621	\| (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
1622	CHECK_RET(paPages[iPage].uPte.pLegacy->u == uPte,
1623	("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, uPte));
1624	if (paPages[iPage].cRefs)
1625	cLoad++;
1626	}
1627	#if defined(IN_RING0) && !defined(PGMRZDYNMAP_STRICT_RELEASE)
1628	else
1629	CHECK_RET(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage],
1630	("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage]));
1631	#endif
1632	}
1633	}
1634	#ifndef IN_RC
1635	else
1636	#endif
1637	{
1638	#ifdef IN_RING0
1639	PCX86PGPAEUINT paSavedPTEs = (PCX86PGPAEUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
1640	#endif
1641	while (iPage-- > 0)
1642	{
1643	CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
1644	if ( paPages[iPage].cRefs == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT
1645	&& paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS)
1646	{
1647	#ifdef PGMR0DYNMAP_GUARD_NP
1648	CHECK_RET(paPages[iPage].uPte.pPae->u == (paSavedPTEs[iPage] & ~(X86PGPAEUINT)X86_PTE_P),
1649	("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage]));
1650	#else
1651	CHECK_RET(paPages[iPage].uPte.pPae->u == PGMR0DYNMAP_GUARD_PAGE_PAE_PTE,
1652	("#%u: %#llx", iPage, paPages[iPage].uPte.pPae->u));
1653	#endif
1654	cGuard++;
1655	}
1656	else if (paPages[iPage].HCPhys != NIL_RTHCPHYS)
1657	{
1658	CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
1659	X86PGPAEUINT uPte = X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1660	#ifdef IN_RING0
1661	\| (paSavedPTEs[iPage] & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1662	#endif
1663	\| (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
1664	CHECK_RET(paPages[iPage].uPte.pPae->u == uPte,
1665	("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pLegacy->u, uPte));
1666	if (paPages[iPage].cRefs)
1667	cLoad++;
1668	}
1669	#ifdef IN_RING0
1670	else
1671	CHECK_RET(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage],
1672	("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage]));
1673	#endif
1674	}
1675	}
1676
1677	CHECK_RET(cLoad == pThis->cLoad, ("%u %u\n", cLoad, pThis->cLoad));
1678	CHECK_RET(cGuard == pThis->cGuardPages, ("%u %u\n", cGuard, pThis->cGuardPages));
1679
1680	#undef CHECK_RET
1681	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1682	return VINF_SUCCESS;
1683	}
1684
1685	#ifdef IN_RING0
1686	/**
1687	* Assert the the integrity of the pool.
1688	*
1689	* @returns VBox status code.
1690	*/
1691	VMMR0DECL(int) PGMR0DynMapAssertIntegrity(void)
1692	{
1693	return pgmRZDynMapAssertIntegrity(g_pPGMR0DynMap);
1694	}
1695	#endif /* IN_RING0 */
1696
1697	#ifdef IN_RC
1698	/**
1699	* Assert the the integrity of the pool.
1700	*
1701	* @returns VBox status code.
1702	*/
1703	VMMRCDECL(int) PGMRCDynMapAssertIntegrity(PVM pVM)
1704	{
1705	return pgmRZDynMapAssertIntegrity((PPGMRZDYNMAP)pVM->pgm.s.pRCDynMap);
1706	}
1707	#endif /* IN_RC */
1708
1709
1710	/**
1711	* As a final resort for a (somewhat) full auto set or full cache, try merge
1712	* duplicate entries and flush the ones we can.
1713	*
1714	* @param pSet The set.
1715	*/
1716	static void pgmDynMapOptimizeAutoSet(PPGMMAPSET pSet)
1717	{
1718	LogFlow(("pgmDynMapOptimizeAutoSet\n"));
1719
1720	for (uint32_t i = 0 ; i < pSet->cEntries; i++)
1721	{
1722	/*
1723	* Try merge entries.
1724	*/
1725	uint16_t const iPage = pSet->aEntries[i].iPage;
1726	uint32_t j = i + 1;
1727	while ( j < pSet->cEntries
1728	&& ( pSet->iSubset == UINT32_MAX
1729	\|\| pSet->iSubset < pSet->cEntries) )
1730	{
1731	if (pSet->aEntries[j].iPage != iPage)
1732	j++;
1733	else
1734	{
1735	uint32_t const cHardRefs = (uint32_t)pSet->aEntries[i].cRefs
1736	+ (uint32_t)pSet->aEntries[j].cRefs;
1737	uint32_t cInlinedRefs = (uint32_t)pSet->aEntries[i].cInlinedRefs
1738	+ (uint32_t)pSet->aEntries[j].cInlinedRefs;
1739	uint32_t cUnrefs = (uint32_t)pSet->aEntries[i].cUnrefs
1740	+ (uint32_t)pSet->aEntries[j].cUnrefs;
1741	uint32_t cSub = RT_MIN(cUnrefs, cInlinedRefs);
1742	cInlinedRefs -= cSub;
1743	cUnrefs -= cSub;
1744
1745	if ( cHardRefs < UINT16_MAX
1746	&& cInlinedRefs < UINT16_MAX
1747	&& cUnrefs < UINT16_MAX)
1748	{
1749	/* merge j into i removing j. */
1750	Log2(("pgmDynMapOptimizeAutoSet: Merging #%u into #%u\n", j, i));
1751	pSet->aEntries[i].cRefs = cHardRefs;
1752	pSet->aEntries[i].cInlinedRefs = cInlinedRefs;
1753	pSet->aEntries[i].cUnrefs = cUnrefs;
1754	pSet->cEntries--;
1755	if (j < pSet->cEntries)
1756	{
1757	pSet->aEntries[j] = pSet->aEntries[pSet->cEntries];
1758	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[pSet->cEntries]);
1759	}
1760	else
1761	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]);
1762	}
1763	#if 0 /* too complicated, skip it. */
1764	else
1765	{
1766	/* migrate the max number of refs from j into i and quit the inner loop. */
1767	uint32_t cMigrate = UINT16_MAX - 1 - pSet->aEntries[i].cRefs;
1768	Assert(pSet->aEntries[j].cRefs > cMigrate);
1769	pSet->aEntries[j].cRefs -= cMigrate;
1770	pSet->aEntries[i].cRefs = UINT16_MAX - 1;
1771	break;
1772	}
1773	#endif
1774	}
1775	}
1776
1777	/*
1778	* Try make use of the unused hinting (cUnrefs) to evict entries
1779	* from both the set as well as the mapping cache.
1780	*/
1781
1782	uint32_t const cTotalRefs = (uint32_t)pSet->aEntries[i].cRefs + pSet->aEntries[i].cInlinedRefs;
1783	Log2(("pgmDynMapOptimizeAutoSet: #%u/%u/%u pvPage=%p iPage=%u cRefs=%u cInlinedRefs=%u cUnrefs=%u cTotalRefs=%u\n",
1784	i,
1785	pSet->iSubset,
1786	pSet->cEntries,
1787	pSet->aEntries[i].pvPage,
1788	pSet->aEntries[i].iPage,
1789	pSet->aEntries[i].cRefs,
1790	pSet->aEntries[i].cInlinedRefs,
1791	pSet->aEntries[i].cUnrefs,
1792	cTotalRefs));
1793	Assert(cTotalRefs >= pSet->aEntries[i].cUnrefs);
1794
1795	if ( cTotalRefs == pSet->aEntries[i].cUnrefs
1796	&& ( pSet->iSubset == UINT32_MAX
1797	\|\| pSet->iSubset < pSet->cEntries)
1798	)
1799	{
1800	Log2(("pgmDynMapOptimizeAutoSet: Releasing iPage=%d/%p\n", pSet->aEntries[i].iPage, pSet->aEntries[i].pvPage));
1801	//LogFlow(("pgmDynMapOptimizeAutoSet: Releasing iPage=%d/%p\n", pSet->aEntries[i].iPage, pSet->aEntries[i].pvPage));
1802	pgmRZDynMapReleasePage(PGMRZDYNMAP_SET_2_DYNMAP(pSet),
1803	pSet->aEntries[i].iPage,
1804	pSet->aEntries[i].cRefs);
1805	pSet->cEntries--;
1806	if (i < pSet->cEntries)
1807	{
1808	pSet->aEntries[i] = pSet->aEntries[pSet->cEntries];
1809	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[pSet->cEntries]);
1810	}
1811
1812	i--;
1813	}
1814	}
1815	}
1816
1817
1818
1819
1820	/**
1821	* Signals the start of a new set of mappings.
1822	*
1823	* Mostly for strictness. PGMDynMapHCPage won't work unless this
1824	* API is called.
1825	*
1826	* @param pVCpu The shared data for the current virtual CPU.
1827	*/
1828	VMMDECL(void) PGMRZDynMapStartAutoSet(PVMCPU pVCpu)
1829	{
1830	LogFlow(("PGMRZDynMapStartAutoSet:\n"));
1831	Assert(pVCpu->pgm.s.AutoSet.cEntries == PGMMAPSET_CLOSED);
1832	Assert(pVCpu->pgm.s.AutoSet.iSubset == UINT32_MAX);
1833	pVCpu->pgm.s.AutoSet.cEntries = 0;
1834	pVCpu->pgm.s.AutoSet.iCpu = PGMRZDYNMAP_CUR_CPU();
1835	}
1836
1837
1838	#ifdef IN_RING0
1839	/**
1840	* Starts or migrates the autoset of a virtual CPU.
1841	*
1842	* This is used by HWACCMR0Enter. When we've longjumped out of the HWACCM
1843	* execution loop with the set open, we'll migrate it when re-entering. While
1844	* under normal circumstances, we'll start it so VMXR0LoadGuestState can access
1845	* guest memory.
1846	*
1847	* @returns @c true if started, @c false if migrated.
1848	* @param pVCpu The shared data for the current virtual CPU.
1849	* @thread EMT
1850	*/
1851	VMMR0DECL(bool) PGMR0DynMapStartOrMigrateAutoSet(PVMCPU pVCpu)
1852	{
1853	bool fStartIt = pVCpu->pgm.s.AutoSet.cEntries == PGMMAPSET_CLOSED;
1854	if (fStartIt)
1855	PGMRZDynMapStartAutoSet(pVCpu);
1856	else
1857	PGMR0DynMapMigrateAutoSet(pVCpu);
1858	return fStartIt;
1859	}
1860	#endif /* IN_RING0 */
1861
1862
1863	/**
1864	* Checks if the set has high load.
1865	*
1866	* @returns true on high load, otherwise false.
1867	* @param pSet The set.
1868	*/
1869	DECLINLINE(bool) pgmRZDynMapHasHighLoad(PPGMMAPSET pSet)
1870	{
1871	#ifdef IN_RC
1872	if (pSet->cEntries < MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE / 2)
1873	return false;
1874	#endif
1875
1876	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
1877	uint32_t cUnusedPages = pThis->cPages - pThis->cLoad;
1878	#ifdef IN_RC
1879	return cUnusedPages <= MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE * 36 / 100;
1880	#else
1881	return cUnusedPages <= PGMR0DYNMAP_PAGES_PER_CPU_MIN;
1882	#endif
1883	}
1884
1885
1886	/**
1887	* Worker that performs the actual flushing of the set.
1888	*
1889	* @param pSet The set to flush.
1890	* @param cEntries The number of entries.
1891	*/
1892	DECLINLINE(void) pgmDynMapFlushAutoSetWorker(PPGMMAPSET pSet, uint32_t cEntries)
1893	{
1894	/*
1895	* Release any pages it's referencing.
1896	*/
1897	if ( cEntries != 0
1898	&& RT_LIKELY(cEntries <= RT_ELEMENTS(pSet->aEntries)))
1899	{
1900	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
1901	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
1902
1903	uint32_t i = cEntries;
1904	while (i-- > 0)
1905	{
1906	uint32_t iPage = pSet->aEntries[i].iPage;
1907	Assert(iPage < pThis->cPages);
1908	int32_t cRefs = pSet->aEntries[i].cRefs;
1909	Assert(cRefs > 0);
1910	pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs);
1911
1912	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[i]);
1913	}
1914
1915	Assert(pThis->cLoad <= pThis->cPages - pThis->cGuardPages);
1916	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1917	}
1918	}
1919
1920
1921	/**
1922	* Releases the dynamic memory mappings made by PGMDynMapHCPage and associates
1923	* since the PGMDynMapStartAutoSet call.
1924	*
1925	* @param pVCpu The shared data for the current virtual CPU.
1926	*/
1927	VMMDECL(void) PGMRZDynMapReleaseAutoSet(PVMCPU pVCpu)
1928	{
1929	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1930
1931	/*
1932	* Close and flush the set.
1933	*/
1934	uint32_t cEntries = pSet->cEntries;
1935	AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
1936	pSet->cEntries = PGMMAPSET_CLOSED;
1937	pSet->iSubset = UINT32_MAX;
1938	pSet->iCpu = -1;
1939
1940	#ifdef IN_RC
1941	if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)
1942	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
1943	else
1944	#endif
1945	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
1946	if (cEntries > RT_ELEMENTS(pSet->aEntries) * 50 / 100)
1947	Log(("PGMRZDynMapReleaseAutoSet: cEntries=%d\n", cEntries));
1948	else
1949	LogFlow(("PGMRZDynMapReleaseAutoSet: cEntries=%d\n", cEntries));
1950
1951	pgmDynMapFlushAutoSetWorker(pSet, cEntries);
1952	}
1953
1954
1955	/**
1956	* Flushes the set if it's above a certain threshold.
1957	*
1958	* @param pVCpu The shared data for the current virtual CPU.
1959	*/
1960	VMMDECL(void) PGMRZDynMapFlushAutoSet(PVMCPU pVCpu)
1961	{
1962	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1963	AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
1964
1965	/*
1966	* Only flush it if it's 45% full.
1967	*/
1968	uint32_t cEntries = pSet->cEntries;
1969	AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
1970	Assert(pSet->iSubset == UINT32_MAX);
1971	#ifdef IN_RC
1972	if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)
1973	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
1974	else
1975	#endif
1976	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
1977	if ( cEntries >= RT_ELEMENTS(pSet->aEntries) * 45 / 100
1978	\|\| pgmRZDynMapHasHighLoad(pSet))
1979	{
1980	pSet->cEntries = 0;
1981	Log(("PGMDynMapFlushAutoSet: cEntries=%d\n", pSet->cEntries));
1982
1983	pgmDynMapFlushAutoSetWorker(pSet, cEntries);
1984	AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
1985	}
1986	}
1987
1988
1989	#ifndef IN_RC
1990	/**
1991	* Migrates the automatic mapping set of the current vCPU if it's active and
1992	* necessary.
1993	*
1994	* This is called when re-entering the hardware assisted execution mode after a
1995	* nip down to ring-3. We run the risk that the CPU might have change and we
1996	* will therefore make sure all the cache entries currently in the auto set will
1997	* be valid on the new CPU. If the cpu didn't change nothing will happen as all
1998	* the entries will have been flagged as invalidated.
1999	*
2000	* @param pVCpu The shared data for the current virtual CPU.
2001	* @thread EMT
2002	*/
2003	VMMR0DECL(void) PGMR0DynMapMigrateAutoSet(PVMCPU pVCpu)
2004	{
2005	LogFlow(("PGMR0DynMapMigrateAutoSet\n"));
2006	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
2007	int32_t iRealCpu = PGMRZDYNMAP_CUR_CPU();
2008	if (pSet->iCpu != iRealCpu)
2009	{
2010	uint32_t i = pSet->cEntries;
2011	if (i != PGMMAPSET_CLOSED)
2012	{
2013	AssertMsg(i <= RT_ELEMENTS(pSet->aEntries), ("%#x (%u)\n", i, i));
2014	if (i != 0 && RT_LIKELY(i <= RT_ELEMENTS(pSet->aEntries)))
2015	{
2016	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
2017	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
2018
2019	while (i-- > 0)
2020	{
2021	Assert(pSet->aEntries[i].cRefs > 0);
2022	uint32_t iPage = pSet->aEntries[i].iPage;
2023	Assert(iPage < pThis->cPages);
2024	if (RTCpuSetIsMemberByIndex(&pThis->paPages[iPage].PendingSet, iRealCpu))
2025	{
2026	RTCpuSetDelByIndex(&pThis->paPages[iPage].PendingSet, iRealCpu);
2027	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
2028
2029	ASMInvalidatePage(pThis->paPages[iPage].pvPage);
2030	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapMigrateInvlPg);
2031
2032	PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis);
2033	}
2034	}
2035
2036	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
2037	}
2038	}
2039	pSet->iCpu = iRealCpu;
2040	}
2041	}
2042	#endif /* !IN_RC */
2043
2044
2045	/**
2046	* Worker function that flushes the current subset.
2047	*
2048	* This is called when the set is popped or when the set
2049	* hash a too high load. As also pointed out elsewhere, the
2050	* whole subset thing is a hack for working around code that
2051	* accesses too many pages. Like PGMPool.
2052	*
2053	* @param pSet The set which subset to flush.
2054	*/
2055	static void pgmDynMapFlushSubset(PPGMMAPSET pSet)
2056	{
2057	uint32_t iSubset = pSet->iSubset;
2058	uint32_t i = pSet->cEntries;
2059	Assert(i <= RT_ELEMENTS(pSet->aEntries));
2060	if ( i > iSubset
2061	&& i <= RT_ELEMENTS(pSet->aEntries))
2062	{
2063	Log(("pgmDynMapFlushSubset: cEntries=%d iSubset=%d\n", pSet->cEntries, iSubset));
2064	pSet->cEntries = iSubset;
2065
2066	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
2067	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
2068
2069	while (i-- > iSubset)
2070	{
2071	uint32_t iPage = pSet->aEntries[i].iPage;
2072	Assert(iPage < pThis->cPages);
2073	int32_t cRefs = pSet->aEntries[i].cRefs;
2074	Assert(cRefs > 0);
2075	pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs);
2076
2077	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[i]);
2078	}
2079
2080	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
2081	}
2082	}
2083
2084
2085	/**
2086	* Creates a subset.
2087	*
2088	* A subset is a hack to avoid having to rewrite code that touches a lot of
2089	* pages. It prevents the mapping set from being overflowed by automatically
2090	* flushing previous mappings when a certain threshold is reached.
2091	*
2092	* Pages mapped after calling this function are only valid until the next page
2093	* is mapped.
2094	*
2095	* @returns The index of the previous subset. Pass this to
2096	* PGMDynMapPopAutoSubset when popping it.
2097	* @param pVCpu Pointer to the virtual cpu data.
2098	*/
2099	VMMDECL(uint32_t) PGMRZDynMapPushAutoSubset(PVMCPU pVCpu)
2100	{
2101	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
2102	AssertReturn(pSet->cEntries != PGMMAPSET_CLOSED, UINT32_MAX);
2103	uint32_t iPrevSubset = pSet->iSubset;
2104	LogFlow(("PGMRZDynMapPushAutoSubset: pVCpu=%p iPrevSubset=%u\n", pVCpu, iPrevSubset));
2105
2106	/*
2107	* If it looks like we're approaching the max set size or mapping space
2108	* optimize the set to drop off unused pages.
2109	*/
2110	if ( pSet->cEntries > RT_ELEMENTS(pSet->aEntries) * 60 / 100
2111	\|\| pgmRZDynMapHasHighLoad(pSet))
2112	{
2113	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize);
2114	pgmDynMapOptimizeAutoSet(pSet);
2115	}
2116
2117	pSet->iSubset = pSet->cEntries;
2118	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSubsets);
2119
2120	AssertMsg(iPrevSubset <= pSet->iSubset \|\| iPrevSubset == UINT32_MAX, ("iPrevSubset=%#x iSubset=%#x\n", iPrevSubset, pSet->iSubset));
2121	return iPrevSubset;
2122	}
2123
2124
2125	/**
2126	* Pops a subset created by a previous call to PGMDynMapPushAutoSubset.
2127	*
2128	* @param pVCpu Pointer to the virtual cpu data.
2129	* @param iPrevSubset What PGMDynMapPushAutoSubset returned.
2130	*/
2131	VMMDECL(void) PGMRZDynMapPopAutoSubset(PVMCPU pVCpu, uint32_t iPrevSubset)
2132	{
2133	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
2134	uint32_t cEntries = pSet->cEntries;
2135	LogFlow(("PGMRZDynMapPopAutoSubset: pVCpu=%p iPrevSubset=%u iSubset=%u cEntries=%u\n", pVCpu, iPrevSubset, pSet->iSubset, cEntries));
2136	AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
2137	AssertMsgReturnVoid(pSet->iSubset >= iPrevSubset \|\| iPrevSubset == UINT32_MAX, ("iPrevSubset=%u iSubset=%u cEntries=%u\n", iPrevSubset, pSet->iSubset, cEntries));
2138	#ifdef IN_RC
2139	if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)
2140	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
2141	else
2142	#endif
2143	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
2144	if ( cEntries >= RT_ELEMENTS(pSet->aEntries) * 40 / 100
2145	&& cEntries != pSet->iSubset)
2146	{
2147	pgmDynMapFlushSubset(pSet);
2148	Assert(pSet->cEntries >= iPrevSubset \|\| iPrevSubset == UINT32_MAX);
2149	}
2150	pSet->iSubset = iPrevSubset;
2151	}
2152
2153
2154	/**
2155	* Indicates that the given page is unused and its mapping can be re-used.
2156	*
2157	* @param pVCpu The current CPU.
2158	* @param pvHint The page that is now unused. This does not have to
2159	* point at the start of the page. NULL is ignored.
2160	*/
2161	#ifdef LOG_ENABLED
2162	void pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint, RT_SRC_POS_DECL)
2163	#else
2164	void pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint)
2165	#endif
2166	{
2167	/*
2168	* Ignore NULL pointers and mask off the page offset bits.
2169	*/
2170	if (pvHint == NULL)
2171	return;
2172	pvHint = (void *)((uintptr_t)pvHint & ~(uintptr_t)PAGE_OFFSET_MASK);
2173
2174	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
2175	uint32_t iEntry = pSet->cEntries;
2176	AssertReturnVoid(iEntry > 0);
2177
2178	/*
2179	* Find the entry in the usual unrolled fashion.
2180	*/
2181	/** @todo add a hint to the set which entry was used last since it's not
2182	* always the last entry? */
2183	#define IS_MATCHING_ENTRY(pSet, iEntry, pvHint) \
2184	( (pSet)->aEntries[(iEntry)].pvPage == (pvHint) \
2185	&& (uint32_t)(pSet)->aEntries[(iEntry)].cRefs + (pSet)->aEntries[(iEntry)].cInlinedRefs \
2186	> (pSet)->aEntries[(iEntry)].cUnrefs )
2187	if ( iEntry >= 1 && IS_MATCHING_ENTRY(pSet, iEntry - 1, pvHint))
2188	iEntry = iEntry - 1;
2189	else if (iEntry >= 2 && IS_MATCHING_ENTRY(pSet, iEntry - 2, pvHint))
2190	iEntry = iEntry - 2;
2191	else if (iEntry >= 3 && IS_MATCHING_ENTRY(pSet, iEntry - 3, pvHint))
2192	iEntry = iEntry - 3;
2193	else if (iEntry >= 4 && IS_MATCHING_ENTRY(pSet, iEntry - 4, pvHint))
2194	iEntry = iEntry - 4;
2195	else if (iEntry >= 5 && IS_MATCHING_ENTRY(pSet, iEntry - 5, pvHint))
2196	iEntry = iEntry - 5;
2197	else if (iEntry >= 6 && IS_MATCHING_ENTRY(pSet, iEntry - 6, pvHint))
2198	iEntry = iEntry - 6;
2199	else if (iEntry >= 7 && IS_MATCHING_ENTRY(pSet, iEntry - 7, pvHint))
2200	iEntry = iEntry - 7;
2201	else
2202	{
2203	/*
2204	* Loop till we find it.
2205	*/
2206	bool fFound = false;
2207	if (iEntry > 7)
2208	{
2209	iEntry -= 7;
2210	while (iEntry-- > 0)
2211	if (IS_MATCHING_ENTRY(pSet, iEntry, pvHint))
2212	{
2213	fFound = true;
2214	break;
2215	}
2216	}
2217	AssertMsgReturnVoid(fFound,
2218	("pvHint=%p cEntries=%#x iSubset=%#x\n"
2219	"aEntries[0] = {%#x, %#x, %#x, %#x, %p}\n"
2220	"aEntries[1] = {%#x, %#x, %#x, %#x, %p}\n"
2221	"aEntries[2] = {%#x, %#x, %#x, %#x, %p}\n"
2222	"aEntries[3] = {%#x, %#x, %#x, %#x, %p}\n"
2223	"aEntries[4] = {%#x, %#x, %#x, %#x, %p}\n"
2224	"aEntries[5] = {%#x, %#x, %#x, %#x, %p}\n"
2225	,
2226	pvHint, pSet->cEntries, pSet->iSubset,
2227	pSet->aEntries[0].iPage, pSet->aEntries[0].cRefs, pSet->aEntries[0].cInlinedRefs, pSet->aEntries[0].cUnrefs, pSet->aEntries[0].pvPage,
2228	pSet->aEntries[1].iPage, pSet->aEntries[1].cRefs, pSet->aEntries[1].cInlinedRefs, pSet->aEntries[1].cUnrefs, pSet->aEntries[1].pvPage,
2229	pSet->aEntries[2].iPage, pSet->aEntries[2].cRefs, pSet->aEntries[2].cInlinedRefs, pSet->aEntries[2].cUnrefs, pSet->aEntries[2].pvPage,
2230	pSet->aEntries[3].iPage, pSet->aEntries[3].cRefs, pSet->aEntries[3].cInlinedRefs, pSet->aEntries[3].cUnrefs, pSet->aEntries[3].pvPage,
2231	pSet->aEntries[4].iPage, pSet->aEntries[4].cRefs, pSet->aEntries[4].cInlinedRefs, pSet->aEntries[4].cUnrefs, pSet->aEntries[4].pvPage,
2232	pSet->aEntries[5].iPage, pSet->aEntries[5].cRefs, pSet->aEntries[5].cInlinedRefs, pSet->aEntries[5].cUnrefs, pSet->aEntries[5].pvPage));
2233	}
2234	#undef IS_MATCHING_ENTRY
2235
2236	/*
2237	* Update it.
2238	*/
2239	uint32_t const cTotalRefs = (uint32_t)pSet->aEntries[iEntry].cRefs + pSet->aEntries[iEntry].cInlinedRefs;
2240	uint32_t const cUnrefs = pSet->aEntries[iEntry].cUnrefs;
2241	LogFlow(("pgmRZDynMapUnusedHint: pvHint=%p #%u cRefs=%d cInlinedRefs=%d cUnrefs=%d (+1) cTotalRefs=%d %s(%d) %s\n",
2242	pvHint, iEntry, pSet->aEntries[iEntry].cRefs, pSet->aEntries[iEntry].cInlinedRefs, cUnrefs, cTotalRefs, pszFile, iLine, pszFunction));
2243	AssertReturnVoid(cTotalRefs > cUnrefs);
2244
2245	if (RT_LIKELY(cUnrefs < UINT16_MAX - 1))
2246	pSet->aEntries[iEntry].cUnrefs++;
2247	else if (pSet->aEntries[iEntry].cInlinedRefs)
2248	{
2249	uint32_t cSub = RT_MIN(pSet->aEntries[iEntry].cInlinedRefs, pSet->aEntries[iEntry].cUnrefs);
2250	pSet->aEntries[iEntry].cInlinedRefs -= cSub;
2251	pSet->aEntries[iEntry].cUnrefs -= cSub;
2252	pSet->aEntries[iEntry].cUnrefs++;
2253	}
2254	else
2255	Log(("pgmRZDynMapUnusedHint: pvHint=%p ignored because of overflow! %s(%d) %s\n", pvHint, pszFile, iLine, pszFunction));
2256
2257	#ifdef PGMRZDYNMAP_STRICT_RELEASE
2258	/*
2259	* Optimize the set to trigger the unmapping and invalidation of the page.
2260	*/
2261	if (cUnrefs + 1 == cTotalRefs)
2262	pgmDynMapOptimizeAutoSet(pSet);
2263	#endif
2264	}
2265
2266
2267	/**
2268	* Common worker code for pgmRZDynMapHCPageInlined, pgmRZDynMapHCPageV2Inlined
2269	* and pgmR0DynMapGCPageOffInlined.
2270	*
2271	* @returns VINF_SUCCESS, bails out to ring-3 on failure.
2272	* @param pSet The set.
2273	* @param HCPhys The physical address of the page.
2274	* @param ppv Where to store the address of the mapping on success.
2275	*
2276	* @remarks This is a very hot path.
2277	*/
2278	int pgmRZDynMapHCPageCommon(PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
2279	{
2280	AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
2281	PVMCPU pVCpu = PGMRZDYNMAP_SET_2_VMCPU(pSet);
2282	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
2283
2284	/*
2285	* Map it.
2286	*/
2287	void *pvPage;
2288	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
2289	uint32_t iPage = pgmR0DynMapPage(pThis, HCPhys, pSet->iCpu, pVCpu, &pvPage);
2290	if (RT_UNLIKELY(iPage == UINT32_MAX))
2291	{
2292	/*
2293	* We're out of mapping space, optimize our set to try remedy the
2294	* situation. (Only works if there are unreference hints.)
2295	*/
2296	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize);
2297	pgmDynMapOptimizeAutoSet(pSet);
2298
2299	iPage = pgmR0DynMapPage(pThis, HCPhys, pSet->iCpu, pVCpu, &pvPage);
2300	if (RT_UNLIKELY(iPage == UINT32_MAX))
2301	{
2302	RTAssertMsg2Weak("pgmRZDynMapHCPageCommon: cLoad=%u/%u cPages=%u cGuardPages=%u\n",
2303	pThis->cLoad, pThis->cMaxLoad, pThis->cPages, pThis->cGuardPages);
2304	if (!g_fPGMR0DynMapTestRunning)
2305	VMMRZCallRing3NoCpu(PGMRZDYNMAP_SET_2_VM(pSet), VMMCALLRING3_VM_R0_ASSERTION, 0);
2306	*ppv = NULL;
2307	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
2308	return VERR_PGM_DYNMAP_FAILED;
2309	}
2310	}
2311
2312	/*
2313	* Add the page to the auto reference set.
2314	*
2315	* The typical usage pattern means that the same pages will be mapped
2316	* several times in the same set. We can catch most of these
2317	* remappings by looking a few pages back into the set. (The searching
2318	* and set optimizing path will hardly ever be used when doing this.)
2319	*/
2320	AssertCompile(RT_ELEMENTS(pSet->aEntries) >= 8);
2321	int32_t i = pSet->cEntries;
2322	if (i-- < 5)
2323	{
2324	unsigned iEntry = pSet->cEntries++;
2325	pSet->aEntries[iEntry].cRefs = 1;
2326	pSet->aEntries[iEntry].cUnrefs = 0;
2327	pSet->aEntries[iEntry].cInlinedRefs = 0;
2328	pSet->aEntries[iEntry].iPage = iPage;
2329	pSet->aEntries[iEntry].pvPage = pvPage;
2330	pSet->aEntries[iEntry].HCPhys = HCPhys;
2331	pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
2332	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/0/0 iPage=%#x [a] %s(%d) %s\n",
2333	pSet, HCPhys, iEntry, iEntry + 1, pvPage, 1, iPage, pszFile, iLine, pszFunction));
2334	}
2335	/* Any of the last 5 pages? */
2336	else if ( pSet->aEntries[i - 0].iPage == iPage
2337	&& pSet->aEntries[i - 0].cRefs < UINT16_MAX - 1)
2338	{
2339	pSet->aEntries[i - 0].cRefs++;
2340	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [0] %s(%d) %s\n", pSet, HCPhys, i - 0, pSet->cEntries, pvPage, pSet->aEntries[i - 0].cRefs, pSet->aEntries[i - 0].cInlinedRefs, pSet->aEntries[i - 0].cUnrefs, iPage, pszFile, iLine, pszFunction));
2341	}
2342	else if ( pSet->aEntries[i - 1].iPage == iPage
2343	&& pSet->aEntries[i - 1].cRefs < UINT16_MAX - 1)
2344	{
2345	pSet->aEntries[i - 1].cRefs++;
2346	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [1] %s(%d) %s\n", pSet, HCPhys, i - 1, pSet->cEntries, pvPage, pSet->aEntries[i - 1].cRefs, pSet->aEntries[i - 1].cInlinedRefs, pSet->aEntries[i - 1].cUnrefs, iPage, pszFile, iLine, pszFunction));
2347	}
2348	else if ( pSet->aEntries[i - 2].iPage == iPage
2349	&& pSet->aEntries[i - 2].cRefs < UINT16_MAX - 1)
2350	{
2351	pSet->aEntries[i - 2].cRefs++;
2352	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [2] %s(%d) %s\n", pSet, HCPhys, i - 2, pSet->cEntries, pvPage, pSet->aEntries[i - 2].cRefs, pSet->aEntries[i - 2].cInlinedRefs, pSet->aEntries[i - 2].cUnrefs, iPage, pszFile, iLine, pszFunction));
2353	}
2354	else if ( pSet->aEntries[i - 3].iPage == iPage
2355	&& pSet->aEntries[i - 3].cRefs < UINT16_MAX - 1)
2356	{
2357	pSet->aEntries[i - 3].cRefs++;
2358	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [4] %s(%d) %s\n", pSet, HCPhys, i - 3, pSet->cEntries, pvPage, pSet->aEntries[i - 3].cRefs, pSet->aEntries[i - 3].cInlinedRefs, pSet->aEntries[i - 3].cUnrefs, iPage, pszFile, iLine, pszFunction));
2359	}
2360	else if ( pSet->aEntries[i - 4].iPage == iPage
2361	&& pSet->aEntries[i - 4].cRefs < UINT16_MAX - 1)
2362	{
2363	pSet->aEntries[i - 4].cRefs++;
2364	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [4] %s(%d) %s\n", pSet, HCPhys, i - 4, pSet->cEntries, pvPage, pSet->aEntries[i - 4].cRefs, pSet->aEntries[i - 4].cInlinedRefs, pSet->aEntries[i - 4].cUnrefs, iPage, pszFile, iLine, pszFunction));
2365	}
2366	/* Don't bother searching unless we're above a 60% load. */
2367	else if (RT_LIKELY(i <= (int32_t)RT_ELEMENTS(pSet->aEntries) * 60 / 100))
2368	{
2369	unsigned iEntry = pSet->cEntries++;
2370	pSet->aEntries[iEntry].cRefs = 1;
2371	pSet->aEntries[iEntry].cUnrefs = 0;
2372	pSet->aEntries[iEntry].cInlinedRefs = 0;
2373	pSet->aEntries[iEntry].iPage = iPage;
2374	pSet->aEntries[iEntry].pvPage = pvPage;
2375	pSet->aEntries[iEntry].HCPhys = HCPhys;
2376	pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
2377	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=1/0/0 iPage=%#x [b] %s(%d) %s\n", pSet, HCPhys, iEntry, pSet->cEntries, pvPage, iPage, pszFile, iLine, pszFunction));
2378	}
2379	else
2380	{
2381	/* Search the rest of the set. */
2382	Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
2383	i -= 4;
2384	while (i-- > 0)
2385	if ( pSet->aEntries[i].iPage == iPage
2386	&& pSet->aEntries[i].cRefs < UINT16_MAX - 1)
2387	{
2388	pSet->aEntries[i].cRefs++;
2389	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchHits);
2390	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [c] %s(%d) %s\n", pSet, HCPhys, i, pSet->cEntries, pvPage, pSet->aEntries[i].cRefs, pSet->aEntries[i].cInlinedRefs, pSet->aEntries[i].cUnrefs, iPage, pszFile, iLine, pszFunction));
2391	break;
2392	}
2393	if (i < 0)
2394	{
2395	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchMisses);
2396	#if 0 /* this is very bogus */
2397	if (pSet->iSubset < pSet->cEntries)
2398	{
2399	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchFlushes);
2400	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(pSet->cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
2401	pgmDynMapFlushSubset(pSet);
2402	}
2403	#endif
2404
2405	if (RT_UNLIKELY(pSet->cEntries >= RT_ELEMENTS(pSet->aEntries)))
2406	{
2407	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize);
2408	pgmDynMapOptimizeAutoSet(pSet);
2409	}
2410
2411	if (RT_LIKELY(pSet->cEntries < RT_ELEMENTS(pSet->aEntries)))
2412	{
2413	unsigned iEntry = pSet->cEntries++;
2414	pSet->aEntries[iEntry].cRefs = 1;
2415	pSet->aEntries[iEntry].cUnrefs = 0;
2416	pSet->aEntries[iEntry].cInlinedRefs = 0;
2417	pSet->aEntries[iEntry].iPage = iPage;
2418	pSet->aEntries[iEntry].pvPage = pvPage;
2419	pSet->aEntries[iEntry].HCPhys = HCPhys;
2420	pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
2421	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=1/0/0 iPage=%#x [d] %s(%d) %s\n", pSet, HCPhys, iEntry, pSet->cEntries, pvPage, iPage, pszFile, iLine, pszFunction));
2422	}
2423	else
2424	{
2425	/* We're screwed. */
2426	pgmRZDynMapReleasePage(pThis, iPage, 1);
2427
2428	RTAssertMsg2Weak("pgmRZDynMapHCPageCommon: set is full!\n");
2429	if (!g_fPGMR0DynMapTestRunning)
2430	VMMRZCallRing3NoCpu(PGMRZDYNMAP_SET_2_VM(pSet), VMMCALLRING3_VM_R0_ASSERTION, 0);
2431	*ppv = NULL;
2432	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
2433	return VERR_PGM_DYNMAP_FULL_SET;
2434	}
2435	}
2436	}
2437
2438	*ppv = pvPage;
2439	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
2440	return VINF_SUCCESS;
2441	}
2442
2443
2444	#if 0 /def DEBUG/
2445	/** For pgmR0DynMapTest3PerCpu. */
2446	typedef struct PGMR0DYNMAPTEST
2447	{
2448	uint32_t u32Expect;
2449	uint32_t *pu32;
2450	uint32_t volatile cFailures;
2451	} PGMR0DYNMAPTEST;
2452	typedef PGMR0DYNMAPTEST *PPGMR0DYNMAPTEST;
2453
2454	/**
2455	* Checks that the content of the page is the same on all CPUs, i.e. that there
2456	* are no CPU specific PTs or similar nasty stuff involved.
2457	*
2458	* @param idCpu The current CPU.
2459	* @param pvUser1 Pointer a PGMR0DYNMAPTEST structure.
2460	* @param pvUser2 Unused, ignored.
2461	*/
2462	static DECLCALLBACK(void) pgmR0DynMapTest3PerCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
2463	{
2464	PPGMR0DYNMAPTEST pTest = (PPGMR0DYNMAPTEST)pvUser1;
2465	ASMInvalidatePage(pTest->pu32);
2466	if (*pTest->pu32 != pTest->u32Expect)
2467	ASMAtomicIncU32(&pTest->cFailures);
2468	NOREF(pvUser2); NOREF(idCpu);
2469	}
2470
2471
2472	/**
2473	* Performs some basic tests in debug builds.
2474	*/
2475	static int pgmR0DynMapTest(PVM pVM)
2476	{
2477	LogRel(("pgmR0DynMapTest: **** START ****\n"));
2478	PPGMMAPSET pSet = &pVM->aCpus[0].pgm.s.AutoSet;
2479	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
2480	uint32_t i;
2481
2482	/*
2483	* Assert internal integrity first.
2484	*/
2485	LogRel(("Test #0\n"));
2486	int rc = PGMR0DynMapAssertIntegrity();
2487	if (RT_FAILURE(rc))
2488	return rc;
2489
2490	void *pvR0DynMapUsedSaved = pVM->pgm.s.pvR0DynMapUsed;
2491	pVM->pgm.s.pvR0DynMapUsed = pThis;
2492	g_fPGMR0DynMapTestRunning = true;
2493
2494	/*
2495	* Simple test, map CR3 twice and check that we're getting the
2496	* same mapping address back.
2497	*/
2498	LogRel(("Test #1\n"));
2499	ASMIntDisable();
2500	PGMRZDynMapStartAutoSet(&pVM->aCpus[0]);
2501
2502	uint64_t cr3 = ASMGetCR3() & ~(uint64_t)PAGE_OFFSET_MASK;
2503	void pv = (void )(intptr_t)-1;
2504	void pv2 = (void )(intptr_t)-2;
2505	rc = pgmRZDynMapHCPageCommon(pVM, cr3, &pv RTLOG_COMMA_SRC_POS);
2506	int rc2 = pgmRZDynMapHCPageCommon(pVM, cr3, &pv2 RTLOG_COMMA_SRC_POS);
2507	ASMIntEnable();
2508	if ( RT_SUCCESS(rc2)
2509	&& RT_SUCCESS(rc)
2510	&& pv == pv2)
2511	{
2512	LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2513	rc = PGMR0DynMapAssertIntegrity();
2514
2515	/*
2516	* Check that the simple set overflow code works by filling it
2517	* with more CR3 mappings.
2518	*/
2519	LogRel(("Test #2\n"));
2520	ASMIntDisable();
2521	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2522	for (i = 0 ; i < UINT16_MAX*2 - 1 && RT_SUCCESS(rc) && pv2 == pv; i++)
2523	{
2524	pv2 = (void *)(intptr_t)-4;
2525	rc = pgmRZDynMapHCPageCommon(pVM, cr3, &pv2 RTLOG_COMMA_SRC_POS);
2526	}
2527	ASMIntEnable();
2528	if (RT_FAILURE(rc) \|\| pv != pv2)
2529	{
2530	LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%p\n", __LINE__, rc, pv, pv2, i));
2531	if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
2532	}
2533	else if (pSet->cEntries != 5)
2534	{
2535	LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries) / 2));
2536	rc = VERR_INTERNAL_ERROR;
2537	}
2538	else if ( pSet->aEntries[4].cRefs != UINT16_MAX - 1
2539	\|\| pSet->aEntries[3].cRefs != UINT16_MAX - 1
2540	\|\| pSet->aEntries[2].cRefs != 1
2541	\|\| pSet->aEntries[1].cRefs != 1
2542	\|\| pSet->aEntries[0].cRefs != 1)
2543	{
2544	LogRel(("failed(%d): bad set dist: ", __LINE__));
2545	for (i = 0; i < pSet->cEntries; i++)
2546	LogRel(("[%d]=%d, ", i, pSet->aEntries[i].cRefs));
2547	LogRel(("\n"));
2548	rc = VERR_INTERNAL_ERROR;
2549	}
2550	if (RT_SUCCESS(rc))
2551	rc = PGMR0DynMapAssertIntegrity();
2552	if (RT_SUCCESS(rc))
2553	{
2554	/*
2555	* Trigger an set optimization run (exactly).
2556	*/
2557	LogRel(("Test #3\n"));
2558	ASMIntDisable();
2559	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2560	pv2 = NULL;
2561	for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) - 5 && RT_SUCCESS(rc) && pv2 != pv; i++)
2562	{
2563	pv2 = (void *)(intptr_t)(-5 - i);
2564	rc = pgmRZDynMapHCPageCommon(pVM, cr3 + PAGE_SIZE * (i + 5), &pv2 RTLOG_COMMA_SRC_POS);
2565	}
2566	ASMIntEnable();
2567	if (RT_FAILURE(rc) \|\| pv == pv2)
2568	{
2569	LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%d\n", __LINE__, rc, pv, pv2, i));
2570	if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
2571	}
2572	else if (pSet->cEntries != RT_ELEMENTS(pSet->aEntries))
2573	{
2574	LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2575	rc = VERR_INTERNAL_ERROR;
2576	}
2577	LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2578	if (RT_SUCCESS(rc))
2579	rc = PGMR0DynMapAssertIntegrity();
2580	if (RT_SUCCESS(rc))
2581	{
2582	/*
2583	* Trigger an overflow error.
2584	*/
2585	LogRel(("Test #4\n"));
2586	ASMIntDisable();
2587	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2588	for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) + 2; i++)
2589	{
2590	rc = pgmRZDynMapHCPageCommon(pVM, cr3 - PAGE_SIZE * (i + 5), &pv2 RTLOG_COMMA_SRC_POS);
2591	if (RT_SUCCESS(rc))
2592	rc = PGMR0DynMapAssertIntegrity();
2593	if (RT_FAILURE(rc))
2594	break;
2595	}
2596	ASMIntEnable();
2597	if (rc == VERR_PGM_DYNMAP_FULL_SET)
2598	{
2599	/* flush the set. */
2600	LogRel(("Test #5\n"));
2601	ASMIntDisable();
2602	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2603	PGMRZDynMapReleaseAutoSet(&pVM->aCpus[0]);
2604	PGMRZDynMapStartAutoSet(&pVM->aCpus[0]);
2605	ASMIntEnable();
2606
2607	rc = PGMR0DynMapAssertIntegrity();
2608	}
2609	else
2610	{
2611	LogRel(("failed(%d): rc=%Rrc, wanted %d ; pv2=%p Set=%u/%u; i=%d\n", __LINE__,
2612	rc, VERR_PGM_DYNMAP_FULL_SET, pv2, pSet->cEntries, RT_ELEMENTS(pSet->aEntries), i));
2613	if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
2614	}
2615	}
2616	}
2617	}
2618	else
2619	{
2620	LogRel(("failed(%d): rc=%Rrc rc2=%Rrc; pv=%p pv2=%p\n", __LINE__, rc, rc2, pv, pv2));
2621	if (RT_SUCCESS(rc))
2622	rc = rc2;
2623	}
2624
2625	/*
2626	* Check that everyone sees the same stuff.
2627	*/
2628	if (RT_SUCCESS(rc))
2629	{
2630	LogRel(("Test #5\n"));
2631	ASMIntDisable();
2632	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2633	RTHCPHYS HCPhysPT = RTR0MemObjGetPagePhysAddr(pThis->pSegHead->ahMemObjPTs[0], 0);
2634	rc = pgmRZDynMapHCPageCommon(pVM, HCPhysPT, &pv RTLOG_COMMA_SRC_POS);
2635	if (RT_SUCCESS(rc))
2636	{
2637	PGMR0DYNMAPTEST Test;
2638	uint32_t *pu32Real = &pThis->paPages[pThis->pSegHead->iPage].uPte.pLegacy->u;
2639	Test.pu32 = (uint32_t *)((uintptr_t)pv \| ((uintptr_t)pu32Real & PAGE_OFFSET_MASK));
2640	Test.u32Expect = *pu32Real;
2641	ASMAtomicWriteU32(&Test.cFailures, 0);
2642	ASMIntEnable();
2643
2644	rc = RTMpOnAll(pgmR0DynMapTest3PerCpu, &Test, NULL);
2645	if (RT_FAILURE(rc))
2646	LogRel(("failed(%d): RTMpOnAll rc=%Rrc\n", __LINE__, rc));
2647	else if (Test.cFailures)
2648	{
2649	LogRel(("failed(%d): cFailures=%d pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n", __LINE__,
2650	Test.cFailures, pu32Real, Test.pu32, Test.u32Expect, *Test.pu32));
2651	rc = VERR_INTERNAL_ERROR;
2652	}
2653	else
2654	LogRel(("pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n",
2655	pu32Real, Test.pu32, Test.u32Expect, *Test.pu32));
2656	}
2657	else
2658	{
2659	ASMIntEnable();
2660	LogRel(("failed(%d): rc=%Rrc\n", rc));
2661	}
2662	}
2663
2664	/*
2665	* Clean up.
2666	*/
2667	LogRel(("Cleanup.\n"));
2668	ASMIntDisable();
2669	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2670	PGMRZDynMapFlushAutoSet(&pVM->aCpus[0]);
2671	PGMRZDynMapReleaseAutoSet(&pVM->aCpus[0]);
2672	ASMIntEnable();
2673
2674	if (RT_SUCCESS(rc))
2675	rc = PGMR0DynMapAssertIntegrity();
2676	else
2677	PGMR0DynMapAssertIntegrity();
2678
2679	g_fPGMR0DynMapTestRunning = false;
2680	LogRel(("Result: rc=%Rrc Load=%u/%u/%u Set=%#x/%u\n", rc,
2681	pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2682	pVM->pgm.s.pvR0DynMapUsed = pvR0DynMapUsedSaved;
2683	LogRel(("pgmR0DynMapTest: **** END ****\n"));
2684	return rc;
2685	}
2686	#endif /* DEBUG */
2687

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMRZ/PGMRZDynMap.cpp@ 36944

Download in other formats: