PGMRZDynMap.cpp@ 61574

Last change on this file since 61574 was 60228, checked in by vboxsync, 9 years ago
asm-amd64-x86.h,*: made ASMInvalidatePage take a flat unsigned pointer instead of a void pointer (simpler for 16-bit mode). Added some of the bits for 1GB pages.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 98.0 KB

Line
1	/* $Id: PGMRZDynMap.cpp 60228 2016-03-28 18:28:22Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, dynamic mapping cache.
4	*/
5
6	/*
7	* Copyright (C) 2008-2015 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Internal Functions *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM_DYNMAP
23	#include <VBox/vmm/pgm.h>
24	#include "PGMInternal.h"
25	#include <VBox/vmm/vm.h>
26	#include "PGMInline.h"
27	#include <VBox/err.h>
28	#include <VBox/param.h>
29	#include <VBox/sup.h>
30	#include <iprt/asm.h>
31	#include <iprt/asm-amd64-x86.h>
32	#include <iprt/assert.h>
33	#ifndef IN_RC
34	# include <iprt/cpuset.h>
35	# include <iprt/mem.h>
36	# include <iprt/memobj.h>
37	# include <iprt/mp.h>
38	# include <iprt/semaphore.h>
39	# include <iprt/spinlock.h>
40	#endif
41	#include <iprt/string.h>
42
43
44	/*********************************************************************************************************************************
45	* Defined Constants And Macros *
46	*********************************************************************************************************************************/
47	#ifdef IN_RING0
48	/** The max size of the mapping cache (in pages). */
49	# define PGMR0DYNMAP_MAX_PAGES ((16*_1M) >> PAGE_SHIFT)
50	/** The small segment size that is adopted on out-of-memory conditions with a
51	* single big segment. */
52	# define PGMR0DYNMAP_SMALL_SEG_PAGES 128
53	/** The number of pages we reserve per CPU. */
54	# define PGMR0DYNMAP_PAGES_PER_CPU 256
55	/** The minimum number of pages we reserve per CPU.
56	* This must be equal or larger than the autoset size. */
57	# define PGMR0DYNMAP_PAGES_PER_CPU_MIN 64
58	/** Calcs the overload threshold (safety margin). Current set at 50%. */
59	# define PGMR0DYNMAP_CALC_OVERLOAD(cPages) ((cPages) / 2)
60	/** The number of guard pages.
61	* @remarks Never do tuning of the hashing or whatnot with a strict build! */
62	# if defined(VBOX_STRICT)
63	# define PGMR0DYNMAP_GUARD_PAGES 1
64	# else
65	# define PGMR0DYNMAP_GUARD_PAGES 0
66	# endif
67	#endif /* IN_RING0 */
68	/** The dummy physical address of guard pages. */
69	#define PGMR0DYNMAP_GUARD_PAGE_HCPHYS UINT32_C(0x7777feed)
70	/** The dummy reference count of guard pages. (Must be non-zero.) */
71	#define PGMR0DYNMAP_GUARD_PAGE_REF_COUNT INT32_C(0x7777feed)
72	#if 0
73	/** Define this to just clear the present bit on guard pages.
74	* The alternative is to replace the entire PTE with an bad not-present
75	* PTE. Either way, XNU will screw us. :-/ */
76	# define PGMR0DYNMAP_GUARD_NP
77	#endif
78	/** The dummy PTE value for a page. */
79	#define PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE X86_PTE_PG_MASK
80	/** The dummy PTE value for a page. */
81	#define PGMR0DYNMAP_GUARD_PAGE_PAE_PTE UINT64_MAX /X86_PTE_PAE_PG_MASK/
82
83	#ifdef IN_RING0 /* Note! Assertions causes panics if preemption is disabled,
84	* disable this to work around that. */
85	/**
86	* Acquire the spinlock.
87	* This will declare a temporary variable and expands to two statements!
88	*/
89	# define PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis) \
90	RTSpinlockAcquire((pThis)->hSpinlock)
91
92	/**
93	* Releases the spinlock.
94	*/
95	# define PGMRZDYNMAP_SPINLOCK_RELEASE(pThis) \
96	RTSpinlockRelease((pThis)->hSpinlock)
97
98	/**
99	* Re-acquires the spinlock.
100	*/
101	# define PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis) \
102	RTSpinlockAcquire((pThis)->hSpinlock)
103	#else
104	# define PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis) do { } while (0)
105	# define PGMRZDYNMAP_SPINLOCK_RELEASE(pThis) do { } while (0)
106	# define PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis) do { } while (0)
107	#endif
108
109
110	/** Converts a PGMCPUM::AutoSet pointer into a PVMCPU. */
111	#define PGMRZDYNMAP_SET_2_VMCPU(pSet) (RT_FROM_MEMBER(pSet, VMCPU, pgm.s.AutoSet))
112
113	/** Converts a PGMCPUM::AutoSet pointer into a PVM. */
114	#define PGMRZDYNMAP_SET_2_VM(pSet) (PGMRZDYNMAP_SET_2_VMCPU(pSet)->CTX_SUFF(pVM))
115
116	/** Converts a PGMCPUM::AutoSet pointer into a PVM. */
117	#ifdef IN_RC
118	# define PGMRZDYNMAP_SET_2_DYNMAP(pSet) (PGMRZDYNMAP_SET_2_VM(pSet)->pgm.s.pRCDynMap)
119	#else
120	# define PGMRZDYNMAP_SET_2_DYNMAP(pSet) (g_pPGMR0DynMap)
121	#endif
122
123	/**
124	* Gets the set index of the current CPU.
125	*
126	* This always returns 0 when in raw-mode context because there is only ever
127	* one EMT in that context (at least presently).
128	*/
129	#ifdef IN_RC
130	# define PGMRZDYNMAP_CUR_CPU() (0)
131	#else
132	# define PGMRZDYNMAP_CUR_CPU() RTMpCurSetIndex()
133	#endif
134
135	/** PGMRZDYNMAP::u32Magic. (Jens Christian Bugge Wesseltoft) */
136	#define PGMRZDYNMAP_MAGIC UINT32_C(0x19640201)
137
138
139	/** Zaps an set entry. */
140	#define PGMRZDYNMAP_ZAP_ENTRY(pEntry) \
141	do \
142	{ \
143	(pEntry)->iPage = UINT16_MAX; \
144	(pEntry)->cRefs = 0; \
145	(pEntry)->cInlinedRefs = 0; \
146	(pEntry)->cUnrefs = 0; \
147	} while (0)
148
149
150	/** @def PGMRZDYNMAP_STRICT_RELEASE
151	* Define this to force pages to be released and make non-present ASAP after
152	* use. This should not normally be enabled as it is a bit expensive. */
153	#if 0 \|\| defined(DOXYGEN_RUNNING)
154	# define PGMRZDYNMAP_STRICT_RELEASE
155	#endif
156
157
158	/*********************************************************************************************************************************
159	* Structures and Typedefs *
160	*********************************************************************************************************************************/
161	#ifdef IN_RING0
162	/**
163	* Ring-0 dynamic mapping cache segment.
164	*
165	* The dynamic mapping cache can be extended with additional segments if the
166	* load is found to be too high. This done the next time a VM is created, under
167	* the protection of the init mutex. The arrays is reallocated and the new
168	* segment is added to the end of these. Nothing is rehashed of course, as the
169	* indexes / addresses must remain unchanged.
170	*
171	* This structure is only modified while owning the init mutex or during module
172	* init / term.
173	*/
174	typedef struct PGMR0DYNMAPSEG
175	{
176	/** Pointer to the next segment. */
177	struct PGMR0DYNMAPSEG *pNext;
178	/** The memory object for the virtual address range that we're abusing. */
179	RTR0MEMOBJ hMemObj;
180	/** The start page in the cache. (I.e. index into the arrays.) */
181	uint16_t iPage;
182	/** The number of pages this segment contributes. */
183	uint16_t cPages;
184	/** The number of page tables. */
185	uint16_t cPTs;
186	/** The memory objects for the page tables. */
187	RTR0MEMOBJ ahMemObjPTs[1];
188	} PGMR0DYNMAPSEG;
189	/** Pointer to a ring-0 dynamic mapping cache segment. */
190	typedef PGMR0DYNMAPSEG *PPGMR0DYNMAPSEG;
191
192
193	/**
194	* Ring-0 dynamic mapping cache entry.
195	*
196	* @sa PGMRZDYNMAPENTRY, PGMRCDYNMAPENTRY.
197	*/
198	typedef struct PGMR0DYNMAPENTRY
199	{
200	/** The physical address of the currently mapped page.
201	* This is duplicate for three reasons: cache locality, cache policy of the PT
202	* mappings and sanity checks. */
203	RTHCPHYS HCPhys;
204	/** Pointer to the page. */
205	void *pvPage;
206	/** The number of references. */
207	int32_t volatile cRefs;
208	/** PTE pointer union. */
209	union PGMR0DYNMAPENTRY_PPTE
210	{
211	/** PTE pointer, 32-bit legacy version. */
212	PX86PTE pLegacy;
213	/** PTE pointer, PAE version. */
214	PX86PTEPAE pPae;
215	/** PTE pointer, the void version. */
216	void *pv;
217	} uPte;
218	/** CPUs that haven't invalidated this entry after it's last update. */
219	RTCPUSET PendingSet;
220	} PGMR0DYNMAPENTRY;
221	/** Pointer a mapping cache entry for the ring-0.
222	* @sa PPGMRZDYNMAPENTRY, PPGMRCDYNMAPENTRY, */
223	typedef PGMR0DYNMAPENTRY *PPGMR0DYNMAPENTRY;
224
225
226	/**
227	* Dynamic mapping cache for ring-0.
228	*
229	* This is initialized during VMMR0 module init but no segments are allocated
230	* at that time. Segments will be added when the first VM is started and
231	* removed again when the last VM shuts down, thus avoid consuming memory while
232	* dormant. At module termination, the remaining bits will be freed up.
233	*
234	* @sa PPGMRZDYNMAP, PGMRCDYNMAP.
235	*/
236	typedef struct PGMR0DYNMAP
237	{
238	/** The usual magic number / eye catcher (PGMRZDYNMAP_MAGIC). */
239	uint32_t u32Magic;
240	/** Spinlock serializing the normal operation of the cache. */
241	RTSPINLOCK hSpinlock;
242	/** Array for tracking and managing the pages. */
243	PPGMR0DYNMAPENTRY paPages;
244	/** The cache size given as a number of pages. */
245	uint32_t cPages;
246	/** Whether it's 32-bit legacy or PAE/AMD64 paging mode. */
247	bool fLegacyMode;
248	/** The current load.
249	* This does not include guard pages. */
250	uint32_t cLoad;
251	/** The max load ever.
252	* This is maintained to trigger the adding of more mapping space. */
253	uint32_t cMaxLoad;
254	/** Initialization / termination lock. */
255	RTSEMFASTMUTEX hInitLock;
256	/** The number of guard pages. */
257	uint32_t cGuardPages;
258	/** The number of users (protected by hInitLock). */
259	uint32_t cUsers;
260	/** Array containing a copy of the original page tables.
261	* The entries are either X86PTE or X86PTEPAE according to fLegacyMode. */
262	void *pvSavedPTEs;
263	/** List of segments. */
264	PPGMR0DYNMAPSEG pSegHead;
265	/** The paging mode. */
266	SUPPAGINGMODE enmPgMode;
267	} PGMR0DYNMAP;
268
269
270	/**
271	* Paging level data.
272	*/
273	typedef struct PGMR0DYNMAPPGLVL
274	{
275	uint32_t cLevels; /*< The number of levels. /
276	struct
277	{
278	RTHCPHYS HCPhys; /**< The address of the page for the current level,
279	* i.e. what hMemObj/hMapObj is currently mapping. */
280	RTHCPHYS fPhysMask; /*< Mask for extracting HCPhys from uEntry. /
281	RTR0MEMOBJ hMemObj; /*< Memory object for HCPhys, PAGE_SIZE. /
282	RTR0MEMOBJ hMapObj; /*< Mapping object for hMemObj. /
283	uint32_t fPtrShift; /*< The pointer shift count. /
284	uint64_t fPtrMask; /*< The mask to apply to the shifted pointer to get the table index. /
285	uint64_t fAndMask; /*< And mask to check entry flags. /
286	uint64_t fResMask; /*< The result from applying fAndMask. /
287	union
288	{
289	void pv; /< hMapObj address. /
290	PX86PGUINT paLegacy; /*< Legacy table view. /
291	PX86PGPAEUINT paPae; /*< PAE/AMD64 table view. /
292	} u;
293	} a[4];
294	} PGMR0DYNMAPPGLVL;
295	/** Pointer to paging level data. */
296	typedef PGMR0DYNMAPPGLVL *PPGMR0DYNMAPPGLVL;
297	#endif
298
299	/** Mapping cache entry for the current context.
300	* @sa PGMR0DYNMAPENTRY, PGMRCDYNMAPENTRY */
301	typedef CTX_MID(PGM,DYNMAPENTRY) PGMRZDYNMAPENTRY;
302	/** Pointer a mapping cache entry for the current context.
303	* @sa PGMR0DYNMAPENTRY, PGMRCDYNMAPENTRY */
304	typedef PGMRZDYNMAPENTRY *PPGMRZDYNMAPENTRY;
305
306	/** Pointer to the mapping cache instance for the current context.
307	* @sa PGMR0DYNMAP, PGMRCDYNMAP */
308	typedef CTX_MID(PGM,DYNMAP) *PPGMRZDYNMAP;
309
310
311
312	/*********************************************************************************************************************************
313	* Global Variables *
314	*********************************************************************************************************************************/
315	#ifdef IN_RING0
316	/** Pointer to the ring-0 dynamic mapping cache. */
317	static PGMR0DYNMAP *g_pPGMR0DynMap;
318	#endif
319	/** For overflow testing. */
320	static bool g_fPGMR0DynMapTestRunning = false;
321
322
323	/*********************************************************************************************************************************
324	* Internal Functions *
325	*********************************************************************************************************************************/
326	static void pgmRZDynMapReleasePage(PPGMRZDYNMAP pThis, uint32_t iPage, uint32_t cRefs);
327	#ifdef IN_RING0
328	static int pgmR0DynMapSetup(PPGMRZDYNMAP pThis);
329	static int pgmR0DynMapExpand(PPGMRZDYNMAP pThis);
330	static void pgmR0DynMapTearDown(PPGMRZDYNMAP pThis);
331	#endif
332	#if 0 /def DEBUG/
333	static int pgmR0DynMapTest(PVM pVM);
334	#endif
335
336
337	/**
338	* Initializes the auto mapping sets for a VM.
339	*
340	* @returns VINF_SUCCESS on success, VERR_PGM_DYNMAP_IPE on failure.
341	* @param pVM The cross context VM structure.
342	*/
343	static int pgmRZDynMapInitAutoSetsForVM(PVM pVM)
344	{
345	VMCPUID idCpu = pVM->cCpus;
346	AssertReturn(idCpu > 0 && idCpu <= VMM_MAX_CPU_COUNT, VERR_PGM_DYNMAP_IPE);
347	while (idCpu-- > 0)
348	{
349	PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
350	uint32_t j = RT_ELEMENTS(pSet->aEntries);
351	while (j-- > 0)
352	{
353	pSet->aEntries[j].pvPage = NULL;
354	pSet->aEntries[j].HCPhys = NIL_RTHCPHYS;
355	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]);
356	}
357	pSet->cEntries = PGMMAPSET_CLOSED;
358	pSet->iSubset = UINT32_MAX;
359	pSet->iCpu = -1;
360	memset(&pSet->aiHashTable[0], 0xff, sizeof(pSet->aiHashTable));
361	}
362
363	return VINF_SUCCESS;
364	}
365
366
367	#ifdef IN_RING0
368
369	/**
370	* Initializes the ring-0 dynamic mapping cache.
371	*
372	* @returns VBox status code.
373	*/
374	VMMR0DECL(int) PGMR0DynMapInit(void)
375	{
376	Assert(!g_pPGMR0DynMap);
377
378	/*
379	* Create and initialize the cache instance.
380	*/
381	PPGMRZDYNMAP pThis = (PPGMRZDYNMAP)RTMemAllocZ(sizeof(*pThis));
382	AssertLogRelReturn(pThis, VERR_NO_MEMORY);
383	int rc = VINF_SUCCESS;
384	pThis->enmPgMode = SUPR0GetPagingMode();
385	switch (pThis->enmPgMode)
386	{
387	case SUPPAGINGMODE_32_BIT:
388	case SUPPAGINGMODE_32_BIT_GLOBAL:
389	pThis->fLegacyMode = false;
390	break;
391	case SUPPAGINGMODE_PAE:
392	case SUPPAGINGMODE_PAE_GLOBAL:
393	case SUPPAGINGMODE_PAE_NX:
394	case SUPPAGINGMODE_PAE_GLOBAL_NX:
395	case SUPPAGINGMODE_AMD64:
396	case SUPPAGINGMODE_AMD64_GLOBAL:
397	case SUPPAGINGMODE_AMD64_NX:
398	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
399	pThis->fLegacyMode = false;
400	break;
401	default:
402	rc = VERR_PGM_DYNMAP_IPE;
403	break;
404	}
405	if (RT_SUCCESS(rc))
406	{
407	rc = RTSemFastMutexCreate(&pThis->hInitLock);
408	if (RT_SUCCESS(rc))
409	{
410	rc = RTSpinlockCreate(&pThis->hSpinlock, RTSPINLOCK_FLAGS_INTERRUPT_UNSAFE, "PGMR0DynMap");
411	if (RT_SUCCESS(rc))
412	{
413	pThis->u32Magic = PGMRZDYNMAP_MAGIC;
414	g_pPGMR0DynMap = pThis;
415	return VINF_SUCCESS;
416	}
417	RTSemFastMutexDestroy(pThis->hInitLock);
418	}
419	}
420	RTMemFree(pThis);
421	return rc;
422	}
423
424
425	/**
426	* Terminates the ring-0 dynamic mapping cache.
427	*/
428	VMMR0DECL(void) PGMR0DynMapTerm(void)
429	{
430	/*
431	* Destroy the cache.
432	*
433	* There is not supposed to be any races here, the loader should
434	* make sure about that. So, don't bother locking anything.
435	*
436	* The VM objects should all be destroyed by now, so there is no
437	* dangling users or anything like that to clean up. This routine
438	* is just a mirror image of PGMR0DynMapInit.
439	*/
440	PPGMRZDYNMAP pThis = g_pPGMR0DynMap;
441	if (pThis)
442	{
443	AssertPtr(pThis);
444	g_pPGMR0DynMap = NULL;
445
446	/* This should never happen, but in case it does try not to leak memory. */
447	AssertLogRelMsg(!pThis->cUsers && !pThis->paPages && !pThis->pvSavedPTEs && !pThis->cPages,
448	("cUsers=%d paPages=%p pvSavedPTEs=%p cPages=%#x\n",
449	pThis->cUsers, pThis->paPages, pThis->pvSavedPTEs, pThis->cPages));
450	if (pThis->paPages)
451	pgmR0DynMapTearDown(pThis);
452
453	/* Free the associated resources. */
454	RTSemFastMutexDestroy(pThis->hInitLock);
455	pThis->hInitLock = NIL_RTSEMFASTMUTEX;
456	RTSpinlockDestroy(pThis->hSpinlock);
457	pThis->hSpinlock = NIL_RTSPINLOCK;
458	pThis->u32Magic = UINT32_MAX;
459	RTMemFree(pThis);
460	}
461	}
462
463
464	/**
465	* Initializes the dynamic mapping cache for a new VM.
466	*
467	* @returns VBox status code.
468	* @param pVM The cross context VM structure.
469	*/
470	VMMR0DECL(int) PGMR0DynMapInitVM(PVM pVM)
471	{
472	AssertMsgReturn(!pVM->pgm.s.pvR0DynMapUsed, ("%p (pThis=%p)\n", pVM->pgm.s.pvR0DynMapUsed, g_pPGMR0DynMap), VERR_WRONG_ORDER);
473
474	/*
475	* Initialize the auto sets.
476	*/
477	int rc = pgmRZDynMapInitAutoSetsForVM(pVM);
478	if (RT_FAILURE(rc))
479	return rc;
480
481	/*
482	* Do we need the cache? Skip the last bit if we don't.
483	*/
484	if (!HMIsEnabled(pVM))
485	return VINF_SUCCESS;
486
487	/*
488	* Reference and if necessary setup or expand the cache.
489	*/
490	PPGMRZDYNMAP pThis = g_pPGMR0DynMap;
491	AssertPtrReturn(pThis, VERR_PGM_DYNMAP_IPE);
492	rc = RTSemFastMutexRequest(pThis->hInitLock);
493	AssertLogRelRCReturn(rc, rc);
494
495	pThis->cUsers++;
496	if (pThis->cUsers == 1)
497	{
498	rc = pgmR0DynMapSetup(pThis);
499	#if 0 /def DEBUG/
500	if (RT_SUCCESS(rc))
501	{
502	rc = pgmR0DynMapTest(pVM);
503	if (RT_FAILURE(rc))
504	pgmR0DynMapTearDown(pThis);
505	}
506	#endif
507	}
508	else if (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(pThis->cPages - pThis->cGuardPages))
509	rc = pgmR0DynMapExpand(pThis);
510	if (RT_SUCCESS(rc))
511	pVM->pgm.s.pvR0DynMapUsed = pThis;
512	else
513	pThis->cUsers--;
514
515	RTSemFastMutexRelease(pThis->hInitLock);
516	return rc;
517	}
518
519
520	/**
521	* Terminates the dynamic mapping cache usage for a VM.
522	*
523	* @param pVM The cross context VM structure.
524	*/
525	VMMR0DECL(void) PGMR0DynMapTermVM(PVM pVM)
526	{
527	/*
528	* Return immediately if we're not using the cache.
529	*/
530	if (!pVM->pgm.s.pvR0DynMapUsed)
531	return;
532
533	PPGMRZDYNMAP pThis = g_pPGMR0DynMap;
534	AssertPtrReturnVoid(pThis);
535
536	int rc = RTSemFastMutexRequest(pThis->hInitLock);
537	AssertLogRelRCReturnVoid(rc);
538
539	if (pVM->pgm.s.pvR0DynMapUsed == pThis)
540	{
541	pVM->pgm.s.pvR0DynMapUsed = NULL;
542
543	#ifdef VBOX_STRICT
544	PGMR0DynMapAssertIntegrity();
545	#endif
546
547	/*
548	* Clean up and check the auto sets.
549	*/
550	VMCPUID idCpu = pVM->cCpus;
551	while (idCpu-- > 0)
552	{
553	PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
554	uint32_t j = pSet->cEntries;
555	if (j <= RT_ELEMENTS(pSet->aEntries))
556	{
557	/*
558	* The set is open, close it.
559	*/
560	while (j-- > 0)
561	{
562	int32_t cRefs = pSet->aEntries[j].cRefs;
563	uint32_t iPage = pSet->aEntries[j].iPage;
564	LogRel(("PGMR0DynMapTermVM: %d dangling refs to %#x\n", cRefs, iPage));
565	if (iPage < pThis->cPages && cRefs > 0)
566	pgmRZDynMapReleasePage(pThis, iPage, cRefs);
567	else
568	AssertLogRelMsgFailed(("cRefs=%d iPage=%#x cPages=%u\n", cRefs, iPage, pThis->cPages));
569
570	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]);
571	}
572	pSet->cEntries = PGMMAPSET_CLOSED;
573	pSet->iSubset = UINT32_MAX;
574	pSet->iCpu = -1;
575	}
576	else
577	AssertMsg(j == PGMMAPSET_CLOSED, ("cEntries=%#x\n", j));
578
579	j = RT_ELEMENTS(pSet->aEntries);
580	while (j-- > 0)
581	{
582	Assert(pSet->aEntries[j].iPage == UINT16_MAX);
583	Assert(!pSet->aEntries[j].cRefs);
584	}
585	}
586
587	/*
588	* Release our reference to the mapping cache.
589	*/
590	Assert(pThis->cUsers > 0);
591	pThis->cUsers--;
592	if (!pThis->cUsers)
593	pgmR0DynMapTearDown(pThis);
594	}
595	else
596	AssertLogRelMsgFailed(("pvR0DynMapUsed=%p pThis=%p\n", pVM->pgm.s.pvR0DynMapUsed, pThis));
597
598	RTSemFastMutexRelease(pThis->hInitLock);
599	}
600
601
602	/**
603	* Shoots down the TLBs for all the cache pages, pgmR0DynMapTearDown helper.
604	*
605	* @param idCpu The current CPU.
606	* @param pvUser1 The dynamic mapping cache instance.
607	* @param pvUser2 Unused, NULL.
608	*/
609	static DECLCALLBACK(void) pgmR0DynMapShootDownTlbs(RTCPUID idCpu, void pvUser1, void pvUser2)
610	{
611	Assert(!pvUser2);
612	PPGMRZDYNMAP pThis = (PPGMRZDYNMAP)pvUser1;
613	Assert(pThis == g_pPGMR0DynMap);
614	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
615	uint32_t iPage = pThis->cPages;
616	while (iPage-- > 0)
617	ASMInvalidatePage((uintptr_t)paPages[iPage].pvPage);
618	}
619
620
621	/**
622	* Shoot down the TLBs for every single cache entry on all CPUs.
623	*
624	* @returns IPRT status code (RTMpOnAll).
625	* @param pThis The dynamic mapping cache instance.
626	*/
627	static int pgmR0DynMapTlbShootDown(PPGMRZDYNMAP pThis)
628	{
629	int rc = RTMpOnAll(pgmR0DynMapShootDownTlbs, pThis, NULL);
630	AssertRC(rc);
631	if (RT_FAILURE(rc))
632	{
633	uint32_t iPage = pThis->cPages;
634	while (iPage-- > 0)
635	ASMInvalidatePage((uintptr_t)pThis->paPages[iPage].pvPage);
636	}
637	return rc;
638	}
639
640
641	/**
642	* Calculate the new cache size based on cMaxLoad statistics.
643	*
644	* @returns Number of pages.
645	* @param pThis The dynamic mapping cache instance.
646	* @param pcMinPages The minimal size in pages.
647	*/
648	static uint32_t pgmR0DynMapCalcNewSize(PPGMRZDYNMAP pThis, uint32_t *pcMinPages)
649	{
650	Assert(pThis->cPages <= PGMR0DYNMAP_MAX_PAGES);
651
652	/* cCpus * PGMR0DYNMAP_PAGES_PER_CPU(_MIN). */
653	RTCPUID cCpus = RTMpGetCount();
654	AssertReturn(cCpus > 0 && cCpus <= RTCPUSET_MAX_CPUS, 0);
655	uint32_t cPages = cCpus * PGMR0DYNMAP_PAGES_PER_CPU;
656	uint32_t cMinPages = cCpus * PGMR0DYNMAP_PAGES_PER_CPU_MIN;
657
658	/* adjust against cMaxLoad. */
659	AssertMsg(pThis->cMaxLoad <= PGMR0DYNMAP_MAX_PAGES, ("%#x\n", pThis->cMaxLoad));
660	if (pThis->cMaxLoad > PGMR0DYNMAP_MAX_PAGES)
661	pThis->cMaxLoad = 0;
662
663	while (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(cPages))
664	cPages += PGMR0DYNMAP_PAGES_PER_CPU;
665
666	if (pThis->cMaxLoad > cMinPages)
667	cMinPages = pThis->cMaxLoad;
668
669	/* adjust against max and current size. */
670	if (cPages < pThis->cPages)
671	cPages = pThis->cPages;
672	cPages *= PGMR0DYNMAP_GUARD_PAGES + 1;
673	if (cPages > PGMR0DYNMAP_MAX_PAGES)
674	cPages = PGMR0DYNMAP_MAX_PAGES;
675
676	if (cMinPages < pThis->cPages)
677	cMinPages = pThis->cPages;
678	cMinPages *= PGMR0DYNMAP_GUARD_PAGES + 1;
679	if (cMinPages > PGMR0DYNMAP_MAX_PAGES)
680	cMinPages = PGMR0DYNMAP_MAX_PAGES;
681
682	Assert(cMinPages);
683	*pcMinPages = cMinPages;
684	return cPages;
685	}
686
687
688	/**
689	* Initializes the paging level data.
690	*
691	* @param pThis The dynamic mapping cache instance.
692	* @param pPgLvl The paging level data.
693	*/
694	void pgmR0DynMapPagingArrayInit(PPGMRZDYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl)
695	{
696	RTCCUINTREG cr4 = ASMGetCR4();
697	switch (pThis->enmPgMode)
698	{
699	case SUPPAGINGMODE_32_BIT:
700	case SUPPAGINGMODE_32_BIT_GLOBAL:
701	pPgLvl->cLevels = 2;
702	pPgLvl->a[0].fPhysMask = X86_CR3_PAGE_MASK;
703	pPgLvl->a[0].fAndMask = X86_PDE_P \| X86_PDE_RW \| (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
704	pPgLvl->a[0].fResMask = X86_PDE_P \| X86_PDE_RW;
705	pPgLvl->a[0].fPtrMask = X86_PD_MASK;
706	pPgLvl->a[0].fPtrShift = X86_PD_SHIFT;
707
708	pPgLvl->a[1].fPhysMask = X86_PDE_PG_MASK;
709	pPgLvl->a[1].fAndMask = X86_PTE_P \| X86_PTE_RW;
710	pPgLvl->a[1].fResMask = X86_PTE_P \| X86_PTE_RW;
711	pPgLvl->a[1].fPtrMask = X86_PT_MASK;
712	pPgLvl->a[1].fPtrShift = X86_PT_SHIFT;
713	break;
714
715	case SUPPAGINGMODE_PAE:
716	case SUPPAGINGMODE_PAE_GLOBAL:
717	case SUPPAGINGMODE_PAE_NX:
718	case SUPPAGINGMODE_PAE_GLOBAL_NX:
719	pPgLvl->cLevels = 3;
720	pPgLvl->a[0].fPhysMask = X86_CR3_PAE_PAGE_MASK;
721	pPgLvl->a[0].fPtrMask = X86_PDPT_MASK_PAE;
722	pPgLvl->a[0].fPtrShift = X86_PDPT_SHIFT;
723	pPgLvl->a[0].fAndMask = X86_PDPE_P;
724	pPgLvl->a[0].fResMask = X86_PDPE_P;
725
726	pPgLvl->a[1].fPhysMask = X86_PDPE_PG_MASK;
727	pPgLvl->a[1].fPtrMask = X86_PD_PAE_MASK;
728	pPgLvl->a[1].fPtrShift = X86_PD_PAE_SHIFT;
729	pPgLvl->a[1].fAndMask = X86_PDE_P \| X86_PDE_RW \| (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
730	pPgLvl->a[1].fResMask = X86_PDE_P \| X86_PDE_RW;
731
732	pPgLvl->a[2].fPhysMask = X86_PDE_PAE_PG_MASK;
733	pPgLvl->a[2].fPtrMask = X86_PT_PAE_MASK;
734	pPgLvl->a[2].fPtrShift = X86_PT_PAE_SHIFT;
735	pPgLvl->a[2].fAndMask = X86_PTE_P \| X86_PTE_RW;
736	pPgLvl->a[2].fResMask = X86_PTE_P \| X86_PTE_RW;
737	break;
738
739	case SUPPAGINGMODE_AMD64:
740	case SUPPAGINGMODE_AMD64_GLOBAL:
741	case SUPPAGINGMODE_AMD64_NX:
742	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
743	pPgLvl->cLevels = 4;
744	pPgLvl->a[0].fPhysMask = X86_CR3_AMD64_PAGE_MASK;
745	pPgLvl->a[0].fPtrShift = X86_PML4_SHIFT;
746	pPgLvl->a[0].fPtrMask = X86_PML4_MASK;
747	pPgLvl->a[0].fAndMask = X86_PML4E_P \| X86_PML4E_RW;
748	pPgLvl->a[0].fResMask = X86_PML4E_P \| X86_PML4E_RW;
749
750	pPgLvl->a[1].fPhysMask = X86_PML4E_PG_MASK;
751	pPgLvl->a[1].fPtrShift = X86_PDPT_SHIFT;
752	pPgLvl->a[1].fPtrMask = X86_PDPT_MASK_AMD64;
753	pPgLvl->a[1].fAndMask = X86_PDPE_P \| X86_PDPE_RW /** @todo check for X86_PDPT_PS support. */;
754	pPgLvl->a[1].fResMask = X86_PDPE_P \| X86_PDPE_RW;
755
756	pPgLvl->a[2].fPhysMask = X86_PDPE_PG_MASK;
757	pPgLvl->a[2].fPtrShift = X86_PD_PAE_SHIFT;
758	pPgLvl->a[2].fPtrMask = X86_PD_PAE_MASK;
759	pPgLvl->a[2].fAndMask = X86_PDE_P \| X86_PDE_RW \| (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
760	pPgLvl->a[2].fResMask = X86_PDE_P \| X86_PDE_RW;
761
762	pPgLvl->a[3].fPhysMask = X86_PDE_PAE_PG_MASK;
763	pPgLvl->a[3].fPtrShift = X86_PT_PAE_SHIFT;
764	pPgLvl->a[3].fPtrMask = X86_PT_PAE_MASK;
765	pPgLvl->a[3].fAndMask = X86_PTE_P \| X86_PTE_RW;
766	pPgLvl->a[3].fResMask = X86_PTE_P \| X86_PTE_RW;
767	break;
768
769	default:
770	AssertFailed();
771	pPgLvl->cLevels = 0;
772	break;
773	}
774
775	for (uint32_t i = 0; i < 4; i++) /* ASSUMING array size. */
776	{
777	pPgLvl->a[i].HCPhys = NIL_RTHCPHYS;
778	pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
779	pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ;
780	pPgLvl->a[i].u.pv = NULL;
781	}
782	}
783
784
785	/**
786	* Maps a PTE.
787	*
788	* This will update the segment structure when new PTs are mapped.
789	*
790	* It also assumes that we (for paranoid reasons) wish to establish a mapping
791	* chain from CR3 to the PT that all corresponds to the processor we're
792	* currently running on, and go about this by running with interrupts disabled
793	* and restarting from CR3 for every change.
794	*
795	* @returns VBox status code, VINF_TRY_AGAIN if we changed any mappings and had
796	* to re-enable interrupts.
797	* @param pThis The dynamic mapping cache instance.
798	* @param pPgLvl The paging level structure.
799	* @param pvPage The page.
800	* @param pSeg The segment.
801	* @param cMaxPTs The max number of PTs expected in the segment.
802	* @param ppvPTE Where to store the PTE address.
803	*/
804	static int pgmR0DynMapPagingArrayMapPte(PPGMRZDYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl, void *pvPage,
805	PPGMR0DYNMAPSEG pSeg, uint32_t cMaxPTs, void **ppvPTE)
806	{
807	Assert(!(ASMGetFlags() & X86_EFL_IF));
808	void *pvEntry = NULL;
809	X86PGPAEUINT uEntry = ASMGetCR3();
810	for (uint32_t i = 0; i < pPgLvl->cLevels; i++)
811	{
812	RTHCPHYS HCPhys = uEntry & pPgLvl->a[i].fPhysMask;
813	if (pPgLvl->a[i].HCPhys != HCPhys)
814	{
815	/*
816	* Need to remap this level.
817	* The final level, the PT, will not be freed since that is what it's all about.
818	*/
819	ASMIntEnable();
820	if (i + 1 == pPgLvl->cLevels)
821	AssertReturn(pSeg->cPTs < cMaxPTs, VERR_PGM_DYNMAP_IPE);
822	else
823	{
824	int rc2 = RTR0MemObjFree(pPgLvl->a[i].hMemObj, true /* fFreeMappings */); AssertRC(rc2);
825	pPgLvl->a[i].hMemObj = pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
826	}
827
828	int rc = RTR0MemObjEnterPhys(&pPgLvl->a[i].hMemObj, HCPhys, PAGE_SIZE, RTMEM_CACHE_POLICY_DONT_CARE);
829	if (RT_SUCCESS(rc))
830	{
831	rc = RTR0MemObjMapKernel(&pPgLvl->a[i].hMapObj, pPgLvl->a[i].hMemObj,
832	(void )-1 / pvFixed /, 0 / cbAlignment */,
833	RTMEM_PROT_WRITE \| RTMEM_PROT_READ);
834	if (RT_SUCCESS(rc))
835	{
836	pPgLvl->a[i].u.pv = RTR0MemObjAddress(pPgLvl->a[i].hMapObj);
837	AssertMsg(((uintptr_t)pPgLvl->a[i].u.pv & ~(uintptr_t)PAGE_OFFSET_MASK), ("%p\n", pPgLvl->a[i].u.pv));
838	pPgLvl->a[i].HCPhys = HCPhys;
839	if (i + 1 == pPgLvl->cLevels)
840	pSeg->ahMemObjPTs[pSeg->cPTs++] = pPgLvl->a[i].hMemObj;
841	ASMIntDisable();
842	return VINF_TRY_AGAIN;
843	}
844
845	pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
846	}
847	else
848	pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ;
849	pPgLvl->a[i].HCPhys = NIL_RTHCPHYS;
850	return rc;
851	}
852
853	/*
854	* The next level.
855	*/
856	uint32_t iEntry = ((uint64_t)(uintptr_t)pvPage >> pPgLvl->a[i].fPtrShift) & pPgLvl->a[i].fPtrMask;
857	if (pThis->fLegacyMode)
858	{
859	pvEntry = &pPgLvl->a[i].u.paLegacy[iEntry];
860	uEntry = pPgLvl->a[i].u.paLegacy[iEntry];
861	}
862	else
863	{
864	pvEntry = &pPgLvl->a[i].u.paPae[iEntry];
865	uEntry = pPgLvl->a[i].u.paPae[iEntry];
866	}
867
868	if ((uEntry & pPgLvl->a[i].fAndMask) != pPgLvl->a[i].fResMask)
869	{
870	LogRel(("PGMR0DynMap: internal error - iPgLvl=%u cLevels=%u uEntry=%#llx fAnd=%#llx fRes=%#llx got=%#llx\n"
871	"PGMR0DynMap: pv=%p pvPage=%p iEntry=%#x fLegacyMode=%RTbool\n",
872	i, pPgLvl->cLevels, uEntry, pPgLvl->a[i].fAndMask, pPgLvl->a[i].fResMask, uEntry & pPgLvl->a[i].fAndMask,
873	pPgLvl->a[i].u.pv, pvPage, iEntry, pThis->fLegacyMode));
874	return VERR_PGM_DYNMAP_IPE;
875	}
876	/Log(("#%d: iEntry=%4d uEntry=%#llx pvEntry=%p HCPhys=%RHp \n", i, iEntry, uEntry, pvEntry, pPgLvl->a[i].HCPhys));/
877	}
878
879	/* made it thru without needing to remap anything. */
880	*ppvPTE = pvEntry;
881	return VINF_SUCCESS;
882	}
883
884
885	/**
886	* Sets up a guard page.
887	*
888	* @param pThis The dynamic mapping cache instance.
889	* @param pPage The page.
890	*/
891	DECLINLINE(void) pgmR0DynMapSetupGuardPage(PPGMRZDYNMAP pThis, PPGMRZDYNMAPENTRY pPage)
892	{
893	memset(pPage->pvPage, 0xfd, PAGE_SIZE);
894	pPage->cRefs = PGMR0DYNMAP_GUARD_PAGE_REF_COUNT;
895	pPage->HCPhys = PGMR0DYNMAP_GUARD_PAGE_HCPHYS;
896	#ifdef PGMR0DYNMAP_GUARD_NP
897	ASMAtomicBitClear(pPage->uPte.pv, X86_PTE_BIT_P);
898	#else
899	if (pThis->fLegacyMode)
900	ASMAtomicWriteU32(&pPage->uPte.pLegacy->u, PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE);
901	else
902	ASMAtomicWriteU64(&pPage->uPte.pPae->u, PGMR0DYNMAP_GUARD_PAGE_PAE_PTE);
903	#endif
904	pThis->cGuardPages++;
905	}
906
907
908	/**
909	* Adds a new segment of the specified size.
910	*
911	* @returns VBox status code.
912	* @param pThis The dynamic mapping cache instance.
913	* @param cPages The size of the new segment, give as a page count.
914	*/
915	static int pgmR0DynMapAddSeg(PPGMRZDYNMAP pThis, uint32_t cPages)
916	{
917	int rc2;
918	AssertReturn(ASMGetFlags() & X86_EFL_IF, VERR_PREEMPT_DISABLED);
919
920	/*
921	* Do the array reallocations first.
922	* (The pages array has to be replaced behind the spinlock of course.)
923	*/
924	void pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) (pThis->cPages + cPages));
925	if (!pvSavedPTEs)
926	return VERR_NO_MEMORY;
927	pThis->pvSavedPTEs = pvSavedPTEs;
928
929	void pvPages = RTMemAllocZ(sizeof(pThis->paPages[0]) (pThis->cPages + cPages));
930	if (!pvPages)
931	{
932	pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) * pThis->cPages);
933	if (pvSavedPTEs)
934	pThis->pvSavedPTEs = pvSavedPTEs;
935	return VERR_NO_MEMORY;
936	}
937
938	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
939
940	memcpy(pvPages, pThis->paPages, sizeof(pThis->paPages[0]) * pThis->cPages);
941	void *pvToFree = pThis->paPages;
942	pThis->paPages = (PPGMRZDYNMAPENTRY)pvPages;
943
944	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
945	RTMemFree(pvToFree);
946
947	/*
948	* Allocate the segment structure and pages of memory, then touch all the pages (paranoia).
949	*/
950	uint32_t cMaxPTs = cPages / (pThis->fLegacyMode ? X86_PG_ENTRIES : X86_PG_PAE_ENTRIES) + 2;
951	PPGMR0DYNMAPSEG pSeg = (PPGMR0DYNMAPSEG)RTMemAllocZ(RT_UOFFSETOF(PGMR0DYNMAPSEG, ahMemObjPTs[cMaxPTs]));
952	if (!pSeg)
953	return VERR_NO_MEMORY;
954	pSeg->pNext = NULL;
955	pSeg->cPages = cPages;
956	pSeg->iPage = pThis->cPages;
957	pSeg->cPTs = 0;
958	int rc = RTR0MemObjAllocPage(&pSeg->hMemObj, cPages << PAGE_SHIFT, false);
959	if (RT_SUCCESS(rc))
960	{
961	uint8_t pbPage = (uint8_t )RTR0MemObjAddress(pSeg->hMemObj);
962	AssertMsg(VALID_PTR(pbPage) && !((uintptr_t)pbPage & PAGE_OFFSET_MASK), ("%p\n", pbPage));
963	memset(pbPage, 0xfe, cPages << PAGE_SHIFT);
964
965	/*
966	* Walk thru the pages and set them up with a mapping of their PTE and everything.
967	*/
968	ASMIntDisable();
969	PGMR0DYNMAPPGLVL PgLvl;
970	pgmR0DynMapPagingArrayInit(pThis, &PgLvl);
971	uint32_t const iEndPage = pSeg->iPage + cPages;
972	for (uint32_t iPage = pSeg->iPage;
973	iPage < iEndPage;
974	iPage++, pbPage += PAGE_SIZE)
975	{
976	/* Initialize the page data. */
977	pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
978	pThis->paPages[iPage].pvPage = pbPage;
979	pThis->paPages[iPage].cRefs = 0;
980	pThis->paPages[iPage].uPte.pPae = 0;
981	#ifndef IN_RC
982	RTCpuSetFill(&pThis->paPages[iPage].PendingSet);
983	#endif
984
985	/* Map its page table, retry until we've got a clean run (paranoia). */
986	do
987	rc = pgmR0DynMapPagingArrayMapPte(pThis, &PgLvl, pbPage, pSeg, cMaxPTs,
988	&pThis->paPages[iPage].uPte.pv);
989	while (rc == VINF_TRY_AGAIN);
990	if (RT_FAILURE(rc))
991	break;
992
993	/* Save the PTE. */
994	if (pThis->fLegacyMode)
995	((PX86PGUINT)pThis->pvSavedPTEs)[iPage] = pThis->paPages[iPage].uPte.pLegacy->u;
996	else
997	((PX86PGPAEUINT)pThis->pvSavedPTEs)[iPage] = pThis->paPages[iPage].uPte.pPae->u;
998
999	#ifdef VBOX_STRICT
1000	/* Check that we've got the right entry. */
1001	RTHCPHYS HCPhysPage = RTR0MemObjGetPagePhysAddr(pSeg->hMemObj, iPage - pSeg->iPage);
1002	RTHCPHYS HCPhysPte = pThis->fLegacyMode
1003	? pThis->paPages[iPage].uPte.pLegacy->u & X86_PTE_PG_MASK
1004	: pThis->paPages[iPage].uPte.pPae->u & X86_PTE_PAE_PG_MASK;
1005	if (HCPhysPage != HCPhysPte)
1006	{
1007	LogRel(("pgmR0DynMapAddSeg: internal error - page #%u HCPhysPage=%RHp HCPhysPte=%RHp pbPage=%p pvPte=%p\n",
1008	iPage - pSeg->iPage, HCPhysPage, HCPhysPte, pbPage, pThis->paPages[iPage].uPte.pv));
1009	rc = VERR_PGM_DYNMAP_IPE;
1010	break;
1011	}
1012	#endif
1013	} /* for each page */
1014	ASMIntEnable();
1015
1016	/* cleanup non-PT mappings */
1017	for (uint32_t i = 0; i < PgLvl.cLevels - 1; i++)
1018	RTR0MemObjFree(PgLvl.a[i].hMemObj, true /* fFreeMappings */);
1019
1020	if (RT_SUCCESS(rc))
1021	{
1022	#if PGMR0DYNMAP_GUARD_PAGES > 0
1023	/*
1024	* Setup guard pages.
1025	* (Note: TLBs will be shot down later on.)
1026	*/
1027	uint32_t iPage = pSeg->iPage;
1028	while (iPage < iEndPage)
1029	{
1030	for (uint32_t iGPg = 0; iGPg < PGMR0DYNMAP_GUARD_PAGES && iPage < iEndPage; iGPg++, iPage++)
1031	pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]);
1032	iPage++; /* the guarded page */
1033	}
1034
1035	/* Make sure the very last page is a guard page too. */
1036	iPage = iEndPage - 1;
1037	if (pThis->paPages[iPage].cRefs != PGMR0DYNMAP_GUARD_PAGE_REF_COUNT)
1038	pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]);
1039	#endif /* PGMR0DYNMAP_GUARD_PAGES > 0 */
1040
1041	/*
1042	* Commit it by adding the segment to the list and updating the page count.
1043	*/
1044	pSeg->pNext = pThis->pSegHead;
1045	pThis->pSegHead = pSeg;
1046	pThis->cPages += cPages;
1047	return VINF_SUCCESS;
1048	}
1049
1050	/*
1051	* Bail out.
1052	*/
1053	while (pSeg->cPTs-- > 0)
1054	{
1055	rc2 = RTR0MemObjFree(pSeg->ahMemObjPTs[pSeg->cPTs], true /* fFreeMappings */);
1056	AssertRC(rc2);
1057	pSeg->ahMemObjPTs[pSeg->cPTs] = NIL_RTR0MEMOBJ;
1058	}
1059
1060	rc2 = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */);
1061	AssertRC(rc2);
1062	pSeg->hMemObj = NIL_RTR0MEMOBJ;
1063	}
1064	else if (rc == VERR_NO_PAGE_MEMORY \|\| rc == VERR_NO_PHYS_MEMORY)
1065	rc = VERR_NO_MEMORY;
1066	RTMemFree(pSeg);
1067
1068	/* Don't bother resizing the arrays, but free them if we're the only user. */
1069	if (!pThis->cPages)
1070	{
1071	RTMemFree(pThis->paPages);
1072	pThis->paPages = NULL;
1073	RTMemFree(pThis->pvSavedPTEs);
1074	pThis->pvSavedPTEs = NULL;
1075	}
1076	return rc;
1077	}
1078
1079
1080	/**
1081	* Called by PGMR0DynMapInitVM under the init lock.
1082	*
1083	* @returns VBox status code.
1084	* @param pThis The dynamic mapping cache instance.
1085	*/
1086	static int pgmR0DynMapSetup(PPGMRZDYNMAP pThis)
1087	{
1088	/*
1089	* Calc the size and add a segment of that size.
1090	*/
1091	uint32_t cMinPages;
1092	uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages);
1093	AssertReturn(cPages, VERR_PGM_DYNMAP_IPE);
1094	int rc = pgmR0DynMapAddSeg(pThis, cPages);
1095	if (rc == VERR_NO_MEMORY)
1096	{
1097	/*
1098	* Try adding smaller segments.
1099	*/
1100	do
1101	rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES);
1102	while (RT_SUCCESS(rc) && pThis->cPages < cPages);
1103	if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages)
1104	rc = VINF_SUCCESS;
1105	if (rc == VERR_NO_MEMORY)
1106	{
1107	if (pThis->cPages)
1108	pgmR0DynMapTearDown(pThis);
1109	rc = VERR_PGM_DYNMAP_SETUP_ERROR;
1110	}
1111	}
1112	Assert(ASMGetFlags() & X86_EFL_IF);
1113
1114	#if PGMR0DYNMAP_GUARD_PAGES > 0
1115	/* paranoia */
1116	if (RT_SUCCESS(rc))
1117	pgmR0DynMapTlbShootDown(pThis);
1118	#endif
1119	return rc;
1120	}
1121
1122
1123	/**
1124	* Called by PGMR0DynMapInitVM under the init lock.
1125	*
1126	* @returns VBox status code.
1127	* @param pThis The dynamic mapping cache instance.
1128	*/
1129	static int pgmR0DynMapExpand(PPGMRZDYNMAP pThis)
1130	{
1131	/*
1132	* Calc the new target size and add a segment of the appropriate size.
1133	*/
1134	uint32_t cMinPages;
1135	uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages);
1136	AssertReturn(cPages, VERR_PGM_DYNMAP_IPE);
1137	if (pThis->cPages >= cPages)
1138	return VINF_SUCCESS;
1139
1140	uint32_t cAdd = cPages - pThis->cPages;
1141	int rc = pgmR0DynMapAddSeg(pThis, cAdd);
1142	if (rc == VERR_NO_MEMORY)
1143	{
1144	/*
1145	* Try adding smaller segments.
1146	*/
1147	do
1148	rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES);
1149	while (RT_SUCCESS(rc) && pThis->cPages < cPages);
1150	if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages)
1151	rc = VINF_SUCCESS;
1152	if (rc == VERR_NO_MEMORY)
1153	rc = VERR_PGM_DYNMAP_EXPAND_ERROR;
1154	}
1155	Assert(ASMGetFlags() & X86_EFL_IF);
1156
1157	#if PGMR0DYNMAP_GUARD_PAGES > 0
1158	/* paranoia */
1159	if (RT_SUCCESS(rc))
1160	pgmR0DynMapTlbShootDown(pThis);
1161	#endif
1162	return rc;
1163	}
1164
1165
1166	/**
1167	* Called by PGMR0DynMapTermVM under the init lock.
1168	*
1169	* @returns VBox status code.
1170	* @param pThis The dynamic mapping cache instance.
1171	*/
1172	static void pgmR0DynMapTearDown(PPGMRZDYNMAP pThis)
1173	{
1174	/*
1175	* Restore the original page table entries
1176	*/
1177	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
1178	uint32_t iPage = pThis->cPages;
1179	if (pThis->fLegacyMode)
1180	{
1181	X86PGUINT const paSavedPTEs = (X86PGUINT const )pThis->pvSavedPTEs;
1182	while (iPage-- > 0)
1183	{
1184	X86PGUINT uOld = paPages[iPage].uPte.pLegacy->u;
1185	X86PGUINT uOld2 = uOld; NOREF(uOld2);
1186	X86PGUINT uNew = paSavedPTEs[iPage];
1187	while (!ASMAtomicCmpXchgExU32(&paPages[iPage].uPte.pLegacy->u, uNew, uOld, &uOld))
1188	AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
1189	Assert(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage]);
1190	}
1191	}
1192	else
1193	{
1194	X86PGPAEUINT const paSavedPTEs = (X86PGPAEUINT const )pThis->pvSavedPTEs;
1195	while (iPage-- > 0)
1196	{
1197	X86PGPAEUINT uOld = paPages[iPage].uPte.pPae->u;
1198	X86PGPAEUINT uOld2 = uOld; NOREF(uOld2);
1199	X86PGPAEUINT uNew = paSavedPTEs[iPage];
1200	while (!ASMAtomicCmpXchgExU64(&paPages[iPage].uPte.pPae->u, uNew, uOld, &uOld))
1201	AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
1202	Assert(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage]);
1203	}
1204	}
1205
1206	/*
1207	* Shoot down the TLBs on all CPUs before freeing them.
1208	*/
1209	pgmR0DynMapTlbShootDown(pThis);
1210
1211	/*
1212	* Free the segments.
1213	*/
1214	while (pThis->pSegHead)
1215	{
1216	int rc;
1217	PPGMR0DYNMAPSEG pSeg = pThis->pSegHead;
1218	pThis->pSegHead = pSeg->pNext;
1219
1220	uint32_t iPT = pSeg->cPTs;
1221	while (iPT-- > 0)
1222	{
1223	rc = RTR0MemObjFree(pSeg->ahMemObjPTs[iPT], true /* fFreeMappings */); AssertRC(rc);
1224	pSeg->ahMemObjPTs[iPT] = NIL_RTR0MEMOBJ;
1225	}
1226	rc = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */); AssertRC(rc);
1227	pSeg->hMemObj = NIL_RTR0MEMOBJ;
1228	pSeg->pNext = NULL;
1229	pSeg->iPage = UINT16_MAX;
1230	pSeg->cPages = 0;
1231	pSeg->cPTs = 0;
1232	RTMemFree(pSeg);
1233	}
1234
1235	/*
1236	* Free the arrays and restore the initial state.
1237	* The cLoadMax value is left behind for the next setup.
1238	*/
1239	RTMemFree(pThis->paPages);
1240	pThis->paPages = NULL;
1241	RTMemFree(pThis->pvSavedPTEs);
1242	pThis->pvSavedPTEs = NULL;
1243	pThis->cPages = 0;
1244	pThis->cLoad = 0;
1245	pThis->cGuardPages = 0;
1246	}
1247
1248	#endif /* IN_RING0 */
1249	#ifdef IN_RC
1250
1251	/**
1252	* Initializes the dynamic mapping cache in raw-mode context.
1253	*
1254	* @returns VBox status code.
1255	* @param pVM The cross context VM structure.
1256	*/
1257	VMMRCDECL(int) PGMRCDynMapInit(PVM pVM)
1258	{
1259	/*
1260	* Allocate and initialize the instance data and page array.
1261	*/
1262	PPGMRZDYNMAP pThis;
1263	size_t const cPages = MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE;
1264	size_t const cb = RT_ALIGN_Z(sizeof(*pThis), 32)
1265	+ sizeof(PGMRZDYNMAPENTRY) * cPages;
1266	int rc = MMHyperAlloc(pVM, cb, 32, MM_TAG_PGM, (void **)&pThis);
1267	if (RT_FAILURE(rc))
1268	return rc;
1269
1270	pThis->u32Magic = PGMRZDYNMAP_MAGIC;
1271	pThis->paPages = RT_ALIGN_PT(pThis + 1, 32, PPGMRZDYNMAPENTRY);
1272	pThis->cPages = cPages;
1273	pThis->cLoad = 0;
1274	pThis->cMaxLoad = 0;
1275	pThis->cGuardPages = 0;
1276	pThis->cUsers = 1;
1277
1278	for (size_t iPage = 0; iPage < cPages; iPage++)
1279	{
1280	pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
1281	pThis->paPages[iPage].pvPage = pVM->pgm.s.pbDynPageMapBaseGC + iPage * PAGE_SIZE;
1282	pThis->paPages[iPage].cRefs = 0;
1283	pThis->paPages[iPage].uPte.pLegacy = &pVM->pgm.s.paDynPageMap32BitPTEsGC[iPage];
1284	pThis->paPages[iPage].uPte.pPae = (PX86PTEPAE)&pVM->pgm.s.paDynPageMapPaePTEsGC[iPage];
1285	}
1286
1287	pVM->pgm.s.pRCDynMap = pThis;
1288
1289	/*
1290	* Initialize the autosets the VM.
1291	*/
1292	rc = pgmRZDynMapInitAutoSetsForVM(pVM);
1293	if (RT_FAILURE(rc))
1294	return rc;
1295
1296	return VINF_SUCCESS;
1297	}
1298
1299	#endif /* IN_RC */
1300
1301	/**
1302	* Release references to a page, caller owns the spin lock.
1303	*
1304	* @param pThis The dynamic mapping cache instance.
1305	* @param iPage The page.
1306	* @param cRefs The number of references to release.
1307	*/
1308	DECLINLINE(void) pgmRZDynMapReleasePageLocked(PPGMRZDYNMAP pThis, uint32_t iPage, int32_t cRefs)
1309	{
1310	cRefs = ASMAtomicSubS32(&pThis->paPages[iPage].cRefs, cRefs) - cRefs;
1311	AssertMsg(cRefs >= 0, ("%d\n", cRefs));
1312	if (!cRefs)
1313	{
1314	pThis->cLoad--;
1315	#ifdef PGMRZDYNMAP_STRICT_RELEASE
1316	pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
1317	ASMAtomicBitClear(pThis->paPages[iPage].uPte.pv, X86_PTE_BIT_P);
1318	ASMInvalidatePage((uintptr_t)pThis->paPages[iPage].pvPage);
1319	#endif
1320	}
1321	}
1322
1323
1324	/**
1325	* Release references to a page, caller does not own the spin lock.
1326	*
1327	* @param pThis The dynamic mapping cache instance.
1328	* @param iPage The page.
1329	* @param cRefs The number of references to release.
1330	*/
1331	static void pgmRZDynMapReleasePage(PPGMRZDYNMAP pThis, uint32_t iPage, uint32_t cRefs)
1332	{
1333	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
1334	pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs);
1335	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1336	}
1337
1338
1339	/**
1340	* pgmR0DynMapPage worker that deals with the tedious bits.
1341	*
1342	* @returns The page index on success, UINT32_MAX on failure.
1343	* @param pThis The dynamic mapping cache instance.
1344	* @param HCPhys The address of the page to be mapped.
1345	* @param iPage The page index pgmR0DynMapPage hashed HCPhys to.
1346	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1347	* For statistics.
1348	* @param pfNew Set to @c true if a new entry was made and @c false if
1349	* an old entry was found and reused.
1350	*/
1351	static uint32_t pgmR0DynMapPageSlow(PPGMRZDYNMAP pThis, RTHCPHYS HCPhys, uint32_t iPage, PVMCPU pVCpu, bool *pfNew)
1352	{
1353	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlow);
1354
1355	/*
1356	* Check if any of the first 3 pages are unreferenced since the caller
1357	* already has made sure they aren't matching.
1358	*/
1359	#ifdef VBOX_WITH_STATISTICS
1360	bool fLooped = false;
1361	#endif
1362	uint32_t const cPages = pThis->cPages;
1363	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
1364	uint32_t iFreePage;
1365	if (!paPages[iPage].cRefs)
1366	iFreePage = iPage;
1367	else if (!paPages[(iPage + 1) % cPages].cRefs)
1368	iFreePage = (iPage + 1) % cPages;
1369	else if (!paPages[(iPage + 2) % cPages].cRefs)
1370	iFreePage = (iPage + 2) % cPages;
1371	else
1372	{
1373	/*
1374	* Search for an unused or matching entry.
1375	*/
1376	iFreePage = (iPage + 3) % cPages;
1377	for (;;)
1378	{
1379	if (paPages[iFreePage].HCPhys == HCPhys)
1380	{
1381	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlowLoopHits);
1382	*pfNew = false;
1383	return iFreePage;
1384	}
1385	if (!paPages[iFreePage].cRefs)
1386	break;
1387
1388	/* advance */
1389	iFreePage = (iFreePage + 1) % cPages;
1390	if (RT_UNLIKELY(iFreePage == iPage))
1391	return UINT32_MAX;
1392	}
1393	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlowLoopMisses);
1394	#ifdef VBOX_WITH_STATISTICS
1395	fLooped = true;
1396	#endif
1397	}
1398	Assert(iFreePage < cPages);
1399
1400	#if 0 //def VBOX_WITH_STATISTICS
1401	/* Check for lost hits. */
1402	if (!fLooped)
1403	for (uint32_t iPage2 = (iPage + 3) % cPages; iPage2 != iPage; iPage2 = (iPage2 + 1) % cPages)
1404	if (paPages[iPage2].HCPhys == HCPhys)
1405	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZDynMapPageSlowLostHits);
1406	#endif
1407
1408	/*
1409	* Setup the new entry.
1410	*/
1411	*pfNew = true;
1412	/Log6(("pgmR0DynMapPageSlow: old - %RHp %#x %#llx\n", paPages[iFreePage].HCPhys, paPages[iFreePage].cRefs, paPages[iFreePage].uPte.pPae->u));/
1413	paPages[iFreePage].HCPhys = HCPhys;
1414	#ifndef IN_RC
1415	RTCpuSetFill(&paPages[iFreePage].PendingSet);
1416
1417	if (pThis->fLegacyMode)
1418	#endif
1419	{
1420	X86PGUINT uOld = paPages[iFreePage].uPte.pLegacy->u;
1421	X86PGUINT uOld2 = uOld; NOREF(uOld2);
1422	X86PGUINT uNew = (uOld & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1423	\| X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1424	\| (HCPhys & X86_PTE_PG_MASK);
1425	while (!ASMAtomicCmpXchgExU32(&paPages[iFreePage].uPte.pLegacy->u, uNew, uOld, &uOld))
1426	AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
1427	Assert(paPages[iFreePage].uPte.pLegacy->u == uNew);
1428	}
1429	#ifndef IN_RC
1430	else
1431	#endif
1432	{
1433	X86PGPAEUINT uOld = paPages[iFreePage].uPte.pPae->u;
1434	X86PGPAEUINT uOld2 = uOld; NOREF(uOld2);
1435	X86PGPAEUINT uNew = (uOld & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1436	\| X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1437	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1438	while (!ASMAtomicCmpXchgExU64(&paPages[iFreePage].uPte.pPae->u, uNew, uOld, &uOld))
1439	AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
1440	Assert(paPages[iFreePage].uPte.pPae->u == uNew);
1441	/Log6(("pgmR0DynMapPageSlow: #%x - %RHp %p %#llx\n", iFreePage, HCPhys, paPages[iFreePage].pvPage, uNew));/
1442	}
1443	return iFreePage;
1444	}
1445
1446
1447	/**
1448	* Maps a page into the pool.
1449	*
1450	* @returns Page index on success, UINT32_MAX on failure.
1451	* @param pThis The dynamic mapping cache instance.
1452	* @param HCPhys The address of the page to be mapped.
1453	* @param iRealCpu The real cpu set index. (optimization)
1454	* @param pVCpu The cross context virtual CPU structure of the calling
1455	* EMT. For statistics.
1456	* @param ppvPage Where to the page address.
1457	*/
1458	DECLINLINE(uint32_t) pgmR0DynMapPage(PPGMRZDYNMAP pThis, RTHCPHYS HCPhys, int32_t iRealCpu, PVMCPU pVCpu, void **ppvPage)
1459	{
1460	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
1461	AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
1462	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPage);
1463
1464	/*
1465	* Find an entry, if possible a matching one. The HCPhys address is hashed
1466	* down to a page index, collisions are handled by linear searching.
1467	* Optimized for a hit in the first 3 pages.
1468	*
1469	* Field easy hits here and defer the tedious searching and inserting
1470	* to pgmR0DynMapPageSlow().
1471	*/
1472	bool fNew = false;
1473	uint32_t const cPages = pThis->cPages;
1474	uint32_t iPage = (HCPhys >> PAGE_SHIFT) % cPages;
1475	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
1476	if (RT_LIKELY(paPages[iPage].HCPhys == HCPhys))
1477	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits0);
1478	else
1479	{
1480	uint32_t iPage2 = (iPage + 1) % cPages;
1481	if (RT_LIKELY(paPages[iPage2].HCPhys == HCPhys))
1482	{
1483	iPage = iPage2;
1484	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits1);
1485	}
1486	else
1487	{
1488	iPage2 = (iPage + 2) % cPages;
1489	if (paPages[iPage2].HCPhys == HCPhys)
1490	{
1491	iPage = iPage2;
1492	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits2);
1493	}
1494	else
1495	{
1496	iPage = pgmR0DynMapPageSlow(pThis, HCPhys, iPage, pVCpu, &fNew);
1497	if (RT_UNLIKELY(iPage == UINT32_MAX))
1498	{
1499	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1500	*ppvPage = NULL;
1501	return iPage;
1502	}
1503	}
1504	}
1505	}
1506
1507	/*
1508	* Reference it, update statistics and get the return address.
1509	*/
1510	int32_t cRefs = ASMAtomicIncS32(&paPages[iPage].cRefs);
1511	if (cRefs == 1)
1512	{
1513	pThis->cLoad++;
1514	if (pThis->cLoad > pThis->cMaxLoad)
1515	pThis->cMaxLoad = pThis->cLoad;
1516	AssertMsg(pThis->cLoad <= pThis->cPages - pThis->cGuardPages, ("%d/%d\n", pThis->cLoad, pThis->cPages - pThis->cGuardPages));
1517	}
1518	else if (RT_UNLIKELY(cRefs <= 0))
1519	{
1520	ASMAtomicDecS32(&paPages[iPage].cRefs);
1521	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1522	*ppvPage = NULL;
1523	AssertLogRelMsgFailedReturn(("cRefs=%d iPage=%p HCPhys=%RHp\n", cRefs, iPage, HCPhys), UINT32_MAX);
1524	}
1525	void *pvPage = paPages[iPage].pvPage;
1526
1527	#ifndef IN_RC
1528	/*
1529	* Invalidate the entry?
1530	*/
1531	bool fInvalidateIt = RTCpuSetIsMemberByIndex(&paPages[iPage].PendingSet, iRealCpu);
1532	if (RT_UNLIKELY(fInvalidateIt))
1533	RTCpuSetDelByIndex(&paPages[iPage].PendingSet, iRealCpu);
1534	#else
1535	NOREF(iRealCpu);
1536	#endif
1537
1538	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1539
1540	/*
1541	* Do the actual invalidation outside the spinlock.
1542	*/
1543	#ifdef IN_RC
1544	if (RT_UNLIKELY(fNew))
1545	#else
1546	if (RT_UNLIKELY(fInvalidateIt))
1547	#endif
1548	{
1549	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageInvlPg);
1550	ASMInvalidatePage((uintptr_t)pvPage);
1551	}
1552
1553	*ppvPage = pvPage;
1554	return iPage;
1555	}
1556
1557
1558	/**
1559	* Assert the integrity of the pool.
1560	*
1561	* @returns VBox status code.
1562	*/
1563	static int pgmRZDynMapAssertIntegrity(PPGMRZDYNMAP pThis)
1564	{
1565	/*
1566	* Basic pool stuff that doesn't require any lock, just assumes we're a user.
1567	*/
1568	if (!pThis)
1569	return VINF_SUCCESS;
1570	AssertPtrReturn(pThis, VERR_INVALID_POINTER);
1571	AssertReturn(pThis->u32Magic == PGMRZDYNMAP_MAGIC, VERR_INVALID_MAGIC);
1572	if (!pThis->cUsers)
1573	return VERR_INVALID_PARAMETER;
1574
1575	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
1576
1577	#define CHECK_RET(expr, a) \
1578	do { \
1579	if (RT_UNLIKELY(!(expr))) \
1580	{ \
1581	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis); \
1582	RTAssertMsg1Weak(#expr, __LINE__, __FILE__, __PRETTY_FUNCTION__); \
1583	RTAssertMsg2Weak a; \
1584	return VERR_PGM_DYNMAP_IPE; \
1585	} \
1586	} while (0)
1587
1588	/*
1589	* Check that the PTEs are correct.
1590	*/
1591	uint32_t cGuard = 0;
1592	uint32_t cLoad = 0;
1593	PPGMRZDYNMAPENTRY paPages = pThis->paPages;
1594
1595	#ifndef IN_RC
1596	if (pThis->fLegacyMode)
1597	#endif
1598	{
1599	#ifdef IN_RING0
1600	PCX86PGUINT paSavedPTEs = (PCX86PGUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
1601	#endif
1602	uint32_t iPage = pThis->cPages;
1603	while (iPage-- > 0)
1604	{
1605	CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
1606	if ( paPages[iPage].cRefs == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT
1607	&& paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS)
1608	{
1609	#ifdef PGMR0DYNMAP_GUARD_NP
1610	CHECK_RET(paPages[iPage].uPte.pLegacy->u == (paSavedPTEs[iPage] & ~(X86PGUINT)X86_PTE_P),
1611	("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage]));
1612	#else
1613	CHECK_RET(paPages[iPage].uPte.pLegacy->u == PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE,
1614	("#%u: %#x", iPage, paPages[iPage].uPte.pLegacy->u));
1615	#endif
1616	cGuard++;
1617	}
1618	else if (paPages[iPage].HCPhys != NIL_RTHCPHYS)
1619	{
1620	CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
1621	X86PGUINT uPte = X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1622	#ifdef IN_RING0
1623	\| (paSavedPTEs[iPage] & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1624	#endif
1625	\| (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
1626	CHECK_RET(paPages[iPage].uPte.pLegacy->u == uPte,
1627	("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, uPte));
1628	if (paPages[iPage].cRefs)
1629	cLoad++;
1630	}
1631	#if defined(IN_RING0) && !defined(PGMRZDYNMAP_STRICT_RELEASE)
1632	else
1633	CHECK_RET(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage],
1634	("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage]));
1635	#endif
1636	}
1637	}
1638	#ifndef IN_RC
1639	else
1640	#endif
1641	{
1642	#ifdef IN_RING0
1643	PCX86PGPAEUINT paSavedPTEs = (PCX86PGPAEUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
1644	#endif
1645	uint32_t iPage = pThis->cPages;
1646	while (iPage-- > 0)
1647	{
1648	CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
1649	if ( paPages[iPage].cRefs == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT
1650	&& paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS)
1651	{
1652	#ifdef PGMR0DYNMAP_GUARD_NP
1653	CHECK_RET(paPages[iPage].uPte.pPae->u == (paSavedPTEs[iPage] & ~(X86PGPAEUINT)X86_PTE_P),
1654	("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage]));
1655	#else
1656	CHECK_RET(paPages[iPage].uPte.pPae->u == PGMR0DYNMAP_GUARD_PAGE_PAE_PTE,
1657	("#%u: %#llx", iPage, paPages[iPage].uPte.pPae->u));
1658	#endif
1659	cGuard++;
1660	}
1661	else if (paPages[iPage].HCPhys != NIL_RTHCPHYS)
1662	{
1663	CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
1664	X86PGPAEUINT uPte = X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1665	#ifdef IN_RING0
1666	\| (paSavedPTEs[iPage] & (X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT))
1667	#endif
1668	\| (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
1669	CHECK_RET(paPages[iPage].uPte.pPae->u == uPte,
1670	("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pLegacy->u, uPte));
1671	if (paPages[iPage].cRefs)
1672	cLoad++;
1673	}
1674	#ifdef IN_RING0
1675	else
1676	CHECK_RET(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage],
1677	("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage]));
1678	#endif
1679	}
1680	}
1681
1682	CHECK_RET(cLoad == pThis->cLoad, ("%u %u\n", cLoad, pThis->cLoad));
1683	CHECK_RET(cGuard == pThis->cGuardPages, ("%u %u\n", cGuard, pThis->cGuardPages));
1684
1685	#undef CHECK_RET
1686	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1687	return VINF_SUCCESS;
1688	}
1689
1690	#ifdef IN_RING0
1691	/**
1692	* Assert the integrity of the pool.
1693	*
1694	* @returns VBox status code.
1695	*/
1696	VMMR0DECL(int) PGMR0DynMapAssertIntegrity(void)
1697	{
1698	return pgmRZDynMapAssertIntegrity(g_pPGMR0DynMap);
1699	}
1700	#endif /* IN_RING0 */
1701
1702	#ifdef IN_RC
1703	/**
1704	* Assert the integrity of the pool.
1705	*
1706	* @returns VBox status code.
1707	*/
1708	VMMRCDECL(int) PGMRCDynMapAssertIntegrity(PVM pVM)
1709	{
1710	return pgmRZDynMapAssertIntegrity((PPGMRZDYNMAP)pVM->pgm.s.pRCDynMap);
1711	}
1712	#endif /* IN_RC */
1713
1714
1715	/**
1716	* As a final resort for a (somewhat) full auto set or full cache, try merge
1717	* duplicate entries and flush the ones we can.
1718	*
1719	* @param pSet The set.
1720	*/
1721	static void pgmDynMapOptimizeAutoSet(PPGMMAPSET pSet)
1722	{
1723	LogFlow(("pgmDynMapOptimizeAutoSet\n"));
1724
1725	for (uint32_t i = 0 ; i < pSet->cEntries; i++)
1726	{
1727	/*
1728	* Try merge entries.
1729	*/
1730	uint16_t const iPage = pSet->aEntries[i].iPage;
1731	uint32_t j = i + 1;
1732	while ( j < pSet->cEntries
1733	&& ( pSet->iSubset == UINT32_MAX
1734	\|\| pSet->iSubset < pSet->cEntries) )
1735	{
1736	if (pSet->aEntries[j].iPage != iPage)
1737	j++;
1738	else
1739	{
1740	uint32_t const cHardRefs = (uint32_t)pSet->aEntries[i].cRefs
1741	+ (uint32_t)pSet->aEntries[j].cRefs;
1742	uint32_t cInlinedRefs = (uint32_t)pSet->aEntries[i].cInlinedRefs
1743	+ (uint32_t)pSet->aEntries[j].cInlinedRefs;
1744	uint32_t cUnrefs = (uint32_t)pSet->aEntries[i].cUnrefs
1745	+ (uint32_t)pSet->aEntries[j].cUnrefs;
1746	uint32_t cSub = RT_MIN(cUnrefs, cInlinedRefs);
1747	cInlinedRefs -= cSub;
1748	cUnrefs -= cSub;
1749
1750	if ( cHardRefs < UINT16_MAX
1751	&& cInlinedRefs < UINT16_MAX
1752	&& cUnrefs < UINT16_MAX)
1753	{
1754	/* merge j into i removing j. */
1755	Log2(("pgmDynMapOptimizeAutoSet: Merging #%u into #%u\n", j, i));
1756	pSet->aEntries[i].cRefs = cHardRefs;
1757	pSet->aEntries[i].cInlinedRefs = cInlinedRefs;
1758	pSet->aEntries[i].cUnrefs = cUnrefs;
1759	pSet->cEntries--;
1760	if (j < pSet->cEntries)
1761	{
1762	pSet->aEntries[j] = pSet->aEntries[pSet->cEntries];
1763	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[pSet->cEntries]);
1764	}
1765	else
1766	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]);
1767	}
1768	#if 0 /* too complicated, skip it. */
1769	else
1770	{
1771	/* migrate the max number of refs from j into i and quit the inner loop. */
1772	uint32_t cMigrate = UINT16_MAX - 1 - pSet->aEntries[i].cRefs;
1773	Assert(pSet->aEntries[j].cRefs > cMigrate);
1774	pSet->aEntries[j].cRefs -= cMigrate;
1775	pSet->aEntries[i].cRefs = UINT16_MAX - 1;
1776	break;
1777	}
1778	#endif
1779	}
1780	}
1781
1782	/*
1783	* Try make use of the unused hinting (cUnrefs) to evict entries
1784	* from both the set as well as the mapping cache.
1785	*/
1786
1787	uint32_t const cTotalRefs = (uint32_t)pSet->aEntries[i].cRefs + pSet->aEntries[i].cInlinedRefs;
1788	Log2(("pgmDynMapOptimizeAutoSet: #%u/%u/%u pvPage=%p iPage=%u cRefs=%u cInlinedRefs=%u cUnrefs=%u cTotalRefs=%u\n",
1789	i,
1790	pSet->iSubset,
1791	pSet->cEntries,
1792	pSet->aEntries[i].pvPage,
1793	pSet->aEntries[i].iPage,
1794	pSet->aEntries[i].cRefs,
1795	pSet->aEntries[i].cInlinedRefs,
1796	pSet->aEntries[i].cUnrefs,
1797	cTotalRefs));
1798	Assert(cTotalRefs >= pSet->aEntries[i].cUnrefs);
1799
1800	if ( cTotalRefs == pSet->aEntries[i].cUnrefs
1801	&& ( pSet->iSubset == UINT32_MAX
1802	\|\| pSet->iSubset < pSet->cEntries)
1803	)
1804	{
1805	Log2(("pgmDynMapOptimizeAutoSet: Releasing iPage=%d/%p\n", pSet->aEntries[i].iPage, pSet->aEntries[i].pvPage));
1806	//LogFlow(("pgmDynMapOptimizeAutoSet: Releasing iPage=%d/%p\n", pSet->aEntries[i].iPage, pSet->aEntries[i].pvPage));
1807	pgmRZDynMapReleasePage(PGMRZDYNMAP_SET_2_DYNMAP(pSet),
1808	pSet->aEntries[i].iPage,
1809	pSet->aEntries[i].cRefs);
1810	pSet->cEntries--;
1811	if (i < pSet->cEntries)
1812	{
1813	pSet->aEntries[i] = pSet->aEntries[pSet->cEntries];
1814	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[pSet->cEntries]);
1815	}
1816
1817	i--;
1818	}
1819	}
1820	}
1821
1822
1823
1824
1825	/**
1826	* Signals the start of a new set of mappings.
1827	*
1828	* Mostly for strictness. PGMDynMapHCPage won't work unless this
1829	* API is called.
1830	*
1831	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1832	*/
1833	VMMDECL(void) PGMRZDynMapStartAutoSet(PVMCPU pVCpu)
1834	{
1835	LogFlow(("PGMRZDynMapStartAutoSet:\n"));
1836	Assert(pVCpu->pgm.s.AutoSet.cEntries == PGMMAPSET_CLOSED);
1837	Assert(pVCpu->pgm.s.AutoSet.iSubset == UINT32_MAX);
1838	pVCpu->pgm.s.AutoSet.cEntries = 0;
1839	pVCpu->pgm.s.AutoSet.iCpu = PGMRZDYNMAP_CUR_CPU();
1840	}
1841
1842
1843	#ifdef IN_RING0
1844	/**
1845	* Starts or migrates the autoset of a virtual CPU.
1846	*
1847	* This is used by HMR0Enter. When we've longjumped out of the HM
1848	* execution loop with the set open, we'll migrate it when re-entering. While
1849	* under normal circumstances, we'll start it so VMXR0LoadGuestState can access
1850	* guest memory.
1851	*
1852	* @returns @c true if started, @c false if migrated.
1853	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1854	* @thread EMT
1855	*/
1856	VMMR0DECL(bool) PGMR0DynMapStartOrMigrateAutoSet(PVMCPU pVCpu)
1857	{
1858	bool fStartIt = pVCpu->pgm.s.AutoSet.cEntries == PGMMAPSET_CLOSED;
1859	if (fStartIt)
1860	PGMRZDynMapStartAutoSet(pVCpu);
1861	else
1862	PGMR0DynMapMigrateAutoSet(pVCpu);
1863	return fStartIt;
1864	}
1865	#endif /* IN_RING0 */
1866
1867
1868	/**
1869	* Checks if the set has high load.
1870	*
1871	* @returns true on high load, otherwise false.
1872	* @param pSet The set.
1873	*/
1874	DECLINLINE(bool) pgmRZDynMapHasHighLoad(PPGMMAPSET pSet)
1875	{
1876	#ifdef IN_RC
1877	if (pSet->cEntries < MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE / 2)
1878	return false;
1879	#endif
1880
1881	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
1882	uint32_t cUnusedPages = pThis->cPages - pThis->cLoad;
1883	#ifdef IN_RC
1884	return cUnusedPages <= MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE * 36 / 100;
1885	#else
1886	return cUnusedPages <= PGMR0DYNMAP_PAGES_PER_CPU_MIN;
1887	#endif
1888	}
1889
1890
1891	/**
1892	* Worker that performs the actual flushing of the set.
1893	*
1894	* @param pSet The set to flush.
1895	* @param cEntries The number of entries.
1896	*/
1897	DECLINLINE(void) pgmDynMapFlushAutoSetWorker(PPGMMAPSET pSet, uint32_t cEntries)
1898	{
1899	/*
1900	* Release any pages it's referencing.
1901	*/
1902	if ( cEntries != 0
1903	&& RT_LIKELY(cEntries <= RT_ELEMENTS(pSet->aEntries)))
1904	{
1905	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
1906	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
1907
1908	uint32_t i = cEntries;
1909	while (i-- > 0)
1910	{
1911	uint32_t iPage = pSet->aEntries[i].iPage;
1912	Assert(iPage < pThis->cPages);
1913	int32_t cRefs = pSet->aEntries[i].cRefs;
1914	Assert(cRefs > 0);
1915	pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs);
1916
1917	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[i]);
1918	}
1919
1920	Assert(pThis->cLoad <= pThis->cPages - pThis->cGuardPages);
1921	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
1922	}
1923	}
1924
1925
1926	/**
1927	* Releases the dynamic memory mappings made by PGMDynMapHCPage and associates
1928	* since the PGMDynMapStartAutoSet call.
1929	*
1930	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1931	*/
1932	VMMDECL(void) PGMRZDynMapReleaseAutoSet(PVMCPU pVCpu)
1933	{
1934	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1935
1936	/*
1937	* Close and flush the set.
1938	*/
1939	uint32_t cEntries = pSet->cEntries;
1940	AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
1941	pSet->cEntries = PGMMAPSET_CLOSED;
1942	pSet->iSubset = UINT32_MAX;
1943	pSet->iCpu = -1;
1944
1945	#ifdef IN_RC
1946	if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)
1947	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
1948	else
1949	#endif
1950	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
1951	if (cEntries > RT_ELEMENTS(pSet->aEntries) * 50 / 100)
1952	Log(("PGMRZDynMapReleaseAutoSet: cEntries=%d\n", cEntries));
1953	else
1954	LogFlow(("PGMRZDynMapReleaseAutoSet: cEntries=%d\n", cEntries));
1955
1956	pgmDynMapFlushAutoSetWorker(pSet, cEntries);
1957	}
1958
1959
1960	/**
1961	* Flushes the set if it's above a certain threshold.
1962	*
1963	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1964	*/
1965	VMMDECL(void) PGMRZDynMapFlushAutoSet(PVMCPU pVCpu)
1966	{
1967	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1968	AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
1969
1970	/*
1971	* Only flush it if it's 45% full.
1972	*/
1973	uint32_t cEntries = pSet->cEntries;
1974	AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
1975	Assert(pSet->iSubset == UINT32_MAX);
1976	#ifdef IN_RC
1977	if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)
1978	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
1979	else
1980	#endif
1981	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
1982	if ( cEntries >= RT_ELEMENTS(pSet->aEntries) * 45 / 100
1983	\|\| pgmRZDynMapHasHighLoad(pSet))
1984	{
1985	pSet->cEntries = 0;
1986	Log(("PGMDynMapFlushAutoSet: cEntries=%d\n", pSet->cEntries));
1987
1988	pgmDynMapFlushAutoSetWorker(pSet, cEntries);
1989	AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
1990	}
1991	}
1992
1993
1994	#ifndef IN_RC
1995	/**
1996	* Migrates the automatic mapping set of the current vCPU if it's active and
1997	* necessary.
1998	*
1999	* This is called when re-entering the hardware assisted execution mode after a
2000	* nip down to ring-3. We run the risk that the CPU might have change and we
2001	* will therefore make sure all the cache entries currently in the auto set will
2002	* be valid on the new CPU. If the cpu didn't change nothing will happen as all
2003	* the entries will have been flagged as invalidated.
2004	*
2005	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2006	* @thread EMT
2007	*/
2008	VMMR0DECL(void) PGMR0DynMapMigrateAutoSet(PVMCPU pVCpu)
2009	{
2010	LogFlow(("PGMR0DynMapMigrateAutoSet\n"));
2011	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
2012	int32_t iRealCpu = PGMRZDYNMAP_CUR_CPU();
2013	if (pSet->iCpu != iRealCpu)
2014	{
2015	uint32_t i = pSet->cEntries;
2016	if (i != PGMMAPSET_CLOSED)
2017	{
2018	AssertMsg(i <= RT_ELEMENTS(pSet->aEntries), ("%#x (%u)\n", i, i));
2019	if (i != 0 && RT_LIKELY(i <= RT_ELEMENTS(pSet->aEntries)))
2020	{
2021	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
2022	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
2023
2024	while (i-- > 0)
2025	{
2026	Assert(pSet->aEntries[i].cRefs > 0);
2027	uint32_t iPage = pSet->aEntries[i].iPage;
2028	Assert(iPage < pThis->cPages);
2029	if (RTCpuSetIsMemberByIndex(&pThis->paPages[iPage].PendingSet, iRealCpu))
2030	{
2031	RTCpuSetDelByIndex(&pThis->paPages[iPage].PendingSet, iRealCpu);
2032	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
2033
2034	ASMInvalidatePage((uintptr_t)pThis->paPages[iPage].pvPage);
2035	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapMigrateInvlPg);
2036
2037	PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis);
2038	}
2039	}
2040
2041	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
2042	}
2043	}
2044	pSet->iCpu = iRealCpu;
2045	}
2046	}
2047	#endif /* !IN_RC */
2048
2049
2050	/**
2051	* Worker function that flushes the current subset.
2052	*
2053	* This is called when the set is popped or when the set
2054	* hash a too high load. As also pointed out elsewhere, the
2055	* whole subset thing is a hack for working around code that
2056	* accesses too many pages. Like PGMPool.
2057	*
2058	* @param pSet The set which subset to flush.
2059	*/
2060	static void pgmDynMapFlushSubset(PPGMMAPSET pSet)
2061	{
2062	uint32_t iSubset = pSet->iSubset;
2063	uint32_t i = pSet->cEntries;
2064	Assert(i <= RT_ELEMENTS(pSet->aEntries));
2065	if ( i > iSubset
2066	&& i <= RT_ELEMENTS(pSet->aEntries))
2067	{
2068	Log(("pgmDynMapFlushSubset: cEntries=%d iSubset=%d\n", pSet->cEntries, iSubset));
2069	pSet->cEntries = iSubset;
2070
2071	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
2072	PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
2073
2074	while (i-- > iSubset)
2075	{
2076	uint32_t iPage = pSet->aEntries[i].iPage;
2077	Assert(iPage < pThis->cPages);
2078	int32_t cRefs = pSet->aEntries[i].cRefs;
2079	Assert(cRefs > 0);
2080	pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs);
2081
2082	PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[i]);
2083	}
2084
2085	PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
2086	}
2087	}
2088
2089
2090	/**
2091	* Creates a subset.
2092	*
2093	* A subset is a hack to avoid having to rewrite code that touches a lot of
2094	* pages. It prevents the mapping set from being overflowed by automatically
2095	* flushing previous mappings when a certain threshold is reached.
2096	*
2097	* Pages mapped after calling this function are only valid until the next page
2098	* is mapped.
2099	*
2100	* @returns The index of the previous subset. Pass this to
2101	* PGMDynMapPopAutoSubset when popping it.
2102	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2103	*/
2104	VMMDECL(uint32_t) PGMRZDynMapPushAutoSubset(PVMCPU pVCpu)
2105	{
2106	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
2107	AssertReturn(pSet->cEntries != PGMMAPSET_CLOSED, UINT32_MAX);
2108	uint32_t iPrevSubset = pSet->iSubset;
2109	LogFlow(("PGMRZDynMapPushAutoSubset: pVCpu=%p iPrevSubset=%u\n", pVCpu, iPrevSubset));
2110
2111	/*
2112	* If it looks like we're approaching the max set size or mapping space
2113	* optimize the set to drop off unused pages.
2114	*/
2115	if ( pSet->cEntries > RT_ELEMENTS(pSet->aEntries) * 60 / 100
2116	\|\| pgmRZDynMapHasHighLoad(pSet))
2117	{
2118	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize);
2119	pgmDynMapOptimizeAutoSet(pSet);
2120	}
2121
2122	pSet->iSubset = pSet->cEntries;
2123	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSubsets);
2124
2125	AssertMsg(iPrevSubset <= pSet->iSubset \|\| iPrevSubset == UINT32_MAX, ("iPrevSubset=%#x iSubset=%#x\n", iPrevSubset, pSet->iSubset));
2126	return iPrevSubset;
2127	}
2128
2129
2130	/**
2131	* Pops a subset created by a previous call to PGMDynMapPushAutoSubset.
2132	*
2133	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2134	* @param iPrevSubset What PGMDynMapPushAutoSubset returned.
2135	*/
2136	VMMDECL(void) PGMRZDynMapPopAutoSubset(PVMCPU pVCpu, uint32_t iPrevSubset)
2137	{
2138	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
2139	uint32_t cEntries = pSet->cEntries;
2140	LogFlow(("PGMRZDynMapPopAutoSubset: pVCpu=%p iPrevSubset=%u iSubset=%u cEntries=%u\n", pVCpu, iPrevSubset, pSet->iSubset, cEntries));
2141	AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
2142	AssertMsgReturnVoid(pSet->iSubset >= iPrevSubset \|\| iPrevSubset == UINT32_MAX, ("iPrevSubset=%u iSubset=%u cEntries=%u\n", iPrevSubset, pSet->iSubset, cEntries));
2143	#ifdef IN_RC
2144	if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)
2145	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
2146	else
2147	#endif
2148	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
2149	if ( cEntries >= RT_ELEMENTS(pSet->aEntries) * 40 / 100
2150	&& cEntries != pSet->iSubset)
2151	{
2152	pgmDynMapFlushSubset(pSet);
2153	Assert(pSet->cEntries >= iPrevSubset \|\| iPrevSubset == UINT32_MAX);
2154	}
2155	pSet->iSubset = iPrevSubset;
2156	}
2157
2158
2159	/**
2160	* Indicates that the given page is unused and its mapping can be re-used.
2161	*
2162	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2163	* @param pvHint The page that is now unused. This does not have to
2164	* point at the start of the page. NULL is ignored.
2165	*/
2166	#ifdef LOG_ENABLED
2167	void pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint, RT_SRC_POS_DECL)
2168	#else
2169	void pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint)
2170	#endif
2171	{
2172	/*
2173	* Ignore NULL pointers and mask off the page offset bits.
2174	*/
2175	if (pvHint == NULL)
2176	return;
2177	pvHint = (void *)((uintptr_t)pvHint & ~(uintptr_t)PAGE_OFFSET_MASK);
2178
2179	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
2180	uint32_t iEntry = pSet->cEntries;
2181	AssertReturnVoid(iEntry > 0);
2182
2183	/*
2184	* Find the entry in the usual unrolled fashion.
2185	*/
2186	/** @todo add a hint to the set which entry was used last since it's not
2187	* always the last entry? */
2188	#define IS_MATCHING_ENTRY(pSet, iEntry, pvHint) \
2189	( (pSet)->aEntries[(iEntry)].pvPage == (pvHint) \
2190	&& (uint32_t)(pSet)->aEntries[(iEntry)].cRefs + (pSet)->aEntries[(iEntry)].cInlinedRefs \
2191	> (pSet)->aEntries[(iEntry)].cUnrefs )
2192	if ( iEntry >= 1 && IS_MATCHING_ENTRY(pSet, iEntry - 1, pvHint))
2193	iEntry = iEntry - 1;
2194	else if (iEntry >= 2 && IS_MATCHING_ENTRY(pSet, iEntry - 2, pvHint))
2195	iEntry = iEntry - 2;
2196	else if (iEntry >= 3 && IS_MATCHING_ENTRY(pSet, iEntry - 3, pvHint))
2197	iEntry = iEntry - 3;
2198	else if (iEntry >= 4 && IS_MATCHING_ENTRY(pSet, iEntry - 4, pvHint))
2199	iEntry = iEntry - 4;
2200	else if (iEntry >= 5 && IS_MATCHING_ENTRY(pSet, iEntry - 5, pvHint))
2201	iEntry = iEntry - 5;
2202	else if (iEntry >= 6 && IS_MATCHING_ENTRY(pSet, iEntry - 6, pvHint))
2203	iEntry = iEntry - 6;
2204	else if (iEntry >= 7 && IS_MATCHING_ENTRY(pSet, iEntry - 7, pvHint))
2205	iEntry = iEntry - 7;
2206	else
2207	{
2208	/*
2209	* Loop till we find it.
2210	*/
2211	bool fFound = false;
2212	if (iEntry > 7)
2213	{
2214	iEntry -= 7;
2215	while (iEntry-- > 0)
2216	if (IS_MATCHING_ENTRY(pSet, iEntry, pvHint))
2217	{
2218	fFound = true;
2219	break;
2220	}
2221	}
2222	AssertMsgReturnVoid(fFound,
2223	("pvHint=%p cEntries=%#x iSubset=%#x\n"
2224	"aEntries[0] = {%#x, %#x, %#x, %#x, %p}\n"
2225	"aEntries[1] = {%#x, %#x, %#x, %#x, %p}\n"
2226	"aEntries[2] = {%#x, %#x, %#x, %#x, %p}\n"
2227	"aEntries[3] = {%#x, %#x, %#x, %#x, %p}\n"
2228	"aEntries[4] = {%#x, %#x, %#x, %#x, %p}\n"
2229	"aEntries[5] = {%#x, %#x, %#x, %#x, %p}\n"
2230	,
2231	pvHint, pSet->cEntries, pSet->iSubset,
2232	pSet->aEntries[0].iPage, pSet->aEntries[0].cRefs, pSet->aEntries[0].cInlinedRefs, pSet->aEntries[0].cUnrefs, pSet->aEntries[0].pvPage,
2233	pSet->aEntries[1].iPage, pSet->aEntries[1].cRefs, pSet->aEntries[1].cInlinedRefs, pSet->aEntries[1].cUnrefs, pSet->aEntries[1].pvPage,
2234	pSet->aEntries[2].iPage, pSet->aEntries[2].cRefs, pSet->aEntries[2].cInlinedRefs, pSet->aEntries[2].cUnrefs, pSet->aEntries[2].pvPage,
2235	pSet->aEntries[3].iPage, pSet->aEntries[3].cRefs, pSet->aEntries[3].cInlinedRefs, pSet->aEntries[3].cUnrefs, pSet->aEntries[3].pvPage,
2236	pSet->aEntries[4].iPage, pSet->aEntries[4].cRefs, pSet->aEntries[4].cInlinedRefs, pSet->aEntries[4].cUnrefs, pSet->aEntries[4].pvPage,
2237	pSet->aEntries[5].iPage, pSet->aEntries[5].cRefs, pSet->aEntries[5].cInlinedRefs, pSet->aEntries[5].cUnrefs, pSet->aEntries[5].pvPage));
2238	}
2239	#undef IS_MATCHING_ENTRY
2240
2241	/*
2242	* Update it.
2243	*/
2244	uint32_t const cTotalRefs = (uint32_t)pSet->aEntries[iEntry].cRefs + pSet->aEntries[iEntry].cInlinedRefs;
2245	uint32_t const cUnrefs = pSet->aEntries[iEntry].cUnrefs;
2246	LogFlow(("pgmRZDynMapUnusedHint: pvHint=%p #%u cRefs=%d cInlinedRefs=%d cUnrefs=%d (+1) cTotalRefs=%d %s(%d) %s\n",
2247	pvHint, iEntry, pSet->aEntries[iEntry].cRefs, pSet->aEntries[iEntry].cInlinedRefs, cUnrefs, cTotalRefs, pszFile, iLine, pszFunction));
2248	AssertReturnVoid(cTotalRefs > cUnrefs);
2249
2250	if (RT_LIKELY(cUnrefs < UINT16_MAX - 1))
2251	pSet->aEntries[iEntry].cUnrefs++;
2252	else if (pSet->aEntries[iEntry].cInlinedRefs)
2253	{
2254	uint32_t cSub = RT_MIN(pSet->aEntries[iEntry].cInlinedRefs, pSet->aEntries[iEntry].cUnrefs);
2255	pSet->aEntries[iEntry].cInlinedRefs -= cSub;
2256	pSet->aEntries[iEntry].cUnrefs -= cSub;
2257	pSet->aEntries[iEntry].cUnrefs++;
2258	}
2259	else
2260	Log(("pgmRZDynMapUnusedHint: pvHint=%p ignored because of overflow! %s(%d) %s\n", pvHint, pszFile, iLine, pszFunction));
2261
2262	#ifdef PGMRZDYNMAP_STRICT_RELEASE
2263	/*
2264	* Optimize the set to trigger the unmapping and invalidation of the page.
2265	*/
2266	if (cUnrefs + 1 == cTotalRefs)
2267	pgmDynMapOptimizeAutoSet(pSet);
2268	#endif
2269	}
2270
2271
2272	/**
2273	* Common worker code for pgmRZDynMapHCPageInlined, pgmRZDynMapHCPageV2Inlined
2274	* and pgmR0DynMapGCPageOffInlined.
2275	*
2276	* @returns VINF_SUCCESS, bails out to ring-3 on failure.
2277	* @param pSet The set.
2278	* @param HCPhys The physical address of the page.
2279	* @param ppv Where to store the address of the mapping on success.
2280	*
2281	* @remarks This is a very hot path.
2282	*/
2283	int pgmRZDynMapHCPageCommon(PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
2284	{
2285	AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
2286	PVMCPU pVCpu = PGMRZDYNMAP_SET_2_VMCPU(pSet);
2287	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
2288
2289	/*
2290	* Map it.
2291	*/
2292	void *pvPage;
2293	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
2294	uint32_t iPage = pgmR0DynMapPage(pThis, HCPhys, pSet->iCpu, pVCpu, &pvPage);
2295	if (RT_UNLIKELY(iPage == UINT32_MAX))
2296	{
2297	/*
2298	* We're out of mapping space, optimize our set to try remedy the
2299	* situation. (Only works if there are unreference hints.)
2300	*/
2301	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize);
2302	pgmDynMapOptimizeAutoSet(pSet);
2303
2304	iPage = pgmR0DynMapPage(pThis, HCPhys, pSet->iCpu, pVCpu, &pvPage);
2305	if (RT_UNLIKELY(iPage == UINT32_MAX))
2306	{
2307	RTAssertMsg2Weak("pgmRZDynMapHCPageCommon: cLoad=%u/%u cPages=%u cGuardPages=%u\n",
2308	pThis->cLoad, pThis->cMaxLoad, pThis->cPages, pThis->cGuardPages);
2309	if (!g_fPGMR0DynMapTestRunning)
2310	VMMRZCallRing3NoCpu(PGMRZDYNMAP_SET_2_VM(pSet), VMMCALLRING3_VM_R0_ASSERTION, 0);
2311	*ppv = NULL;
2312	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
2313	return VERR_PGM_DYNMAP_FAILED;
2314	}
2315	}
2316
2317	/*
2318	* Add the page to the auto reference set.
2319	*
2320	* The typical usage pattern means that the same pages will be mapped
2321	* several times in the same set. We can catch most of these
2322	* remappings by looking a few pages back into the set. (The searching
2323	* and set optimizing path will hardly ever be used when doing this.)
2324	*/
2325	AssertCompile(RT_ELEMENTS(pSet->aEntries) >= 8);
2326	int32_t i = pSet->cEntries;
2327	if (i-- < 5)
2328	{
2329	unsigned iEntry = pSet->cEntries++;
2330	pSet->aEntries[iEntry].cRefs = 1;
2331	pSet->aEntries[iEntry].cUnrefs = 0;
2332	pSet->aEntries[iEntry].cInlinedRefs = 0;
2333	pSet->aEntries[iEntry].iPage = iPage;
2334	pSet->aEntries[iEntry].pvPage = pvPage;
2335	pSet->aEntries[iEntry].HCPhys = HCPhys;
2336	pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
2337	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/0/0 iPage=%#x [a] %s(%d) %s\n",
2338	pSet, HCPhys, iEntry, iEntry + 1, pvPage, 1, iPage, pszFile, iLine, pszFunction));
2339	}
2340	/* Any of the last 5 pages? */
2341	else if ( pSet->aEntries[i - 0].iPage == iPage
2342	&& pSet->aEntries[i - 0].cRefs < UINT16_MAX - 1)
2343	{
2344	pSet->aEntries[i - 0].cRefs++;
2345	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [0] %s(%d) %s\n", pSet, HCPhys, i - 0, pSet->cEntries, pvPage, pSet->aEntries[i - 0].cRefs, pSet->aEntries[i - 0].cInlinedRefs, pSet->aEntries[i - 0].cUnrefs, iPage, pszFile, iLine, pszFunction));
2346	}
2347	else if ( pSet->aEntries[i - 1].iPage == iPage
2348	&& pSet->aEntries[i - 1].cRefs < UINT16_MAX - 1)
2349	{
2350	pSet->aEntries[i - 1].cRefs++;
2351	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [1] %s(%d) %s\n", pSet, HCPhys, i - 1, pSet->cEntries, pvPage, pSet->aEntries[i - 1].cRefs, pSet->aEntries[i - 1].cInlinedRefs, pSet->aEntries[i - 1].cUnrefs, iPage, pszFile, iLine, pszFunction));
2352	}
2353	else if ( pSet->aEntries[i - 2].iPage == iPage
2354	&& pSet->aEntries[i - 2].cRefs < UINT16_MAX - 1)
2355	{
2356	pSet->aEntries[i - 2].cRefs++;
2357	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [2] %s(%d) %s\n", pSet, HCPhys, i - 2, pSet->cEntries, pvPage, pSet->aEntries[i - 2].cRefs, pSet->aEntries[i - 2].cInlinedRefs, pSet->aEntries[i - 2].cUnrefs, iPage, pszFile, iLine, pszFunction));
2358	}
2359	else if ( pSet->aEntries[i - 3].iPage == iPage
2360	&& pSet->aEntries[i - 3].cRefs < UINT16_MAX - 1)
2361	{
2362	pSet->aEntries[i - 3].cRefs++;
2363	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [4] %s(%d) %s\n", pSet, HCPhys, i - 3, pSet->cEntries, pvPage, pSet->aEntries[i - 3].cRefs, pSet->aEntries[i - 3].cInlinedRefs, pSet->aEntries[i - 3].cUnrefs, iPage, pszFile, iLine, pszFunction));
2364	}
2365	else if ( pSet->aEntries[i - 4].iPage == iPage
2366	&& pSet->aEntries[i - 4].cRefs < UINT16_MAX - 1)
2367	{
2368	pSet->aEntries[i - 4].cRefs++;
2369	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [4] %s(%d) %s\n", pSet, HCPhys, i - 4, pSet->cEntries, pvPage, pSet->aEntries[i - 4].cRefs, pSet->aEntries[i - 4].cInlinedRefs, pSet->aEntries[i - 4].cUnrefs, iPage, pszFile, iLine, pszFunction));
2370	}
2371	/* Don't bother searching unless we're above a 60% load. */
2372	else if (RT_LIKELY(i <= (int32_t)RT_ELEMENTS(pSet->aEntries) * 60 / 100))
2373	{
2374	unsigned iEntry = pSet->cEntries++;
2375	pSet->aEntries[iEntry].cRefs = 1;
2376	pSet->aEntries[iEntry].cUnrefs = 0;
2377	pSet->aEntries[iEntry].cInlinedRefs = 0;
2378	pSet->aEntries[iEntry].iPage = iPage;
2379	pSet->aEntries[iEntry].pvPage = pvPage;
2380	pSet->aEntries[iEntry].HCPhys = HCPhys;
2381	pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
2382	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=1/0/0 iPage=%#x [b] %s(%d) %s\n", pSet, HCPhys, iEntry, pSet->cEntries, pvPage, iPage, pszFile, iLine, pszFunction));
2383	}
2384	else
2385	{
2386	/* Search the rest of the set. */
2387	Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
2388	i -= 4;
2389	while (i-- > 0)
2390	if ( pSet->aEntries[i].iPage == iPage
2391	&& pSet->aEntries[i].cRefs < UINT16_MAX - 1)
2392	{
2393	pSet->aEntries[i].cRefs++;
2394	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchHits);
2395	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [c] %s(%d) %s\n", pSet, HCPhys, i, pSet->cEntries, pvPage, pSet->aEntries[i].cRefs, pSet->aEntries[i].cInlinedRefs, pSet->aEntries[i].cUnrefs, iPage, pszFile, iLine, pszFunction));
2396	break;
2397	}
2398	if (i < 0)
2399	{
2400	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchMisses);
2401	#if 0 /* this is very bogus */
2402	if (pSet->iSubset < pSet->cEntries)
2403	{
2404	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchFlushes);
2405	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(pSet->cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
2406	pgmDynMapFlushSubset(pSet);
2407	}
2408	#endif
2409
2410	if (RT_UNLIKELY(pSet->cEntries >= RT_ELEMENTS(pSet->aEntries)))
2411	{
2412	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize);
2413	pgmDynMapOptimizeAutoSet(pSet);
2414	}
2415
2416	if (RT_LIKELY(pSet->cEntries < RT_ELEMENTS(pSet->aEntries)))
2417	{
2418	unsigned iEntry = pSet->cEntries++;
2419	pSet->aEntries[iEntry].cRefs = 1;
2420	pSet->aEntries[iEntry].cUnrefs = 0;
2421	pSet->aEntries[iEntry].cInlinedRefs = 0;
2422	pSet->aEntries[iEntry].iPage = iPage;
2423	pSet->aEntries[iEntry].pvPage = pvPage;
2424	pSet->aEntries[iEntry].HCPhys = HCPhys;
2425	pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
2426	LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=1/0/0 iPage=%#x [d] %s(%d) %s\n", pSet, HCPhys, iEntry, pSet->cEntries, pvPage, iPage, pszFile, iLine, pszFunction));
2427	}
2428	else
2429	{
2430	/* We're screwed. */
2431	pgmRZDynMapReleasePage(pThis, iPage, 1);
2432
2433	RTAssertMsg2Weak("pgmRZDynMapHCPageCommon: set is full!\n");
2434	if (!g_fPGMR0DynMapTestRunning)
2435	VMMRZCallRing3NoCpu(PGMRZDYNMAP_SET_2_VM(pSet), VMMCALLRING3_VM_R0_ASSERTION, 0);
2436	*ppv = NULL;
2437	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
2438	return VERR_PGM_DYNMAP_FULL_SET;
2439	}
2440	}
2441	}
2442
2443	*ppv = pvPage;
2444	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
2445	return VINF_SUCCESS;
2446	}
2447
2448
2449	#if 0 /def DEBUG/
2450	/** For pgmR0DynMapTest3PerCpu. */
2451	typedef struct PGMR0DYNMAPTEST
2452	{
2453	uint32_t u32Expect;
2454	uint32_t *pu32;
2455	uint32_t volatile cFailures;
2456	} PGMR0DYNMAPTEST;
2457	typedef PGMR0DYNMAPTEST *PPGMR0DYNMAPTEST;
2458
2459	/**
2460	* Checks that the content of the page is the same on all CPUs, i.e. that there
2461	* are no CPU specific PTs or similar nasty stuff involved.
2462	*
2463	* @param idCpu The current CPU.
2464	* @param pvUser1 Pointer a PGMR0DYNMAPTEST structure.
2465	* @param pvUser2 Unused, ignored.
2466	*/
2467	static DECLCALLBACK(void) pgmR0DynMapTest3PerCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
2468	{
2469	PPGMR0DYNMAPTEST pTest = (PPGMR0DYNMAPTEST)pvUser1;
2470	ASMInvalidatePage(pTest->pu32);
2471	if (*pTest->pu32 != pTest->u32Expect)
2472	ASMAtomicIncU32(&pTest->cFailures);
2473	NOREF(pvUser2); NOREF(idCpu);
2474	}
2475
2476
2477	/**
2478	* Performs some basic tests in debug builds.
2479	*/
2480	static int pgmR0DynMapTest(PVM pVM)
2481	{
2482	LogRel(("pgmR0DynMapTest: **** START ****\n"));
2483	PPGMMAPSET pSet = &pVM->aCpus[0].pgm.s.AutoSet;
2484	PPGMRZDYNMAP pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
2485	uint32_t i;
2486
2487	/*
2488	* Assert internal integrity first.
2489	*/
2490	LogRel(("Test #0\n"));
2491	int rc = PGMR0DynMapAssertIntegrity();
2492	if (RT_FAILURE(rc))
2493	return rc;
2494
2495	void *pvR0DynMapUsedSaved = pVM->pgm.s.pvR0DynMapUsed;
2496	pVM->pgm.s.pvR0DynMapUsed = pThis;
2497	g_fPGMR0DynMapTestRunning = true;
2498
2499	/*
2500	* Simple test, map CR3 twice and check that we're getting the
2501	* same mapping address back.
2502	*/
2503	LogRel(("Test #1\n"));
2504	ASMIntDisable();
2505	PGMRZDynMapStartAutoSet(&pVM->aCpus[0]);
2506
2507	uint64_t cr3 = ASMGetCR3() & ~(uint64_t)PAGE_OFFSET_MASK;
2508	void pv = (void )(intptr_t)-1;
2509	void pv2 = (void )(intptr_t)-2;
2510	rc = pgmRZDynMapHCPageCommon(pVM, cr3, &pv RTLOG_COMMA_SRC_POS);
2511	int rc2 = pgmRZDynMapHCPageCommon(pVM, cr3, &pv2 RTLOG_COMMA_SRC_POS);
2512	ASMIntEnable();
2513	if ( RT_SUCCESS(rc2)
2514	&& RT_SUCCESS(rc)
2515	&& pv == pv2)
2516	{
2517	LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2518	rc = PGMR0DynMapAssertIntegrity();
2519
2520	/*
2521	* Check that the simple set overflow code works by filling it
2522	* with more CR3 mappings.
2523	*/
2524	LogRel(("Test #2\n"));
2525	ASMIntDisable();
2526	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2527	for (i = 0 ; i < UINT16_MAX*2 - 1 && RT_SUCCESS(rc) && pv2 == pv; i++)
2528	{
2529	pv2 = (void *)(intptr_t)-4;
2530	rc = pgmRZDynMapHCPageCommon(pVM, cr3, &pv2 RTLOG_COMMA_SRC_POS);
2531	}
2532	ASMIntEnable();
2533	if (RT_FAILURE(rc) \|\| pv != pv2)
2534	{
2535	LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%p\n", __LINE__, rc, pv, pv2, i));
2536	if (RT_SUCCESS(rc)) rc = VERR_PGM_DYNMAP_IPE;
2537	}
2538	else if (pSet->cEntries != 5)
2539	{
2540	LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries) / 2));
2541	rc = VERR_PGM_DYNMAP_IPE;
2542	}
2543	else if ( pSet->aEntries[4].cRefs != UINT16_MAX - 1
2544	\|\| pSet->aEntries[3].cRefs != UINT16_MAX - 1
2545	\|\| pSet->aEntries[2].cRefs != 1
2546	\|\| pSet->aEntries[1].cRefs != 1
2547	\|\| pSet->aEntries[0].cRefs != 1)
2548	{
2549	LogRel(("failed(%d): bad set dist: ", __LINE__));
2550	for (i = 0; i < pSet->cEntries; i++)
2551	LogRel(("[%d]=%d, ", i, pSet->aEntries[i].cRefs));
2552	LogRel(("\n"));
2553	rc = VERR_PGM_DYNMAP_IPE;
2554	}
2555	if (RT_SUCCESS(rc))
2556	rc = PGMR0DynMapAssertIntegrity();
2557	if (RT_SUCCESS(rc))
2558	{
2559	/*
2560	* Trigger an set optimization run (exactly).
2561	*/
2562	LogRel(("Test #3\n"));
2563	ASMIntDisable();
2564	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2565	pv2 = NULL;
2566	for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) - 5 && RT_SUCCESS(rc) && pv2 != pv; i++)
2567	{
2568	pv2 = (void *)(intptr_t)(-5 - i);
2569	rc = pgmRZDynMapHCPageCommon(pVM, cr3 + PAGE_SIZE * (i + 5), &pv2 RTLOG_COMMA_SRC_POS);
2570	}
2571	ASMIntEnable();
2572	if (RT_FAILURE(rc) \|\| pv == pv2)
2573	{
2574	LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%d\n", __LINE__, rc, pv, pv2, i));
2575	if (RT_SUCCESS(rc)) rc = VERR_PGM_DYNMAP_IPE;
2576	}
2577	else if (pSet->cEntries != RT_ELEMENTS(pSet->aEntries))
2578	{
2579	LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2580	rc = VERR_PGM_DYNMAP_IPE;
2581	}
2582	LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2583	if (RT_SUCCESS(rc))
2584	rc = PGMR0DynMapAssertIntegrity();
2585	if (RT_SUCCESS(rc))
2586	{
2587	/*
2588	* Trigger an overflow error.
2589	*/
2590	LogRel(("Test #4\n"));
2591	ASMIntDisable();
2592	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2593	for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) + 2; i++)
2594	{
2595	rc = pgmRZDynMapHCPageCommon(pVM, cr3 - PAGE_SIZE * (i + 5), &pv2 RTLOG_COMMA_SRC_POS);
2596	if (RT_SUCCESS(rc))
2597	rc = PGMR0DynMapAssertIntegrity();
2598	if (RT_FAILURE(rc))
2599	break;
2600	}
2601	ASMIntEnable();
2602	if (rc == VERR_PGM_DYNMAP_FULL_SET)
2603	{
2604	/* flush the set. */
2605	LogRel(("Test #5\n"));
2606	ASMIntDisable();
2607	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2608	PGMRZDynMapReleaseAutoSet(&pVM->aCpus[0]);
2609	PGMRZDynMapStartAutoSet(&pVM->aCpus[0]);
2610	ASMIntEnable();
2611
2612	rc = PGMR0DynMapAssertIntegrity();
2613	}
2614	else
2615	{
2616	LogRel(("failed(%d): rc=%Rrc, wanted %d ; pv2=%p Set=%u/%u; i=%d\n", __LINE__,
2617	rc, VERR_PGM_DYNMAP_FULL_SET, pv2, pSet->cEntries, RT_ELEMENTS(pSet->aEntries), i));
2618	if (RT_SUCCESS(rc)) rc = VERR_PGM_DYNMAP_IPE;
2619	}
2620	}
2621	}
2622	}
2623	else
2624	{
2625	LogRel(("failed(%d): rc=%Rrc rc2=%Rrc; pv=%p pv2=%p\n", __LINE__, rc, rc2, pv, pv2));
2626	if (RT_SUCCESS(rc))
2627	rc = rc2;
2628	}
2629
2630	/*
2631	* Check that everyone sees the same stuff.
2632	*/
2633	if (RT_SUCCESS(rc))
2634	{
2635	LogRel(("Test #5\n"));
2636	ASMIntDisable();
2637	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2638	RTHCPHYS HCPhysPT = RTR0MemObjGetPagePhysAddr(pThis->pSegHead->ahMemObjPTs[0], 0);
2639	rc = pgmRZDynMapHCPageCommon(pVM, HCPhysPT, &pv RTLOG_COMMA_SRC_POS);
2640	if (RT_SUCCESS(rc))
2641	{
2642	PGMR0DYNMAPTEST Test;
2643	uint32_t *pu32Real = &pThis->paPages[pThis->pSegHead->iPage].uPte.pLegacy->u;
2644	Test.pu32 = (uint32_t *)((uintptr_t)pv \| ((uintptr_t)pu32Real & PAGE_OFFSET_MASK));
2645	Test.u32Expect = *pu32Real;
2646	ASMAtomicWriteU32(&Test.cFailures, 0);
2647	ASMIntEnable();
2648
2649	rc = RTMpOnAll(pgmR0DynMapTest3PerCpu, &Test, NULL);
2650	if (RT_FAILURE(rc))
2651	LogRel(("failed(%d): RTMpOnAll rc=%Rrc\n", __LINE__, rc));
2652	else if (Test.cFailures)
2653	{
2654	LogRel(("failed(%d): cFailures=%d pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n", __LINE__,
2655	Test.cFailures, pu32Real, Test.pu32, Test.u32Expect, *Test.pu32));
2656	rc = VERR_PGM_DYNMAP_IPE;
2657	}
2658	else
2659	LogRel(("pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n",
2660	pu32Real, Test.pu32, Test.u32Expect, *Test.pu32));
2661	}
2662	else
2663	{
2664	ASMIntEnable();
2665	LogRel(("failed(%d): rc=%Rrc\n", rc));
2666	}
2667	}
2668
2669	/*
2670	* Clean up.
2671	*/
2672	LogRel(("Cleanup.\n"));
2673	ASMIntDisable();
2674	PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
2675	PGMRZDynMapFlushAutoSet(&pVM->aCpus[0]);
2676	PGMRZDynMapReleaseAutoSet(&pVM->aCpus[0]);
2677	ASMIntEnable();
2678
2679	if (RT_SUCCESS(rc))
2680	rc = PGMR0DynMapAssertIntegrity();
2681	else
2682	PGMR0DynMapAssertIntegrity();
2683
2684	g_fPGMR0DynMapTestRunning = false;
2685	LogRel(("Result: rc=%Rrc Load=%u/%u/%u Set=%#x/%u\n", rc,
2686	pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2687	pVM->pgm.s.pvR0DynMapUsed = pvR0DynMapUsedSaved;
2688	LogRel(("pgmR0DynMapTest: **** END ****\n"));
2689	return rc;
2690	}
2691	#endif /* DEBUG */
2692

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMRZ/PGMRZDynMap.cpp@ 61574

Download in other formats: