PGMR0DynMap.cpp@ 14663

Last change on this file since 14663 was 14663, checked in by vboxsync, 16 years ago
PGMR0DynMap: logging/hacking.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 57.8 KB

Line
1	/* $Id: PGMR0DynMap.cpp 14663 2008-11-26 21:08:09Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, ring-0 dynamic mapping cache.
4	*/
5
6	/*
7	* Copyright (C) 2008 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22	/*******************************************************************************
23	* Internal Functions *
24	*******************************************************************************/
25	#define LOG_GROUP LOG_GROUP_PGM
26	#include <VBox/pgm.h>
27	#include "../PGMInternal.h"
28	#include <VBox/vm.h>
29	#include <VBox/sup.h>
30	#include <VBox/err.h>
31	#include <iprt/asm.h>
32	#include <iprt/alloc.h>
33	#include <iprt/assert.h>
34	#include <iprt/cpuset.h>
35	#include <iprt/memobj.h>
36	#include <iprt/mp.h>
37	#include <iprt/semaphore.h>
38	#include <iprt/spinlock.h>
39	#include <iprt/string.h>
40
41
42	/*******************************************************************************
43	* Defined Constants And Macros *
44	*******************************************************************************/
45	/** The max size of the mapping cache (in pages). */
46	#define PGMR0DYNMAP_MAX_PAGES ((8*_1M) >> PAGE_SHIFT)
47	/** The small segment size that is adopted on out-of-memory conditions with a
48	* single big segment. */
49	#define PGMR0DYNMAP_SMALL_SEG_PAGES 128
50	/** The number of pages we reserve per CPU. */
51	#define PGMR0DYNMAP_PAGES_PER_CPU 64
52	/** Calcs the overload threshold. Current set at 50%. */
53	#define PGMR0DYNMAP_CALC_OVERLOAD(cPages) ((cPages) / 2)
54
55	/* Assertions causes panics if preemption is disabled, this can be used to work aroudn that. */
56	/*#define RTSpinlockAcquire(a,b) do {} while (0)
57	#define RTSpinlockRelease(a,b) do {} while (0) */
58
59
60
61	/*******************************************************************************
62	* Structures and Typedefs *
63	*******************************************************************************/
64	/**
65	* Ring-0 dynamic mapping cache segment.
66	*
67	* The dynamic mapping cache can be extended with additional segments if the
68	* load is found to be too high. This done the next time a VM is created, under
69	* the protection of the init mutex. The arrays is reallocated and the new
70	* segment is added to the end of these. Nothing is rehashed of course, as the
71	* indexes / addresses must remain unchanged.
72	*
73	* This structure is only modified while owning the init mutex or during module
74	* init / term.
75	*/
76	typedef struct PGMR0DYNMAPSEG
77	{
78	/** Pointer to the next segment. */
79	struct PGMR0DYNMAPSEG *pNext;
80	/** The memory object for the virtual address range that we're abusing. */
81	RTR0MEMOBJ hMemObj;
82	/** The start page in the cache. (I.e. index into the arrays.) */
83	uint16_t iPage;
84	/** The number of pages this segment contributes. */
85	uint16_t cPages;
86	/** The number of page tables. */
87	uint16_t cPTs;
88	/** The memory objects for the page tables. */
89	RTR0MEMOBJ ahMemObjPTs[1];
90	} PGMR0DYNMAPSEG;
91	/** Pointer to a ring-0 dynamic mapping cache segment. */
92	typedef PGMR0DYNMAPSEG *PPGMR0DYNMAPSEG;
93
94
95	/**
96	* Ring-0 dynamic mapping cache entry.
97	*
98	* This structure tracks
99	*/
100	typedef struct PGMR0DYNMAPENTRY
101	{
102	/** The physical address of the currently mapped page.
103	* This is duplicate for three reasons: cache locality, cache policy of the PT
104	* mappings and sanity checks. */
105	RTHCPHYS HCPhys;
106	/** Pointer to the page. */
107	void *pvPage;
108	/** The number of references. */
109	int32_t volatile cRefs;
110	/** PTE pointer union. */
111	union PGMR0DYNMAPENTRY_PPTE
112	{
113	/** PTE pointer, 32-bit legacy version. */
114	PX86PTE pLegacy;
115	/** PTE pointer, PAE version. */
116	PX86PTEPAE pPae;
117	/** PTE pointer, the void version. */
118	void *pv;
119	} uPte;
120	/** CPUs that haven't invalidated this entry after it's last update. */
121	RTCPUSET PendingSet;
122	} PGMR0DYNMAPENTRY;
123	/** Pointer to a ring-0 dynamic mapping cache entry. */
124	typedef PGMR0DYNMAPENTRY *PPGMR0DYNMAPENTRY;
125
126
127	/**
128	* Ring-0 dynamic mapping cache.
129	*
130	* This is initialized during VMMR0 module init but no segments are allocated at
131	* that time. Segments will be added when the first VM is started and removed
132	* again when the last VM shuts down, thus avoid consuming memory while dormant.
133	* At module termination, the remaining bits will be freed up.
134	*/
135	typedef struct PGMR0DYNMAP
136	{
137	/** The usual magic number / eye catcher (PGMR0DYNMAP_MAGIC). */
138	uint32_t u32Magic;
139	/** Spinlock serializing the normal operation of the cache. */
140	RTSPINLOCK hSpinlock;
141	/** Array for tracking and managing the pages. */
142	PPGMR0DYNMAPENTRY paPages;
143	/** The cache size given as a number of pages. */
144	uint32_t cPages;
145	/** Whether it's 32-bit legacy or PAE/AMD64 paging mode. */
146	bool fLegacyMode;
147	/** The current load. */
148	uint32_t cLoad;
149	/** The max load ever.
150	* This is maintained to get trigger adding of more mapping space. */
151	uint32_t cMaxLoad;
152	/** Initialization / termination lock. */
153	RTSEMFASTMUTEX hInitLock;
154	/** The number of users (protected by hInitLock). */
155	uint32_t cUsers;
156	/** Array containing a copy of the original page tables.
157	* The entries are either X86PTE or X86PTEPAE according to fLegacyMode. */
158	void *pvSavedPTEs;
159	/** List of segments. */
160	PPGMR0DYNMAPSEG pSegHead;
161	/** The paging mode. */
162	SUPPAGINGMODE enmPgMode;
163	} PGMR0DYNMAP;
164	/** Pointer to the ring-0 dynamic mapping cache */
165	typedef PGMR0DYNMAP *PPGMR0DYNMAP;
166
167	/** PGMR0DYNMAP::u32Magic. (Jens Christian Bugge Wesseltoft) */
168	#define PGMR0DYNMAP_MAGIC 0x19640201
169
170
171	/**
172	* Paging level data.
173	*/
174	typedef struct PGMR0DYNMAPPGLVL
175	{
176	uint32_t cLevels; /*< The number of levels. /
177	struct
178	{
179	RTHCPHYS HCPhys; /**< The address of the page for the current level,
180	* i.e. what hMemObj/hMapObj is currently mapping. */
181	RTHCPHYS fPhysMask; /*< Mask for extracting HCPhys from uEntry. /
182	RTR0MEMOBJ hMemObj; /*< Memory object for HCPhys, PAGE_SIZE. /
183	RTR0MEMOBJ hMapObj; /*< Mapping object for hMemObj. /
184	uint32_t fPtrShift; /*< The pointer shift count. /
185	uint64_t fPtrMask; /*< The mask to apply to the shifted pointer to get the table index. /
186	uint64_t fAndMask; /*< And mask to check entry flags. /
187	uint64_t fResMask; /*< The result from applying fAndMask. /
188	union
189	{
190	void pv; /< hMapObj address. /
191	PX86PGUINT paLegacy; /*< Legacy table view. /
192	PX86PGPAEUINT paPae; /*< PAE/AMD64 table view. /
193	} u;
194	} a[4];
195	} PGMR0DYNMAPPGLVL;
196	/** Pointer to paging level data. */
197	typedef PGMR0DYNMAPPGLVL *PPGMR0DYNMAPPGLVL;
198
199
200	/*******************************************************************************
201	* Global Variables *
202	*******************************************************************************/
203	/** Pointer to the ring-0 dynamic mapping cache. */
204	static PPGMR0DYNMAP g_pPGMR0DynMap;
205
206
207	/*******************************************************************************
208	* Internal Functions *
209	*******************************************************************************/
210	static void pgmR0DynMapReleasePage(PPGMR0DYNMAP pThis, uint32_t iPage, uint32_t cRefs);
211	static int pgmR0DynMapSetup(PPGMR0DYNMAP pThis);
212	static int pgmR0DynMapExpand(PPGMR0DYNMAP pThis);
213	static void pgmR0DynMapTearDown(PPGMR0DYNMAP pThis);
214	#ifdef DEBUG
215	static int pgmR0DynMapTest(PVM pVM);
216	#endif
217
218
219	/**
220	* Initializes the ring-0 dynamic mapping cache.
221	*
222	* @returns VBox status code.
223	*/
224	VMMR0DECL(int) PGMR0DynMapInit(void)
225	{
226	Assert(!g_pPGMR0DynMap);
227
228	/*
229	* Create and initialize the cache instance.
230	*/
231	PPGMR0DYNMAP pThis = (PPGMR0DYNMAP)RTMemAllocZ(sizeof(*pThis));
232	AssertLogRelReturn(pThis, VERR_NO_MEMORY);
233	int rc = VINF_SUCCESS;
234	pThis->enmPgMode = SUPR0GetPagingMode();
235	switch (pThis->enmPgMode)
236	{
237	case SUPPAGINGMODE_32_BIT:
238	case SUPPAGINGMODE_32_BIT_GLOBAL:
239	pThis->fLegacyMode = false;
240	break;
241	case SUPPAGINGMODE_PAE:
242	case SUPPAGINGMODE_PAE_GLOBAL:
243	case SUPPAGINGMODE_PAE_NX:
244	case SUPPAGINGMODE_PAE_GLOBAL_NX:
245	case SUPPAGINGMODE_AMD64:
246	case SUPPAGINGMODE_AMD64_GLOBAL:
247	case SUPPAGINGMODE_AMD64_NX:
248	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
249	pThis->fLegacyMode = false;
250	break;
251	default:
252	rc = VERR_INTERNAL_ERROR;
253	break;
254	}
255	if (RT_SUCCESS(rc))
256	{
257	rc = RTSemFastMutexCreate(&pThis->hInitLock);
258	if (RT_SUCCESS(rc))
259	{
260	rc = RTSpinlockCreate(&pThis->hSpinlock);
261	if (RT_SUCCESS(rc))
262	{
263	pThis->u32Magic = PGMR0DYNMAP_MAGIC;
264	g_pPGMR0DynMap = pThis;
265	return VINF_SUCCESS;
266	}
267	RTSemFastMutexDestroy(pThis->hInitLock);
268	}
269	}
270	RTMemFree(pThis);
271	return rc;
272	}
273
274
275	/**
276	* Terminates the ring-0 dynamic mapping cache.
277	*/
278	VMMR0DECL(void) PGMR0DynMapTerm(void)
279	{
280	/*
281	* Destroy the cache.
282	*
283	* There is not supposed to be any races here, the loader should
284	* make sure about that. So, don't bother locking anything.
285	*
286	* The VM objects should all be destroyed by now, so there is no
287	* dangling users or anything like that to clean up. This routine
288	* is just a mirror image of PGMR0DynMapInit.
289	*/
290	PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
291	if (pThis)
292	{
293	AssertPtr(pThis);
294	g_pPGMR0DynMap = NULL;
295
296	AssertLogRelMsg(!pThis->cUsers && !pThis->paPages && !pThis->pvSavedPTEs && !pThis->cPages,
297	("cUsers=%d paPages=%p pvSavedPTEs=%p cPages=%#x\n",
298	pThis->cUsers, pThis->paPages, pThis->pvSavedPTEs, pThis->cPages));
299
300	/* Free the associated resources. */
301	RTSemFastMutexDestroy(pThis->hInitLock);
302	pThis->hInitLock = NIL_RTSEMFASTMUTEX;
303	RTSpinlockDestroy(pThis->hSpinlock);
304	pThis->hSpinlock = NIL_RTSPINLOCK;
305	pThis->u32Magic = UINT32_MAX;
306	RTMemFree(pThis);
307	}
308	}
309
310
311	/**
312	* Initializes the dynamic mapping cache for a new VM.
313	*
314	* @returns VBox status code.
315	* @param pVM Pointer to the shared VM structure.
316	*/
317	VMMR0DECL(int) PGMR0DynMapInitVM(PVM pVM)
318	{
319	AssertMsgReturn(!pVM->pgm.s.pvR0DynMapUsed, ("%p (pThis=%p)\n", pVM->pgm.s.pvR0DynMapUsed, g_pPGMR0DynMap), VERR_WRONG_ORDER);
320
321	/*
322	* Initialize the auto sets.
323	*/
324	VMCPUID idCpu = pVM->cCPUs;
325	AssertReturn(idCpu > 0 && idCpu <= VMCPU_MAX_CPU_COUNT, VERR_INTERNAL_ERROR);
326	while (idCpu-- > 0)
327	{
328	PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
329	uint32_t j = RT_ELEMENTS(pSet->aEntries);
330	while (j-- > 0)
331	{
332	pSet->aEntries[j].iPage = UINT16_MAX;
333	pSet->aEntries[j].cRefs = 0;
334	}
335	pSet->cEntries = PGMMAPSET_CLOSED;
336	}
337
338	/*
339	* Do we need the cache? Skip the last bit if we don't.
340	*/
341	#if 1
342	if (!VMMIsHwVirtExtForced(pVM))
343	return VINF_SUCCESS;
344	#endif
345
346	/*
347	* Reference and if necessary setup or expand the cache.
348	*/
349	PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
350	AssertPtrReturn(pThis, VERR_INTERNAL_ERROR);
351	int rc = RTSemFastMutexRequest(pThis->hInitLock);
352	AssertLogRelRCReturn(rc, rc);
353
354	pThis->cUsers++;
355	if (pThis->cUsers == 1)
356	{
357	rc = pgmR0DynMapSetup(pThis);
358	#ifdef DEBUG
359	if (RT_SUCCESS(rc))
360	{
361	rc = pgmR0DynMapTest(pVM);
362	if (RT_FAILURE(rc))
363	pgmR0DynMapTearDown(pThis);
364	}
365	#endif
366	}
367	else if (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(pThis->cPages))
368	rc = pgmR0DynMapExpand(pThis);
369	if (RT_SUCCESS(rc))
370	pVM->pgm.s.pvR0DynMapUsed = pThis;
371	else
372	pThis->cUsers--;
373
374	RTSemFastMutexRelease(pThis->hInitLock);
375	return rc;
376	}
377
378
379	/**
380	* Terminates the dynamic mapping cache usage for a VM.
381	*
382	* @param pVM Pointer to the shared VM structure.
383	*/
384	VMMR0DECL(void) PGMR0DynMapTermVM(PVM pVM)
385	{
386	/*
387	* Return immediately if we're not using the cache.
388	*/
389	if (!pVM->pgm.s.pvR0DynMapUsed)
390	return;
391
392	PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
393	AssertPtrReturnVoid(pThis);
394
395	int rc = RTSemFastMutexRequest(pThis->hInitLock);
396	AssertLogRelRCReturnVoid(rc);
397
398	if (pVM->pgm.s.pvR0DynMapUsed == pThis)
399	{
400	pVM->pgm.s.pvR0DynMapUsed = NULL;
401
402	/*
403	* Clean up and check the auto sets.
404	*/
405	VMCPUID idCpu = pVM->cCPUs;
406	while (idCpu-- > 0)
407	{
408	PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
409	uint32_t j = pSet->cEntries;
410	if (j <= RT_ELEMENTS(pSet->aEntries))
411	{
412	/*
413	* The set is open, close it.
414	*/
415	while (j-- > 0)
416	{
417	int32_t cRefs = pSet->aEntries[j].cRefs;
418	uint32_t iPage = pSet->aEntries[j].iPage;
419	LogRel(("PGMR0DynMapTermVM: %d dangling refs to %#x\n", cRefs, iPage));
420	if (iPage < pThis->cPages && cRefs > 0)
421	pgmR0DynMapReleasePage(pThis, iPage, cRefs);
422	else
423	AssertLogRelMsgFailed(("cRefs=%d iPage=%#x cPages=%u\n", cRefs, iPage, pThis->cPages));
424
425	pSet->aEntries[j].iPage = UINT16_MAX;
426	pSet->aEntries[j].cRefs = 0;
427	}
428	pSet->cEntries = PGMMAPSET_CLOSED;
429	}
430	else
431	AssertMsg(j == PGMMAPSET_CLOSED, ("cEntries=%#x\n", j));
432
433	j = RT_ELEMENTS(pSet->aEntries);
434	while (j-- > 0)
435	{
436	Assert(pSet->aEntries[j].iPage == UINT16_MAX);
437	Assert(!pSet->aEntries[j].cRefs);
438	}
439	}
440
441	/*
442	* Release our reference to the mapping cache.
443	*/
444	Assert(pThis->cUsers > 0);
445	pThis->cUsers--;
446	if (!pThis->cUsers)
447	pgmR0DynMapTearDown(pThis);
448	}
449	else
450	AssertLogRelMsgFailed(("pvR0DynMapUsed=%p pThis=%p\n", pVM->pgm.s.pvR0DynMapUsed, pThis));
451
452	RTSemFastMutexRelease(pThis->hInitLock);
453	}
454
455
456	/**
457	* Calculate the new cache size based on cMaxLoad statistics.
458	*
459	* @returns Number of pages.
460	* @param pThis The dynamic mapping cache instance.
461	* @param pcMinPages The minimal size in pages.
462	*/
463	static uint32_t pgmR0DynMapCalcNewSize(PPGMR0DYNMAP pThis, uint32_t *pcMinPages)
464	{
465	Assert(pThis->cPages <= PGMR0DYNMAP_MAX_PAGES);
466
467	/* cCpus * PGMR0DYNMAP_PAGES_PER_CPU (/2). */
468	RTCPUID cCpus = RTMpGetCount();
469	AssertReturn(cCpus > 0 && cCpus <= RTCPUSET_MAX_CPUS, 0);
470	uint32_t cPages = cCpus * PGMR0DYNMAP_PAGES_PER_CPU;
471	uint32_t cMinPages = cCpus * (PGMR0DYNMAP_PAGES_PER_CPU / 2);
472
473	/* adjust against cMaxLoad. */
474	AssertMsg(pThis->cMaxLoad <= PGMR0DYNMAP_MAX_PAGES, ("%#x\n", pThis->cMaxLoad));
475	if (pThis->cMaxLoad > PGMR0DYNMAP_MAX_PAGES)
476	pThis->cMaxLoad = 0;
477
478	while (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(cPages))
479	cPages += PGMR0DYNMAP_PAGES_PER_CPU;
480
481	if (pThis->cMaxLoad > cMinPages)
482	cMinPages = pThis->cMaxLoad;
483
484	/* adjust against max and current size. */
485	if (cPages < pThis->cPages)
486	cPages = pThis->cPages;
487	if (cPages > PGMR0DYNMAP_MAX_PAGES)
488	cPages = PGMR0DYNMAP_MAX_PAGES;
489
490	if (cMinPages < pThis->cPages)
491	cMinPages = pThis->cPages;
492	if (cMinPages > PGMR0DYNMAP_MAX_PAGES)
493	cMinPages = PGMR0DYNMAP_MAX_PAGES;
494
495	Assert(cMinPages);
496	*pcMinPages = cMinPages;
497	return cPages;
498	}
499
500
501	/**
502	* Initializes the paging level data.
503	*
504	* @param pThis The dynamic mapping cache instance.
505	* @param pPgLvl The paging level data.
506	*/
507	void pgmR0DynMapPagingArrayInit(PPGMR0DYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl)
508	{
509	RTCCUINTREG cr4 = ASMGetCR4();
510	switch (pThis->enmPgMode)
511	{
512	case SUPPAGINGMODE_32_BIT:
513	case SUPPAGINGMODE_32_BIT_GLOBAL:
514	pPgLvl->cLevels = 2;
515	pPgLvl->a[0].fPhysMask = X86_CR3_PAGE_MASK;
516	pPgLvl->a[0].fAndMask = X86_PDE_P \| X86_PDE_RW \| (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
517	pPgLvl->a[0].fResMask = X86_PDE_P \| X86_PDE_RW;
518	pPgLvl->a[0].fPtrMask = X86_PD_MASK;
519	pPgLvl->a[0].fPtrShift = X86_PD_SHIFT;
520
521	pPgLvl->a[1].fPhysMask = X86_PDE_PG_MASK;
522	pPgLvl->a[1].fAndMask = X86_PTE_P \| X86_PTE_RW;
523	pPgLvl->a[1].fResMask = X86_PTE_P \| X86_PTE_RW;
524	pPgLvl->a[1].fPtrMask = X86_PT_MASK;
525	pPgLvl->a[1].fPtrShift = X86_PT_SHIFT;
526	break;
527
528	case SUPPAGINGMODE_PAE:
529	case SUPPAGINGMODE_PAE_GLOBAL:
530	case SUPPAGINGMODE_PAE_NX:
531	case SUPPAGINGMODE_PAE_GLOBAL_NX:
532	pPgLvl->cLevels = 3;
533	pPgLvl->a[0].fPhysMask = X86_CR3_PAE_PAGE_MASK;
534	pPgLvl->a[0].fPtrMask = X86_PDPT_MASK_PAE;
535	pPgLvl->a[0].fPtrShift = X86_PDPT_SHIFT;
536	pPgLvl->a[0].fAndMask = X86_PDPE_P;
537	pPgLvl->a[0].fResMask = X86_PDPE_P;
538
539	pPgLvl->a[1].fPhysMask = X86_PDPE_PG_MASK;
540	pPgLvl->a[1].fPtrMask = X86_PD_PAE_MASK;
541	pPgLvl->a[1].fPtrShift = X86_PD_PAE_SHIFT;
542	pPgLvl->a[1].fAndMask = X86_PDE_P \| X86_PDE_RW \| (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
543	pPgLvl->a[1].fResMask = X86_PDE_P \| X86_PDE_RW;
544
545	pPgLvl->a[2].fPhysMask = X86_PDE_PAE_PG_MASK;
546	pPgLvl->a[2].fPtrMask = X86_PT_PAE_MASK;
547	pPgLvl->a[2].fPtrShift = X86_PT_PAE_SHIFT;
548	pPgLvl->a[2].fAndMask = X86_PTE_P \| X86_PTE_RW;
549	pPgLvl->a[2].fResMask = X86_PTE_P \| X86_PTE_RW;
550	break;
551
552	case SUPPAGINGMODE_AMD64:
553	case SUPPAGINGMODE_AMD64_GLOBAL:
554	case SUPPAGINGMODE_AMD64_NX:
555	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
556	pPgLvl->cLevels = 4;
557	pPgLvl->a[0].fPhysMask = X86_CR3_AMD64_PAGE_MASK;
558	pPgLvl->a[0].fPtrShift = X86_PML4_SHIFT;
559	pPgLvl->a[0].fPtrMask = X86_PML4_MASK;
560	pPgLvl->a[0].fAndMask = X86_PML4E_P \| X86_PML4E_RW;
561	pPgLvl->a[0].fResMask = X86_PML4E_P \| X86_PML4E_RW;
562
563	pPgLvl->a[1].fPhysMask = X86_PML4E_PG_MASK;
564	pPgLvl->a[1].fPtrShift = X86_PDPT_SHIFT;
565	pPgLvl->a[1].fPtrMask = X86_PDPT_MASK_AMD64;
566	pPgLvl->a[1].fAndMask = X86_PDPE_P \| X86_PDPE_RW /** @todo check for X86_PDPT_PS support. */;
567	pPgLvl->a[1].fResMask = X86_PDPE_P \| X86_PDPE_RW;
568
569	pPgLvl->a[2].fPhysMask = X86_PDPE_PG_MASK;
570	pPgLvl->a[2].fPtrShift = X86_PD_PAE_SHIFT;
571	pPgLvl->a[2].fPtrMask = X86_PD_PAE_MASK;
572	pPgLvl->a[2].fAndMask = X86_PDE_P \| X86_PDE_RW \| (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
573	pPgLvl->a[2].fResMask = X86_PDE_P \| X86_PDE_RW;
574
575	pPgLvl->a[3].fPhysMask = X86_PDE_PAE_PG_MASK;
576	pPgLvl->a[3].fPtrShift = X86_PT_PAE_SHIFT;
577	pPgLvl->a[3].fPtrMask = X86_PT_PAE_MASK;
578	pPgLvl->a[3].fAndMask = X86_PTE_P \| X86_PTE_RW;
579	pPgLvl->a[3].fResMask = X86_PTE_P \| X86_PTE_RW;
580	break;
581
582	default:
583	AssertFailed();
584	pPgLvl->cLevels = 0;
585	break;
586	}
587
588	for (uint32_t i = 0; i < 4; i++) /* ASSUMING array size. */
589	{
590	pPgLvl->a[i].HCPhys = NIL_RTHCPHYS;
591	pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
592	pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ;
593	pPgLvl->a[i].u.pv = NULL;
594	}
595	}
596
597
598	/**
599	* Maps a PTE.
600	*
601	* This will update the segment structure when new PTs are mapped.
602	*
603	* It also assumes that we (for paranoid reasons) wish to establish a mapping
604	* chain from CR3 to the PT that all corresponds to the processor we're
605	* currently running on, and go about this by running with interrupts disabled
606	* and restarting from CR3 for every change.
607	*
608	* @returns VBox status code, VINF_TRY_AGAIN if we changed any mappings and had
609	* to re-enable interrupts.
610	* @param pThis The dynamic mapping cache instance.
611	* @param pPgLvl The paging level structure.
612	* @param pvPage The page.
613	* @param pSeg The segment.
614	* @param cMaxPTs The max number of PTs expected in the segment.
615	* @param ppvPTE Where to store the PTE address.
616	*/
617	static int pgmR0DynMapPagingArrayMapPte(PPGMR0DYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl, void *pvPage,
618	PPGMR0DYNMAPSEG pSeg, uint32_t cMaxPTs, void **ppvPTE)
619	{
620	Assert(!(ASMGetFlags() & X86_EFL_IF));
621	void *pvEntry = NULL;
622	X86PGPAEUINT uEntry = ASMGetCR3();
623	for (uint32_t i = 0; i < pPgLvl->cLevels; i++)
624	{
625	RTHCPHYS HCPhys = uEntry & pPgLvl->a[i].fPhysMask;
626	if (pPgLvl->a[i].HCPhys != HCPhys)
627	{
628	/*
629	* Need to remap this level.
630	* The final level, the PT, will not be freed since that is what it's all about.
631	*/
632	ASMIntEnable();
633	if (i + 1 == pPgLvl->cLevels)
634	AssertReturn(pSeg->cPTs < cMaxPTs, VERR_INTERNAL_ERROR);
635	else
636	{
637	int rc2 = RTR0MemObjFree(pPgLvl->a[i].hMemObj, true /* fFreeMappings */); AssertRC(rc2);
638	pPgLvl->a[i].hMemObj = pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
639	}
640
641	int rc = RTR0MemObjEnterPhys(&pPgLvl->a[i].hMemObj, HCPhys, PAGE_SIZE);
642	if (RT_SUCCESS(rc))
643	{
644	rc = RTR0MemObjMapKernel(&pPgLvl->a[i].hMapObj, pPgLvl->a[i].hMemObj,
645	(void )-1 / pvFixed /, 0 / cbAlignment */,
646	RTMEM_PROT_WRITE \| RTMEM_PROT_READ);
647	if (RT_SUCCESS(rc))
648	{
649	pPgLvl->a[i].u.pv = RTR0MemObjAddress(pPgLvl->a[i].hMapObj);
650	AssertMsg(((uintptr_t)pPgLvl->a[i].u.pv & ~(uintptr_t)PAGE_OFFSET_MASK), ("%p\n", pPgLvl->a[i].u.pv));
651	pPgLvl->a[i].HCPhys = HCPhys;
652	if (i + 1 == pPgLvl->cLevels)
653	pSeg->ahMemObjPTs[pSeg->cPTs++] = pPgLvl->a[i].hMemObj;
654	ASMIntDisable();
655	return VINF_TRY_AGAIN;
656	}
657
658	pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
659	}
660	else
661	pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ;
662	pPgLvl->a[i].HCPhys = NIL_RTHCPHYS;
663	return rc;
664	}
665
666	/*
667	* The next level.
668	*/
669	uint32_t iEntry = ((uint64_t)(uintptr_t)pvPage >> pPgLvl->a[i].fPtrShift) & pPgLvl->a[i].fPtrMask;
670	if (pThis->fLegacyMode)
671	{
672	pvEntry = &pPgLvl->a[i].u.paLegacy[iEntry];
673	uEntry = pPgLvl->a[i].u.paLegacy[iEntry];
674	}
675	else
676	{
677	pvEntry = &pPgLvl->a[i].u.paPae[iEntry];
678	uEntry = pPgLvl->a[i].u.paPae[iEntry];
679	}
680
681	if ((uEntry & pPgLvl->a[i].fAndMask) != pPgLvl->a[i].fResMask)
682	{
683	LogRel(("PGMR0DynMap: internal error - iPgLvl=%u cLevels=%u uEntry=%#llx fAnd=%#llx fRes=%#llx got=%#llx\n"
684	"PGMR0DynMap: pv=%p pvPage=%p iEntry=%#x fLegacyMode=%RTbool\n",
685	i, pPgLvl->cLevels, uEntry, pPgLvl->a[i].fAndMask, pPgLvl->a[i].fResMask, uEntry & pPgLvl->a[i].fAndMask,
686	pPgLvl->a[i].u.pv, pvPage, iEntry, pThis->fLegacyMode));
687	return VERR_INTERNAL_ERROR;
688	}
689	Log(("#%d: iEntry=%4d uEntry=%#llx pvEntry=%p HCPhys=%RHp \n", i, iEntry, uEntry, pvEntry, pPgLvl->a[i].HCPhys));
690	}
691
692	/* made it thru without needing to remap anything. */
693	*ppvPTE = pvEntry;
694	return VINF_SUCCESS;
695	}
696
697
698	/**
699	* Adds a new segment of the specified size.
700	*
701	* @returns VBox status code.
702	* @param pThis The dynamic mapping cache instance.
703	* @param cPages The size of the new segment, give as a page count.
704	*/
705	static int pgmR0DynMapAddSeg(PPGMR0DYNMAP pThis, uint32_t cPages)
706	{
707	int rc2;
708	AssertReturn(ASMGetFlags() & X86_EFL_IF, VERR_PREEMPT_DISABLED);
709
710	/*
711	* Do the array rellocation first.
712	* (The pages array has to be replaced behind the spinlock of course.)
713	*/
714	void pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) (pThis->cPages + cPages));
715	if (!pvSavedPTEs)
716	return VERR_NO_MEMORY;
717	pThis->pvSavedPTEs = pvSavedPTEs;
718
719	void pvPages = RTMemAllocZ(sizeof(pThis->paPages[0]) (pThis->cPages + cPages));
720	if (!pvPages)
721	{
722	pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) * pThis->cPages);
723	if (pvSavedPTEs)
724	pThis->pvSavedPTEs = pvSavedPTEs;
725	return VERR_NO_MEMORY;
726	}
727
728	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
729	RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
730
731	memcpy(pvPages, pThis->paPages, sizeof(pThis->paPages[0]) * pThis->cPages);
732	void *pvToFree = pThis->paPages;
733	pThis->paPages = (PPGMR0DYNMAPENTRY)pvPages;
734
735	RTSpinlockRelease(pThis->hSpinlock, &Tmp);
736	RTMemFree(pvToFree);
737
738	/*
739	* Allocate the segment structure and pages of memory, then touch all the pages (paranoia).
740	*/
741	uint32_t cMaxPTs = cPages / (pThis->fLegacyMode ? X86_PG_ENTRIES : X86_PG_PAE_ENTRIES) + 2;
742	PPGMR0DYNMAPSEG pSeg = (PPGMR0DYNMAPSEG)RTMemAllocZ(RT_UOFFSETOF(PGMR0DYNMAPSEG, ahMemObjPTs[cMaxPTs]));
743	if (!pSeg)
744	return VERR_NO_MEMORY;
745	pSeg->pNext = NULL;
746	pSeg->cPages = cPages;
747	pSeg->iPage = pThis->cPages;
748	pSeg->cPTs = 0;
749	int rc = RTR0MemObjAllocPage(&pSeg->hMemObj, cPages << PAGE_SHIFT, false);
750	if (RT_SUCCESS(rc))
751	{
752	uint8_t pbPage = (uint8_t )RTR0MemObjAddress(pSeg->hMemObj);
753	AssertMsg(VALID_PTR(pbPage) && !((uintptr_t)pbPage & PAGE_OFFSET_MASK), ("%p\n", pbPage));
754	memset(pbPage, 0xfe, cPages << PAGE_SHIFT);
755
756	/*
757	* Walk thru the pages and set them up with a mapping of their PTE and everything.
758	*/
759	ASMIntDisable();
760	PGMR0DYNMAPPGLVL PgLvl;
761	pgmR0DynMapPagingArrayInit(pThis, &PgLvl);
762	uint32_t iEndPage = pThis->cPages + cPages;
763	for (uint32_t iPage = pThis->cPages;
764	iPage < iEndPage;
765	iPage++, pbPage += PAGE_SIZE)
766	{
767	/* Initialize the page data. */
768	pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
769	pThis->paPages[iPage].pvPage = pbPage;
770	pThis->paPages[iPage].cRefs = 0;
771	pThis->paPages[iPage].uPte.pPae = 0;
772	RTCpuSetFill(&pThis->paPages[iPage].PendingSet);
773
774	/* Map its page table, retry until we've got a clean run (paranoia). */
775	do
776	rc = pgmR0DynMapPagingArrayMapPte(pThis, &PgLvl, pbPage, pSeg, cMaxPTs,
777	&pThis->paPages[iPage].uPte.pv);
778	while (rc == VINF_TRY_AGAIN);
779	if (RT_FAILURE(rc))
780	break;
781
782	/* Save the PTE. */
783	if (pThis->fLegacyMode)
784	((PX86PGUINT)pThis->pvSavedPTEs)[iPage] = pThis->paPages[iPage].uPte.pLegacy->u;
785	else
786	((PX86PGPAEUINT)pThis->pvSavedPTEs)[iPage] = pThis->paPages[iPage].uPte.pPae->u;
787
788	#ifdef VBOX_STRICT
789	/* Check that we've got the right entry. */
790	RTHCPHYS HCPhysPage = RTR0MemObjGetPagePhysAddr(pSeg->hMemObj, iPage - pSeg->iPage);
791	RTHCPHYS HCPhysPte = pThis->fLegacyMode
792	? pThis->paPages[iPage].uPte.pLegacy->u & X86_PTE_PG_MASK
793	: pThis->paPages[iPage].uPte.pPae->u & X86_PTE_PAE_PG_MASK;
794	if (HCPhysPage != HCPhysPte)
795	{
796	LogRel(("pgmR0DynMapAddSeg: internal error - page #%u HCPhysPage=%RHp HCPhysPte=%RHp pbPage=%p pvPte=%p\n",
797	iPage - pSeg->iPage, HCPhysPage, HCPhysPte, pbPage, pThis->paPages[iPage].uPte.pv));
798	rc = VERR_INTERNAL_ERROR;
799	break;
800	}
801	#endif
802	} /* for each page */
803	ASMIntEnable();
804
805	/* cleanup non-PT mappings */
806	for (uint32_t i = 0; i < PgLvl.cLevels - 1; i++)
807	RTR0MemObjFree(PgLvl.a[i].hMemObj, true /* fFreeMappings */);
808
809	if (RT_SUCCESS(rc))
810	{
811	/** @todo setup guard pages here later (strict builds should leave every
812	* second page and the start/end pages not present). */
813
814	/*
815	* Commit it by adding the segment to the list and updating the page count.
816	*/
817	pSeg->pNext = pThis->pSegHead;
818	pThis->pSegHead = pSeg;
819	pThis->cPages += cPages;
820	return VINF_SUCCESS;
821	}
822
823	/*
824	* Bail out.
825	*/
826	while (pSeg->cPTs-- > 0)
827	{
828	rc2 = RTR0MemObjFree(pSeg->ahMemObjPTs[pSeg->cPTs], true /* fFreeMappings */);
829	AssertRC(rc2);
830	pSeg->ahMemObjPTs[pSeg->cPTs] = NIL_RTR0MEMOBJ;
831	}
832
833	rc2 = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */);
834	AssertRC(rc2);
835	pSeg->hMemObj = NIL_RTR0MEMOBJ;
836	}
837	RTMemFree(pSeg);
838
839	/* Don't bother resizing the arrays, but free them if we're the only user. */
840	if (!pThis->cPages)
841	{
842	RTMemFree(pThis->paPages);
843	pThis->paPages = NULL;
844	RTMemFree(pThis->pvSavedPTEs);
845	pThis->pvSavedPTEs = NULL;
846	}
847	return rc;
848	}
849
850
851	/**
852	* Called by PGMR0DynMapInitVM under the init lock.
853	*
854	* @returns VBox status code.
855	* @param pThis The dynamic mapping cache instance.
856	*/
857	static int pgmR0DynMapSetup(PPGMR0DYNMAP pThis)
858	{
859	/*
860	* Calc the size and add a segment of that size.
861	*/
862	uint32_t cMinPages;
863	uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages);
864	AssertReturn(cPages, VERR_INTERNAL_ERROR);
865	int rc = pgmR0DynMapAddSeg(pThis, cPages);
866	if (rc == VERR_NO_MEMORY)
867	{
868	/*
869	* Try adding smaller segments.
870	*/
871	do
872	rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES);
873	while (RT_SUCCESS(rc) && pThis->cPages < cPages);
874	if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages)
875	rc = VINF_SUCCESS;
876	if (rc == VERR_NO_MEMORY)
877	{
878	if (pThis->cPages)
879	pgmR0DynMapTearDown(pThis);
880	rc = VERR_PGM_DYNMAP_SETUP_ERROR;
881	}
882	}
883	Assert(ASMGetFlags() & X86_EFL_IF);
884	return rc;
885	}
886
887
888	/**
889	* Called by PGMR0DynMapInitVM under the init lock.
890	*
891	* @returns VBox status code.
892	* @param pThis The dynamic mapping cache instance.
893	*/
894	static int pgmR0DynMapExpand(PPGMR0DYNMAP pThis)
895	{
896	/*
897	* Calc the new target size and add a segment of the appropriate size.
898	*/
899	uint32_t cMinPages;
900	uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages);
901	AssertReturn(cPages, VERR_INTERNAL_ERROR);
902	if (pThis->cPages >= cPages)
903	return VINF_SUCCESS;
904
905	uint32_t cAdd = cPages - pThis->cPages;
906	int rc = pgmR0DynMapAddSeg(pThis, cAdd);
907	if (rc == VERR_NO_MEMORY)
908	{
909	/*
910	* Try adding smaller segments.
911	*/
912	do
913	rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES);
914	while (RT_SUCCESS(rc) && pThis->cPages < cPages);
915	if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages)
916	rc = VINF_SUCCESS;
917	if (rc == VERR_NO_MEMORY)
918	rc = VERR_PGM_DYNMAP_EXPAND_ERROR;
919	}
920	Assert(ASMGetFlags() & X86_EFL_IF);
921	return rc;
922	}
923
924
925	/**
926	* Shoots down the TLBs for all the cache pages, pgmR0DynMapTearDown helper.
927	*
928	* @param idCpu The current CPU.
929	* @param pvUser1 The dynamic mapping cache instance.
930	* @param pvUser2 Unused, NULL.
931	*/
932	static DECLCALLBACK(void) pgmR0DynMapShootDownTlbs(RTCPUID idCpu, void pvUser1, void pvUser2)
933	{
934	Assert(!pvUser2);
935	PPGMR0DYNMAP pThis = (PPGMR0DYNMAP)pvUser1;
936	Assert(pThis == g_pPGMR0DynMap);
937	PPGMR0DYNMAPENTRY paPages = pThis->paPages;
938	uint32_t iPage = pThis->cPages;
939	while (iPage-- > 0)
940	ASMInvalidatePage(paPages[iPage].pvPage);
941	}
942
943
944	/**
945	* Called by PGMR0DynMapTermVM under the init lock.
946	*
947	* @returns VBox status code.
948	* @param pThis The dynamic mapping cache instance.
949	*/
950	static void pgmR0DynMapTearDown(PPGMR0DYNMAP pThis)
951	{
952	/*
953	* Restore the original page table entries
954	*/
955	PPGMR0DYNMAPENTRY paPages = pThis->paPages;
956	uint32_t iPage = pThis->cPages;
957	if (pThis->fLegacyMode)
958	{
959	X86PGUINT const paSavedPTEs = (X86PGUINT const )pThis->pvSavedPTEs;
960	while (iPage-- > 0)
961	{
962	X86PGUINT uOld = paPages[iPage].uPte.pLegacy->u;
963	X86PGUINT uOld2 = uOld; NOREF(uOld2);
964	X86PGUINT uNew = paSavedPTEs[iPage];
965	while (!ASMAtomicCmpXchgExU32(&paPages[iPage].uPte.pLegacy->u, uNew, uOld, &uOld))
966	AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
967	}
968	}
969	else
970	{
971	X86PGPAEUINT const paSavedPTEs = (X86PGPAEUINT const )pThis->pvSavedPTEs;
972	while (iPage-- > 0)
973	{
974	X86PGPAEUINT uOld = paPages[iPage].uPte.pPae->u;
975	X86PGPAEUINT uOld2 = uOld; NOREF(uOld2);
976	X86PGPAEUINT uNew = paSavedPTEs[iPage];
977	while (!ASMAtomicCmpXchgExU64(&paPages[iPage].uPte.pPae->u, uNew, uOld, &uOld))
978	AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
979	}
980	}
981
982	/*
983	* Shoot down the TLBs on all CPUs before freeing them.
984	* If RTMpOnAll fails, make sure the TLBs are invalidated on the current CPU at least.
985	*/
986	int rc = RTMpOnAll(pgmR0DynMapShootDownTlbs, pThis, NULL);
987	AssertRC(rc);
988	if (RT_FAILURE(rc))
989	{
990	iPage = pThis->cPages;
991	while (iPage-- > 0)
992	ASMInvalidatePage(paPages[iPage].pvPage);
993	}
994
995	/*
996	* Free the segments.
997	*/
998	while (pThis->pSegHead)
999	{
1000	PPGMR0DYNMAPSEG pSeg = pThis->pSegHead;
1001	pThis->pSegHead = pSeg->pNext;
1002
1003	uint32_t iPT = pSeg->cPTs;
1004	while (iPT-- > 0)
1005	{
1006	rc = RTR0MemObjFree(pSeg->ahMemObjPTs[iPT], true /* fFreeMappings */); AssertRC(rc);
1007	pSeg->ahMemObjPTs[iPT] = NIL_RTR0MEMOBJ;
1008	}
1009	rc = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */); AssertRC(rc);
1010	pSeg->hMemObj = NIL_RTR0MEMOBJ;
1011	pSeg->pNext = NULL;
1012	pSeg->iPage = UINT16_MAX;
1013	pSeg->cPages = 0;
1014	pSeg->cPTs = 0;
1015	RTMemFree(pSeg);
1016	}
1017
1018	/*
1019	* Free the arrays and restore the initial state.
1020	* The cLoadMax value is left behind for the next setup.
1021	*/
1022	RTMemFree(pThis->paPages);
1023	pThis->paPages = NULL;
1024	RTMemFree(pThis->pvSavedPTEs);
1025	pThis->pvSavedPTEs = NULL;
1026	pThis->cPages = 0;
1027	pThis->cLoad = 0;
1028	}
1029
1030
1031	/**
1032	* Release references to a page, caller owns the spin lock.
1033	*
1034	* @param pThis The dynamic mapping cache instance.
1035	* @param iPage The page.
1036	* @param cRefs The number of references to release.
1037	*/
1038	DECLINLINE(void) pgmR0DynMapReleasePageLocked(PPGMR0DYNMAP pThis, uint32_t iPage, int32_t cRefs)
1039	{
1040	cRefs = ASMAtomicSubS32(&pThis->paPages[iPage].cRefs, cRefs);
1041	AssertMsg(cRefs >= 0, ("%d\n", cRefs));
1042	if (!cRefs)
1043	pThis->cLoad--;
1044	}
1045
1046
1047	/**
1048	* Release references to a page, caller does not own the spin lock.
1049	*
1050	* @param pThis The dynamic mapping cache instance.
1051	* @param iPage The page.
1052	* @param cRefs The number of references to release.
1053	*/
1054	static void pgmR0DynMapReleasePage(PPGMR0DYNMAP pThis, uint32_t iPage, uint32_t cRefs)
1055	{
1056	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1057	RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1058	pgmR0DynMapReleasePageLocked(pThis, iPage, cRefs);
1059	RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1060	}
1061
1062
1063	/**
1064	* pgmR0DynMapPage worker that deals with the tedious bits.
1065	*
1066	* @returns The page index on success, UINT32_MAX on failure.
1067	* @param pThis The dynamic mapping cache instance.
1068	* @param HCPhys The address of the page to be mapped.
1069	* @param iPage The page index pgmR0DynMapPage hashed HCPhys to.
1070	*/
1071	static uint32_t pgmR0DynMapPageSlow(PPGMR0DYNMAP pThis, RTHCPHYS HCPhys, uint32_t iPage)
1072	{
1073	/*
1074	* Check if any of the first 5 pages are unreferenced since the caller
1075	* already has made sure they aren't matching.
1076	*/
1077	uint32_t const cPages = pThis->cPages;
1078	PPGMR0DYNMAPENTRY paPages = pThis->paPages;
1079	uint32_t iFreePage;
1080	if (!paPages[iPage].cRefs)
1081	iFreePage = iPage;
1082	else if (!paPages[(iPage + 1) % cPages].cRefs)
1083	iFreePage = iPage;
1084	else if (!paPages[(iPage + 2) % cPages].cRefs)
1085	iFreePage = iPage;
1086	else if (!paPages[(iPage + 3) % cPages].cRefs)
1087	iFreePage = iPage;
1088	else if (!paPages[(iPage + 4) % cPages].cRefs)
1089	iFreePage = iPage;
1090	else
1091	{
1092	/*
1093	* Search for an unused or matching entry.
1094	*/
1095	iFreePage = (iPage + 5) % pThis->cPages;
1096	for (;;)
1097	{
1098	if (paPages[iFreePage].HCPhys == HCPhys)
1099	return iFreePage;
1100	if (!paPages[iFreePage].cRefs)
1101	break;
1102
1103	/* advance */
1104	iFreePage = (iFreePage + 1) % cPages;
1105	if (RT_UNLIKELY(iFreePage != iPage))
1106	return UINT32_MAX;
1107	}
1108	}
1109
1110	/*
1111	* Setup the new entry.
1112	*/
1113	paPages[iFreePage].HCPhys = HCPhys;
1114	RTCpuSetFill(&paPages[iFreePage].PendingSet);
1115	if (pThis->fLegacyMode)
1116	{
1117	X86PGUINT uOld = paPages[iFreePage].uPte.pLegacy->u;
1118	X86PGUINT uOld2 = uOld; NOREF(uOld2);
1119	X86PGUINT uNew = (uOld & X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT)
1120	\| X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1121	\| (HCPhys & X86_PTE_PG_MASK);
1122	while (!ASMAtomicCmpXchgExU32(&paPages[iFreePage].uPte.pLegacy->u, uNew, uOld, &uOld))
1123	AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
1124	}
1125	else
1126	{
1127	X86PGPAEUINT uOld = paPages[iFreePage].uPte.pPae->u;
1128	X86PGPAEUINT uOld2 = uOld; NOREF(uOld2);
1129	X86PGPAEUINT uNew = (uOld & X86_PTE_G \| X86_PTE_PAT \| X86_PTE_PCD \| X86_PTE_PWT)
1130	\| X86_PTE_P \| X86_PTE_RW \| X86_PTE_A \| X86_PTE_D
1131	\| (HCPhys & X86_PTE_PAE_PG_MASK);
1132	while (!ASMAtomicCmpXchgExU64(&paPages[iFreePage].uPte.pPae->u, uNew, uOld, &uOld))
1133	AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
1134	/Log6(("pgmR0DynMapPageSlow: #%x - %RHp %p %#llx\n", iFreePage, HCPhys, paPages[iFreePage].pvPage, uNew));/
1135	}
1136	return iFreePage;
1137	}
1138
1139
1140	/**
1141	* Maps a page into the pool.
1142	*
1143	* @returns Pointer to the mapping.
1144	* @param pThis The dynamic mapping cache instance.
1145	* @param HCPhys The address of the page to be mapped.
1146	* @param piPage Where to store the page index.
1147	*/
1148	DECLINLINE(void ) pgmR0DynMapPage(PPGMR0DYNMAP pThis, RTHCPHYS HCPhys, uint32_t piPage)
1149	{
1150	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1151	RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1152	AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
1153
1154	/*
1155	* Find an entry, if possible a matching one. The HCPhys address is hashed
1156	* down to a page index, collisions are handled by linear searching. Optimize
1157	* for a hit in the first 5 pages.
1158	*
1159	* To the cheap hits here and defer the tedious searching and inserting
1160	* to a helper function.
1161	*/
1162	uint32_t const cPages = pThis->cPages;
1163	uint32_t iPage = (HCPhys >> PAGE_SHIFT) % cPages;
1164	PPGMR0DYNMAPENTRY paPages = pThis->paPages;
1165	if (paPages[iPage].HCPhys != HCPhys)
1166	{
1167	uint32_t iPage2 = (iPage + 1) % cPages;
1168	if (paPages[iPage2].HCPhys != HCPhys)
1169	{
1170	iPage2 = (iPage + 2) % cPages;
1171	if (paPages[iPage2].HCPhys != HCPhys)
1172	{
1173	iPage2 = (iPage + 3) % cPages;
1174	if (paPages[iPage2].HCPhys != HCPhys)
1175	{
1176	iPage2 = (iPage + 4) % cPages;
1177	if (paPages[iPage2].HCPhys != HCPhys)
1178	{
1179	iPage = pgmR0DynMapPageSlow(pThis, HCPhys, iPage);
1180	if (RT_UNLIKELY(iPage == UINT32_MAX))
1181	{
1182	RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1183	return NULL;
1184	}
1185	}
1186	else
1187	iPage = iPage2;
1188	}
1189	else
1190	iPage = iPage2;
1191	}
1192	else
1193	iPage = iPage2;
1194	}
1195	else
1196	iPage = iPage2;
1197	}
1198
1199	/*
1200	* Reference it, update statistics and get the return address.
1201	*/
1202	int32_t cRefs = ASMAtomicIncS32(&paPages[iPage].cRefs);
1203	if (cRefs == 1)
1204	{
1205	pThis->cLoad++;
1206	if (pThis->cLoad > pThis->cMaxLoad)
1207	pThis->cMaxLoad = pThis->cLoad;
1208	AssertMsg(pThis->cLoad <= pThis->cPages, ("%d/%d\n", pThis->cLoad, pThis->cPages));
1209	}
1210	else if (RT_UNLIKELY(cRefs <= 0))
1211	{
1212	ASMAtomicDecS32(&paPages[iPage].cRefs);
1213	RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1214	AssertLogRelMsgFailedReturn(("cRefs=%d iPage=%p HCPhys=%RHp\n", cRefs, iPage, HCPhys), NULL);
1215	}
1216	void *pvPage = paPages[iPage].pvPage;
1217
1218	/*
1219	* Invalidate the entry?
1220	*/
1221	RTCPUID idRealCpu = RTMpCpuId();
1222	bool fInvalidateIt = RTCpuSetIsMember(&paPages[iPage].PendingSet, idRealCpu);
1223	if (fInvalidateIt)
1224	RTCpuSetDel(&paPages[iPage].PendingSet, idRealCpu);
1225
1226	RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1227
1228	/*
1229	* Do the actual invalidation outside the spinlock.
1230	*/
1231	ASMInvalidatePage(pvPage);
1232
1233	*piPage = iPage;
1234	return pvPage;
1235	}
1236
1237
1238	/**
1239	* Signals the start of a new set of mappings.
1240	*
1241	* Mostly for strictness. PGMDynMapHCPage won't work unless this
1242	* API is called.
1243	*
1244	* @param pVCpu The shared data for the current virtual CPU.
1245	*/
1246	VMMDECL(void) PGMDynMapStartAutoSet(PVMCPU pVCpu)
1247	{
1248	Assert(pVCpu->pgm.s.AutoSet.cEntries == PGMMAPSET_CLOSED);
1249	pVCpu->pgm.s.AutoSet.cEntries = 0;
1250	}
1251
1252
1253	/**
1254	* Releases the dynamic memory mappings made by PGMDynMapHCPage and associates
1255	* since the PGMDynMapStartAutoSet call.
1256	*
1257	* @param pVCpu The shared data for the current virtual CPU.
1258	*/
1259	VMMDECL(void) PGMDynMapReleaseAutoSet(PVMCPU pVCpu)
1260	{
1261	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1262
1263	/* close the set */
1264	uint32_t i = pSet->cEntries;
1265	AssertMsg(i <= RT_ELEMENTS(pSet->aEntries), ("%#x (%u)\n", i, i));
1266	pSet->cEntries = PGMMAPSET_CLOSED;
1267
1268	/* release any pages we're referencing. */
1269	if (i != 0 && RT_LIKELY(i <= RT_ELEMENTS(pSet->aEntries)))
1270	{
1271	PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
1272	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1273	RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1274
1275	while (i-- > 0)
1276	{
1277	uint32_t iPage = pSet->aEntries[i].iPage;
1278	Assert(iPage < pThis->cPages);
1279	int32_t cRefs = pSet->aEntries[i].cRefs;
1280	Assert(cRefs > 0);
1281	pgmR0DynMapReleasePageLocked(pThis, iPage, cRefs);
1282
1283	pSet->aEntries[i].iPage = UINT16_MAX;
1284	pSet->aEntries[i].cRefs = 0;
1285	}
1286
1287	Assert(pThis->cLoad <= pThis->cPages);
1288	RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1289	}
1290	}
1291
1292
1293	/**
1294	* Migrates the automatic mapping set of the current vCPU if it's active and
1295	* necessary.
1296	*
1297	* This is called when re-entering the hardware assisted execution mode after a
1298	* nip down to ring-3. We run the risk that the CPU might have change and we
1299	* will therefore make sure all the cache entries currently in the auto set will
1300	* be valid on the new CPU. If the cpu didn't change nothing will happen as all
1301	* the entries will have been flagged as invalidated.
1302	*
1303	* @param pVCpu The shared data for the current virtual CPU.
1304	* @thread EMT
1305	*/
1306	VMMDECL(void) PGMDynMapMigrateAutoSet(PVMCPU pVCpu)
1307	{
1308	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1309	uint32_t i = pSet->cEntries;
1310	if (i != PGMMAPSET_CLOSED)
1311	{
1312	AssertMsg(i <= RT_ELEMENTS(pSet->aEntries), ("%#x (%u)\n", i, i));
1313	if (i != 0 && RT_LIKELY(i <= RT_ELEMENTS(pSet->aEntries)))
1314	{
1315	PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
1316	RTCPUID idRealCpu = RTMpCpuId();
1317
1318	while (i-- > 0)
1319	{
1320	Assert(pSet->aEntries[i].cRefs > 0);
1321	uint32_t iPage = pSet->aEntries[i].iPage;
1322	Assert(iPage < pThis->cPages);
1323	if (RTCpuSetIsMember(&pThis->paPages[iPage].PendingSet, idRealCpu))
1324	{
1325	RTCpuSetDel(&pThis->paPages[iPage].PendingSet, idRealCpu);
1326	ASMInvalidatePage(pThis->paPages[iPage].pvPage);
1327	}
1328	}
1329	}
1330	}
1331	}
1332
1333
1334	/**
1335	* As a final resort for a full auto set, try merge duplicate entries.
1336	*
1337	* @param pSet The set.
1338	*/
1339	static void pgmDynMapOptimizeAutoSet(PPGMMAPSET pSet)
1340	{
1341	for (uint32_t i = 0 ; i < pSet->cEntries; i++)
1342	{
1343	uint16_t const iPage = pSet->aEntries[i].iPage;
1344	uint32_t j = i + 1;
1345	while (j < pSet->cEntries)
1346	{
1347	if (pSet->aEntries[j].iPage != iPage)
1348	j++;
1349	else if ((uint32_t)pSet->aEntries[i].cRefs + (uint32_t)pSet->aEntries[j].cRefs < UINT16_MAX)
1350	{
1351	/* merge j into i removing j. */
1352	pSet->aEntries[i].cRefs += pSet->aEntries[j].cRefs;
1353	pSet->cEntries--;
1354	if (j < pSet->cEntries)
1355	{
1356	pSet->aEntries[j] = pSet->aEntries[pSet->cEntries];
1357	pSet->aEntries[pSet->cEntries].iPage = UINT16_MAX;
1358	pSet->aEntries[pSet->cEntries].cRefs = 0;
1359	}
1360	else
1361	{
1362	pSet->aEntries[j].iPage = UINT16_MAX;
1363	pSet->aEntries[j].cRefs = 0;
1364	}
1365	}
1366	else
1367	{
1368	/* migrate the max number of refs from j into i and quit the inner loop. */
1369	uint32_t cMigrate = UINT16_MAX - 1 - pSet->aEntries[i].cRefs;
1370	Assert(pSet->aEntries[j].cRefs > cMigrate);
1371	pSet->aEntries[j].cRefs -= cMigrate;
1372	pSet->aEntries[i].cRefs = UINT16_MAX - 1;
1373	break;
1374	}
1375	}
1376	}
1377	}
1378
1379
1380	/* documented elsewhere - a bit of a mess. */
1381	VMMDECL(int) PGMDynMapHCPage(PVM pVM, RTHCPHYS HCPhys, void **ppv)
1382	{
1383	/*
1384	* Validate state.
1385	*/
1386	AssertPtr(ppv);
1387	*ppv = NULL;
1388	AssertMsgReturn(pVM->pgm.s.pvR0DynMapUsed == g_pPGMR0DynMap,
1389	("%p != %p\n", pVM->pgm.s.pvR0DynMapUsed, g_pPGMR0DynMap),
1390	VERR_ACCESS_DENIED);
1391	AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
1392	PVMCPU pVCpu = VMMGetCpu(pVM);
1393	PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
1394	AssertPtrReturn(pVCpu, VERR_INTERNAL_ERROR);
1395	AssertMsgReturn(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries),
1396	("%#x (%u)\n", pSet->cEntries, pSet->cEntries), VERR_WRONG_ORDER);
1397
1398	/*
1399	* Map it.
1400	*/
1401	uint32_t iPage;
1402	void *pvPage = pgmR0DynMapPage(g_pPGMR0DynMap, HCPhys, &iPage);
1403	if (RT_UNLIKELY(!pvPage))
1404	{
1405	static uint32_t s_cBitched = 0;
1406	if (++s_cBitched < 10)
1407	LogRel(("PGMDynMapHCPage: cLoad=%u/%u cPages=%u\n",
1408	g_pPGMR0DynMap->cLoad, g_pPGMR0DynMap->cMaxLoad, g_pPGMR0DynMap->cPages));
1409	return VERR_PGM_DYNMAP_FAILED;
1410	}
1411	*ppv = pvPage;
1412
1413	/*
1414	* Add the page to the auto reference set.
1415	* If it's less than half full, don't bother looking for duplicates.
1416	*/
1417	if (pSet->cEntries < RT_ELEMENTS(pSet->aEntries) / 2)
1418	{
1419	pSet->aEntries[pSet->cEntries].cRefs = 1;
1420	pSet->aEntries[pSet->cEntries].iPage = iPage;
1421	pSet->cEntries++;
1422	}
1423	else
1424	{
1425	Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
1426	int32_t i = pSet->cEntries;
1427	while (i-- > 0)
1428	if ( pSet->aEntries[i].iPage == iPage
1429	&& pSet->aEntries[i].cRefs < UINT16_MAX - 1)
1430	{
1431	pSet->aEntries[i].cRefs++;
1432	break;
1433	}
1434	if (i < 0)
1435	{
1436	if (RT_UNLIKELY(pSet->cEntries >= RT_ELEMENTS(pSet->aEntries)))
1437	pgmDynMapOptimizeAutoSet(pSet);
1438	if (RT_LIKELY(pSet->cEntries < RT_ELEMENTS(pSet->aEntries)))
1439	{
1440	pSet->aEntries[pSet->cEntries].cRefs = 1;
1441	pSet->aEntries[pSet->cEntries].iPage = iPage;
1442	pSet->cEntries++;
1443	}
1444	else
1445	{
1446	/* We're screwed. */
1447	pgmR0DynMapReleasePage(g_pPGMR0DynMap, iPage, 1);
1448
1449	static uint32_t s_cBitched = 0;
1450	if (++s_cBitched < 10)
1451	LogRel(("PGMDynMapHCPage: set is full!\n"));
1452	*ppv = NULL;
1453	return VERR_PGM_DYNMAP_FULL_SET;
1454	}
1455	}
1456	}
1457
1458	return VINF_SUCCESS;
1459	}
1460
1461
1462	#ifdef DEBUG
1463	/** For pgmR0DynMapTest3PerCpu. */
1464	typedef struct PGMR0DYNMAPTEST
1465	{
1466	uint32_t u32Expect;
1467	uint32_t *pu32;
1468	uint32_t volatile cFailures;
1469	} PGMR0DYNMAPTEST;
1470	typedef PGMR0DYNMAPTEST *PPGMR0DYNMAPTEST;
1471
1472	/**
1473	* Checks that the content of the page is the same on all CPUs, i.e. that there
1474	* are no CPU specfic PTs or similar nasty stuff involved.
1475	*
1476	* @param idCpu The current CPU.
1477	* @param pvUser1 Pointer a PGMR0DYNMAPTEST structure.
1478	* @param pvUser2 Unused, ignored.
1479	*/
1480	static DECLCALLBACK(void) pgmR0DynMapTest3PerCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
1481	{
1482	PPGMR0DYNMAPTEST pTest = (PPGMR0DYNMAPTEST)pvUser1;
1483	ASMInvalidatePage(pTest->pu32);
1484	if (*pTest->pu32 != pTest->u32Expect)
1485	ASMAtomicIncU32(&pTest->cFailures);
1486	NOREF(pvUser2); NOREF(idCpu);
1487	}
1488
1489
1490	/**
1491	* Performs some basic tests in debug builds.
1492	*/
1493	static int pgmR0DynMapTest(PVM pVM)
1494	{
1495	LogRel(("pgmR0DynMapTest: **** START ****\n"));
1496	PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
1497	PPGMMAPSET pSet = &pVM->aCpus[0].pgm.s.AutoSet;
1498	uint32_t i;
1499	void *pvR0DynMapUsedSaved = pVM->pgm.s.pvR0DynMapUsed;
1500	pVM->pgm.s.pvR0DynMapUsed = pThis;
1501
1502	/*
1503	* Simple test, map CR3 twice and check that we're getting the
1504	* same mapping address back.
1505	*/
1506	LogRel(("Test #1\n"));
1507	ASMIntDisable();
1508	PGMDynMapStartAutoSet(&pVM->aCpus[0]);
1509
1510	uint64_t cr3 = ASMGetCR3() & ~(uint64_t)PAGE_OFFSET_MASK;
1511	void pv = (void )(intptr_t)-1;
1512	void pv2 = (void )(intptr_t)-2;
1513	int rc = PGMDynMapHCPage(pVM, cr3, &pv);
1514	int rc2 = PGMDynMapHCPage(pVM, cr3, &pv2);
1515	ASMIntEnable();
1516	if ( RT_SUCCESS(rc2)
1517	&& RT_SUCCESS(rc)
1518	&& pv == pv2)
1519	{
1520	LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
1521
1522	/*
1523	* Check that the simple set overflow code works by filling it
1524	* with more CR3 mappings.
1525	*/
1526	LogRel(("Test #2\n"));
1527	ASMIntDisable();
1528	for (i = 0 ; i < UINT16_MAX*2 + RT_ELEMENTS(pSet->aEntries) / 2 && RT_SUCCESS(rc) && pv2 == pv; i++)
1529	{
1530	pv2 = (void *)(intptr_t)-4;
1531	rc = PGMDynMapHCPage(pVM, cr3, &pv2);
1532	}
1533	ASMIntEnable();
1534	if (RT_FAILURE(rc) \|\| pv != pv2)
1535	{
1536	LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%p\n", __LINE__, rc, pv, pv2, i));
1537	if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
1538	}
1539	else if (pSet->cEntries != RT_ELEMENTS(pSet->aEntries) / 2)
1540	{
1541	LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries) / 2));
1542	rc = VERR_INTERNAL_ERROR;
1543	}
1544	else if ( pSet->aEntries[(RT_ELEMENTS(pSet->aEntries) / 2) - 1].cRefs != UINT16_MAX - 1
1545	\|\| pSet->aEntries[(RT_ELEMENTS(pSet->aEntries) / 2) - 2].cRefs != UINT16_MAX - 1
1546	\|\| pSet->aEntries[(RT_ELEMENTS(pSet->aEntries) / 2) - 3].cRefs != 2+2+3
1547	\|\| pSet->aEntries[(RT_ELEMENTS(pSet->aEntries) / 2) - 4].cRefs != 1)
1548	{
1549	LogRel(("failed(%d): bad set dist: ", __LINE__));
1550	for (i = 0; i < pSet->cEntries; i++)
1551	LogRel(("[%d]=%d, ", i, pSet->aEntries[i].cRefs));
1552	LogRel(("\n"));
1553	rc = VERR_INTERNAL_ERROR;
1554	}
1555	if (RT_SUCCESS(rc))
1556	{
1557	/*
1558	* Trigger an set optimization run (exactly).
1559	*/
1560	LogRel(("Test #3\n"));
1561	ASMIntDisable();
1562	pv2 = NULL;
1563	for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) / 2 && RT_SUCCESS(rc) && pv2 != pv; i++)
1564	{
1565	pv2 = (void *)(intptr_t)(-5 - i);
1566	rc = PGMDynMapHCPage(pVM, cr3 + PAGE_SIZE * (i + 5), &pv2);
1567	}
1568	ASMIntEnable();
1569	if (RT_FAILURE(rc) \|\| pv == pv2)
1570	{
1571	LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%d\n", __LINE__, rc, pv, pv2, i));
1572	if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
1573	}
1574	else if (pSet->cEntries != RT_ELEMENTS(pSet->aEntries))
1575	{
1576	LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
1577	rc = VERR_INTERNAL_ERROR;
1578	}
1579	LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
1580	if (RT_SUCCESS(rc))
1581	{
1582	/*
1583	* Trigger an overflow error.
1584	*/
1585	LogRel(("Test #4\n"));
1586	ASMIntDisable();
1587	for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) / 2 - 3 + 1 && RT_SUCCESS(rc) && pv2 != pv; i++)
1588	rc = PGMDynMapHCPage(pVM, cr3 + PAGE_SIZE * -(i + 5), &pv2);
1589	ASMIntEnable();
1590	if (rc == VERR_PGM_DYNMAP_FULL_SET)
1591	{
1592	rc = VINF_SUCCESS;
1593
1594	/* flush the set. */
1595	ASMIntDisable();
1596	PGMDynMapMigrateAutoSet(&pVM->aCpus[0]);
1597	PGMDynMapReleaseAutoSet(&pVM->aCpus[0]);
1598	PGMDynMapStartAutoSet(&pVM->aCpus[0]);
1599	ASMIntEnable();
1600	}
1601	else
1602	{
1603	LogRel(("failed(%d): rc=%Rrc, wanted %d ; pv2=%p Set=%u/%u\n", __LINE__,
1604	rc, VERR_PGM_DYNMAP_FULL_SET, pv2, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
1605	if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
1606	}
1607	}
1608	}
1609	}
1610	else
1611	{
1612	LogRel(("failed(%d): rc=%Rrc rc2=%Rrc; pv=%p pv2=%p\n", __LINE__, rc, rc2, pv, pv2));
1613	if (RT_SUCCESS(rc))
1614	rc = rc2;
1615	}
1616
1617	/*
1618	* Check that everyone sees the same stuff.
1619	*/
1620	if (RT_SUCCESS(rc))
1621	{
1622	LogRel(("Test #5\n"));
1623	ASMIntDisable();
1624	RTHCPHYS HCPhysPT = RTR0MemObjGetPagePhysAddr(pThis->pSegHead->ahMemObjPTs[0], 0);
1625	rc = PGMDynMapHCPage(pVM, HCPhysPT, &pv);
1626	if (RT_SUCCESS(rc))
1627	{
1628	PGMR0DYNMAPTEST Test;
1629	uint32_t *pu32Real = &pThis->paPages[pThis->pSegHead->iPage].uPte.pLegacy->u;
1630	Test.pu32 = (uint32_t *)((uintptr_t)pv \| ((uintptr_t)pu32Real & PAGE_OFFSET_MASK));
1631	Test.u32Expect = *pu32Real;
1632	ASMAtomicWriteU32(&Test.cFailures, 0);
1633	ASMIntEnable();
1634
1635	rc = RTMpOnAll(pgmR0DynMapTest3PerCpu, &Test, NULL);
1636	if (RT_FAILURE(rc))
1637	LogRel(("failed(%d): RTMpOnAll rc=%Rrc\n", __LINE__, rc));
1638	else if (Test.cFailures)
1639	{
1640	LogRel(("failed(%d): cFailures=%d pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n", __LINE__,
1641	Test.cFailures, pu32Real, Test.pu32, Test.u32Expect, *Test.pu32));
1642	rc = VERR_INTERNAL_ERROR;
1643	}
1644	else
1645	LogRel(("pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n",
1646	pu32Real, Test.pu32, Test.u32Expect, *Test.pu32));
1647	}
1648	else
1649	{
1650	ASMIntEnable();
1651	LogRel(("failed(%d): rc=%Rrc\n", rc));
1652	}
1653	}
1654
1655	/*
1656	* Clean up.
1657	*/
1658	LogRel(("Cleanup.\n"));
1659	ASMIntDisable();
1660	PGMDynMapMigrateAutoSet(&pVM->aCpus[0]);
1661	PGMDynMapReleaseAutoSet(&pVM->aCpus[0]);
1662	ASMIntEnable();
1663
1664	LogRel(("Result: rc=%Rrc Load=%u/%u/%u Set=%#x/%u\n", rc,
1665	pThis->cLoad, pThis->cMaxLoad, pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
1666	pVM->pgm.s.pvR0DynMapUsed = pvR0DynMapUsedSaved;
1667	LogRel(("pgmR0DynMapTest: **** END ****\n"));
1668	return rc;
1669	}
1670	#endif /* DEBUG */
1671

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/PGMR0DynMap.cpp@ 14663

Download in other formats: