VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/PGMR0DynMap.cpp@ 18983

Last change on this file since 18983 was 18983, checked in by vboxsync, 16 years ago

More
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 77.2 KB
Line 
1/* $Id: PGMR0DynMap.cpp 18983 2009-04-17 08:45:48Z vboxsync $ */
2/** @file
3 * PGM - Page Manager and Monitor, ring-0 dynamic mapping cache.
4 */
5
6/*
7 * Copyright (C) 2008 Sun Microsystems, Inc.
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18 * Clara, CA 95054 USA or visit http://www.sun.com if you need
19 * additional information or have any questions.
20 */
21
22/*******************************************************************************
23* Internal Functions *
24*******************************************************************************/
25#define LOG_GROUP LOG_GROUP_PGM
26#include <VBox/pgm.h>
27#include "../PGMInternal.h"
28#include <VBox/vm.h>
29#include <VBox/sup.h>
30#include <VBox/err.h>
31#include <iprt/asm.h>
32#include <iprt/alloc.h>
33#include <iprt/assert.h>
34#include <iprt/cpuset.h>
35#include <iprt/memobj.h>
36#include <iprt/mp.h>
37#include <iprt/semaphore.h>
38#include <iprt/spinlock.h>
39#include <iprt/string.h>
40
41
42/*******************************************************************************
43* Defined Constants And Macros *
44*******************************************************************************/
45/** The max size of the mapping cache (in pages). */
46#define PGMR0DYNMAP_MAX_PAGES ((16*_1M) >> PAGE_SHIFT)
47/** The small segment size that is adopted on out-of-memory conditions with a
48 * single big segment. */
49#define PGMR0DYNMAP_SMALL_SEG_PAGES 128
50/** The number of pages we reserve per CPU. */
51#define PGMR0DYNMAP_PAGES_PER_CPU 256
52/** The minimum number of pages we reserve per CPU.
53 * This must be equal or larger than the autoset size. */
54#define PGMR0DYNMAP_PAGES_PER_CPU_MIN 64
55/** The number of guard pages.
56 * @remarks Never do tuning of the hashing or whatnot with a strict build! */
57#if defined(VBOX_STRICT)
58# define PGMR0DYNMAP_GUARD_PAGES 1
59#else
60# define PGMR0DYNMAP_GUARD_PAGES 0
61#endif
62/** The dummy physical address of guard pages. */
63#define PGMR0DYNMAP_GUARD_PAGE_HCPHYS UINT32_C(0x7777feed)
64/** The dummy reference count of guard pages. (Must be non-zero.) */
65#define PGMR0DYNMAP_GUARD_PAGE_REF_COUNT INT32_C(0x7777feed)
66#if 0
67/** Define this to just clear the present bit on guard pages.
68 * The alternative is to replace the entire PTE with an bad not-present
69 * PTE. Either way, XNU will screw us. :-/ */
70#define PGMR0DYNMAP_GUARD_NP
71#endif
72/** The dummy PTE value for a page. */
73#define PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE X86_PTE_PG_MASK
74/** The dummy PTE value for a page. */
75#define PGMR0DYNMAP_GUARD_PAGE_PAE_PTE UINT64_MAX /*X86_PTE_PAE_PG_MASK*/
76/** Calcs the overload threshold. Current set at 50%. */
77#define PGMR0DYNMAP_CALC_OVERLOAD(cPages) ((cPages) / 2)
78
79#if 0
80/* Assertions causes panics if preemption is disabled, this can be used to work aroudn that. */
81//#define RTSpinlockAcquire(a,b) do {} while (0)
82//#define RTSpinlockRelease(a,b) do {} while (0)
83#endif
84
85
86/*******************************************************************************
87* Structures and Typedefs *
88*******************************************************************************/
89/**
90 * Ring-0 dynamic mapping cache segment.
91 *
92 * The dynamic mapping cache can be extended with additional segments if the
93 * load is found to be too high. This done the next time a VM is created, under
94 * the protection of the init mutex. The arrays is reallocated and the new
95 * segment is added to the end of these. Nothing is rehashed of course, as the
96 * indexes / addresses must remain unchanged.
97 *
98 * This structure is only modified while owning the init mutex or during module
99 * init / term.
100 */
101typedef struct PGMR0DYNMAPSEG
102{
103 /** Pointer to the next segment. */
104 struct PGMR0DYNMAPSEG *pNext;
105 /** The memory object for the virtual address range that we're abusing. */
106 RTR0MEMOBJ hMemObj;
107 /** The start page in the cache. (I.e. index into the arrays.) */
108 uint16_t iPage;
109 /** The number of pages this segment contributes. */
110 uint16_t cPages;
111 /** The number of page tables. */
112 uint16_t cPTs;
113 /** The memory objects for the page tables. */
114 RTR0MEMOBJ ahMemObjPTs[1];
115} PGMR0DYNMAPSEG;
116/** Pointer to a ring-0 dynamic mapping cache segment. */
117typedef PGMR0DYNMAPSEG *PPGMR0DYNMAPSEG;
118
119
120/**
121 * Ring-0 dynamic mapping cache entry.
122 *
123 * This structure tracks
124 */
125typedef struct PGMR0DYNMAPENTRY
126{
127 /** The physical address of the currently mapped page.
128 * This is duplicate for three reasons: cache locality, cache policy of the PT
129 * mappings and sanity checks. */
130 RTHCPHYS HCPhys;
131 /** Pointer to the page. */
132 void *pvPage;
133 /** The number of references. */
134 int32_t volatile cRefs;
135 /** PTE pointer union. */
136 union PGMR0DYNMAPENTRY_PPTE
137 {
138 /** PTE pointer, 32-bit legacy version. */
139 PX86PTE pLegacy;
140 /** PTE pointer, PAE version. */
141 PX86PTEPAE pPae;
142 /** PTE pointer, the void version. */
143 void *pv;
144 } uPte;
145 /** CPUs that haven't invalidated this entry after it's last update. */
146 RTCPUSET PendingSet;
147} PGMR0DYNMAPENTRY;
148/** Pointer to a ring-0 dynamic mapping cache entry. */
149typedef PGMR0DYNMAPENTRY *PPGMR0DYNMAPENTRY;
150
151
152/**
153 * Ring-0 dynamic mapping cache.
154 *
155 * This is initialized during VMMR0 module init but no segments are allocated at
156 * that time. Segments will be added when the first VM is started and removed
157 * again when the last VM shuts down, thus avoid consuming memory while dormant.
158 * At module termination, the remaining bits will be freed up.
159 */
160typedef struct PGMR0DYNMAP
161{
162 /** The usual magic number / eye catcher (PGMR0DYNMAP_MAGIC). */
163 uint32_t u32Magic;
164 /** Spinlock serializing the normal operation of the cache. */
165 RTSPINLOCK hSpinlock;
166 /** Array for tracking and managing the pages. */
167 PPGMR0DYNMAPENTRY paPages;
168 /** The cache size given as a number of pages. */
169 uint32_t cPages;
170 /** Whether it's 32-bit legacy or PAE/AMD64 paging mode. */
171 bool fLegacyMode;
172 /** The current load.
173 * This does not include guard pages. */
174 uint32_t cLoad;
175 /** The max load ever.
176 * This is maintained to get trigger adding of more mapping space. */
177 uint32_t cMaxLoad;
178 /** Initialization / termination lock. */
179 RTSEMFASTMUTEX hInitLock;
180 /** The number of guard pages. */
181 uint32_t cGuardPages;
182 /** The number of users (protected by hInitLock). */
183 uint32_t cUsers;
184 /** Array containing a copy of the original page tables.
185 * The entries are either X86PTE or X86PTEPAE according to fLegacyMode. */
186 void *pvSavedPTEs;
187 /** List of segments. */
188 PPGMR0DYNMAPSEG pSegHead;
189 /** The paging mode. */
190 SUPPAGINGMODE enmPgMode;
191} PGMR0DYNMAP;
192/** Pointer to the ring-0 dynamic mapping cache */
193typedef PGMR0DYNMAP *PPGMR0DYNMAP;
194
195/** PGMR0DYNMAP::u32Magic. (Jens Christian Bugge Wesseltoft) */
196#define PGMR0DYNMAP_MAGIC 0x19640201
197
198
199/**
200 * Paging level data.
201 */
202typedef struct PGMR0DYNMAPPGLVL
203{
204 uint32_t cLevels; /**< The number of levels. */
205 struct
206 {
207 RTHCPHYS HCPhys; /**< The address of the page for the current level,
208 * i.e. what hMemObj/hMapObj is currently mapping. */
209 RTHCPHYS fPhysMask; /**< Mask for extracting HCPhys from uEntry. */
210 RTR0MEMOBJ hMemObj; /**< Memory object for HCPhys, PAGE_SIZE. */
211 RTR0MEMOBJ hMapObj; /**< Mapping object for hMemObj. */
212 uint32_t fPtrShift; /**< The pointer shift count. */
213 uint64_t fPtrMask; /**< The mask to apply to the shifted pointer to get the table index. */
214 uint64_t fAndMask; /**< And mask to check entry flags. */
215 uint64_t fResMask; /**< The result from applying fAndMask. */
216 union
217 {
218 void *pv; /**< hMapObj address. */
219 PX86PGUINT paLegacy; /**< Legacy table view. */
220 PX86PGPAEUINT paPae; /**< PAE/AMD64 table view. */
221 } u;
222 } a[4];
223} PGMR0DYNMAPPGLVL;
224/** Pointer to paging level data. */
225typedef PGMR0DYNMAPPGLVL *PPGMR0DYNMAPPGLVL;
226
227
228/*******************************************************************************
229* Global Variables *
230*******************************************************************************/
231/** Pointer to the ring-0 dynamic mapping cache. */
232static PPGMR0DYNMAP g_pPGMR0DynMap;
233/** For overflow testing. */
234static bool g_fPGMR0DynMapTestRunning = false;
235
236
237/*******************************************************************************
238* Internal Functions *
239*******************************************************************************/
240static void pgmR0DynMapReleasePage(PPGMR0DYNMAP pThis, uint32_t iPage, uint32_t cRefs);
241static int pgmR0DynMapSetup(PPGMR0DYNMAP pThis);
242static int pgmR0DynMapExpand(PPGMR0DYNMAP pThis);
243static void pgmR0DynMapTearDown(PPGMR0DYNMAP pThis);
244#if 0 /*def DEBUG*/
245static int pgmR0DynMapTest(PVM pVM);
246#endif
247
248
249/**
250 * Initializes the ring-0 dynamic mapping cache.
251 *
252 * @returns VBox status code.
253 */
254VMMR0DECL(int) PGMR0DynMapInit(void)
255{
256 Assert(!g_pPGMR0DynMap);
257
258 /*
259 * Create and initialize the cache instance.
260 */
261 PPGMR0DYNMAP pThis = (PPGMR0DYNMAP)RTMemAllocZ(sizeof(*pThis));
262 AssertLogRelReturn(pThis, VERR_NO_MEMORY);
263 int rc = VINF_SUCCESS;
264 pThis->enmPgMode = SUPR0GetPagingMode();
265 switch (pThis->enmPgMode)
266 {
267 case SUPPAGINGMODE_32_BIT:
268 case SUPPAGINGMODE_32_BIT_GLOBAL:
269 pThis->fLegacyMode = false;
270 break;
271 case SUPPAGINGMODE_PAE:
272 case SUPPAGINGMODE_PAE_GLOBAL:
273 case SUPPAGINGMODE_PAE_NX:
274 case SUPPAGINGMODE_PAE_GLOBAL_NX:
275 case SUPPAGINGMODE_AMD64:
276 case SUPPAGINGMODE_AMD64_GLOBAL:
277 case SUPPAGINGMODE_AMD64_NX:
278 case SUPPAGINGMODE_AMD64_GLOBAL_NX:
279 pThis->fLegacyMode = false;
280 break;
281 default:
282 rc = VERR_INTERNAL_ERROR;
283 break;
284 }
285 if (RT_SUCCESS(rc))
286 {
287 rc = RTSemFastMutexCreate(&pThis->hInitLock);
288 if (RT_SUCCESS(rc))
289 {
290 rc = RTSpinlockCreate(&pThis->hSpinlock);
291 if (RT_SUCCESS(rc))
292 {
293 pThis->u32Magic = PGMR0DYNMAP_MAGIC;
294 g_pPGMR0DynMap = pThis;
295 return VINF_SUCCESS;
296 }
297 RTSemFastMutexDestroy(pThis->hInitLock);
298 }
299 }
300 RTMemFree(pThis);
301 return rc;
302}
303
304
305/**
306 * Terminates the ring-0 dynamic mapping cache.
307 */
308VMMR0DECL(void) PGMR0DynMapTerm(void)
309{
310 /*
311 * Destroy the cache.
312 *
313 * There is not supposed to be any races here, the loader should
314 * make sure about that. So, don't bother locking anything.
315 *
316 * The VM objects should all be destroyed by now, so there is no
317 * dangling users or anything like that to clean up. This routine
318 * is just a mirror image of PGMR0DynMapInit.
319 */
320 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
321 if (pThis)
322 {
323 AssertPtr(pThis);
324 g_pPGMR0DynMap = NULL;
325
326 /* This should *never* happen, but in case it does try not to leak memory. */
327 AssertLogRelMsg(!pThis->cUsers && !pThis->paPages && !pThis->pvSavedPTEs && !pThis->cPages,
328 ("cUsers=%d paPages=%p pvSavedPTEs=%p cPages=%#x\n",
329 pThis->cUsers, pThis->paPages, pThis->pvSavedPTEs, pThis->cPages));
330 if (pThis->paPages)
331 pgmR0DynMapTearDown(pThis);
332
333 /* Free the associated resources. */
334 RTSemFastMutexDestroy(pThis->hInitLock);
335 pThis->hInitLock = NIL_RTSEMFASTMUTEX;
336 RTSpinlockDestroy(pThis->hSpinlock);
337 pThis->hSpinlock = NIL_RTSPINLOCK;
338 pThis->u32Magic = UINT32_MAX;
339 RTMemFree(pThis);
340 }
341}
342
343
344/**
345 * Initializes the dynamic mapping cache for a new VM.
346 *
347 * @returns VBox status code.
348 * @param pVM Pointer to the shared VM structure.
349 */
350VMMR0DECL(int) PGMR0DynMapInitVM(PVM pVM)
351{
352 AssertMsgReturn(!pVM->pgm.s.pvR0DynMapUsed, ("%p (pThis=%p)\n", pVM->pgm.s.pvR0DynMapUsed, g_pPGMR0DynMap), VERR_WRONG_ORDER);
353
354 /*
355 * Initialize the auto sets.
356 */
357 PPGMMAPSET pSet = &pVM->pgm.s.AutoSet;
358 uint32_t j = RT_ELEMENTS(pSet->aEntries);
359 while (j-- > 0)
360 {
361 pSet->aEntries[j].iPage = UINT16_MAX;
362 pSet->aEntries[j].cRefs = 0;
363 pSet->aEntries[j].pvPage = NULL;
364 pSet->aEntries[j].HCPhys = NIL_RTHCPHYS;
365 }
366 pSet->cEntries = PGMMAPSET_CLOSED;
367 pSet->iSubset = UINT32_MAX;
368 pSet->iCpu = -1;
369 memset(&pSet->aiHashTable[0], 0xff, sizeof(pSet->aiHashTable));
370
371 /*
372 * Do we need the cache? Skip the last bit if we don't.
373 */
374 if (!VMMIsHwVirtExtForced(pVM))
375 return VINF_SUCCESS;
376
377 /*
378 * Reference and if necessary setup or expand the cache.
379 */
380 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
381 AssertPtrReturn(pThis, VERR_INTERNAL_ERROR);
382 int rc = RTSemFastMutexRequest(pThis->hInitLock);
383 AssertLogRelRCReturn(rc, rc);
384
385 pThis->cUsers++;
386 if (pThis->cUsers == 1)
387 {
388 rc = pgmR0DynMapSetup(pThis);
389#if 0 /*def DEBUG*/
390 if (RT_SUCCESS(rc))
391 {
392 rc = pgmR0DynMapTest(pVM);
393 if (RT_FAILURE(rc))
394 pgmR0DynMapTearDown(pThis);
395 }
396#endif
397 }
398 else if (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(pThis->cPages - pThis->cGuardPages))
399 rc = pgmR0DynMapExpand(pThis);
400 if (RT_SUCCESS(rc))
401 pVM->pgm.s.pvR0DynMapUsed = pThis;
402 else
403 pThis->cUsers--;
404
405 RTSemFastMutexRelease(pThis->hInitLock);
406 return rc;
407}
408
409
410/**
411 * Terminates the dynamic mapping cache usage for a VM.
412 *
413 * @param pVM Pointer to the shared VM structure.
414 */
415VMMR0DECL(void) PGMR0DynMapTermVM(PVM pVM)
416{
417 /*
418 * Return immediately if we're not using the cache.
419 */
420 if (!pVM->pgm.s.pvR0DynMapUsed)
421 return;
422
423 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
424 AssertPtrReturnVoid(pThis);
425
426 int rc = RTSemFastMutexRequest(pThis->hInitLock);
427 AssertLogRelRCReturnVoid(rc);
428
429 if (pVM->pgm.s.pvR0DynMapUsed == pThis)
430 {
431 pVM->pgm.s.pvR0DynMapUsed = NULL;
432
433#ifdef VBOX_STRICT
434 PGMR0DynMapAssertIntegrity();
435#endif
436
437 /*
438 * Clean up and check the auto sets.
439 */
440 PPGMMAPSET pSet = &pVM->pgm.s.AutoSet;
441 uint32_t j = pSet->cEntries;
442 if (j <= RT_ELEMENTS(pSet->aEntries))
443 {
444 /*
445 * The set is open, close it.
446 */
447 while (j-- > 0)
448 {
449 int32_t cRefs = pSet->aEntries[j].cRefs;
450 uint32_t iPage = pSet->aEntries[j].iPage;
451 LogRel(("PGMR0DynMapTermVM: %d dangling refs to %#x\n", cRefs, iPage));
452 if (iPage < pThis->cPages && cRefs > 0)
453 pgmR0DynMapReleasePage(pThis, iPage, cRefs);
454 else
455 AssertLogRelMsgFailed(("cRefs=%d iPage=%#x cPages=%u\n", cRefs, iPage, pThis->cPages));
456
457 pSet->aEntries[j].iPage = UINT16_MAX;
458 pSet->aEntries[j].cRefs = 0;
459 pSet->aEntries[j].pvPage = NULL;
460 pSet->aEntries[j].HCPhys = NIL_RTHCPHYS;
461 }
462 pSet->cEntries = PGMMAPSET_CLOSED;
463 pSet->iSubset = UINT32_MAX;
464 pSet->iCpu = -1;
465 }
466 else
467 AssertMsg(j == PGMMAPSET_CLOSED, ("cEntries=%#x\n", j));
468
469 j = RT_ELEMENTS(pSet->aEntries);
470 while (j-- > 0)
471 {
472 Assert(pSet->aEntries[j].iPage == UINT16_MAX);
473 Assert(!pSet->aEntries[j].cRefs);
474 }
475
476 /*
477 * Release our reference to the mapping cache.
478 */
479 Assert(pThis->cUsers > 0);
480 pThis->cUsers--;
481 if (!pThis->cUsers)
482 pgmR0DynMapTearDown(pThis);
483 }
484 else
485 AssertLogRelMsgFailed(("pvR0DynMapUsed=%p pThis=%p\n", pVM->pgm.s.pvR0DynMapUsed, pThis));
486
487 RTSemFastMutexRelease(pThis->hInitLock);
488}
489
490
491/**
492 * Shoots down the TLBs for all the cache pages, pgmR0DynMapTearDown helper.
493 *
494 * @param idCpu The current CPU.
495 * @param pvUser1 The dynamic mapping cache instance.
496 * @param pvUser2 Unused, NULL.
497 */
498static DECLCALLBACK(void) pgmR0DynMapShootDownTlbs(RTCPUID idCpu, void *pvUser1, void *pvUser2)
499{
500 Assert(!pvUser2);
501 PPGMR0DYNMAP pThis = (PPGMR0DYNMAP)pvUser1;
502 Assert(pThis == g_pPGMR0DynMap);
503 PPGMR0DYNMAPENTRY paPages = pThis->paPages;
504 uint32_t iPage = pThis->cPages;
505 while (iPage-- > 0)
506 ASMInvalidatePage(paPages[iPage].pvPage);
507}
508
509
510/**
511 * Shoot down the TLBs for every single cache entry on all CPUs.
512 *
513 * @returns IPRT status code (RTMpOnAll).
514 * @param pThis The dynamic mapping cache instance.
515 */
516static int pgmR0DynMapTlbShootDown(PPGMR0DYNMAP pThis)
517{
518 int rc = RTMpOnAll(pgmR0DynMapShootDownTlbs, pThis, NULL);
519 AssertRC(rc);
520 if (RT_FAILURE(rc))
521 {
522 uint32_t iPage = pThis->cPages;
523 while (iPage-- > 0)
524 ASMInvalidatePage(pThis->paPages[iPage].pvPage);
525 }
526 return rc;
527}
528
529
530/**
531 * Calculate the new cache size based on cMaxLoad statistics.
532 *
533 * @returns Number of pages.
534 * @param pThis The dynamic mapping cache instance.
535 * @param pcMinPages The minimal size in pages.
536 */
537static uint32_t pgmR0DynMapCalcNewSize(PPGMR0DYNMAP pThis, uint32_t *pcMinPages)
538{
539 Assert(pThis->cPages <= PGMR0DYNMAP_MAX_PAGES);
540
541 /* cCpus * PGMR0DYNMAP_PAGES_PER_CPU(_MIN). */
542 RTCPUID cCpus = RTMpGetCount();
543 AssertReturn(cCpus > 0 && cCpus <= RTCPUSET_MAX_CPUS, 0);
544 uint32_t cPages = cCpus * PGMR0DYNMAP_PAGES_PER_CPU;
545 uint32_t cMinPages = cCpus * PGMR0DYNMAP_PAGES_PER_CPU_MIN;
546
547 /* adjust against cMaxLoad. */
548 AssertMsg(pThis->cMaxLoad <= PGMR0DYNMAP_MAX_PAGES, ("%#x\n", pThis->cMaxLoad));
549 if (pThis->cMaxLoad > PGMR0DYNMAP_MAX_PAGES)
550 pThis->cMaxLoad = 0;
551
552 while (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(cPages))
553 cPages += PGMR0DYNMAP_PAGES_PER_CPU;
554
555 if (pThis->cMaxLoad > cMinPages)
556 cMinPages = pThis->cMaxLoad;
557
558 /* adjust against max and current size. */
559 if (cPages < pThis->cPages)
560 cPages = pThis->cPages;
561 cPages *= PGMR0DYNMAP_GUARD_PAGES + 1;
562 if (cPages > PGMR0DYNMAP_MAX_PAGES)
563 cPages = PGMR0DYNMAP_MAX_PAGES;
564
565 if (cMinPages < pThis->cPages)
566 cMinPages = pThis->cPages;
567 cMinPages *= PGMR0DYNMAP_GUARD_PAGES + 1;
568 if (cMinPages > PGMR0DYNMAP_MAX_PAGES)
569 cMinPages = PGMR0DYNMAP_MAX_PAGES;
570
571 Assert(cMinPages);
572 *pcMinPages = cMinPages;
573 return cPages;
574}
575
576
577/**
578 * Initializes the paging level data.
579 *
580 * @param pThis The dynamic mapping cache instance.
581 * @param pPgLvl The paging level data.
582 */
583void pgmR0DynMapPagingArrayInit(PPGMR0DYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl)
584{
585 RTCCUINTREG cr4 = ASMGetCR4();
586 switch (pThis->enmPgMode)
587 {
588 case SUPPAGINGMODE_32_BIT:
589 case SUPPAGINGMODE_32_BIT_GLOBAL:
590 pPgLvl->cLevels = 2;
591 pPgLvl->a[0].fPhysMask = X86_CR3_PAGE_MASK;
592 pPgLvl->a[0].fAndMask = X86_PDE_P | X86_PDE_RW | (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
593 pPgLvl->a[0].fResMask = X86_PDE_P | X86_PDE_RW;
594 pPgLvl->a[0].fPtrMask = X86_PD_MASK;
595 pPgLvl->a[0].fPtrShift = X86_PD_SHIFT;
596
597 pPgLvl->a[1].fPhysMask = X86_PDE_PG_MASK;
598 pPgLvl->a[1].fAndMask = X86_PTE_P | X86_PTE_RW;
599 pPgLvl->a[1].fResMask = X86_PTE_P | X86_PTE_RW;
600 pPgLvl->a[1].fPtrMask = X86_PT_MASK;
601 pPgLvl->a[1].fPtrShift = X86_PT_SHIFT;
602 break;
603
604 case SUPPAGINGMODE_PAE:
605 case SUPPAGINGMODE_PAE_GLOBAL:
606 case SUPPAGINGMODE_PAE_NX:
607 case SUPPAGINGMODE_PAE_GLOBAL_NX:
608 pPgLvl->cLevels = 3;
609 pPgLvl->a[0].fPhysMask = X86_CR3_PAE_PAGE_MASK;
610 pPgLvl->a[0].fPtrMask = X86_PDPT_MASK_PAE;
611 pPgLvl->a[0].fPtrShift = X86_PDPT_SHIFT;
612 pPgLvl->a[0].fAndMask = X86_PDPE_P;
613 pPgLvl->a[0].fResMask = X86_PDPE_P;
614
615 pPgLvl->a[1].fPhysMask = X86_PDPE_PG_MASK;
616 pPgLvl->a[1].fPtrMask = X86_PD_PAE_MASK;
617 pPgLvl->a[1].fPtrShift = X86_PD_PAE_SHIFT;
618 pPgLvl->a[1].fAndMask = X86_PDE_P | X86_PDE_RW | (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
619 pPgLvl->a[1].fResMask = X86_PDE_P | X86_PDE_RW;
620
621 pPgLvl->a[2].fPhysMask = X86_PDE_PAE_PG_MASK;
622 pPgLvl->a[2].fPtrMask = X86_PT_PAE_MASK;
623 pPgLvl->a[2].fPtrShift = X86_PT_PAE_SHIFT;
624 pPgLvl->a[2].fAndMask = X86_PTE_P | X86_PTE_RW;
625 pPgLvl->a[2].fResMask = X86_PTE_P | X86_PTE_RW;
626 break;
627
628 case SUPPAGINGMODE_AMD64:
629 case SUPPAGINGMODE_AMD64_GLOBAL:
630 case SUPPAGINGMODE_AMD64_NX:
631 case SUPPAGINGMODE_AMD64_GLOBAL_NX:
632 pPgLvl->cLevels = 4;
633 pPgLvl->a[0].fPhysMask = X86_CR3_AMD64_PAGE_MASK;
634 pPgLvl->a[0].fPtrShift = X86_PML4_SHIFT;
635 pPgLvl->a[0].fPtrMask = X86_PML4_MASK;
636 pPgLvl->a[0].fAndMask = X86_PML4E_P | X86_PML4E_RW;
637 pPgLvl->a[0].fResMask = X86_PML4E_P | X86_PML4E_RW;
638
639 pPgLvl->a[1].fPhysMask = X86_PML4E_PG_MASK;
640 pPgLvl->a[1].fPtrShift = X86_PDPT_SHIFT;
641 pPgLvl->a[1].fPtrMask = X86_PDPT_MASK_AMD64;
642 pPgLvl->a[1].fAndMask = X86_PDPE_P | X86_PDPE_RW /** @todo check for X86_PDPT_PS support. */;
643 pPgLvl->a[1].fResMask = X86_PDPE_P | X86_PDPE_RW;
644
645 pPgLvl->a[2].fPhysMask = X86_PDPE_PG_MASK;
646 pPgLvl->a[2].fPtrShift = X86_PD_PAE_SHIFT;
647 pPgLvl->a[2].fPtrMask = X86_PD_PAE_MASK;
648 pPgLvl->a[2].fAndMask = X86_PDE_P | X86_PDE_RW | (cr4 & X86_CR4_PSE ? X86_PDE_PS : 0);
649 pPgLvl->a[2].fResMask = X86_PDE_P | X86_PDE_RW;
650
651 pPgLvl->a[3].fPhysMask = X86_PDE_PAE_PG_MASK;
652 pPgLvl->a[3].fPtrShift = X86_PT_PAE_SHIFT;
653 pPgLvl->a[3].fPtrMask = X86_PT_PAE_MASK;
654 pPgLvl->a[3].fAndMask = X86_PTE_P | X86_PTE_RW;
655 pPgLvl->a[3].fResMask = X86_PTE_P | X86_PTE_RW;
656 break;
657
658 default:
659 AssertFailed();
660 pPgLvl->cLevels = 0;
661 break;
662 }
663
664 for (uint32_t i = 0; i < 4; i++) /* ASSUMING array size. */
665 {
666 pPgLvl->a[i].HCPhys = NIL_RTHCPHYS;
667 pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
668 pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ;
669 pPgLvl->a[i].u.pv = NULL;
670 }
671}
672
673
674/**
675 * Maps a PTE.
676 *
677 * This will update the segment structure when new PTs are mapped.
678 *
679 * It also assumes that we (for paranoid reasons) wish to establish a mapping
680 * chain from CR3 to the PT that all corresponds to the processor we're
681 * currently running on, and go about this by running with interrupts disabled
682 * and restarting from CR3 for every change.
683 *
684 * @returns VBox status code, VINF_TRY_AGAIN if we changed any mappings and had
685 * to re-enable interrupts.
686 * @param pThis The dynamic mapping cache instance.
687 * @param pPgLvl The paging level structure.
688 * @param pvPage The page.
689 * @param pSeg The segment.
690 * @param cMaxPTs The max number of PTs expected in the segment.
691 * @param ppvPTE Where to store the PTE address.
692 */
693static int pgmR0DynMapPagingArrayMapPte(PPGMR0DYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl, void *pvPage,
694 PPGMR0DYNMAPSEG pSeg, uint32_t cMaxPTs, void **ppvPTE)
695{
696 Assert(!(ASMGetFlags() & X86_EFL_IF));
697 void *pvEntry = NULL;
698 X86PGPAEUINT uEntry = ASMGetCR3();
699 for (uint32_t i = 0; i < pPgLvl->cLevels; i++)
700 {
701 RTHCPHYS HCPhys = uEntry & pPgLvl->a[i].fPhysMask;
702 if (pPgLvl->a[i].HCPhys != HCPhys)
703 {
704 /*
705 * Need to remap this level.
706 * The final level, the PT, will not be freed since that is what it's all about.
707 */
708 ASMIntEnable();
709 if (i + 1 == pPgLvl->cLevels)
710 AssertReturn(pSeg->cPTs < cMaxPTs, VERR_INTERNAL_ERROR);
711 else
712 {
713 int rc2 = RTR0MemObjFree(pPgLvl->a[i].hMemObj, true /* fFreeMappings */); AssertRC(rc2);
714 pPgLvl->a[i].hMemObj = pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
715 }
716
717 int rc = RTR0MemObjEnterPhys(&pPgLvl->a[i].hMemObj, HCPhys, PAGE_SIZE);
718 if (RT_SUCCESS(rc))
719 {
720 rc = RTR0MemObjMapKernel(&pPgLvl->a[i].hMapObj, pPgLvl->a[i].hMemObj,
721 (void *)-1 /* pvFixed */, 0 /* cbAlignment */,
722 RTMEM_PROT_WRITE | RTMEM_PROT_READ);
723 if (RT_SUCCESS(rc))
724 {
725 pPgLvl->a[i].u.pv = RTR0MemObjAddress(pPgLvl->a[i].hMapObj);
726 AssertMsg(((uintptr_t)pPgLvl->a[i].u.pv & ~(uintptr_t)PAGE_OFFSET_MASK), ("%p\n", pPgLvl->a[i].u.pv));
727 pPgLvl->a[i].HCPhys = HCPhys;
728 if (i + 1 == pPgLvl->cLevels)
729 pSeg->ahMemObjPTs[pSeg->cPTs++] = pPgLvl->a[i].hMemObj;
730 ASMIntDisable();
731 return VINF_TRY_AGAIN;
732 }
733
734 pPgLvl->a[i].hMapObj = NIL_RTR0MEMOBJ;
735 }
736 else
737 pPgLvl->a[i].hMemObj = NIL_RTR0MEMOBJ;
738 pPgLvl->a[i].HCPhys = NIL_RTHCPHYS;
739 return rc;
740 }
741
742 /*
743 * The next level.
744 */
745 uint32_t iEntry = ((uint64_t)(uintptr_t)pvPage >> pPgLvl->a[i].fPtrShift) & pPgLvl->a[i].fPtrMask;
746 if (pThis->fLegacyMode)
747 {
748 pvEntry = &pPgLvl->a[i].u.paLegacy[iEntry];
749 uEntry = pPgLvl->a[i].u.paLegacy[iEntry];
750 }
751 else
752 {
753 pvEntry = &pPgLvl->a[i].u.paPae[iEntry];
754 uEntry = pPgLvl->a[i].u.paPae[iEntry];
755 }
756
757 if ((uEntry & pPgLvl->a[i].fAndMask) != pPgLvl->a[i].fResMask)
758 {
759 LogRel(("PGMR0DynMap: internal error - iPgLvl=%u cLevels=%u uEntry=%#llx fAnd=%#llx fRes=%#llx got=%#llx\n"
760 "PGMR0DynMap: pv=%p pvPage=%p iEntry=%#x fLegacyMode=%RTbool\n",
761 i, pPgLvl->cLevels, uEntry, pPgLvl->a[i].fAndMask, pPgLvl->a[i].fResMask, uEntry & pPgLvl->a[i].fAndMask,
762 pPgLvl->a[i].u.pv, pvPage, iEntry, pThis->fLegacyMode));
763 return VERR_INTERNAL_ERROR;
764 }
765 /*Log(("#%d: iEntry=%4d uEntry=%#llx pvEntry=%p HCPhys=%RHp \n", i, iEntry, uEntry, pvEntry, pPgLvl->a[i].HCPhys));*/
766 }
767
768 /* made it thru without needing to remap anything. */
769 *ppvPTE = pvEntry;
770 return VINF_SUCCESS;
771}
772
773
774/**
775 * Sets up a guard page.
776 *
777 * @param pThis The dynamic mapping cache instance.
778 * @param pPage The page.
779 */
780DECLINLINE(void) pgmR0DynMapSetupGuardPage(PPGMR0DYNMAP pThis, PPGMR0DYNMAPENTRY pPage)
781{
782 memset(pPage->pvPage, 0xfd, PAGE_SIZE);
783 pPage->cRefs = PGMR0DYNMAP_GUARD_PAGE_REF_COUNT;
784 pPage->HCPhys = PGMR0DYNMAP_GUARD_PAGE_HCPHYS;
785#ifdef PGMR0DYNMAP_GUARD_NP
786 ASMAtomicBitClear(pPage->uPte.pv, X86_PTE_BIT_P);
787#else
788 if (pThis->fLegacyMode)
789 ASMAtomicWriteU32(&pPage->uPte.pLegacy->u, PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE);
790 else
791 ASMAtomicWriteU64(&pPage->uPte.pPae->u, PGMR0DYNMAP_GUARD_PAGE_PAE_PTE);
792#endif
793 pThis->cGuardPages++;
794}
795
796
797/**
798 * Adds a new segment of the specified size.
799 *
800 * @returns VBox status code.
801 * @param pThis The dynamic mapping cache instance.
802 * @param cPages The size of the new segment, give as a page count.
803 */
804static int pgmR0DynMapAddSeg(PPGMR0DYNMAP pThis, uint32_t cPages)
805{
806 int rc2;
807 AssertReturn(ASMGetFlags() & X86_EFL_IF, VERR_PREEMPT_DISABLED);
808
809 /*
810 * Do the array reallocations first.
811 * (The pages array has to be replaced behind the spinlock of course.)
812 */
813 void *pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) * (pThis->cPages + cPages));
814 if (!pvSavedPTEs)
815 return VERR_NO_MEMORY;
816 pThis->pvSavedPTEs = pvSavedPTEs;
817
818 void *pvPages = RTMemAllocZ(sizeof(pThis->paPages[0]) * (pThis->cPages + cPages));
819 if (!pvPages)
820 {
821 pvSavedPTEs = RTMemRealloc(pThis->pvSavedPTEs, (pThis->fLegacyMode ? sizeof(X86PGUINT) : sizeof(X86PGPAEUINT)) * pThis->cPages);
822 if (pvSavedPTEs)
823 pThis->pvSavedPTEs = pvSavedPTEs;
824 return VERR_NO_MEMORY;
825 }
826
827 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
828 RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
829
830 memcpy(pvPages, pThis->paPages, sizeof(pThis->paPages[0]) * pThis->cPages);
831 void *pvToFree = pThis->paPages;
832 pThis->paPages = (PPGMR0DYNMAPENTRY)pvPages;
833
834 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
835 RTMemFree(pvToFree);
836
837 /*
838 * Allocate the segment structure and pages of memory, then touch all the pages (paranoia).
839 */
840 uint32_t cMaxPTs = cPages / (pThis->fLegacyMode ? X86_PG_ENTRIES : X86_PG_PAE_ENTRIES) + 2;
841 PPGMR0DYNMAPSEG pSeg = (PPGMR0DYNMAPSEG)RTMemAllocZ(RT_UOFFSETOF(PGMR0DYNMAPSEG, ahMemObjPTs[cMaxPTs]));
842 if (!pSeg)
843 return VERR_NO_MEMORY;
844 pSeg->pNext = NULL;
845 pSeg->cPages = cPages;
846 pSeg->iPage = pThis->cPages;
847 pSeg->cPTs = 0;
848 int rc = RTR0MemObjAllocPage(&pSeg->hMemObj, cPages << PAGE_SHIFT, false);
849 if (RT_SUCCESS(rc))
850 {
851 uint8_t *pbPage = (uint8_t *)RTR0MemObjAddress(pSeg->hMemObj);
852 AssertMsg(VALID_PTR(pbPage) && !((uintptr_t)pbPage & PAGE_OFFSET_MASK), ("%p\n", pbPage));
853 memset(pbPage, 0xfe, cPages << PAGE_SHIFT);
854
855 /*
856 * Walk thru the pages and set them up with a mapping of their PTE and everything.
857 */
858 ASMIntDisable();
859 PGMR0DYNMAPPGLVL PgLvl;
860 pgmR0DynMapPagingArrayInit(pThis, &PgLvl);
861 uint32_t const iEndPage = pSeg->iPage + cPages;
862 for (uint32_t iPage = pSeg->iPage;
863 iPage < iEndPage;
864 iPage++, pbPage += PAGE_SIZE)
865 {
866 /* Initialize the page data. */
867 pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
868 pThis->paPages[iPage].pvPage = pbPage;
869 pThis->paPages[iPage].cRefs = 0;
870 pThis->paPages[iPage].uPte.pPae = 0;
871 RTCpuSetFill(&pThis->paPages[iPage].PendingSet);
872
873 /* Map its page table, retry until we've got a clean run (paranoia). */
874 do
875 rc = pgmR0DynMapPagingArrayMapPte(pThis, &PgLvl, pbPage, pSeg, cMaxPTs,
876 &pThis->paPages[iPage].uPte.pv);
877 while (rc == VINF_TRY_AGAIN);
878 if (RT_FAILURE(rc))
879 break;
880
881 /* Save the PTE. */
882 if (pThis->fLegacyMode)
883 ((PX86PGUINT)pThis->pvSavedPTEs)[iPage] = pThis->paPages[iPage].uPte.pLegacy->u;
884 else
885 ((PX86PGPAEUINT)pThis->pvSavedPTEs)[iPage] = pThis->paPages[iPage].uPte.pPae->u;
886
887#ifdef VBOX_STRICT
888 /* Check that we've got the right entry. */
889 RTHCPHYS HCPhysPage = RTR0MemObjGetPagePhysAddr(pSeg->hMemObj, iPage - pSeg->iPage);
890 RTHCPHYS HCPhysPte = pThis->fLegacyMode
891 ? pThis->paPages[iPage].uPte.pLegacy->u & X86_PTE_PG_MASK
892 : pThis->paPages[iPage].uPte.pPae->u & X86_PTE_PAE_PG_MASK;
893 if (HCPhysPage != HCPhysPte)
894 {
895 LogRel(("pgmR0DynMapAddSeg: internal error - page #%u HCPhysPage=%RHp HCPhysPte=%RHp pbPage=%p pvPte=%p\n",
896 iPage - pSeg->iPage, HCPhysPage, HCPhysPte, pbPage, pThis->paPages[iPage].uPte.pv));
897 rc = VERR_INTERNAL_ERROR;
898 break;
899 }
900#endif
901 } /* for each page */
902 ASMIntEnable();
903
904 /* cleanup non-PT mappings */
905 for (uint32_t i = 0; i < PgLvl.cLevels - 1; i++)
906 RTR0MemObjFree(PgLvl.a[i].hMemObj, true /* fFreeMappings */);
907
908 if (RT_SUCCESS(rc))
909 {
910#if PGMR0DYNMAP_GUARD_PAGES > 0
911 /*
912 * Setup guard pages.
913 * (Note: TLBs will be shot down later on.)
914 */
915 uint32_t iPage = pSeg->iPage;
916 while (iPage < iEndPage)
917 {
918 for (uint32_t iGPg = 0; iGPg < PGMR0DYNMAP_GUARD_PAGES && iPage < iEndPage; iGPg++, iPage++)
919 pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]);
920 iPage++; /* the guarded page */
921 }
922
923 /* Make sure the very last page is a guard page too. */
924 iPage = iEndPage - 1;
925 if (pThis->paPages[iPage].cRefs != PGMR0DYNMAP_GUARD_PAGE_REF_COUNT)
926 pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]);
927#endif /* PGMR0DYNMAP_GUARD_PAGES > 0 */
928
929 /*
930 * Commit it by adding the segment to the list and updating the page count.
931 */
932 pSeg->pNext = pThis->pSegHead;
933 pThis->pSegHead = pSeg;
934 pThis->cPages += cPages;
935 return VINF_SUCCESS;
936 }
937
938 /*
939 * Bail out.
940 */
941 while (pSeg->cPTs-- > 0)
942 {
943 rc2 = RTR0MemObjFree(pSeg->ahMemObjPTs[pSeg->cPTs], true /* fFreeMappings */);
944 AssertRC(rc2);
945 pSeg->ahMemObjPTs[pSeg->cPTs] = NIL_RTR0MEMOBJ;
946 }
947
948 rc2 = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */);
949 AssertRC(rc2);
950 pSeg->hMemObj = NIL_RTR0MEMOBJ;
951 }
952 RTMemFree(pSeg);
953
954 /* Don't bother resizing the arrays, but free them if we're the only user. */
955 if (!pThis->cPages)
956 {
957 RTMemFree(pThis->paPages);
958 pThis->paPages = NULL;
959 RTMemFree(pThis->pvSavedPTEs);
960 pThis->pvSavedPTEs = NULL;
961 }
962 return rc;
963}
964
965
966/**
967 * Called by PGMR0DynMapInitVM under the init lock.
968 *
969 * @returns VBox status code.
970 * @param pThis The dynamic mapping cache instance.
971 */
972static int pgmR0DynMapSetup(PPGMR0DYNMAP pThis)
973{
974 /*
975 * Calc the size and add a segment of that size.
976 */
977 uint32_t cMinPages;
978 uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages);
979 AssertReturn(cPages, VERR_INTERNAL_ERROR);
980 int rc = pgmR0DynMapAddSeg(pThis, cPages);
981 if (rc == VERR_NO_MEMORY)
982 {
983 /*
984 * Try adding smaller segments.
985 */
986 do
987 rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES);
988 while (RT_SUCCESS(rc) && pThis->cPages < cPages);
989 if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages)
990 rc = VINF_SUCCESS;
991 if (rc == VERR_NO_MEMORY)
992 {
993 if (pThis->cPages)
994 pgmR0DynMapTearDown(pThis);
995 rc = VERR_PGM_DYNMAP_SETUP_ERROR;
996 }
997 }
998 Assert(ASMGetFlags() & X86_EFL_IF);
999
1000#if PGMR0DYNMAP_GUARD_PAGES > 0
1001 /* paranoia */
1002 if (RT_SUCCESS(rc))
1003 pgmR0DynMapTlbShootDown(pThis);
1004#endif
1005 return rc;
1006}
1007
1008
1009/**
1010 * Called by PGMR0DynMapInitVM under the init lock.
1011 *
1012 * @returns VBox status code.
1013 * @param pThis The dynamic mapping cache instance.
1014 */
1015static int pgmR0DynMapExpand(PPGMR0DYNMAP pThis)
1016{
1017 /*
1018 * Calc the new target size and add a segment of the appropriate size.
1019 */
1020 uint32_t cMinPages;
1021 uint32_t cPages = pgmR0DynMapCalcNewSize(pThis, &cMinPages);
1022 AssertReturn(cPages, VERR_INTERNAL_ERROR);
1023 if (pThis->cPages >= cPages)
1024 return VINF_SUCCESS;
1025
1026 uint32_t cAdd = cPages - pThis->cPages;
1027 int rc = pgmR0DynMapAddSeg(pThis, cAdd);
1028 if (rc == VERR_NO_MEMORY)
1029 {
1030 /*
1031 * Try adding smaller segments.
1032 */
1033 do
1034 rc = pgmR0DynMapAddSeg(pThis, PGMR0DYNMAP_SMALL_SEG_PAGES);
1035 while (RT_SUCCESS(rc) && pThis->cPages < cPages);
1036 if (rc == VERR_NO_MEMORY && pThis->cPages >= cMinPages)
1037 rc = VINF_SUCCESS;
1038 if (rc == VERR_NO_MEMORY)
1039 rc = VERR_PGM_DYNMAP_EXPAND_ERROR;
1040 }
1041 Assert(ASMGetFlags() & X86_EFL_IF);
1042
1043#if PGMR0DYNMAP_GUARD_PAGES > 0
1044 /* paranoia */
1045 if (RT_SUCCESS(rc))
1046 pgmR0DynMapTlbShootDown(pThis);
1047#endif
1048 return rc;
1049}
1050
1051
1052/**
1053 * Called by PGMR0DynMapTermVM under the init lock.
1054 *
1055 * @returns VBox status code.
1056 * @param pThis The dynamic mapping cache instance.
1057 */
1058static void pgmR0DynMapTearDown(PPGMR0DYNMAP pThis)
1059{
1060 /*
1061 * Restore the original page table entries
1062 */
1063 PPGMR0DYNMAPENTRY paPages = pThis->paPages;
1064 uint32_t iPage = pThis->cPages;
1065 if (pThis->fLegacyMode)
1066 {
1067 X86PGUINT const *paSavedPTEs = (X86PGUINT const *)pThis->pvSavedPTEs;
1068 while (iPage-- > 0)
1069 {
1070 X86PGUINT uOld = paPages[iPage].uPte.pLegacy->u;
1071 X86PGUINT uOld2 = uOld; NOREF(uOld2);
1072 X86PGUINT uNew = paSavedPTEs[iPage];
1073 while (!ASMAtomicCmpXchgExU32(&paPages[iPage].uPte.pLegacy->u, uNew, uOld, &uOld))
1074 AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
1075 Assert(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage]);
1076 }
1077 }
1078 else
1079 {
1080 X86PGPAEUINT const *paSavedPTEs = (X86PGPAEUINT const *)pThis->pvSavedPTEs;
1081 while (iPage-- > 0)
1082 {
1083 X86PGPAEUINT uOld = paPages[iPage].uPte.pPae->u;
1084 X86PGPAEUINT uOld2 = uOld; NOREF(uOld2);
1085 X86PGPAEUINT uNew = paSavedPTEs[iPage];
1086 while (!ASMAtomicCmpXchgExU64(&paPages[iPage].uPte.pPae->u, uNew, uOld, &uOld))
1087 AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
1088 Assert(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage]);
1089 }
1090 }
1091
1092 /*
1093 * Shoot down the TLBs on all CPUs before freeing them.
1094 */
1095 pgmR0DynMapTlbShootDown(pThis);
1096
1097 /*
1098 * Free the segments.
1099 */
1100 while (pThis->pSegHead)
1101 {
1102 int rc;
1103 PPGMR0DYNMAPSEG pSeg = pThis->pSegHead;
1104 pThis->pSegHead = pSeg->pNext;
1105
1106 uint32_t iPT = pSeg->cPTs;
1107 while (iPT-- > 0)
1108 {
1109 rc = RTR0MemObjFree(pSeg->ahMemObjPTs[iPT], true /* fFreeMappings */); AssertRC(rc);
1110 pSeg->ahMemObjPTs[iPT] = NIL_RTR0MEMOBJ;
1111 }
1112 rc = RTR0MemObjFree(pSeg->hMemObj, true /* fFreeMappings */); AssertRC(rc);
1113 pSeg->hMemObj = NIL_RTR0MEMOBJ;
1114 pSeg->pNext = NULL;
1115 pSeg->iPage = UINT16_MAX;
1116 pSeg->cPages = 0;
1117 pSeg->cPTs = 0;
1118 RTMemFree(pSeg);
1119 }
1120
1121 /*
1122 * Free the arrays and restore the initial state.
1123 * The cLoadMax value is left behind for the next setup.
1124 */
1125 RTMemFree(pThis->paPages);
1126 pThis->paPages = NULL;
1127 RTMemFree(pThis->pvSavedPTEs);
1128 pThis->pvSavedPTEs = NULL;
1129 pThis->cPages = 0;
1130 pThis->cLoad = 0;
1131 pThis->cGuardPages = 0;
1132}
1133
1134
1135/**
1136 * Release references to a page, caller owns the spin lock.
1137 *
1138 * @param pThis The dynamic mapping cache instance.
1139 * @param iPage The page.
1140 * @param cRefs The number of references to release.
1141 */
1142DECLINLINE(void) pgmR0DynMapReleasePageLocked(PPGMR0DYNMAP pThis, uint32_t iPage, int32_t cRefs)
1143{
1144 cRefs = ASMAtomicSubS32(&pThis->paPages[iPage].cRefs, cRefs) - cRefs;
1145 AssertMsg(cRefs >= 0, ("%d\n", cRefs));
1146 if (!cRefs)
1147 pThis->cLoad--;
1148}
1149
1150
1151/**
1152 * Release references to a page, caller does not own the spin lock.
1153 *
1154 * @param pThis The dynamic mapping cache instance.
1155 * @param iPage The page.
1156 * @param cRefs The number of references to release.
1157 */
1158static void pgmR0DynMapReleasePage(PPGMR0DYNMAP pThis, uint32_t iPage, uint32_t cRefs)
1159{
1160 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1161 RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1162 pgmR0DynMapReleasePageLocked(pThis, iPage, cRefs);
1163 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1164}
1165
1166
1167/**
1168 * pgmR0DynMapPage worker that deals with the tedious bits.
1169 *
1170 * @returns The page index on success, UINT32_MAX on failure.
1171 * @param pThis The dynamic mapping cache instance.
1172 * @param HCPhys The address of the page to be mapped.
1173 * @param iPage The page index pgmR0DynMapPage hashed HCPhys to.
1174 * @param pVM The shared VM structure, for statistics only.
1175 */
1176static uint32_t pgmR0DynMapPageSlow(PPGMR0DYNMAP pThis, RTHCPHYS HCPhys, uint32_t iPage, PVM pVM)
1177{
1178 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageSlow);
1179
1180 /*
1181 * Check if any of the first 3 pages are unreferenced since the caller
1182 * already has made sure they aren't matching.
1183 */
1184#ifdef VBOX_WITH_STATISTICS
1185 bool fLooped = false;
1186#endif
1187 uint32_t const cPages = pThis->cPages;
1188 PPGMR0DYNMAPENTRY paPages = pThis->paPages;
1189 uint32_t iFreePage;
1190 if (!paPages[iPage].cRefs)
1191 iFreePage = iPage;
1192 else if (!paPages[(iPage + 1) % cPages].cRefs)
1193 iFreePage = (iPage + 1) % cPages;
1194 else if (!paPages[(iPage + 2) % cPages].cRefs)
1195 iFreePage = (iPage + 2) % cPages;
1196 else
1197 {
1198 /*
1199 * Search for an unused or matching entry.
1200 */
1201 iFreePage = (iPage + 3) % cPages;
1202 for (;;)
1203 {
1204 if (paPages[iFreePage].HCPhys == HCPhys)
1205 {
1206 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageSlowLoopHits);
1207 return iFreePage;
1208 }
1209 if (!paPages[iFreePage].cRefs)
1210 break;
1211
1212 /* advance */
1213 iFreePage = (iFreePage + 1) % cPages;
1214 if (RT_UNLIKELY(iFreePage == iPage))
1215 return UINT32_MAX;
1216 }
1217 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageSlowLoopMisses);
1218#ifdef VBOX_WITH_STATISTICS
1219 fLooped = true;
1220#endif
1221 }
1222 Assert(iFreePage < cPages);
1223
1224#if 0 //def VBOX_WITH_STATISTICS
1225 /* Check for lost hits. */
1226 if (!fLooped)
1227 for (uint32_t iPage2 = (iPage + 3) % cPages; iPage2 != iPage; iPage2 = (iPage2 + 1) % cPages)
1228 if (paPages[iPage2].HCPhys == HCPhys)
1229 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageSlowLostHits);
1230#endif
1231
1232 /*
1233 * Setup the new entry.
1234 */
1235 /*Log6(("pgmR0DynMapPageSlow: old - %RHp %#x %#llx\n", paPages[iFreePage].HCPhys, paPages[iFreePage].cRefs, paPages[iFreePage].uPte.pPae->u));*/
1236 paPages[iFreePage].HCPhys = HCPhys;
1237 RTCpuSetFill(&paPages[iFreePage].PendingSet);
1238 if (pThis->fLegacyMode)
1239 {
1240 X86PGUINT uOld = paPages[iFreePage].uPte.pLegacy->u;
1241 X86PGUINT uOld2 = uOld; NOREF(uOld2);
1242 X86PGUINT uNew = (uOld & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
1243 | X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D
1244 | (HCPhys & X86_PTE_PG_MASK);
1245 while (!ASMAtomicCmpXchgExU32(&paPages[iFreePage].uPte.pLegacy->u, uNew, uOld, &uOld))
1246 AssertMsgFailed(("uOld=%#x uOld2=%#x uNew=%#x\n", uOld, uOld2, uNew));
1247 Assert(paPages[iFreePage].uPte.pLegacy->u == uNew);
1248 }
1249 else
1250 {
1251 X86PGPAEUINT uOld = paPages[iFreePage].uPte.pPae->u;
1252 X86PGPAEUINT uOld2 = uOld; NOREF(uOld2);
1253 X86PGPAEUINT uNew = (uOld & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
1254 | X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D
1255 | (HCPhys & X86_PTE_PAE_PG_MASK);
1256 while (!ASMAtomicCmpXchgExU64(&paPages[iFreePage].uPte.pPae->u, uNew, uOld, &uOld))
1257 AssertMsgFailed(("uOld=%#llx uOld2=%#llx uNew=%#llx\n", uOld, uOld2, uNew));
1258 Assert(paPages[iFreePage].uPte.pPae->u == uNew);
1259 /*Log6(("pgmR0DynMapPageSlow: #%x - %RHp %p %#llx\n", iFreePage, HCPhys, paPages[iFreePage].pvPage, uNew));*/
1260 }
1261 return iFreePage;
1262}
1263
1264
1265/**
1266 * Maps a page into the pool.
1267 *
1268 * @returns Page index on success, UINT32_MAX on failure.
1269 * @param pThis The dynamic mapping cache instance.
1270 * @param HCPhys The address of the page to be mapped.
1271 * @param iRealCpu The real cpu set index. (optimization)
1272 * @param pVM The shared VM structure, for statistics only.
1273 * @param ppvPage Where to the page address.
1274 */
1275DECLINLINE(uint32_t) pgmR0DynMapPage(PPGMR0DYNMAP pThis, RTHCPHYS HCPhys, int32_t iRealCpu, PVM pVM, void **ppvPage)
1276{
1277 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1278 RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1279 AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
1280 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPage);
1281
1282 /*
1283 * Find an entry, if possible a matching one. The HCPhys address is hashed
1284 * down to a page index, collisions are handled by linear searching.
1285 * Optimized for a hit in the first 3 pages.
1286 *
1287 * To the cheap hits here and defer the tedious searching and inserting
1288 * to a helper function.
1289 */
1290 uint32_t const cPages = pThis->cPages;
1291 uint32_t iPage = (HCPhys >> PAGE_SHIFT) % cPages;
1292 PPGMR0DYNMAPENTRY paPages = pThis->paPages;
1293 if (RT_LIKELY(paPages[iPage].HCPhys == HCPhys))
1294 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageHits0);
1295 else
1296 {
1297 uint32_t iPage2 = (iPage + 1) % cPages;
1298 if (RT_LIKELY(paPages[iPage2].HCPhys == HCPhys))
1299 {
1300 iPage = iPage2;
1301 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageHits1);
1302 }
1303 else
1304 {
1305 iPage2 = (iPage + 2) % cPages;
1306 if (paPages[iPage2].HCPhys == HCPhys)
1307 {
1308 iPage = iPage2;
1309 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageHits2);
1310 }
1311 else
1312 {
1313 iPage = pgmR0DynMapPageSlow(pThis, HCPhys, iPage, pVM);
1314 if (RT_UNLIKELY(iPage == UINT32_MAX))
1315 {
1316 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1317 *ppvPage = NULL;
1318 return iPage;
1319 }
1320 }
1321 }
1322 }
1323
1324 /*
1325 * Reference it, update statistics and get the return address.
1326 */
1327 int32_t cRefs = ASMAtomicIncS32(&paPages[iPage].cRefs);
1328 if (cRefs == 1)
1329 {
1330 pThis->cLoad++;
1331 if (pThis->cLoad > pThis->cMaxLoad)
1332 pThis->cMaxLoad = pThis->cLoad;
1333 AssertMsg(pThis->cLoad <= pThis->cPages - pThis->cGuardPages, ("%d/%d\n", pThis->cLoad, pThis->cPages - pThis->cGuardPages));
1334 }
1335 else if (RT_UNLIKELY(cRefs <= 0))
1336 {
1337 ASMAtomicDecS32(&paPages[iPage].cRefs);
1338 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1339 *ppvPage = NULL;
1340 AssertLogRelMsgFailedReturn(("cRefs=%d iPage=%p HCPhys=%RHp\n", cRefs, iPage, HCPhys), UINT32_MAX);
1341 }
1342 void *pvPage = paPages[iPage].pvPage;
1343
1344 /*
1345 * Invalidate the entry?
1346 */
1347 bool fInvalidateIt = RTCpuSetIsMemberByIndex(&paPages[iPage].PendingSet, iRealCpu);
1348 if (RT_UNLIKELY(fInvalidateIt))
1349 RTCpuSetDelByIndex(&paPages[iPage].PendingSet, iRealCpu);
1350
1351 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1352
1353 /*
1354 * Do the actual invalidation outside the spinlock.
1355 */
1356 if (RT_UNLIKELY(fInvalidateIt))
1357 {
1358 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageInvlPg);
1359 ASMInvalidatePage(pvPage);
1360 }
1361
1362 *ppvPage = pvPage;
1363 return iPage;
1364}
1365
1366
1367/**
1368 * Assert the the integrity of the pool.
1369 *
1370 * @returns VBox status code.
1371 */
1372VMMR0DECL(int) PGMR0DynMapAssertIntegrity(void)
1373{
1374 /*
1375 * Basic pool stuff that doesn't require any lock, just assumes we're a user.
1376 */
1377 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
1378 if (!pThis)
1379 return VINF_SUCCESS;
1380 AssertPtrReturn(pThis, VERR_INVALID_POINTER);
1381 AssertReturn(pThis->u32Magic == PGMR0DYNMAP_MAGIC, VERR_INVALID_MAGIC);
1382 if (!pThis->cUsers)
1383 return VERR_INVALID_PARAMETER;
1384
1385
1386 int rc = VINF_SUCCESS;
1387 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1388 RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1389
1390#define CHECK_RET(expr, a) \
1391 do { \
1392 if (RT_UNLIKELY(!(expr))) \
1393 { \
1394 RTSpinlockRelease(pThis->hSpinlock, &Tmp); \
1395 AssertMsg1(#expr, __LINE__, __FILE__, __PRETTY_FUNCTION__); \
1396 AssertMsg2 a; \
1397 return VERR_INTERNAL_ERROR; \
1398 } \
1399 } while (0)
1400
1401 /*
1402 * Check that the PTEs are correct.
1403 */
1404 uint32_t cGuard = 0;
1405 uint32_t cLoad = 0;
1406 PPGMR0DYNMAPENTRY paPages = pThis->paPages;
1407 uint32_t iPage = pThis->cPages;
1408 if (pThis->fLegacyMode)
1409 {
1410 PCX86PGUINT paSavedPTEs = (PCX86PGUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
1411 while (iPage-- > 0)
1412 {
1413 CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
1414 if ( paPages[iPage].cRefs == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT
1415 && paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS)
1416 {
1417#ifdef PGMR0DYNMAP_GUARD_NP
1418 CHECK_RET(paPages[iPage].uPte.pLegacy->u == (paSavedPTEs[iPage] & ~(X86PGUINT)X86_PTE_P),
1419 ("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage]));
1420#else
1421 CHECK_RET(paPages[iPage].uPte.pLegacy->u == PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE,
1422 ("#%u: %#x", iPage, paPages[iPage].uPte.pLegacy->u));
1423#endif
1424 cGuard++;
1425 }
1426 else if (paPages[iPage].HCPhys != NIL_RTHCPHYS)
1427 {
1428 CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
1429 X86PGUINT uPte = (paSavedPTEs[iPage] & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
1430 | X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D
1431 | (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
1432 CHECK_RET(paPages[iPage].uPte.pLegacy->u == uPte,
1433 ("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, uPte));
1434 if (paPages[iPage].cRefs)
1435 cLoad++;
1436 }
1437 else
1438 CHECK_RET(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage],
1439 ("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage]));
1440 }
1441 }
1442 else
1443 {
1444 PCX86PGPAEUINT paSavedPTEs = (PCX86PGPAEUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
1445 while (iPage-- > 0)
1446 {
1447 CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
1448 if ( paPages[iPage].cRefs == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT
1449 && paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS)
1450 {
1451#ifdef PGMR0DYNMAP_GUARD_NP
1452 CHECK_RET(paPages[iPage].uPte.pPae->u == (paSavedPTEs[iPage] & ~(X86PGPAEUINT)X86_PTE_P),
1453 ("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage]));
1454#else
1455 CHECK_RET(paPages[iPage].uPte.pPae->u == PGMR0DYNMAP_GUARD_PAGE_PAE_PTE,
1456 ("#%u: %#llx", iPage, paPages[iPage].uPte.pPae->u));
1457#endif
1458 cGuard++;
1459 }
1460 else if (paPages[iPage].HCPhys != NIL_RTHCPHYS)
1461 {
1462 CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
1463 X86PGPAEUINT uPte = (paSavedPTEs[iPage] & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
1464 | X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D
1465 | (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
1466 CHECK_RET(paPages[iPage].uPte.pPae->u == uPte,
1467 ("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pLegacy->u, uPte));
1468 if (paPages[iPage].cRefs)
1469 cLoad++;
1470 }
1471 else
1472 CHECK_RET(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage],
1473 ("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage]));
1474 }
1475 }
1476
1477 CHECK_RET(cLoad == pThis->cLoad, ("%u %u\n", cLoad, pThis->cLoad));
1478 CHECK_RET(cGuard == pThis->cGuardPages, ("%u %u\n", cGuard, pThis->cGuardPages));
1479
1480#undef CHECK_RET
1481 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1482 return VINF_SUCCESS;
1483}
1484
1485
1486/**
1487 * Signals the start of a new set of mappings.
1488 *
1489 * Mostly for strictness. PGMDynMapHCPage won't work unless this
1490 * API is called.
1491 *
1492 * @param pVM Pointer to the shared VM structure.
1493 */
1494VMMDECL(void) PGMDynMapStartAutoSet(PVM pVM)
1495{
1496 Assert(pVM->pgm.s.AutoSet.cEntries == PGMMAPSET_CLOSED);
1497 Assert(pVM->pgm.s.AutoSet.iSubset == UINT32_MAX);
1498 pVM->pgm.s.AutoSet.cEntries = 0;
1499 pVM->pgm.s.AutoSet.iCpu = RTMpCpuIdToSetIndex(RTMpCpuId());
1500}
1501
1502
1503/**
1504 * Worker that performs the actual flushing of the set.
1505 *
1506 * @param pSet The set to flush.
1507 * @param cEntries The number of entries.
1508 */
1509DECLINLINE(void) pgmDynMapFlushAutoSetWorker(PPGMMAPSET pSet, uint32_t cEntries)
1510{
1511 /*
1512 * Release any pages it's referencing.
1513 */
1514 if ( cEntries != 0
1515 && RT_LIKELY(cEntries <= RT_ELEMENTS(pSet->aEntries)))
1516 {
1517 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
1518 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1519 RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1520
1521 uint32_t i = cEntries;
1522 while (i-- > 0)
1523 {
1524 uint32_t iPage = pSet->aEntries[i].iPage;
1525 Assert(iPage < pThis->cPages);
1526 int32_t cRefs = pSet->aEntries[i].cRefs;
1527 Assert(cRefs > 0);
1528 pgmR0DynMapReleasePageLocked(pThis, iPage, cRefs);
1529
1530 pSet->aEntries[i].iPage = UINT16_MAX;
1531 pSet->aEntries[i].cRefs = 0;
1532 }
1533
1534 Assert(pThis->cLoad <= pThis->cPages - pThis->cGuardPages);
1535 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1536 }
1537}
1538
1539
1540/**
1541 * Releases the dynamic memory mappings made by PGMDynMapHCPage and associates
1542 * since the PGMDynMapStartAutoSet call.
1543 *
1544 * @param pVM Pointer to the shared VM structure.
1545 */
1546VMMDECL(void) PGMDynMapReleaseAutoSet(PVM pVM)
1547{
1548 PPGMMAPSET pSet = &pVM->pgm.s.AutoSet;
1549
1550 /*
1551 * Close and flush the set.
1552 */
1553 uint32_t cEntries = pSet->cEntries;
1554 AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
1555 pSet->cEntries = PGMMAPSET_CLOSED;
1556 pSet->iSubset = UINT32_MAX;
1557 pSet->iCpu = -1;
1558
1559 STAM_COUNTER_INC(&pVM->pgm.s.aStatR0DynMapSetSize[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
1560 AssertMsg(cEntries < PGMMAPSET_MAX_FILL, ("%u\n", cEntries));
1561 if (cEntries > RT_ELEMENTS(pSet->aEntries) * 50 / 100)
1562 Log(("PGMDynMapReleaseAutoSet: cEntries=%d\n", pSet->cEntries));
1563
1564 pgmDynMapFlushAutoSetWorker(pSet, cEntries);
1565}
1566
1567
1568/**
1569 * Flushes the set if it's above a certain threshold.
1570 *
1571 * @param pVM Pointer to the shared VM structure.
1572 */
1573VMMDECL(void) PGMDynMapFlushAutoSet(PVM pVM)
1574{
1575 PPGMMAPSET pSet = &pVM->pgm.s.AutoSet;
1576 AssertMsg(pSet->iCpu == RTMpCpuIdToSetIndex(RTMpCpuId()), ("%d %d(%d) efl=%#x\n", pSet->iCpu, RTMpCpuIdToSetIndex(RTMpCpuId()), RTMpCpuId(), ASMGetFlags()));
1577
1578 /*
1579 * Only flush it if it's 45% full.
1580 */
1581 uint32_t cEntries = pSet->cEntries;
1582 AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
1583 STAM_COUNTER_INC(&pVM->pgm.s.aStatR0DynMapSetSize[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
1584 if (cEntries >= RT_ELEMENTS(pSet->aEntries) * 45 / 100)
1585 {
1586 pSet->cEntries = 0;
1587
1588 AssertMsg(cEntries < PGMMAPSET_MAX_FILL, ("%u\n", cEntries));
1589 Log(("PGMDynMapFlushAutoSet: cEntries=%d\n", pSet->cEntries));
1590
1591 pgmDynMapFlushAutoSetWorker(pSet, cEntries);
1592 AssertMsg(pSet->iCpu == RTMpCpuIdToSetIndex(RTMpCpuId()), ("%d %d(%d) efl=%#x\n", pSet->iCpu, RTMpCpuIdToSetIndex(RTMpCpuId()), RTMpCpuId(), ASMGetFlags()));
1593 }
1594}
1595
1596
1597/**
1598 * Migrates the automatic mapping set of the current vCPU if it's active and
1599 * necessary.
1600 *
1601 * This is called when re-entering the hardware assisted execution mode after a
1602 * nip down to ring-3. We run the risk that the CPU might have change and we
1603 * will therefore make sure all the cache entries currently in the auto set will
1604 * be valid on the new CPU. If the cpu didn't change nothing will happen as all
1605 * the entries will have been flagged as invalidated.
1606 *
1607 * @param pVM Pointer to the shared VM structure.
1608 * @thread EMT
1609 */
1610VMMDECL(void) PGMDynMapMigrateAutoSet(PVM pVM)
1611{
1612 PPGMMAPSET pSet = &pVM->pgm.s.AutoSet;
1613 int32_t iRealCpu = RTMpCpuIdToSetIndex(RTMpCpuId());
1614 if (pSet->iCpu != iRealCpu)
1615 {
1616 uint32_t i = pSet->cEntries;
1617 if (i != PGMMAPSET_CLOSED)
1618 {
1619 AssertMsg(i <= RT_ELEMENTS(pSet->aEntries), ("%#x (%u)\n", i, i));
1620 if (i != 0 && RT_LIKELY(i <= RT_ELEMENTS(pSet->aEntries)))
1621 {
1622 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
1623 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1624 RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1625
1626 while (i-- > 0)
1627 {
1628 Assert(pSet->aEntries[i].cRefs > 0);
1629 uint32_t iPage = pSet->aEntries[i].iPage;
1630 Assert(iPage < pThis->cPages);
1631 if (RTCpuSetIsMemberByIndex(&pThis->paPages[iPage].PendingSet, iRealCpu))
1632 {
1633 RTCpuSetDelByIndex(&pThis->paPages[iPage].PendingSet, iRealCpu);
1634 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1635
1636 ASMInvalidatePage(pThis->paPages[iPage].pvPage);
1637 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapMigrateInvlPg);
1638
1639 RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1640 }
1641 }
1642
1643 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1644 }
1645 }
1646 pSet->iCpu = iRealCpu;
1647 }
1648}
1649
1650
1651/**
1652 * Worker function that flushes the current subset.
1653 *
1654 * This is called when the set is popped or when the set
1655 * hash a too high load. As also pointed out elsewhere, the
1656 * whole subset thing is a hack for working around code that
1657 * accesses too many pages. Like PGMPool.
1658 *
1659 * @param pSet The set which subset to flush.
1660 */
1661static void pgmDynMapFlushSubset(PPGMMAPSET pSet)
1662{
1663 uint32_t iSubset = pSet->iSubset;
1664 uint32_t i = pSet->cEntries;
1665 Assert(i <= RT_ELEMENTS(pSet->aEntries));
1666 if ( i > iSubset
1667 && i <= RT_ELEMENTS(pSet->aEntries))
1668 {
1669 Log(("pgmDynMapFlushSubset: cEntries=%d iSubset=%d\n", pSet->cEntries, iSubset));
1670 pSet->cEntries = iSubset;
1671
1672 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
1673 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1674 RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
1675
1676 while (i-- > iSubset)
1677 {
1678 uint32_t iPage = pSet->aEntries[i].iPage;
1679 Assert(iPage < pThis->cPages);
1680 int32_t cRefs = pSet->aEntries[i].cRefs;
1681 Assert(cRefs > 0);
1682 pgmR0DynMapReleasePageLocked(pThis, iPage, cRefs);
1683
1684 pSet->aEntries[i].iPage = UINT16_MAX;
1685 pSet->aEntries[i].cRefs = 0;
1686 }
1687
1688 RTSpinlockRelease(pThis->hSpinlock, &Tmp);
1689 }
1690}
1691
1692
1693/**
1694 * Creates a subset.
1695 *
1696 * A subset is a hack to avoid having to rewrite code that touches a lot of
1697 * pages. It prevents the mapping set from being overflowed by automatically
1698 * flushing previous mappings when a certain threshold is reached.
1699 *
1700 * Pages mapped after calling this function are only valid until the next page
1701 * is mapped.
1702 *
1703 * @returns The index of the previous subset. Pass this to
1704 * PGMDynMapPopAutoSubset when poping it.
1705 * @param pVM Pointer to the shared VM structure.
1706 */
1707VMMDECL(uint32_t) PGMDynMapPushAutoSubset(PVM pVM)
1708{
1709 PPGMMAPSET pSet = &pVM->pgm.s.AutoSet;
1710 AssertReturn(pSet->cEntries != PGMMAPSET_CLOSED, UINT32_MAX);
1711 uint32_t iPrevSubset = pSet->iSubset;
1712Assert(iPrevSubset == UINT32_MAX);
1713 pSet->iSubset = pSet->cEntries;
1714 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapSubsets);
1715 return iPrevSubset;
1716}
1717
1718
1719/**
1720 * Pops a subset created by a previous call to PGMDynMapPushAutoSubset.
1721 *
1722 * @param pVM Pointer to the shared VM structure.
1723 * @param iPrevSubset What PGMDynMapPushAutoSubset returned.
1724 */
1725VMMDECL(void) PGMDynMapPopAutoSubset(PVM pVM, uint32_t iPrevSubset)
1726{
1727 PPGMMAPSET pSet = &pVM->pgm.s.AutoSet;
1728 uint32_t cEntries = pSet->cEntries;
1729 AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
1730 AssertReturnVoid(pSet->iSubset <= iPrevSubset || iPrevSubset == UINT32_MAX);
1731 Assert(iPrevSubset == UINT32_MAX);
1732 STAM_COUNTER_INC(&pVM->pgm.s.aStatR0DynMapSetSize[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
1733 if ( cEntries >= RT_ELEMENTS(pSet->aEntries) * 40 / 100
1734 && cEntries != pSet->iSubset)
1735 {
1736 AssertMsg(cEntries < PGMMAPSET_MAX_FILL, ("%u\n", cEntries));
1737 pgmDynMapFlushSubset(pSet);
1738 }
1739 pSet->iSubset = iPrevSubset;
1740}
1741
1742
1743/**
1744 * As a final resort for a full auto set, try merge duplicate entries.
1745 *
1746 * @param pSet The set.
1747 */
1748static void pgmDynMapOptimizeAutoSet(PPGMMAPSET pSet)
1749{
1750 for (uint32_t i = 0 ; i < pSet->cEntries; i++)
1751 {
1752 uint16_t const iPage = pSet->aEntries[i].iPage;
1753 uint32_t j = i + 1;
1754 while (j < pSet->cEntries)
1755 {
1756 if (pSet->aEntries[j].iPage != iPage)
1757 j++;
1758 else if ((uint32_t)pSet->aEntries[i].cRefs + (uint32_t)pSet->aEntries[j].cRefs < UINT16_MAX)
1759 {
1760 /* merge j into i removing j. */
1761 pSet->aEntries[i].cRefs += pSet->aEntries[j].cRefs;
1762 pSet->cEntries--;
1763 if (j < pSet->cEntries)
1764 {
1765 pSet->aEntries[j] = pSet->aEntries[pSet->cEntries];
1766 pSet->aEntries[pSet->cEntries].iPage = UINT16_MAX;
1767 pSet->aEntries[pSet->cEntries].cRefs = 0;
1768 }
1769 else
1770 {
1771 pSet->aEntries[j].iPage = UINT16_MAX;
1772 pSet->aEntries[j].cRefs = 0;
1773 }
1774 }
1775 else
1776 {
1777 /* migrate the max number of refs from j into i and quit the inner loop. */
1778 uint32_t cMigrate = UINT16_MAX - 1 - pSet->aEntries[i].cRefs;
1779 Assert(pSet->aEntries[j].cRefs > cMigrate);
1780 pSet->aEntries[j].cRefs -= cMigrate;
1781 pSet->aEntries[i].cRefs = UINT16_MAX - 1;
1782 break;
1783 }
1784 }
1785 }
1786}
1787
1788
1789/**
1790 * Common worker code for PGMDynMapHCPhys, pgmR0DynMapHCPageInlined and
1791 * pgmR0DynMapGCPageInlined.
1792 *
1793 * @returns VINF_SUCCESS, bails out to ring-3 on failure.
1794 * @param pVM The shared VM structure (for statistics).
1795 * @param pSet The set.
1796 * @param HCPhys The physical address of the page.
1797 * @param ppv Where to store the address of the mapping on success.
1798 *
1799 * @remarks This is a very hot path.
1800 */
1801int pgmR0DynMapHCPageCommon(PVM pVM, PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv)
1802{
1803 AssertMsg(pSet->iCpu == RTMpCpuIdToSetIndex(RTMpCpuId()), ("%d %d(%d) efl=%#x\n", pSet->iCpu, RTMpCpuIdToSetIndex(RTMpCpuId()), RTMpCpuId(), ASMGetFlags()));
1804
1805 /*
1806 * Map it.
1807 */
1808 void *pvPage;
1809 uint32_t const iPage = pgmR0DynMapPage(g_pPGMR0DynMap, HCPhys, pSet->iCpu, pVM, &pvPage);
1810 if (RT_UNLIKELY(iPage == UINT32_MAX))
1811 {
1812 AssertMsg2("PGMDynMapHCPage: cLoad=%u/%u cPages=%u cGuardPages=%u\n",
1813 g_pPGMR0DynMap->cLoad, g_pPGMR0DynMap->cMaxLoad, g_pPGMR0DynMap->cPages, g_pPGMR0DynMap->cGuardPages);
1814 if (!g_fPGMR0DynMapTestRunning)
1815 VMMR0CallHost(pVM, VMMCALLHOST_VM_R0_ASSERTION, 0);
1816 *ppv = NULL;
1817 return VERR_PGM_DYNMAP_FAILED;
1818 }
1819
1820 /*
1821 * Add the page to the auto reference set.
1822 *
1823 * The typical usage pattern means that the same pages will be mapped
1824 * several times in the same set. We can catch most of these
1825 * remappings by looking a few pages back into the set. (The searching
1826 * and set optimizing path will hardly ever be used when doing this.)
1827 */
1828 AssertCompile(RT_ELEMENTS(pSet->aEntries) >= 8);
1829 int32_t i = pSet->cEntries;
1830 if (i-- < 5)
1831 {
1832 unsigned iEntry = pSet->cEntries++;
1833 pSet->aEntries[iEntry].cRefs = 1;
1834 pSet->aEntries[iEntry].iPage = iPage;
1835 pSet->aEntries[iEntry].pvPage = pvPage;
1836 pSet->aEntries[iEntry].HCPhys = HCPhys;
1837 pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
1838 }
1839 /* Any of the last 5 pages? */
1840 else if ( pSet->aEntries[i - 0].iPage == iPage
1841 && pSet->aEntries[i - 0].cRefs < UINT16_MAX - 1)
1842 pSet->aEntries[i - 0].cRefs++;
1843 else if ( pSet->aEntries[i - 1].iPage == iPage
1844 && pSet->aEntries[i - 1].cRefs < UINT16_MAX - 1)
1845 pSet->aEntries[i - 1].cRefs++;
1846 else if ( pSet->aEntries[i - 2].iPage == iPage
1847 && pSet->aEntries[i - 2].cRefs < UINT16_MAX - 1)
1848 pSet->aEntries[i - 2].cRefs++;
1849 else if ( pSet->aEntries[i - 3].iPage == iPage
1850 && pSet->aEntries[i - 3].cRefs < UINT16_MAX - 1)
1851 pSet->aEntries[i - 3].cRefs++;
1852 else if ( pSet->aEntries[i - 4].iPage == iPage
1853 && pSet->aEntries[i - 4].cRefs < UINT16_MAX - 1)
1854 pSet->aEntries[i - 4].cRefs++;
1855 /* Don't bother searching unless we're above a 60% load. */
1856 else if (RT_LIKELY(i <= (int32_t)RT_ELEMENTS(pSet->aEntries) * 60 / 100))
1857 {
1858 unsigned iEntry = pSet->cEntries++;
1859 pSet->aEntries[iEntry].cRefs = 1;
1860 pSet->aEntries[iEntry].iPage = iPage;
1861 pSet->aEntries[iEntry].pvPage = pvPage;
1862 pSet->aEntries[iEntry].HCPhys = HCPhys;
1863 pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
1864 }
1865 else
1866 {
1867 /* Search the rest of the set. */
1868 Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
1869 i -= 4;
1870 while (i-- > 0)
1871 if ( pSet->aEntries[i].iPage == iPage
1872 && pSet->aEntries[i].cRefs < UINT16_MAX - 1)
1873 {
1874 pSet->aEntries[i].cRefs++;
1875 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapSetSearchHits);
1876 break;
1877 }
1878 if (i < 0)
1879 {
1880 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapSetSearchMisses);
1881 if (pSet->iSubset < pSet->cEntries)
1882 {
1883 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapSetSearchFlushes);
1884 STAM_COUNTER_INC(&pVM->pgm.s.aStatR0DynMapSetSize[(pSet->cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
1885 AssertMsg(pSet->cEntries < PGMMAPSET_MAX_FILL, ("%u\n", pSet->cEntries));
1886 pgmDynMapFlushSubset(pSet);
1887 }
1888
1889 if (RT_UNLIKELY(pSet->cEntries >= RT_ELEMENTS(pSet->aEntries)))
1890 {
1891 STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapSetOptimize);
1892 pgmDynMapOptimizeAutoSet(pSet);
1893 }
1894
1895 if (RT_LIKELY(pSet->cEntries < RT_ELEMENTS(pSet->aEntries)))
1896 {
1897 unsigned iEntry = pSet->cEntries++;
1898 pSet->aEntries[iEntry].cRefs = 1;
1899 pSet->aEntries[iEntry].iPage = iPage;
1900 pSet->aEntries[iEntry].pvPage = pvPage;
1901 pSet->aEntries[iEntry].HCPhys = HCPhys;
1902 pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
1903 }
1904 else
1905 {
1906 /* We're screwed. */
1907 pgmR0DynMapReleasePage(g_pPGMR0DynMap, iPage, 1);
1908
1909 AssertMsg2("PGMDynMapHCPage: set is full!\n");
1910 if (!g_fPGMR0DynMapTestRunning)
1911 VMMR0CallHost(pVM, VMMCALLHOST_VM_R0_ASSERTION, 0);
1912 *ppv = NULL;
1913 return VERR_PGM_DYNMAP_FULL_SET;
1914 }
1915 }
1916 }
1917
1918 *ppv = pvPage;
1919 return VINF_SUCCESS;
1920}
1921
1922
1923#if 0 /* Not used in R0, should internalized the other PGMDynMapHC/GCPage too. */
1924/* documented elsewhere - a bit of a mess. */
1925VMMDECL(int) PGMDynMapHCPage(PVM pVM, RTHCPHYS HCPhys, void **ppv)
1926{
1927 /*
1928 * Validate state.
1929 */
1930 STAM_PROFILE_START(&pVM->pgm.s.StatR0DynMapHCPage, a);
1931 AssertPtr(ppv);
1932 AssertMsg(pVM->pgm.s.pvR0DynMapUsed == g_pPGMR0DynMap,
1933 ("%p != %p\n", pVM->pgm.s.pvR0DynMapUsed, g_pPGMR0DynMap));
1934 AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
1935 PPGMMAPSET pSet = &pVM->pgm.s.AutoSet;
1936 AssertMsg(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries),
1937 ("%#x (%u)\n", pSet->cEntries, pSet->cEntries));
1938
1939 /*
1940 * Call common code.
1941 */
1942 int rc = pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
1943
1944 STAM_PROFILE_STOP(&pVM->pgm.s.StatR0DynMapHCPage, a);
1945 return rc;
1946}
1947#endif
1948
1949
1950#if 0 /*def DEBUG*/
1951/** For pgmR0DynMapTest3PerCpu. */
1952typedef struct PGMR0DYNMAPTEST
1953{
1954 uint32_t u32Expect;
1955 uint32_t *pu32;
1956 uint32_t volatile cFailures;
1957} PGMR0DYNMAPTEST;
1958typedef PGMR0DYNMAPTEST *PPGMR0DYNMAPTEST;
1959
1960/**
1961 * Checks that the content of the page is the same on all CPUs, i.e. that there
1962 * are no CPU specfic PTs or similar nasty stuff involved.
1963 *
1964 * @param idCpu The current CPU.
1965 * @param pvUser1 Pointer a PGMR0DYNMAPTEST structure.
1966 * @param pvUser2 Unused, ignored.
1967 */
1968static DECLCALLBACK(void) pgmR0DynMapTest3PerCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
1969{
1970 PPGMR0DYNMAPTEST pTest = (PPGMR0DYNMAPTEST)pvUser1;
1971 ASMInvalidatePage(pTest->pu32);
1972 if (*pTest->pu32 != pTest->u32Expect)
1973 ASMAtomicIncU32(&pTest->cFailures);
1974 NOREF(pvUser2); NOREF(idCpu);
1975}
1976
1977
1978/**
1979 * Performs some basic tests in debug builds.
1980 */
1981static int pgmR0DynMapTest(PVM pVM)
1982{
1983 LogRel(("pgmR0DynMapTest: ****** START ******\n"));
1984 PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
1985 PPGMMAPSET pSet = &pVM->pgm.s.AutoSet;
1986 uint32_t i;
1987
1988 /*
1989 * Assert internal integrity first.
1990 */
1991 LogRel(("Test #0\n"));
1992 int rc = PGMR0DynMapAssertIntegrity();
1993 if (RT_FAILURE(rc))
1994 return rc;
1995
1996 void *pvR0DynMapUsedSaved = pVM->pgm.s.pvR0DynMapUsed;
1997 pVM->pgm.s.pvR0DynMapUsed = pThis;
1998 g_fPGMR0DynMapTestRunning = true;
1999
2000 /*
2001 * Simple test, map CR3 twice and check that we're getting the
2002 * same mapping address back.
2003 */
2004 LogRel(("Test #1\n"));
2005 ASMIntDisable();
2006 PGMDynMapStartAutoSet(pVM);
2007
2008 uint64_t cr3 = ASMGetCR3() & ~(uint64_t)PAGE_OFFSET_MASK;
2009 void *pv = (void *)(intptr_t)-1;
2010 void *pv2 = (void *)(intptr_t)-2;
2011 rc = PGMDynMapHCPage(pVM, cr3, &pv);
2012 int rc2 = PGMDynMapHCPage(pVM, cr3, &pv2);
2013 ASMIntEnable();
2014 if ( RT_SUCCESS(rc2)
2015 && RT_SUCCESS(rc)
2016 && pv == pv2)
2017 {
2018 LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2019 rc = PGMR0DynMapAssertIntegrity();
2020
2021 /*
2022 * Check that the simple set overflow code works by filling it
2023 * with more CR3 mappings.
2024 */
2025 LogRel(("Test #2\n"));
2026 ASMIntDisable();
2027 PGMDynMapMigrateAutoSet(pVM);
2028 for (i = 0 ; i < UINT16_MAX*2 - 1 && RT_SUCCESS(rc) && pv2 == pv; i++)
2029 {
2030 pv2 = (void *)(intptr_t)-4;
2031 rc = PGMDynMapHCPage(pVM, cr3, &pv2);
2032 }
2033 ASMIntEnable();
2034 if (RT_FAILURE(rc) || pv != pv2)
2035 {
2036 LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%p\n", __LINE__, rc, pv, pv2, i));
2037 if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
2038 }
2039 else if (pSet->cEntries != 5)
2040 {
2041 LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries) / 2));
2042 rc = VERR_INTERNAL_ERROR;
2043 }
2044 else if ( pSet->aEntries[4].cRefs != UINT16_MAX - 1
2045 || pSet->aEntries[3].cRefs != UINT16_MAX - 1
2046 || pSet->aEntries[2].cRefs != 1
2047 || pSet->aEntries[1].cRefs != 1
2048 || pSet->aEntries[0].cRefs != 1)
2049 {
2050 LogRel(("failed(%d): bad set dist: ", __LINE__));
2051 for (i = 0; i < pSet->cEntries; i++)
2052 LogRel(("[%d]=%d, ", i, pSet->aEntries[i].cRefs));
2053 LogRel(("\n"));
2054 rc = VERR_INTERNAL_ERROR;
2055 }
2056 if (RT_SUCCESS(rc))
2057 rc = PGMR0DynMapAssertIntegrity();
2058 if (RT_SUCCESS(rc))
2059 {
2060 /*
2061 * Trigger an set optimization run (exactly).
2062 */
2063 LogRel(("Test #3\n"));
2064 ASMIntDisable();
2065 PGMDynMapMigrateAutoSet(pVM);
2066 pv2 = NULL;
2067 for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) - 5 && RT_SUCCESS(rc) && pv2 != pv; i++)
2068 {
2069 pv2 = (void *)(intptr_t)(-5 - i);
2070 rc = PGMDynMapHCPage(pVM, cr3 + PAGE_SIZE * (i + 5), &pv2);
2071 }
2072 ASMIntEnable();
2073 if (RT_FAILURE(rc) || pv == pv2)
2074 {
2075 LogRel(("failed(%d): rc=%Rrc; pv=%p pv2=%p i=%d\n", __LINE__, rc, pv, pv2, i));
2076 if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
2077 }
2078 else if (pSet->cEntries != RT_ELEMENTS(pSet->aEntries))
2079 {
2080 LogRel(("failed(%d): cEntries=%d expected %d\n", __LINE__, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2081 rc = VERR_INTERNAL_ERROR;
2082 }
2083 LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2084 if (RT_SUCCESS(rc))
2085 rc = PGMR0DynMapAssertIntegrity();
2086 if (RT_SUCCESS(rc))
2087 {
2088 /*
2089 * Trigger an overflow error.
2090 */
2091 LogRel(("Test #4\n"));
2092 ASMIntDisable();
2093 PGMDynMapMigrateAutoSet(pVM);
2094 for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) + 2; i++)
2095 {
2096 rc = PGMDynMapHCPage(pVM, cr3 - PAGE_SIZE * (i + 5), &pv2);
2097 if (RT_SUCCESS(rc))
2098 rc = PGMR0DynMapAssertIntegrity();
2099 if (RT_FAILURE(rc))
2100 break;
2101 }
2102 ASMIntEnable();
2103 if (rc == VERR_PGM_DYNMAP_FULL_SET)
2104 {
2105 /* flush the set. */
2106 LogRel(("Test #5\n"));
2107 ASMIntDisable();
2108 PGMDynMapMigrateAutoSet(pVM);
2109 PGMDynMapReleaseAutoSet(pVM);
2110 PGMDynMapStartAutoSet(pVM);
2111 ASMIntEnable();
2112
2113 rc = PGMR0DynMapAssertIntegrity();
2114 }
2115 else
2116 {
2117 LogRel(("failed(%d): rc=%Rrc, wanted %d ; pv2=%p Set=%u/%u; i=%d\n", __LINE__,
2118 rc, VERR_PGM_DYNMAP_FULL_SET, pv2, pSet->cEntries, RT_ELEMENTS(pSet->aEntries), i));
2119 if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
2120 }
2121 }
2122 }
2123 }
2124 else
2125 {
2126 LogRel(("failed(%d): rc=%Rrc rc2=%Rrc; pv=%p pv2=%p\n", __LINE__, rc, rc2, pv, pv2));
2127 if (RT_SUCCESS(rc))
2128 rc = rc2;
2129 }
2130
2131 /*
2132 * Check that everyone sees the same stuff.
2133 */
2134 if (RT_SUCCESS(rc))
2135 {
2136 LogRel(("Test #5\n"));
2137 ASMIntDisable();
2138 PGMDynMapMigrateAutoSet(pVM);
2139 RTHCPHYS HCPhysPT = RTR0MemObjGetPagePhysAddr(pThis->pSegHead->ahMemObjPTs[0], 0);
2140 rc = PGMDynMapHCPage(pVM, HCPhysPT, &pv);
2141 if (RT_SUCCESS(rc))
2142 {
2143 PGMR0DYNMAPTEST Test;
2144 uint32_t *pu32Real = &pThis->paPages[pThis->pSegHead->iPage].uPte.pLegacy->u;
2145 Test.pu32 = (uint32_t *)((uintptr_t)pv | ((uintptr_t)pu32Real & PAGE_OFFSET_MASK));
2146 Test.u32Expect = *pu32Real;
2147 ASMAtomicWriteU32(&Test.cFailures, 0);
2148 ASMIntEnable();
2149
2150 rc = RTMpOnAll(pgmR0DynMapTest3PerCpu, &Test, NULL);
2151 if (RT_FAILURE(rc))
2152 LogRel(("failed(%d): RTMpOnAll rc=%Rrc\n", __LINE__, rc));
2153 else if (Test.cFailures)
2154 {
2155 LogRel(("failed(%d): cFailures=%d pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n", __LINE__,
2156 Test.cFailures, pu32Real, Test.pu32, Test.u32Expect, *Test.pu32));
2157 rc = VERR_INTERNAL_ERROR;
2158 }
2159 else
2160 LogRel(("pu32Real=%p pu32=%p u32Expect=%#x *pu32=%#x\n",
2161 pu32Real, Test.pu32, Test.u32Expect, *Test.pu32));
2162 }
2163 else
2164 {
2165 ASMIntEnable();
2166 LogRel(("failed(%d): rc=%Rrc\n", rc));
2167 }
2168 }
2169
2170 /*
2171 * Clean up.
2172 */
2173 LogRel(("Cleanup.\n"));
2174 ASMIntDisable();
2175 PGMDynMapMigrateAutoSet(pVM);
2176 PGMDynMapFlushAutoSet(pVM);
2177 PGMDynMapReleaseAutoSet(pVM);
2178 ASMIntEnable();
2179
2180 if (RT_SUCCESS(rc))
2181 rc = PGMR0DynMapAssertIntegrity();
2182 else
2183 PGMR0DynMapAssertIntegrity();
2184
2185 g_fPGMR0DynMapTestRunning = false;
2186 LogRel(("Result: rc=%Rrc Load=%u/%u/%u Set=%#x/%u\n", rc,
2187 pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
2188 pVM->pgm.s.pvR0DynMapUsed = pvR0DynMapUsedSaved;
2189 LogRel(("pgmR0DynMapTest: ****** END ******\n"));
2190 return rc;
2191}
2192#endif /* DEBUG */
2193
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette