Changeset 14748 in vbox for trunk/src/VBox/VMM/VMMR0/PGMR0DynMap.cpp

Timestamp:

Nov 28, 2008 12:34:24 AM (16 years ago)

Author:

vboxsync

Message:

PGMR0DynMap: a couple of bugs, guard pages and PGMR0DynMapAssertIntegrity.

File:

• trunk/src/VBox/VMM/VMMR0/PGMR0DynMap.cpp (modified) (37 diffs)

trunk/src/VBox/VMM/VMMR0/PGMR0DynMap.cpp

@@ -50 +50 @@
 /** The number of pages we reserve per CPU. */
 #define PGMR0DYNMAP_PAGES_PER_CPU           64
+/** The number of guard pages. */
+#if defined(VBOX_STRICT)
+# define PGMR0DYNMAP_GUARD_PAGES            7
+#else
+# define PGMR0DYNMAP_GUARD_PAGES            0
+#endif
+/** The dummy physical address of guard pages. */
+#define PGMR0DYNMAP_GUARD_PAGE_HCPHYS       UINT32_C(0x7777feed)
+/** The dummy reference count of guard pages. (Must be non-zero.) */
+#define PGMR0DYNMAP_GUARD_PAGE_REF_COUNT    INT32_C(0x7777feed)
+#if 0
+/** Define this to just clear the present bit on guard pages.
+ * The alternative is to replace the entire PTE with an bad not-present
+ * PTE. Either way, XNU will screw us. :-/   */
+#define PGMR0DYNMAP_GUARD_NP
+#endif
+/** The dummy PTE value for a page. */
+#define PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE   X86_PTE_PG_MASK
+/** The dummy PTE value for a page. */
+#define PGMR0DYNMAP_GUARD_PAGE_PAE_PTE      UINT64_MAX /*X86_PTE_PAE_PG_MASK*/
 /** Calcs the overload threshold. Current set at 50%. */
 #define PGMR0DYNMAP_CALC_OVERLOAD(cPages)   ((cPages) / 2)
 
+#if 0
 /* Assertions causes panics if preemption is disabled, this can be used to work aroudn that. */
-/*#define RTSpinlockAcquire(a,b) do {} while (0)
-#define RTSpinlockRelease(a,b) do {} while (0) */
-
+#define RTSpinlockAcquire(a,b) do {} while (0)
+#define RTSpinlockRelease(a,b) do {} while (0)
+#endif
 
 
@@ -145 +166 @@
     /** Whether it's 32-bit legacy or PAE/AMD64 paging mode. */
     bool                        fLegacyMode;
-    /** The current load. */
+    /** The current load.
+     * This does not include guard pages. */
     uint32_t                    cLoad;
     /** The max load ever.
@@ -152 +174 @@
     /** Initialization / termination lock. */
     RTSEMFASTMUTEX              hInitLock;
+    /** The number of guard pages. */
+    uint32_t                    cGuardPages;
     /** The number of users (protected by hInitLock). */
     uint32_t                    cUsers;
@@ -365 +389 @@
 #endif
     }
-    else if (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(pThis->cPages))
+    else if (pThis->cMaxLoad > PGMR0DYNMAP_CALC_OVERLOAD(pThis->cPages - pThis->cGuardPages))
         rc = pgmR0DynMapExpand(pThis);
     if (RT_SUCCESS(rc))
@@ -399 +423 @@
     {
         pVM->pgm.s.pvR0DynMapUsed = NULL;
+
+#ifdef VBOX_STRICT
+        PGMR0DynMapAssertIntegrity();
+#endif
 
         /*
@@ -455 +483 @@
 
 /**
+ * Shoots down the TLBs for all the cache pages, pgmR0DynMapTearDown helper.
+ *
+ * @param   idCpu           The current CPU.
+ * @param   pvUser1         The dynamic mapping cache instance.
+ * @param   pvUser2         Unused, NULL.
+ */
+static DECLCALLBACK(void) pgmR0DynMapShootDownTlbs(RTCPUID idCpu, void *pvUser1, void *pvUser2)
+{
+    Assert(!pvUser2);
+    PPGMR0DYNMAP        pThis   = (PPGMR0DYNMAP)pvUser1;
+    Assert(pThis == g_pPGMR0DynMap);
+    PPGMR0DYNMAPENTRY   paPages = pThis->paPages;
+    uint32_t            iPage   = pThis->cPages;
+    while (iPage-- > 0)
+        ASMInvalidatePage(paPages[iPage].pvPage);
+}
+
+
+/**
+ * Shoot down the TLBs for every single cache entry on all CPUs.
+ *
+ * @returns IPRT status code (RTMpOnAll).
+ * @param   pThis       The dynamic mapping cache instance.
+ */
+static int pgmR0DynMapTlbShootDown(PPGMR0DYNMAP pThis)
+{
+    int rc = RTMpOnAll(pgmR0DynMapShootDownTlbs, pThis, NULL);
+    AssertRC(rc);
+    if (RT_FAILURE(rc))
+    {
+        uint32_t iPage = pThis->cPages;
+        while (iPage-- > 0)
+            ASMInvalidatePage(pThis->paPages[iPage].pvPage);
+    }
+    return rc;
+}
+
+
+/**
  * Calculate the new cache size based on cMaxLoad statistics.
  *
@@ -468 +535 @@
     RTCPUID     cCpus     = RTMpGetCount();
     AssertReturn(cCpus > 0 && cCpus <= RTCPUSET_MAX_CPUS, 0);
-    uint32_t    cPages    = cCpus * PGMR0DYNMAP_PAGES_PER_CPU;
+    uint32_t    cPages    = cCpus *  PGMR0DYNMAP_PAGES_PER_CPU;
     uint32_t    cMinPages = cCpus * (PGMR0DYNMAP_PAGES_PER_CPU / 2);
 
@@ -485 +552 @@
     if (cPages < pThis->cPages)
         cPages = pThis->cPages;
+    cPages *= PGMR0DYNMAP_GUARD_PAGES + 1;
     if (cPages > PGMR0DYNMAP_MAX_PAGES)
         cPages = PGMR0DYNMAP_MAX_PAGES;
@@ -490 +558 @@
     if (cMinPages < pThis->cPages)
         cMinPages = pThis->cPages;
+    cMinPages *= PGMR0DYNMAP_GUARD_PAGES + 1;
     if (cMinPages > PGMR0DYNMAP_MAX_PAGES)
         cMinPages = PGMR0DYNMAP_MAX_PAGES;
@@ -687 +756 @@
             return VERR_INTERNAL_ERROR;
         }
-        Log(("#%d: iEntry=%4d uEntry=%#llx pvEntry=%p HCPhys=%RHp \n", i, iEntry, uEntry, pvEntry, pPgLvl->a[i].HCPhys));
+        /*Log(("#%d: iEntry=%4d uEntry=%#llx pvEntry=%p HCPhys=%RHp \n", i, iEntry, uEntry, pvEntry, pPgLvl->a[i].HCPhys));*/
     }
 
@@ -697 +766 @@
 
 /**
+ * Sets up a guard page.
+ *
+ * @param   pThis       The dynamic mapping cache instance.
+ * @param   pPage       The page.
+ */
+DECLINLINE(void) pgmR0DynMapSetupGuardPage(PPGMR0DYNMAP pThis, PPGMR0DYNMAPENTRY pPage)
+{
+    memset(pPage->pvPage, 0xfd, PAGE_SIZE);
+    pPage->cRefs  = PGMR0DYNMAP_GUARD_PAGE_REF_COUNT;
+    pPage->HCPhys = PGMR0DYNMAP_GUARD_PAGE_HCPHYS;
+#ifdef PGMR0DYNMAP_GUARD_NP
+    ASMAtomicBitClear(pPage->uPte.pv, X86_PTE_BIT_P);
+#else
+    if (pThis->fLegacyMode)
+        ASMAtomicWriteU32(&pPage->uPte.pLegacy->u, PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE);
+    else
+        ASMAtomicWriteU64(&pPage->uPte.pPae->u,    PGMR0DYNMAP_GUARD_PAGE_PAE_PTE);
+#endif
+    pThis->cGuardPages++;
+}
+
+
+/**
  * Adds a new segment of the specified size.
  *
@@ -709 +801 @@
 
     /*
-     * Do the array rellocation first.
+     * Do the array reallocations first.
      * (The pages array has to be replaced behind the spinlock of course.)
      */
@@ -760 +852 @@
         PGMR0DYNMAPPGLVL    PgLvl;
         pgmR0DynMapPagingArrayInit(pThis, &PgLvl);
-        uint32_t            iEndPage = pThis->cPages + cPages;
+        uint32_t const      iEndPage = pThis->cPages + cPages;
         for (uint32_t iPage = pThis->cPages;
              iPage < iEndPage;
@@ -809 +901 @@
         if (RT_SUCCESS(rc))
         {
-            /** @todo setup guard pages here later (strict builds should leave every
-             *        second page and the start/end pages not present).  */
+#if PGMR0DYNMAP_GUARD_PAGES > 0
+            /*
+             * Setup guard pages.
+             * (Note: TLBs will be shot down later on.)
+             */
+            uint32_t iPage = pThis->cPages;
+            while (iPage < iEndPage)
+            {
+                for (uint32_t iGPg = 0; iGPg < PGMR0DYNMAP_GUARD_PAGES && iPage < iEndPage; iGPg++, iPage++)
+                    pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]);
+                iPage++; /* the guarded page */
+            }
+
+            /* Make sure the very last page is a guard page too. */
+            iPage = iEndPage - 1;
+            if (pThis->paPages[iPage].cRefs != PGMR0DYNMAP_GUARD_PAGE_REF_COUNT)
+                pgmR0DynMapSetupGuardPage(pThis, &pThis->paPages[iPage]);
+#endif /* PGMR0DYNMAP_GUARD_PAGES > 0 */
 
             /*
@@ -882 +990 @@
     }
     Assert(ASMGetFlags() & X86_EFL_IF);
+
+#if PGMR0DYNMAP_GUARD_PAGES > 0
+    /* paranoia */
+    if (RT_SUCCESS(rc))
+        pgmR0DynMapTlbShootDown(pThis);
+#endif
     return rc;
 }
@@ -919 +1033 @@
     }
     Assert(ASMGetFlags() & X86_EFL_IF);
+
+#if PGMR0DYNMAP_GUARD_PAGES > 0
+    /* paranoia */
+    if (RT_SUCCESS(rc))
+        pgmR0DynMapTlbShootDown(pThis);
+#endif
     return rc;
-}
-
-
-/**
- * Shoots down the TLBs for all the cache pages, pgmR0DynMapTearDown helper.
- *
- * @param   idCpu           The current CPU.
- * @param   pvUser1         The dynamic mapping cache instance.
- * @param   pvUser2         Unused, NULL.
- */
-static DECLCALLBACK(void) pgmR0DynMapShootDownTlbs(RTCPUID idCpu, void *pvUser1, void *pvUser2)
-{
-    Assert(!pvUser2);
-    PPGMR0DYNMAP        pThis   = (PPGMR0DYNMAP)pvUser1;
-    Assert(pThis == g_pPGMR0DynMap);
-    PPGMR0DYNMAPENTRY   paPages = pThis->paPages;
-    uint32_t            iPage   = pThis->cPages;
-    while (iPage-- > 0)
-        ASMInvalidatePage(paPages[iPage].pvPage);
 }
 
@@ -982 +1083 @@
     /*
      * Shoot down the TLBs on all CPUs before freeing them.
-     * If RTMpOnAll fails, make sure the TLBs are invalidated on the current CPU at least.
-     */
-    int rc = RTMpOnAll(pgmR0DynMapShootDownTlbs, pThis, NULL);
-    AssertRC(rc);
-    if (RT_FAILURE(rc))
-    {
-        iPage = pThis->cPages;
-        while (iPage-- > 0)
-            ASMInvalidatePage(paPages[iPage].pvPage);
-    }
+     */
+    pgmR0DynMapTlbShootDown(pThis);
 
     /*
@@ -998 +1091 @@
     while (pThis->pSegHead)
     {
+        int             rc;
         PPGMR0DYNMAPSEG pSeg = pThis->pSegHead;
         pThis->pSegHead = pSeg->pNext;
@@ -1081 +1175 @@
         iFreePage = iPage;
     else if (!paPages[(iPage + 1) % cPages].cRefs)
-        iFreePage = iPage;
+        iFreePage = iPage + 1;
     else if (!paPages[(iPage + 2) % cPages].cRefs)
-        iFreePage = iPage;
+        iFreePage = iPage + 2;
     else if (!paPages[(iPage + 3) % cPages].cRefs)
-        iFreePage = iPage;
+        iFreePage = iPage + 3;
     else if (!paPages[(iPage + 4) % cPages].cRefs)
-        iFreePage = iPage;
+        iFreePage = iPage + 4;
     else
     {
@@ -1093 +1187 @@
          * Search for an unused or matching entry.
          */
-        iFreePage = (iPage + 5) % pThis->cPages;
+        iFreePage = (iPage + 5) % cPages;
         for (;;)
         {
@@ -1103 +1197 @@
             /* advance */
             iFreePage = (iFreePage + 1) % cPages;
-            if (RT_UNLIKELY(iFreePage != iPage))
+            if (RT_UNLIKELY(iFreePage == iPage))
                 return UINT32_MAX;
         }
@@ -1111 +1205 @@
      * Setup the new entry.
      */
+    /*Log6(("pgmR0DynMapPageSlow: old - %RHp %#x %#llx\n", paPages[iFreePage].HCPhys, paPages[iFreePage].cRefs, paPages[iFreePage].uPte.pPae->u));*/
     paPages[iFreePage].HCPhys = HCPhys;
     RTCpuSetFill(&paPages[iFreePage].PendingSet);
@@ -1141 +1236 @@
  * Maps a page into the pool.
  *
- * @returns Pointer to the mapping.
+ * @returns Page index on success, UINT32_MAX on failure.
  * @param   pThis       The dynamic mapping cache instance.
  * @param   HCPhys      The address of the page to be mapped.
- * @param   piPage      Where to store the page index.
- */
-DECLINLINE(void *) pgmR0DynMapPage(PPGMR0DYNMAP pThis, RTHCPHYS HCPhys, uint32_t *piPage)
+ * @param   ppvPage      Where to the page address.
+ */
+DECLINLINE(uint32_t) pgmR0DynMapPage(PPGMR0DYNMAP pThis, RTHCPHYS HCPhys, void **ppvPage)
 {
     RTSPINLOCKTMP   Tmp       = RTSPINLOCKTMP_INITIALIZER;
@@ -1181 +1276 @@
                         {
                             RTSpinlockRelease(pThis->hSpinlock, &Tmp);
-                            return NULL;
+                            return iPage;
                         }
                     }
@@ -1206 +1301 @@
         if (pThis->cLoad > pThis->cMaxLoad)
             pThis->cMaxLoad = pThis->cLoad;
-        AssertMsg(pThis->cLoad <= pThis->cPages, ("%d/%d\n", pThis->cLoad, pThis->cPages));
+        AssertMsg(pThis->cLoad <= pThis->cPages - pThis->cGuardPages, ("%d/%d\n", pThis->cLoad, pThis->cPages - pThis->cGuardPages));
     }
     else if (RT_UNLIKELY(cRefs <= 0))
@@ -1212 +1307 @@
         ASMAtomicDecS32(&paPages[iPage].cRefs);
         RTSpinlockRelease(pThis->hSpinlock, &Tmp);
-        AssertLogRelMsgFailedReturn(("cRefs=%d iPage=%p HCPhys=%RHp\n", cRefs, iPage, HCPhys), NULL);
+        AssertLogRelMsgFailedReturn(("cRefs=%d iPage=%p HCPhys=%RHp\n", cRefs, iPage, HCPhys), UINT32_MAX);
     }
     void *pvPage = paPages[iPage].pvPage;
@@ -1231 +1326 @@
     ASMInvalidatePage(pvPage);
 
-    *piPage = iPage;
-    return pvPage;
+    *ppvPage = pvPage;
+    return iPage;
+}
+
+
+/**
+ * Assert the the integrity of the pool.
+ *
+ * @returns VBox status code.
+ */
+VMMR0DECL(int) PGMR0DynMapAssertIntegrity(void)
+{
+    /*
+     * Basic pool stuff that doesn't require any lock, just assumes we're a user.
+     */
+    PPGMR0DYNMAP    pThis = g_pPGMR0DynMap;
+    if (!pThis)
+        return VINF_SUCCESS;
+    AssertPtrReturn(pThis, VERR_INVALID_POINTER);
+    AssertReturn(pThis->u32Magic == PGMR0DYNMAP_MAGIC, VERR_INVALID_MAGIC);
+    if (!pThis->cUsers)
+        return VERR_INVALID_PARAMETER;
+
+
+    int             rc = VINF_SUCCESS;
+    RTSPINLOCKTMP   Tmp = RTSPINLOCKTMP_INITIALIZER;
+    RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
+#define CHECK_RET(expr, a) \
+    do { \
+        if (!(expr)) \
+        { \
+            RTSpinlockRelease(pThis->hSpinlock, &Tmp); \
+            AssertMsg1(#expr, __LINE__, __FILE__, __PRETTY_FUNCTION__); \
+            AssertMsg2 a; \
+            return VERR_INTERNAL_ERROR; \
+        } \
+    } while (0)
+
+    /*
+     * Check that the PTEs are correct.
+     */
+    uint32_t            cGuard      = 0;
+    uint32_t            cLoad       = 0;
+    PPGMR0DYNMAPENTRY   paPages     = pThis->paPages;
+    uint32_t            iPage       = pThis->cPages;
+    if (pThis->fLegacyMode)
+    {
+        PCX86PGUINT     paSavedPTEs = (PCX86PGUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
+        while (iPage-- > 0)
+        {
+            CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
+            if (    paPages[iPage].cRefs  == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT
+                &&  paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS)
+            {
+#ifdef PGMR0DYNMAP_GUARD_NP
+                CHECK_RET(paPages[iPage].uPte.pLegacy->u == (paSavedPTEs[iPage] & ~(X86PGUINT)X86_PTE_P),
+                          ("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage]));
+#else
+                CHECK_RET(paPages[iPage].uPte.pLegacy->u == PGMR0DYNMAP_GUARD_PAGE_LEGACY_PTE,
+                          ("#%u: %#x", iPage, paPages[iPage].uPte.pLegacy->u));
+#endif
+                cGuard++;
+            }
+            else if (paPages[iPage].HCPhys != NIL_RTHCPHYS)
+            {
+                CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
+                X86PGUINT uPte = (paSavedPTEs[iPage] & X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT)
+                               | X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D
+                               | (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
+                CHECK_RET(paPages[iPage].uPte.pLegacy->u == uPte,
+                          ("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, uPte));
+                if (paPages[iPage].cRefs)
+                    cLoad++;
+            }
+            else
+                CHECK_RET(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage],
+                          ("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage]));
+        }
+    }
+    else
+    {
+        PCX86PGPAEUINT  paSavedPTEs = (PCX86PGPAEUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
+        while (iPage-- > 0)
+        {
+            CHECK_RET(!((uintptr_t)paPages[iPage].pvPage & PAGE_OFFSET_MASK), ("#%u: %p\n", iPage, paPages[iPage].pvPage));
+            if (    paPages[iPage].cRefs  == PGMR0DYNMAP_GUARD_PAGE_REF_COUNT
+                &&  paPages[iPage].HCPhys == PGMR0DYNMAP_GUARD_PAGE_HCPHYS)
+            {
+#ifdef PGMR0DYNMAP_GUARD_NP
+                CHECK_RET(paPages[iPage].uPte.pPae->u == (paSavedPTEs[iPage] & ~(X86PGPAEUINT)X86_PTE_P),
+                          ("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage]));
+#else
+                CHECK_RET(paPages[iPage].uPte.pPae->u == PGMR0DYNMAP_GUARD_PAGE_PAE_PTE,
+                          ("#%u: %#llx", iPage, paPages[iPage].uPte.pPae->u));
+#endif
+                cGuard++;
+            }
+            else if (paPages[iPage].HCPhys != NIL_RTHCPHYS)
+            {
+                CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
+                X86PGPAEUINT uPte = (paSavedPTEs[iPage] & X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT)
+                                  | X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D
+                                  | (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
+                CHECK_RET(paPages[iPage].uPte.pPae->u == uPte,
+                          ("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pLegacy->u, uPte));
+                if (paPages[iPage].cRefs)
+                    cLoad++;
+            }
+            else
+                CHECK_RET(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage],
+                          ("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage]));
+        }
+    }
+
+    CHECK_RET(cLoad == pThis->cLoad, ("%u %u\n", cLoad, pThis->cLoad));
+    CHECK_RET(cGuard == pThis->cGuardPages, ("%u %u\n", cGuard, pThis->cGuardPages));
+
+#undef CHECK_RET
+    RTSpinlockRelease(pThis->hSpinlock, &Tmp);
+    return VINF_SUCCESS;
 }
 
@@ -1285 +1498 @@
         }
 
-        Assert(pThis->cLoad <= pThis->cPages);
+        Assert(pThis->cLoad <= pThis->cPages - pThis->cGuardPages);
         RTSpinlockRelease(pThis->hSpinlock, &Tmp);
     }
@@ -1399 +1612 @@
      * Map it.
      */
-    uint32_t        iPage;
-    void           *pvPage  = pgmR0DynMapPage(g_pPGMR0DynMap, HCPhys, &iPage);
-    if (RT_UNLIKELY(!pvPage))
+    uint32_t const  iPage = pgmR0DynMapPage(g_pPGMR0DynMap, HCPhys, ppv);
+    if (RT_UNLIKELY(iPage == UINT32_MAX))
     {
         static uint32_t s_cBitched = 0;
         if (++s_cBitched < 10)
-            LogRel(("PGMDynMapHCPage: cLoad=%u/%u cPages=%u\n",
-                    g_pPGMR0DynMap->cLoad, g_pPGMR0DynMap->cMaxLoad, g_pPGMR0DynMap->cPages));
+            LogRel(("PGMDynMapHCPage: cLoad=%u/%u cPages=%u cGuardPages=%u\n",
+                    g_pPGMR0DynMap->cLoad, g_pPGMR0DynMap->cMaxLoad, g_pPGMR0DynMap->cPages, g_pPGMR0DynMap->cGuardPages));
         return VERR_PGM_DYNMAP_FAILED;
     }
-    *ppv = pvPage;
 
     /*
@@ -1497 +1708 @@
     PPGMMAPSET      pSet  = &pVM->aCpus[0].pgm.s.AutoSet;
     uint32_t        i;
+
+    /*
+     * Assert internal integrity first.
+     */
+    LogRel(("Test #0\n"));
+    int rc = PGMR0DynMapAssertIntegrity();
+    if (RT_FAILURE(rc))
+        return rc;
+
     void           *pvR0DynMapUsedSaved = pVM->pgm.s.pvR0DynMapUsed;
     pVM->pgm.s.pvR0DynMapUsed = pThis;
@@ -1511 +1731 @@
     void    *pv  = (void *)(intptr_t)-1;
     void    *pv2 = (void *)(intptr_t)-2;
-    int      rc  = PGMDynMapHCPage(pVM, cr3, &pv);
+    rc           = PGMDynMapHCPage(pVM, cr3, &pv);
     int      rc2 = PGMDynMapHCPage(pVM, cr3, &pv2);
     ASMIntEnable();
@@ -1518 +1738 @@
         &&  pv == pv2)
     {
-        LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
+        LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
+        rc = PGMR0DynMapAssertIntegrity();
 
         /*
@@ -1554 +1775 @@
         }
         if (RT_SUCCESS(rc))
+            rc = PGMR0DynMapAssertIntegrity();
+        if (RT_SUCCESS(rc))
         {
             /*
@@ -1577 +1800 @@
                 rc = VERR_INTERNAL_ERROR;
             }
-            LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
+            LogRel(("Load=%u/%u/%u Set=%u/%u\n", pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
+            if (RT_SUCCESS(rc))
+                rc = PGMR0DynMapAssertIntegrity();
             if (RT_SUCCESS(rc))
             {
@@ -1585 +1810 @@
                 LogRel(("Test #4\n"));
                 ASMIntDisable();
-                for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) / 2 - 3 + 1 && RT_SUCCESS(rc) && pv2 != pv; i++)
-                    rc = PGMDynMapHCPage(pVM, cr3 + PAGE_SIZE * -(i + 5), &pv2);
+                for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) / 2 - 3 + 1 && pv2 != pv; i++)
+                {
+                    rc = PGMDynMapHCPage(pVM, cr3 - PAGE_SIZE * (i + 5), &pv2);
+                    if (RT_SUCCESS(rc))
+                        rc = PGMR0DynMapAssertIntegrity();
+                    if (RT_FAILURE(rc))
+                        break;
+                }
                 ASMIntEnable();
                 if (rc == VERR_PGM_DYNMAP_FULL_SET)
                 {
-                    rc = VINF_SUCCESS;
-
                     /* flush the set. */
+                    LogRel(("Test #5\n"));
                     ASMIntDisable();
                     PGMDynMapMigrateAutoSet(&pVM->aCpus[0]);
@@ -1598 +1828 @@
                     PGMDynMapStartAutoSet(&pVM->aCpus[0]);
                     ASMIntEnable();
+
+                    rc = PGMR0DynMapAssertIntegrity();
                 }
                 else
                 {
-                    LogRel(("failed(%d): rc=%Rrc, wanted %d ; pv2=%p Set=%u/%u\n", __LINE__,
-                            rc, VERR_PGM_DYNMAP_FULL_SET, pv2, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
+                    LogRel(("failed(%d): rc=%Rrc, wanted %d ; pv2=%p Set=%u/%u; i=%d\n", __LINE__,
+                            rc, VERR_PGM_DYNMAP_FULL_SET, pv2, pSet->cEntries, RT_ELEMENTS(pSet->aEntries), i));
                     if (RT_SUCCESS(rc)) rc = VERR_INTERNAL_ERROR;
                 }
@@ -1662 +1894 @@
     ASMIntEnable();
 
+    if (RT_SUCCESS(rc))
+        rc = PGMR0DynMapAssertIntegrity();
+    else
+        PGMR0DynMapAssertIntegrity();
+
     LogRel(("Result: rc=%Rrc Load=%u/%u/%u Set=%#x/%u\n", rc,
-            pThis->cLoad, pThis->cMaxLoad, pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
+            pThis->cLoad, pThis->cMaxLoad, pThis->cPages - pThis->cPages, pSet->cEntries, RT_ELEMENTS(pSet->aEntries)));
     pVM->pgm.s.pvR0DynMapUsed = pvR0DynMapUsedSaved;
     LogRel(("pgmR0DynMapTest: ****** END ******\n"));