Changeset 31402 in vbox for trunk/src/VBox/VMM/VMMRZ

Timestamp:

Aug 5, 2010 12:28:18 PM (14 years ago)

Author:

vboxsync

Message:

PGM: Replaced the hazzardous raw-mode context dynamic mapping code with the PGMR0DynMap code used by darwin/x86. This is a risky change but it should pay off once stable by providing 100% certainty that dynamically mapped pages aren't resued behind our back (this has been observed in seemingly benign code paths recently).

File:

• trunk/src/VBox/VMM/VMMRZ/PGMRZDynMap.cpp (moved) (moved from trunk/src/VBox/VMM/VMMR0/PGMR0DynMap.cpp ) (90 diffs)

trunk/src/VBox/VMM/VMMRZ/PGMRZDynMap.cpp

@@ -1 +1 @@
 /* $Id$ */
 /** @file
- * PGM - Page Manager and Monitor, ring-0 dynamic mapping cache.
+ * PGM - Page Manager and Monitor, dynamic mapping cache.
  */
 
 /*
- * Copyright (C) 2008 Oracle Corporation
+ * Copyright (C) 2008-2010 Oracle Corporation
  *
  * This file is part of VirtualBox Open Source Edition (OSE), as
@@ -16 +16 @@
  */
 
+
 /*******************************************************************************
 *   Internal Functions                                                         *
 *******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PGM
+#define LOG_GROUP LOG_GROUP_PGM_DYNMAP
 #include <VBox/pgm.h>
 #include "../PGMInternal.h"
 #include <VBox/vm.h>
 #include "../PGMInline.h"
+#include <VBox/err.h>
+#include <VBox/param.h>
 #include <VBox/sup.h>
-#include <VBox/err.h>
 #include <iprt/asm.h>
 #include <iprt/asm-amd64-x86.h>
-#include <iprt/alloc.h>
 #include <iprt/assert.h>
-#include <iprt/cpuset.h>
-#include <iprt/memobj.h>
-#include <iprt/mp.h>
-#include <iprt/semaphore.h>
-#include <iprt/spinlock.h>
+#ifndef IN_RC
+# include <iprt/cpuset.h>
+# include <iprt/mem.h>
+# include <iprt/memobj.h>
+# include <iprt/mp.h>
+# include <iprt/semaphore.h>
+# include <iprt/spinlock.h>
+#endif
 #include <iprt/string.h>
 
@@ -41 +45 @@
 *   Defined Constants And Macros                                               *
 *******************************************************************************/
+#ifdef IN_RING0
 /** The max size of the mapping cache (in pages). */
-#define PGMR0DYNMAP_MAX_PAGES               ((16*_1M) >> PAGE_SHIFT)
+# define PGMR0DYNMAP_MAX_PAGES              ((16*_1M) >> PAGE_SHIFT)
 /** The small segment size that is adopted on out-of-memory conditions with a
  * single big segment. */
-#define PGMR0DYNMAP_SMALL_SEG_PAGES         128
+# define PGMR0DYNMAP_SMALL_SEG_PAGES        128
 /** The number of pages we reserve per CPU. */
-#define PGMR0DYNMAP_PAGES_PER_CPU           256
+# define PGMR0DYNMAP_PAGES_PER_CPU          256
 /** The minimum number of pages we reserve per CPU.
  * This must be equal or larger than the autoset size.  */
-#define PGMR0DYNMAP_PAGES_PER_CPU_MIN       64
+# define PGMR0DYNMAP_PAGES_PER_CPU_MIN      64
+/** Calcs the overload threshold (safety margin).  Current set at 50%. */
+# define PGMR0DYNMAP_CALC_OVERLOAD(cPages)  ((cPages) / 2)
 /** The number of guard pages.
  * @remarks Never do tuning of the hashing or whatnot with a strict build!  */
-#if defined(VBOX_STRICT)
-# define PGMR0DYNMAP_GUARD_PAGES            1
-#else
-# define PGMR0DYNMAP_GUARD_PAGES            0
-#endif
+# if defined(VBOX_STRICT)
+#  define PGMR0DYNMAP_GUARD_PAGES           1
+# else
+#  define PGMR0DYNMAP_GUARD_PAGES           0
+# endif
+#endif /* IN_RING0 */
 /** The dummy physical address of guard pages. */
 #define PGMR0DYNMAP_GUARD_PAGE_HCPHYS       UINT32_C(0x7777feed)
@@ -66 +74 @@
  * The alternative is to replace the entire PTE with an bad not-present
  * PTE. Either way, XNU will screw us. :-/   */
-#define PGMR0DYNMAP_GUARD_NP
+# define PGMR0DYNMAP_GUARD_NP
 #endif
 /** The dummy PTE value for a page. */
@@ -72 +80 @@
 /** The dummy PTE value for a page. */
 #define PGMR0DYNMAP_GUARD_PAGE_PAE_PTE      UINT64_MAX /*X86_PTE_PAE_PG_MASK*/
-/** Calcs the overload threshold. Current set at 50%. */
-#define PGMR0DYNMAP_CALC_OVERLOAD(cPages)   ((cPages) / 2)
-
-#if 0
-/* Assertions causes panics if preemption is disabled, this can be used to work around that. */
-//#define RTSpinlockAcquire(a,b) do {} while (0)
-//#define RTSpinlockRelease(a,b) do {} while (0)
-#endif
+
+#ifdef IN_RING0 /* Note! Assertions causes panics if preemption is disabled,
+                 *       disable this to work around that. */
+/**
+ * Acquire the spinlock.
+ * This will declare a temporary variable and expands to two statements!
+ */
+# define PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis) \
+    RTSPINLOCKTMP   MySpinlockTmp = RTSPINLOCKTMP_INITIALIZER; \
+    RTSpinlockAcquire((pThis)->hSpinlock, &MySpinlockTmp)
+/**
+ * Releases the spinlock.
+ */
+# define PGMRZDYNMAP_SPINLOCK_RELEASE(pThis) \
+    RTSpinlockRelease((pThis)->hSpinlock, &MySpinlockTmp)
+
+/**
+ * Re-acquires the spinlock.
+ */
+# define PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis) \
+    RTSpinlockAcquire((pThis)->hSpinlock, &MySpinlockTmp)
+#else
+# define PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis)   do { } while (0)
+# define PGMRZDYNMAP_SPINLOCK_RELEASE(pThis)   do { } while (0)
+# define PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis) do { } while (0)
+#endif
+
 
 /** Converts a PGMCPUM::AutoSet pointer into a PVMCPU. */
-#define PGMR0DYNMAP_2_VMCPU(pSet)           (RT_FROM_MEMBER(pSet, VMCPU, pgm.s.AutoSet))
+#define PGMRZDYNMAP_SET_2_VMCPU(pSet)       (RT_FROM_MEMBER(pSet, VMCPU, pgm.s.AutoSet))
 
 /** Converts a PGMCPUM::AutoSet pointer into a PVM. */
-#define PGMR0DYNMAP_2_VM(pSet)              (PGMR0DYNMAP_2_VMCPU(pSet)->CTX_SUFF(pVM))
+#define PGMRZDYNMAP_SET_2_VM(pSet)          (PGMRZDYNMAP_SET_2_VMCPU(pSet)->CTX_SUFF(pVM))
+
+/** Converts a PGMCPUM::AutoSet pointer into a PVM. */
+#ifdef IN_RC
+# define PGMRZDYNMAP_SET_2_DYNMAP(pSet)     (PGMRZDYNMAP_SET_2_VM(pSet)->pgm.s.pRCDynMap)
+#else
+# define PGMRZDYNMAP_SET_2_DYNMAP(pSet)     (g_pPGMR0DynMap)
+#endif
+
+/**
+ * Gets the set index of the current CPU.
+ *
+ * This always returns 0 when in raw-mode context because there is only ever
+ * one EMT in that context (at least presently).
+ */
+#ifdef IN_RC
+# define PGMRZDYNMAP_CUR_CPU()              (0)
+#else
+# define PGMRZDYNMAP_CUR_CPU()              RTMpCpuIdToSetIndex(RTMpCpuId())
+#endif
+
+/** PGMRZDYNMAP::u32Magic. (Jens Christian Bugge Wesseltoft) */
+#define PGMRZDYNMAP_MAGIC                   UINT32_C(0x19640201)
+
+
+/** Zaps an set entry. */
+#define PGMRZDYNMAP_ZAP_ENTRY(pEntry) \
+    do \
+    { \
+        (pEntry)->iPage        = UINT16_MAX; \
+        (pEntry)->cRefs        = 0; \
+        (pEntry)->cInlinedRefs = 0; \
+        (pEntry)->cUnrefs      = 0; \
+    } while (0)
 
 
@@ -91 +151 @@
 *   Structures and Typedefs                                                    *
 *******************************************************************************/
+#ifdef IN_RING0
 /**
  * Ring-0 dynamic mapping cache segment.
@@ -125 +186 @@
  * Ring-0 dynamic mapping cache entry.
  *
- * This structure tracks
+ * @sa PGMRZDYNMAPENTRY, PGMRCDYNMAPENTRY.
  */
 typedef struct PGMR0DYNMAPENTRY
@@ -147 +208 @@
         void                   *pv;
     } uPte;
+# ifndef IN_RC
     /** CPUs that haven't invalidated this entry after it's last update. */
     RTCPUSET                    PendingSet;
+# endif
 } PGMR0DYNMAPENTRY;
-/** Pointer to a ring-0 dynamic mapping cache entry. */
+/** Pointer a mapping cache entry for the ring-0.
+ * @sa PPGMRZDYNMAPENTRY, PPGMRCDYNMAPENTRY,  */
 typedef PGMR0DYNMAPENTRY *PPGMR0DYNMAPENTRY;
 
 
 /**
- * Ring-0 dynamic mapping cache.
- *
- * This is initialized during VMMR0 module init but no segments are allocated at
- * that time.  Segments will be added when the first VM is started and removed
- * again when the last VM shuts down, thus avoid consuming memory while dormant.
- * At module termination, the remaining bits will be freed up.
+ * Dynamic mapping cache for ring-0.
+ *
+ * This is initialized during VMMR0 module init but no segments are allocated
+ * at that time.  Segments will be added when the first VM is started and
+ * removed again when the last VM shuts down, thus avoid consuming memory while
+ * dormant. At module termination, the remaining bits will be freed up.
+ *
+ * @sa PPGMRZDYNMAP, PGMRCDYNMAP.
  */
 typedef struct PGMR0DYNMAP
 {
-    /** The usual magic number / eye catcher (PGMR0DYNMAP_MAGIC). */
+    /** The usual magic number / eye catcher (PGMRZDYNMAP_MAGIC). */
     uint32_t                    u32Magic;
+# ifndef IN_RC
     /** Spinlock serializing the normal operation of the cache. */
     RTSPINLOCK                  hSpinlock;
+# endif
     /** Array for tracking and managing the pages.  */
     PPGMR0DYNMAPENTRY           paPages;
@@ -180 +248 @@
      * This is maintained to get trigger adding of more mapping space. */
     uint32_t                    cMaxLoad;
+# ifndef IN_RC
     /** Initialization / termination lock. */
     RTSEMFASTMUTEX              hInitLock;
+# endif
     /** The number of guard pages. */
     uint32_t                    cGuardPages;
     /** The number of users (protected by hInitLock). */
     uint32_t                    cUsers;
+# ifndef IN_RC
     /** Array containing a copy of the original page tables.
      * The entries are either X86PTE or X86PTEPAE according to fLegacyMode. */
@@ -193 +264 @@
     /** The paging mode. */
     SUPPAGINGMODE               enmPgMode;
+# endif
 } PGMR0DYNMAP;
-/** Pointer to the ring-0 dynamic mapping cache */
-typedef PGMR0DYNMAP *PPGMR0DYNMAP;
-
-/** PGMR0DYNMAP::u32Magic. (Jens Christian Bugge Wesseltoft) */
-#define PGMR0DYNMAP_MAGIC       0x19640201
 
 
@@ -228 +295 @@
 /** Pointer to paging level data. */
 typedef PGMR0DYNMAPPGLVL *PPGMR0DYNMAPPGLVL;
+#endif
+
+/** Mapping cache entry for the current context.
+ * @sa PGMR0DYNMAPENTRY, PGMRCDYNMAPENTRY  */
+typedef CTX_MID(PGM,DYNMAPENTRY) PGMRZDYNMAPENTRY;
+/** Pointer a mapping cache entry for the current context.
+ * @sa PGMR0DYNMAPENTRY, PGMRCDYNMAPENTRY  */
+typedef PGMRZDYNMAPENTRY *PPGMRZDYNMAPENTRY;
+
+/** Pointer the mapping cache instance for the current context.
+ * @sa PGMR0DYNMAP, PGMRCDYNMAP  */
+typedef CTX_MID(PGM,DYNMAP) *PPGMRZDYNMAP;
+
 
 
@@ -233 +313 @@
 *   Global Variables                                                           *
 *******************************************************************************/
+#ifdef IN_RING0
 /** Pointer to the ring-0 dynamic mapping cache. */
-static PPGMR0DYNMAP g_pPGMR0DynMap;
+static PGMR0DYNMAP *g_pPGMR0DynMap;
+#endif
 /** For overflow testing. */
 static bool         g_fPGMR0DynMapTestRunning = false;
@@ -242 +324 @@
 *   Internal Functions                                                         *
 *******************************************************************************/
-static void pgmR0DynMapReleasePage(PPGMR0DYNMAP pThis, uint32_t iPage, uint32_t cRefs);
-static int  pgmR0DynMapSetup(PPGMR0DYNMAP pThis);
-static int  pgmR0DynMapExpand(PPGMR0DYNMAP pThis);
-static void pgmR0DynMapTearDown(PPGMR0DYNMAP pThis);
+static void pgmRZDynMapReleasePage(PPGMRZDYNMAP pThis, uint32_t iPage, uint32_t cRefs);
+#ifdef IN_RING0
+static int  pgmR0DynMapSetup(PPGMRZDYNMAP pThis);
+static int  pgmR0DynMapExpand(PPGMRZDYNMAP pThis);
+static void pgmR0DynMapTearDown(PPGMRZDYNMAP pThis);
+#endif
 #if 0 /*def DEBUG*/
 static int  pgmR0DynMapTest(PVM pVM);
@@ -252 +336 @@
 
 /**
+ * Initializes the auto mapping sets for a VM.
+ *
+ * @returns VINF_SUCCESS on success, VERR_INTERNAL_ERROR on failure.
+ * @param   pVM         The VM in question.
+ */
+static int pgmRZDynMapInitAutoSetsForVM(PVM pVM)
+{
+    VMCPUID idCpu = pVM->cCpus;
+    AssertReturn(idCpu > 0 && idCpu <= VMM_MAX_CPU_COUNT, VERR_INTERNAL_ERROR);
+    while (idCpu-- > 0)
+    {
+        PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
+        uint32_t j = RT_ELEMENTS(pSet->aEntries);
+        while (j-- > 0)
+        {
+            pSet->aEntries[j].pvPage        = NULL;
+            pSet->aEntries[j].HCPhys        = NIL_RTHCPHYS;
+            PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]);
+        }
+        pSet->cEntries = PGMMAPSET_CLOSED;
+        pSet->iSubset = UINT32_MAX;
+        pSet->iCpu = -1;
+        memset(&pSet->aiHashTable[0], 0xff, sizeof(pSet->aiHashTable));
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+#ifdef IN_RING0
+
+/**
  * Initializes the ring-0 dynamic mapping cache.
  *
@@ -263 +379 @@
      * Create and initialize the cache instance.
      */
-    PPGMR0DYNMAP pThis = (PPGMR0DYNMAP)RTMemAllocZ(sizeof(*pThis));
+    PPGMRZDYNMAP pThis = (PPGMRZDYNMAP)RTMemAllocZ(sizeof(*pThis));
     AssertLogRelReturn(pThis, VERR_NO_MEMORY);
     int             rc = VINF_SUCCESS;
@@ -295 +411 @@
             if (RT_SUCCESS(rc))
             {
-                pThis->u32Magic = PGMR0DYNMAP_MAGIC;
+                pThis->u32Magic = PGMRZDYNMAP_MAGIC;
                 g_pPGMR0DynMap = pThis;
                 return VINF_SUCCESS;
@@ -322 +438 @@
      * is just a mirror image of PGMR0DynMapInit.
      */
-    PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
+    PPGMRZDYNMAP pThis = g_pPGMR0DynMap;
     if (pThis)
     {
@@ -359 +475 @@
      * Initialize the auto sets.
      */
-    VMCPUID idCpu = pVM->cCpus;
-    AssertReturn(idCpu > 0 && idCpu <= VMM_MAX_CPU_COUNT, VERR_INTERNAL_ERROR);
-    while (idCpu-- > 0)
-    {
-        PPGMMAPSET pSet = &pVM->aCpus[idCpu].pgm.s.AutoSet;
-        uint32_t j = RT_ELEMENTS(pSet->aEntries);
-        while (j-- > 0)
-        {
-            pSet->aEntries[j].iPage  = UINT16_MAX;
-            pSet->aEntries[j].cRefs  = 0;
-            pSet->aEntries[j].pvPage = NULL;
-            pSet->aEntries[j].HCPhys = NIL_RTHCPHYS;
-        }
-        pSet->cEntries = PGMMAPSET_CLOSED;
-        pSet->iSubset = UINT32_MAX;
-        pSet->iCpu = -1;
-        memset(&pSet->aiHashTable[0], 0xff, sizeof(pSet->aiHashTable));
-    }
+    int rc = pgmRZDynMapInitAutoSetsForVM(pVM);
+    if (RT_FAILURE(rc))
+        return rc;
 
     /*
@@ -387 +488 @@
      * Reference and if necessary setup or expand the cache.
      */
-    PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
+    PPGMRZDYNMAP pThis = g_pPGMR0DynMap;
     AssertPtrReturn(pThis, VERR_INTERNAL_ERROR);
-    int rc = RTSemFastMutexRequest(pThis->hInitLock);
+    rc = RTSemFastMutexRequest(pThis->hInitLock);
     AssertLogRelRCReturn(rc, rc);
 
@@ -430 +531 @@
         return;
 
-    PPGMR0DYNMAP pThis = g_pPGMR0DynMap;
+    PPGMRZDYNMAP pThis = g_pPGMR0DynMap;
     AssertPtrReturnVoid(pThis);
 
@@ -463 +564 @@
                     LogRel(("PGMR0DynMapTermVM: %d dangling refs to %#x\n", cRefs, iPage));
                     if (iPage < pThis->cPages && cRefs > 0)
-                        pgmR0DynMapReleasePage(pThis, iPage, cRefs);
+                        pgmRZDynMapReleasePage(pThis, iPage, cRefs);
                     else
                         AssertLogRelMsgFailed(("cRefs=%d iPage=%#x cPages=%u\n", cRefs, iPage, pThis->cPages));
 
-                    pSet->aEntries[j].iPage  = UINT16_MAX;
-                    pSet->aEntries[j].cRefs  = 0;
-                    pSet->aEntries[j].pvPage = NULL;
-                    pSet->aEntries[j].HCPhys = NIL_RTHCPHYS;
+                    PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]);
                 }
                 pSet->cEntries = PGMMAPSET_CLOSED;
@@ -512 +610 @@
 {
     Assert(!pvUser2);
-    PPGMR0DYNMAP        pThis   = (PPGMR0DYNMAP)pvUser1;
+    PPGMRZDYNMAP        pThis   = (PPGMRZDYNMAP)pvUser1;
     Assert(pThis == g_pPGMR0DynMap);
-    PPGMR0DYNMAPENTRY   paPages = pThis->paPages;
+    PPGMRZDYNMAPENTRY   paPages = pThis->paPages;
     uint32_t            iPage   = pThis->cPages;
     while (iPage-- > 0)
@@ -527 +625 @@
  * @param   pThis       The dynamic mapping cache instance.
  */
-static int pgmR0DynMapTlbShootDown(PPGMR0DYNMAP pThis)
+static int pgmR0DynMapTlbShootDown(PPGMRZDYNMAP pThis)
 {
     int rc = RTMpOnAll(pgmR0DynMapShootDownTlbs, pThis, NULL);
@@ -548 +646 @@
  * @param   pcMinPages  The minimal size in pages.
  */
-static uint32_t pgmR0DynMapCalcNewSize(PPGMR0DYNMAP pThis, uint32_t *pcMinPages)
+static uint32_t pgmR0DynMapCalcNewSize(PPGMRZDYNMAP pThis, uint32_t *pcMinPages)
 {
     Assert(pThis->cPages <= PGMR0DYNMAP_MAX_PAGES);
@@ -594 +692 @@
  * @param   pPgLvl      The paging level data.
  */
-void pgmR0DynMapPagingArrayInit(PPGMR0DYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl)
+void pgmR0DynMapPagingArrayInit(PPGMRZDYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl)
 {
     RTCCUINTREG     cr4 = ASMGetCR4();
@@ -704 +802 @@
  * @param   ppvPTE      Where to store the PTE address.
  */
-static int pgmR0DynMapPagingArrayMapPte(PPGMR0DYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl, void *pvPage,
+static int pgmR0DynMapPagingArrayMapPte(PPGMRZDYNMAP pThis, PPGMR0DYNMAPPGLVL pPgLvl, void *pvPage,
                                         PPGMR0DYNMAPSEG pSeg, uint32_t cMaxPTs, void **ppvPTE)
 {
@@ -791 +889 @@
  * @param   pPage       The page.
  */
-DECLINLINE(void) pgmR0DynMapSetupGuardPage(PPGMR0DYNMAP pThis, PPGMR0DYNMAPENTRY pPage)
+DECLINLINE(void) pgmR0DynMapSetupGuardPage(PPGMRZDYNMAP pThis, PPGMRZDYNMAPENTRY pPage)
 {
     memset(pPage->pvPage, 0xfd, PAGE_SIZE);
@@ -815 +913 @@
  * @param   cPages      The size of the new segment, give as a page count.
  */
-static int pgmR0DynMapAddSeg(PPGMR0DYNMAP pThis, uint32_t cPages)
+static int pgmR0DynMapAddSeg(PPGMRZDYNMAP pThis, uint32_t cPages)
 {
     int rc2;
@@ -838 +936 @@
     }
 
-    RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
-    RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
+    PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
 
     memcpy(pvPages, pThis->paPages, sizeof(pThis->paPages[0]) * pThis->cPages);
     void *pvToFree = pThis->paPages;
-    pThis->paPages = (PPGMR0DYNMAPENTRY)pvPages;
-
-    RTSpinlockRelease(pThis->hSpinlock, &Tmp);
+    pThis->paPages = (PPGMRZDYNMAPENTRY)pvPages;
+
+    PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
     RTMemFree(pvToFree);
 
@@ -882 +979 @@
             pThis->paPages[iPage].cRefs  = 0;
             pThis->paPages[iPage].uPte.pPae = 0;
+#ifndef IN_RC
             RTCpuSetFill(&pThis->paPages[iPage].PendingSet);
+#endif
 
             /* Map its page table, retry until we've got a clean run (paranoia). */
@@ -983 +1082 @@
  * @param   pThis       The dynamic mapping cache instance.
  */
-static int pgmR0DynMapSetup(PPGMR0DYNMAP pThis)
+static int pgmR0DynMapSetup(PPGMRZDYNMAP pThis)
 {
     /*
@@ -1026 +1125 @@
  * @param   pThis       The dynamic mapping cache instance.
  */
-static int pgmR0DynMapExpand(PPGMR0DYNMAP pThis)
+static int pgmR0DynMapExpand(PPGMRZDYNMAP pThis)
 {
     /*
@@ -1069 +1168 @@
  * @param   pThis       The dynamic mapping cache instance.
  */
-static void pgmR0DynMapTearDown(PPGMR0DYNMAP pThis)
+static void pgmR0DynMapTearDown(PPGMRZDYNMAP pThis)
 {
     /*
      * Restore the original page table entries
      */
-    PPGMR0DYNMAPENTRY   paPages = pThis->paPages;
+    PPGMRZDYNMAPENTRY   paPages = pThis->paPages;
     uint32_t            iPage   = pThis->cPages;
     if (pThis->fLegacyMode)
@@ -1145 +1244 @@
 }
 
+#endif /* IN_RING0 */
+#ifdef IN_RC
+
+/**
+ * Initializes the dynamic mapping cache in raw-mode context.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ */
+VMMRCDECL(int) PGMRCDynMapInit(PVM pVM)
+{
+    /*
+     * Allocate and initialize the instance data and page array.
+     */
+    PPGMRZDYNMAP    pThis;
+    size_t const    cPages = MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE;
+    size_t const    cb     = RT_ALIGN_Z(sizeof(*pThis), 32)
+                           + sizeof(PGMRZDYNMAPENTRY) * cPages;
+    int rc = MMHyperAlloc(pVM, cb, 32, MM_TAG_PGM, (void **)&pThis);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    pThis->u32Magic     = PGMRZDYNMAP_MAGIC;
+    pThis->paPages      = RT_ALIGN_PT(pThis + 1, 32, PPGMRZDYNMAPENTRY);
+    pThis->cPages       = cPages;
+    pThis->fLegacyMode  = PGMGetHostMode(pVM) == PGMMODE_32_BIT;
+    pThis->cLoad        = 0;
+    pThis->cMaxLoad     = 0;
+    pThis->cGuardPages  = 0;
+    pThis->cUsers       = 1;
+
+    for (size_t iPage = 0; iPage < cPages; iPage++)
+    {
+        pThis->paPages[iPage].HCPhys = NIL_RTHCPHYS;
+        pThis->paPages[iPage].pvPage = pVM->pgm.s.pbDynPageMapBaseGC + iPage * PAGE_SIZE;
+        pThis->paPages[iPage].cRefs  = 0;
+        if (pThis->fLegacyMode)
+            pThis->paPages[iPage].uPte.pLegacy = &pVM->pgm.s.paDynPageMap32BitPTEsGC[iPage];
+        else
+            pThis->paPages[iPage].uPte.pPae    = &pVM->pgm.s.paDynPageMapPaePTEsGC[iPage];
+    }
+
+    pVM->pgm.s.pRCDynMap = pThis;
+
+    /*
+     * Initialize the autosets the VM.
+     */
+    rc = pgmRZDynMapInitAutoSetsForVM(pVM);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    return VINF_SUCCESS;
+}
+
+#endif /* IN_RC */
 
 /**
@@ -1153 +1307 @@
  * @param   cRefs       The number of references to release.
  */
-DECLINLINE(void) pgmR0DynMapReleasePageLocked(PPGMR0DYNMAP pThis, uint32_t iPage, int32_t cRefs)
+DECLINLINE(void) pgmRZDynMapReleasePageLocked(PPGMRZDYNMAP pThis, uint32_t iPage, int32_t cRefs)
 {
     cRefs = ASMAtomicSubS32(&pThis->paPages[iPage].cRefs, cRefs) - cRefs;
@@ -1169 +1323 @@
  * @param   cRefs       The number of references to release.
  */
-static void pgmR0DynMapReleasePage(PPGMR0DYNMAP pThis, uint32_t iPage, uint32_t cRefs)
-{
-    RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
-    RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
-    pgmR0DynMapReleasePageLocked(pThis, iPage, cRefs);
-    RTSpinlockRelease(pThis->hSpinlock, &Tmp);
+static void pgmRZDynMapReleasePage(PPGMRZDYNMAP pThis, uint32_t iPage, uint32_t cRefs)
+{
+    PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
+    pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs);
+    PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
 }
 
@@ -1186 +1339 @@
  * @param   iPage       The page index pgmR0DynMapPage hashed HCPhys to.
  * @param   pVCpu       The current CPU, for statistics.
- */
-static uint32_t pgmR0DynMapPageSlow(PPGMR0DYNMAP pThis, RTHCPHYS HCPhys, uint32_t iPage, PVMCPU pVCpu)
-{
-    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatR0DynMapPageSlow);
+ * @param   pfNew       Set to @c true if a new entry was made and @c false if
+ *                      an old entry was found and reused.
+ */
+static uint32_t pgmR0DynMapPageSlow(PPGMRZDYNMAP pThis, RTHCPHYS HCPhys, uint32_t iPage, PVMCPU pVCpu, bool *pfNew)
+{
+    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlow);
 
     /*
@@ -1199 +1354 @@
 #endif
     uint32_t const      cPages  = pThis->cPages;
-    PPGMR0DYNMAPENTRY   paPages = pThis->paPages;
+    PPGMRZDYNMAPENTRY   paPages = pThis->paPages;
     uint32_t            iFreePage;
     if (!paPages[iPage].cRefs)
@@ -1217 +1372 @@
             if (paPages[iFreePage].HCPhys == HCPhys)
             {
-                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatR0DynMapPageSlowLoopHits);
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlowLoopHits);
+                *pfNew = false;
                 return iFreePage;
             }
@@ -1228 +1384 @@
                 return UINT32_MAX;
         }
-        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatR0DynMapPageSlowLoopMisses);
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageSlowLoopMisses);
 #ifdef VBOX_WITH_STATISTICS
         fLooped = true;
@@ -1240 +1396 @@
         for (uint32_t iPage2 = (iPage + 3) % cPages; iPage2 != iPage; iPage2 = (iPage2 + 1) % cPages)
             if (paPages[iPage2].HCPhys == HCPhys)
-                STAM_COUNTER_INC(&pVCpu->pgm.s.StatR0DynMapPageSlowLostHits);
+                STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZDynMapPageSlowLostHits);
 #endif
 
@@ -1246 +1402 @@
      * Setup the new entry.
      */
+    *pfNew = true;
     /*Log6(("pgmR0DynMapPageSlow: old - %RHp %#x %#llx\n", paPages[iFreePage].HCPhys, paPages[iFreePage].cRefs, paPages[iFreePage].uPte.pPae->u));*/
     paPages[iFreePage].HCPhys = HCPhys;
+#ifndef IN_RC
     RTCpuSetFill(&paPages[iFreePage].PendingSet);
+#endif
     if (pThis->fLegacyMode)
     {
@@ -1286 +1445 @@
  * @param   ppvPage     Where to the page address.
  */
-DECLINLINE(uint32_t) pgmR0DynMapPage(PPGMR0DYNMAP pThis, RTHCPHYS HCPhys, int32_t iRealCpu, PVMCPU pVCpu, void **ppvPage)
-{
-    RTSPINLOCKTMP       Tmp     = RTSPINLOCKTMP_INITIALIZER;
-    RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
+DECLINLINE(uint32_t) pgmR0DynMapPage(PPGMRZDYNMAP pThis, RTHCPHYS HCPhys, int32_t iRealCpu, PVMCPU pVCpu, void **ppvPage)
+{
+    PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
     AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
-    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatR0DynMapPage);
+    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPage);
 
     /*
@@ -1301 +1459 @@
      * to pgmR0DynMapPageSlow().
      */
+    bool                fNew    = false;
     uint32_t const      cPages  = pThis->cPages;
     uint32_t            iPage   = (HCPhys >> PAGE_SHIFT) % cPages;
-    PPGMR0DYNMAPENTRY   paPages = pThis->paPages;
+    PPGMRZDYNMAPENTRY   paPages = pThis->paPages;
     if (RT_LIKELY(paPages[iPage].HCPhys == HCPhys))
-        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatR0DynMapPageHits0);
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits0);
     else
     {
@@ -1312 +1471 @@
         {
             iPage = iPage2;
-            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatR0DynMapPageHits1);
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits1);
         }
         else
@@ -1320 +1479 @@
             {
                 iPage = iPage2;
-                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatR0DynMapPageHits2);
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageHits2);
             }
             else
             {
-                iPage = pgmR0DynMapPageSlow(pThis, HCPhys, iPage, pVCpu);
+                iPage = pgmR0DynMapPageSlow(pThis, HCPhys, iPage, pVCpu, &fNew);
                 if (RT_UNLIKELY(iPage == UINT32_MAX))
                 {
-                    RTSpinlockRelease(pThis->hSpinlock, &Tmp);
+                    PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
                     *ppvPage = NULL;
                     return iPage;
@@ -1349 +1508 @@
     {
         ASMAtomicDecS32(&paPages[iPage].cRefs);
-        RTSpinlockRelease(pThis->hSpinlock, &Tmp);
+        PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
         *ppvPage = NULL;
         AssertLogRelMsgFailedReturn(("cRefs=%d iPage=%p HCPhys=%RHp\n", cRefs, iPage, HCPhys), UINT32_MAX);
@@ -1355 +1514 @@
     void *pvPage = paPages[iPage].pvPage;
 
+#ifndef IN_RC
     /*
      * Invalidate the entry?
@@ -1361 +1521 @@
     if (RT_UNLIKELY(fInvalidateIt))
         RTCpuSetDelByIndex(&paPages[iPage].PendingSet, iRealCpu);
-
-    RTSpinlockRelease(pThis->hSpinlock, &Tmp);
+#endif
+
+    PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
 
     /*
      * Do the actual invalidation outside the spinlock.
      */
+#ifdef IN_RC
+    if (RT_UNLIKELY(fNew))
+#else
     if (RT_UNLIKELY(fInvalidateIt))
-    {
-        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatR0DynMapPageInvlPg);
+#endif
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapPageInvlPg);
         ASMInvalidatePage(pvPage);
     }
@@ -1383 +1548 @@
  * @returns VBox status code.
  */
-VMMR0DECL(int) PGMR0DynMapAssertIntegrity(void)
+static int pgmRZDynMapAssertIntegrity(PPGMRZDYNMAP pThis)
 {
     /*
      * Basic pool stuff that doesn't require any lock, just assumes we're a user.
      */
-    PPGMR0DYNMAP        pThis       = g_pPGMR0DynMap;
     if (!pThis)
         return VINF_SUCCESS;
     AssertPtrReturn(pThis, VERR_INVALID_POINTER);
-    AssertReturn(pThis->u32Magic == PGMR0DYNMAP_MAGIC, VERR_INVALID_MAGIC);
+    AssertReturn(pThis->u32Magic == PGMRZDYNMAP_MAGIC, VERR_INVALID_MAGIC);
     if (!pThis->cUsers)
         return VERR_INVALID_PARAMETER;
@@ -1398 +1562 @@
 
     int                 rc          = VINF_SUCCESS;
-    RTSPINLOCKTMP       Tmp         = RTSPINLOCKTMP_INITIALIZER;
-    RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
+    PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
 
 #define CHECK_RET(expr, a) \
@@ -1405 +1568 @@
         if (RT_UNLIKELY(!(expr))) \
         { \
-            RTSpinlockRelease(pThis->hSpinlock, &Tmp); \
+            PGMRZDYNMAP_SPINLOCK_RELEASE(pThis); \
             RTAssertMsg1Weak(#expr, __LINE__, __FILE__, __PRETTY_FUNCTION__); \
             RTAssertMsg2Weak a; \
@@ -1417 +1580 @@
     uint32_t            cGuard      = 0;
     uint32_t            cLoad       = 0;
-    PPGMR0DYNMAPENTRY   paPages     = pThis->paPages;
+    PPGMRZDYNMAPENTRY   paPages     = pThis->paPages;
     uint32_t            iPage       = pThis->cPages;
     if (pThis->fLegacyMode)
     {
+#ifdef IN_RING0
         PCX86PGUINT     paSavedPTEs = (PCX86PGUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
+#endif
         while (iPage-- > 0)
         {
@@ -1440 +1605 @@
             {
                 CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
-                X86PGUINT uPte = (paSavedPTEs[iPage] & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
-                               | X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D
+                X86PGUINT uPte = X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D
+#ifdef IN_RING0
+                               | (paSavedPTEs[iPage] & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
+#endif
                                | (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
                 CHECK_RET(paPages[iPage].uPte.pLegacy->u == uPte,
@@ -1448 +1615 @@
                     cLoad++;
             }
+#ifdef IN_RING0
             else
                 CHECK_RET(paPages[iPage].uPte.pLegacy->u == paSavedPTEs[iPage],
                           ("#%u: %#x %#x", iPage, paPages[iPage].uPte.pLegacy->u, paSavedPTEs[iPage]));
+#endif
         }
     }
     else
     {
+#ifdef IN_RING0
         PCX86PGPAEUINT  paSavedPTEs = (PCX86PGPAEUINT)pThis->pvSavedPTEs; NOREF(paSavedPTEs);
+#endif
         while (iPage-- > 0)
         {
@@ -1474 +1645 @@
             {
                 CHECK_RET(!(paPages[iPage].HCPhys & PAGE_OFFSET_MASK), ("#%u: %RHp\n", iPage, paPages[iPage].HCPhys));
-                X86PGPAEUINT uPte = (paSavedPTEs[iPage] & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
-                                  | X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D
+                X86PGPAEUINT uPte = X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D
+#ifdef IN_RING0
+                                  | (paSavedPTEs[iPage] & (X86_PTE_G | X86_PTE_PAT | X86_PTE_PCD | X86_PTE_PWT))
+#endif
                                   | (paPages[iPage].HCPhys & X86_PTE_PAE_PG_MASK);
                 CHECK_RET(paPages[iPage].uPte.pPae->u == uPte,
@@ -1482 +1655 @@
                     cLoad++;
             }
+#ifdef IN_RING0
             else
                 CHECK_RET(paPages[iPage].uPte.pPae->u == paSavedPTEs[iPage],
                           ("#%u: %#llx %#llx", iPage, paPages[iPage].uPte.pPae->u, paSavedPTEs[iPage]));
+#endif
         }
     }
@@ -1492 +1667 @@
 
 #undef CHECK_RET
-    RTSpinlockRelease(pThis->hSpinlock, &Tmp);
+    PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
     return VINF_SUCCESS;
 }
+
+#ifdef IN_RING0
+/**
+ * Assert the the integrity of the pool.
+ *
+ * @returns VBox status code.
+ */
+VMMR0DECL(int) PGMR0DynMapAssertIntegrity(void)
+{
+    return pgmRZDynMapAssertIntegrity(g_pPGMR0DynMap);
+}
+#endif /* IN_RING0 */
+
+#ifdef IN_RC
+/**
+ * Assert the the integrity of the pool.
+ *
+ * @returns VBox status code.
+ */
+VMMRCDECL(int) PGMRCDynMapAssertIntegrity(PVM pVM)
+{
+    return pgmRZDynMapAssertIntegrity((PPGMRZDYNMAP)pVM->pgm.s.pRCDynMap);
+}
+#endif /* IN_RC */
+
+
+/**
+ * As a final resort for a (somewhat) full auto set or full cache, try merge
+ * duplicate entries and flush the ones we can.
+ *
+ * @param   pSet        The set.
+ */
+static void pgmDynMapOptimizeAutoSet(PPGMMAPSET pSet)
+{
+    LogFlow(("pgmDynMapOptimizeAutoSet\n"));
+
+    for (uint32_t i = 0 ; i < pSet->cEntries; i++)
+    {
+        /*
+         * Try merge entries.
+         */
+        uint16_t const  iPage = pSet->aEntries[i].iPage;
+        uint32_t        j     = i + 1;
+        while (   j < pSet->cEntries
+               && (   pSet->iSubset == UINT32_MAX
+                   || pSet->iSubset < pSet->cEntries) )
+        {
+            if (pSet->aEntries[j].iPage != iPage)
+                j++;
+            else
+            {
+                uint32_t const  cHardRefs    = (uint32_t)pSet->aEntries[i].cRefs
+                                             + (uint32_t)pSet->aEntries[j].cRefs;
+                uint32_t        cInlinedRefs = (uint32_t)pSet->aEntries[i].cInlinedRefs
+                                             + (uint32_t)pSet->aEntries[j].cInlinedRefs;
+                uint32_t        cUnrefs      = (uint32_t)pSet->aEntries[i].cUnrefs
+                                             + (uint32_t)pSet->aEntries[j].cUnrefs;
+                uint32_t        cSub         = RT_MIN(cUnrefs, cInlinedRefs);
+                cInlinedRefs -= cSub;
+                cUnrefs      -= cSub;
+
+                if (    cHardRefs    < UINT16_MAX
+                    &&  cInlinedRefs < UINT16_MAX
+                    &&  cUnrefs      < UINT16_MAX)
+                {
+                    /* merge j into i removing j. */
+                    Log2(("pgmDynMapOptimizeAutoSet: Merging #%u into #%u\n", j, i));
+                    pSet->aEntries[i].cRefs        = cHardRefs;
+                    pSet->aEntries[i].cInlinedRefs = cInlinedRefs;
+                    pSet->aEntries[i].cUnrefs      = cUnrefs;
+                    pSet->cEntries--;
+                    if (j < pSet->cEntries)
+                    {
+                        pSet->aEntries[j] = pSet->aEntries[pSet->cEntries];
+                        PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[pSet->cEntries]);
+                    }
+                    else
+                        PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[j]);
+                }
+#if 0 /* too complicated, skip it. */
+                else
+                {
+                    /* migrate the max number of refs from j into i and quit the inner loop. */
+                    uint32_t cMigrate = UINT16_MAX - 1 - pSet->aEntries[i].cRefs;
+                    Assert(pSet->aEntries[j].cRefs > cMigrate);
+                    pSet->aEntries[j].cRefs -= cMigrate;
+                    pSet->aEntries[i].cRefs = UINT16_MAX - 1;
+                    break;
+                }
+#endif
+            }
+        }
+
+        /*
+         * Try make use of the unused hinting (cUnrefs) to evict entries
+         * from both the set as well as the mapping cache.
+         */
+
+        uint32_t const cTotalRefs = (uint32_t)pSet->aEntries[i].cRefs + pSet->aEntries[i].cInlinedRefs;
+        Log2(("pgmDynMapOptimizeAutoSet: #%u/%u/%u pvPage=%p iPage=%u cRefs=%u cInlinedRefs=%u cUnrefs=%u cTotalRefs=%u\n",
+              i,
+              pSet->iSubset,
+              pSet->cEntries,
+              pSet->aEntries[i].pvPage,
+              pSet->aEntries[i].iPage,
+              pSet->aEntries[i].cRefs,
+              pSet->aEntries[i].cInlinedRefs,
+              pSet->aEntries[i].cUnrefs,
+              cTotalRefs));
+        Assert(cTotalRefs >= pSet->aEntries[i].cUnrefs);
+
+        if (    cTotalRefs == pSet->aEntries[i].cUnrefs
+            &&  (   pSet->iSubset == UINT32_MAX
+                 || pSet->iSubset < pSet->cEntries)
+           )
+        {
+            Log2(("pgmDynMapOptimizeAutoSet: Releasing iPage=%d/%p\n", pSet->aEntries[i].iPage, pSet->aEntries[i].pvPage));
+            //LogFlow(("pgmDynMapOptimizeAutoSet: Releasing iPage=%d/%p\n", pSet->aEntries[i].iPage, pSet->aEntries[i].pvPage));
+            pgmRZDynMapReleasePage(PGMRZDYNMAP_SET_2_DYNMAP(pSet),
+                                   pSet->aEntries[i].iPage,
+                                   pSet->aEntries[i].cRefs);
+            pSet->cEntries--;
+            if (i < pSet->cEntries)
+            {
+                pSet->aEntries[i] = pSet->aEntries[pSet->cEntries];
+                PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[pSet->cEntries]);
+            }
+
+            i--;
+        }
+    }
+}
+
+
 
 
@@ -1505 +1814 @@
  * @param   pVCpu       The shared data for the current virtual CPU.
  */
-VMMDECL(void) PGMDynMapStartAutoSet(PVMCPU pVCpu)
-{
+VMMDECL(void) PGMRZDynMapStartAutoSet(PVMCPU pVCpu)
+{
+    LogFlow(("PGMRZDynMapStartAutoSet:\n"));
     Assert(pVCpu->pgm.s.AutoSet.cEntries == PGMMAPSET_CLOSED);
     Assert(pVCpu->pgm.s.AutoSet.iSubset == UINT32_MAX);
     pVCpu->pgm.s.AutoSet.cEntries = 0;
-    pVCpu->pgm.s.AutoSet.iCpu = RTMpCpuIdToSetIndex(RTMpCpuId());
-}
-
-
+    pVCpu->pgm.s.AutoSet.iCpu = PGMRZDYNMAP_CUR_CPU();
+}
+
+
+#ifdef IN_RING0
 /**
  * Starts or migrates the autoset of a virtual CPU.
@@ -1526 +1837 @@
  * @thread  EMT
  */
-VMMDECL(bool) PGMDynMapStartOrMigrateAutoSet(PVMCPU pVCpu)
+VMMR0DECL(bool) PGMR0DynMapStartOrMigrateAutoSet(PVMCPU pVCpu)
 {
     bool fStartIt = pVCpu->pgm.s.AutoSet.cEntries == PGMMAPSET_CLOSED;
     if (fStartIt)
-        PGMDynMapStartAutoSet(pVCpu);
+        PGMRZDynMapStartAutoSet(pVCpu);
     else
-        PGMDynMapMigrateAutoSet(pVCpu);
+        PGMR0DynMapMigrateAutoSet(pVCpu);
     return fStartIt;
 }
+#endif /* IN_RING0 */
 
 
@@ -1551 +1863 @@
         &&  RT_LIKELY(cEntries <= RT_ELEMENTS(pSet->aEntries)))
     {
-        PPGMR0DYNMAP    pThis   = g_pPGMR0DynMap;
-        RTSPINLOCKTMP   Tmp     = RTSPINLOCKTMP_INITIALIZER;
-        RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
+        PPGMRZDYNMAP    pThis   = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
+        PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
 
         uint32_t i = cEntries;
@@ -1562 +1873 @@
             int32_t  cRefs = pSet->aEntries[i].cRefs;
             Assert(cRefs > 0);
-            pgmR0DynMapReleasePageLocked(pThis, iPage, cRefs);
-
-            pSet->aEntries[i].iPage = UINT16_MAX;
-            pSet->aEntries[i].cRefs = 0;
+            pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs);
+
+            PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[i]);
         }
 
         Assert(pThis->cLoad <= pThis->cPages - pThis->cGuardPages);
-        RTSpinlockRelease(pThis->hSpinlock, &Tmp);
+        PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
     }
 }
@@ -1580 +1890 @@
  * @param   pVCpu       The shared data for the current virtual CPU.
  */
-VMMDECL(void) PGMDynMapReleaseAutoSet(PVMCPU pVCpu)
+VMMDECL(void) PGMRZDynMapReleaseAutoSet(PVMCPU pVCpu)
 {
     PPGMMAPSET  pSet = &pVCpu->pgm.s.AutoSet;
@@ -1593 +1903 @@
     pSet->iCpu = -1;
 
-    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatR0DynMapSetSize[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
+#ifdef IN_RC
+    if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
+    else
+#endif
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
     AssertMsg(cEntries < PGMMAPSET_MAX_FILL, ("%u\n", cEntries));
     if (cEntries > RT_ELEMENTS(pSet->aEntries) * 50 / 100)
-        Log(("PGMDynMapReleaseAutoSet: cEntries=%d\n", pSet->cEntries));
+        Log(("PGMRZDynMapReleaseAutoSet: cEntries=%d\n", cEntries));
+    else
+        LogFlow(("PGMRZDynMapReleaseAutoSet: cEntries=%d\n", cEntries));
 
     pgmDynMapFlushAutoSetWorker(pSet, cEntries);
@@ -1607 +1924 @@
  * @param   pVCpu       The shared data for the current virtual CPU.
  */
-VMMDECL(void) PGMDynMapFlushAutoSet(PVMCPU pVCpu)
+VMMDECL(void) PGMRZDynMapFlushAutoSet(PVMCPU pVCpu)
 {
     PPGMMAPSET  pSet = &pVCpu->pgm.s.AutoSet;
-    AssertMsg(pSet->iCpu == RTMpCpuIdToSetIndex(RTMpCpuId()), ("%d %d(%d) efl=%#x\n", pSet->iCpu, RTMpCpuIdToSetIndex(RTMpCpuId()), RTMpCpuId(), ASMGetFlags()));
+    AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
 
     /*
@@ -1617 +1934 @@
     uint32_t cEntries = pSet->cEntries;
     AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
-    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatR0DynMapSetSize[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
+#ifdef IN_RC
+    if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
+    else
+#endif
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
     if (cEntries >= RT_ELEMENTS(pSet->aEntries) * 45 / 100)
     {
@@ -1626 +1948 @@
 
         pgmDynMapFlushAutoSetWorker(pSet, cEntries);
-        AssertMsg(pSet->iCpu == RTMpCpuIdToSetIndex(RTMpCpuId()), ("%d %d(%d) efl=%#x\n", pSet->iCpu, RTMpCpuIdToSetIndex(RTMpCpuId()), RTMpCpuId(), ASMGetFlags()));
-    }
-}
-
-
+        AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
+    }
+}
+
+
+#ifndef IN_RC
 /**
  * Migrates the automatic mapping set of the current vCPU if it's active and
@@ -1644 +1967 @@
  * @thread  EMT
  */
-VMMDECL(void) PGMDynMapMigrateAutoSet(PVMCPU pVCpu)
-{
+VMMR0DECL(void) PGMR0DynMapMigrateAutoSet(PVMCPU pVCpu)
+{
+    LogFlow(("PGMR0DynMapMigrateAutoSet\n"));
     PPGMMAPSET      pSet     = &pVCpu->pgm.s.AutoSet;
-    int32_t         iRealCpu = RTMpCpuIdToSetIndex(RTMpCpuId());
+    int32_t         iRealCpu = PGMRZDYNMAP_CUR_CPU();
     if (pSet->iCpu != iRealCpu)
     {
@@ -1656 +1980 @@
             if (i != 0 && RT_LIKELY(i <= RT_ELEMENTS(pSet->aEntries)))
             {
-                PPGMR0DYNMAP    pThis  = g_pPGMR0DynMap;
-                RTSPINLOCKTMP   Tmp    = RTSPINLOCKTMP_INITIALIZER;
-                RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
+                PPGMRZDYNMAP    pThis  = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
+                PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
 
                 while (i-- > 0)
@@ -1668 +1991 @@
                     {
                         RTCpuSetDelByIndex(&pThis->paPages[iPage].PendingSet, iRealCpu);
-                        RTSpinlockRelease(pThis->hSpinlock, &Tmp);
+                        PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
 
                         ASMInvalidatePage(pThis->paPages[iPage].pvPage);
-                        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatR0DynMapMigrateInvlPg);
-
-                        RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
+                        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapMigrateInvlPg);
+
+                        PGMRZDYNMAP_SPINLOCK_REACQUIRE(pThis);
                     }
                 }
 
-                RTSpinlockRelease(pThis->hSpinlock, &Tmp);
+                PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
             }
         }
@@ -1683 +2006 @@
     }
 }
+#endif /* !IN_RC */
 
 
@@ -1706 +2030 @@
         pSet->cEntries = iSubset;
 
-        PPGMR0DYNMAP    pThis = g_pPGMR0DynMap;
-        RTSPINLOCKTMP   Tmp   = RTSPINLOCKTMP_INITIALIZER;
-        RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
+        PPGMRZDYNMAP    pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
+        PGMRZDYNMAP_SPINLOCK_ACQUIRE(pThis);
 
         while (i-- > iSubset)
@@ -1716 +2039 @@
             int32_t  cRefs = pSet->aEntries[i].cRefs;
             Assert(cRefs > 0);
-            pgmR0DynMapReleasePageLocked(pThis, iPage, cRefs);
-
-            pSet->aEntries[i].iPage = UINT16_MAX;
-            pSet->aEntries[i].cRefs = 0;
-        }
-
-        RTSpinlockRelease(pThis->hSpinlock, &Tmp);
+            pgmRZDynMapReleasePageLocked(pThis, iPage, cRefs);
+
+            PGMRZDYNMAP_ZAP_ENTRY(&pSet->aEntries[i]);
+        }
+
+        PGMRZDYNMAP_SPINLOCK_RELEASE(pThis);
     }
 }
@@ -1738 +2060 @@
  *
  * @returns The index of the previous subset. Pass this to
- *        PGMDynMapPopAutoSubset when poping it.
+ *          PGMDynMapPopAutoSubset when popping it.
  * @param   pVCpu           Pointer to the virtual cpu data.
  */
-VMMDECL(uint32_t) PGMDynMapPushAutoSubset(PVMCPU pVCpu)
+VMMDECL(uint32_t) PGMRZDynMapPushAutoSubset(PVMCPU pVCpu)
 {
     PPGMMAPSET      pSet = &pVCpu->pgm.s.AutoSet;
     AssertReturn(pSet->cEntries != PGMMAPSET_CLOSED, UINT32_MAX);
     uint32_t        iPrevSubset = pSet->iSubset;
-    LogFlow(("PGMDynMapPushAutoSubset: pVCpu=%p iPrevSubset=%u\n", pVCpu, iPrevSubset));
+    LogFlow(("PGMRZDynMapPushAutoSubset: pVCpu=%p iPrevSubset=%u\n", pVCpu, iPrevSubset));
+
+#ifdef IN_RC
+    /* kludge */
+    if (pSet->cEntries > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE / 2)
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize);
+        pgmDynMapOptimizeAutoSet(pSet);
+    }
+#endif
 
     pSet->iSubset = pSet->cEntries;
-    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatR0DynMapSubsets);
+    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSubsets);
+
     return iPrevSubset;
 }
@@ -1760 +2092 @@
  * @param   iPrevSubset     What PGMDynMapPushAutoSubset returned.
  */
-VMMDECL(void) PGMDynMapPopAutoSubset(PVMCPU pVCpu, uint32_t iPrevSubset)
+VMMDECL(void) PGMRZDynMapPopAutoSubset(PVMCPU pVCpu, uint32_t iPrevSubset)
 {
     PPGMMAPSET      pSet = &pVCpu->pgm.s.AutoSet;
     uint32_t        cEntries = pSet->cEntries;
-    LogFlow(("PGMDynMapPopAutoSubset: pVCpu=%p iPrevSubset=%u iSubset=%u cEntries=%u\n", pVCpu, iPrevSubset, pSet->iSubset, cEntries));
+    LogFlow(("PGMRZDynMapPopAutoSubset: pVCpu=%p iPrevSubset=%u iSubset=%u cEntries=%u\n", pVCpu, iPrevSubset, pSet->iSubset, cEntries));
     AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
     AssertReturnVoid(pSet->iSubset >= iPrevSubset || iPrevSubset == UINT32_MAX);
-    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatR0DynMapSetSize[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
+#ifdef IN_RC
+    if (RT_ELEMENTS(pSet->aEntries) > MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / (MM_HYPER_DYNAMIC_SIZE / PAGE_SIZE)) % 11]);
+    else
+#endif
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
     if (    cEntries >= RT_ELEMENTS(pSet->aEntries) * 40 / 100
         &&  cEntries != pSet->iSubset)
@@ -1779 +2116 @@
 
 /**
- * As a final resort for a full auto set, try merge duplicate entries.
- *
- * @param   pSet        The set.
- */
-static void pgmDynMapOptimizeAutoSet(PPGMMAPSET pSet)
-{
-    for (uint32_t i = 0 ; i < pSet->cEntries; i++)
-    {
-        uint16_t const  iPage = pSet->aEntries[i].iPage;
-        uint32_t        j     = i + 1;
-        while (j < pSet->cEntries)
-        {
-            if (pSet->aEntries[j].iPage != iPage)
-                j++;
-            else if ((uint32_t)pSet->aEntries[i].cRefs + (uint32_t)pSet->aEntries[j].cRefs < UINT16_MAX)
-            {
-                /* merge j into i removing j. */
-                pSet->aEntries[i].cRefs += pSet->aEntries[j].cRefs;
-                pSet->cEntries--;
-                if (j < pSet->cEntries)
+ * Indicates that the given page is unused and its mapping can be re-used.
+ *
+ * @param   pVCpu           The current CPU.
+ * @param   pvHint          The page that is now unused.  This does not have to
+ *                          point at the start of the page.  NULL is ignored.
+ */
+#ifdef LOG_ENABLED
+void pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint, RT_SRC_POS_DECL)
+#else
+void pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint)
+#endif
+{
+    /*
+     * Ignore NULL pointers and mask off the page offset bits.
+     */
+    if (pvHint == NULL)
+        return;
+    pvHint = (void *)((uintptr_t)pvHint & ~(uintptr_t)PAGE_OFFSET_MASK);
+
+    PPGMMAPSET  pSet    = &pVCpu->pgm.s.AutoSet;
+    uint32_t    iEntry  = pSet->cEntries;
+    AssertReturnVoid(iEntry > 0);
+
+    /*
+     * Find the entry in the usual unrolled fashion.
+     */
+#define IS_MATCHING_ENTRY(pSet, iEntry, pvHint) \
+        (   (pSet)->aEntries[(iEntry)].pvPage == (pvHint) \
+         &&   (uint32_t)(pSet)->aEntries[(iEntry)].cRefs + (pSet)->aEntries[(iEntry)].cInlinedRefs \
+            > (pSet)->aEntries[(iEntry)].cUnrefs )
+    if (     iEntry >= 1 && IS_MATCHING_ENTRY(pSet, iEntry - 1, pvHint))
+        iEntry = iEntry - 1;
+    else if (iEntry >= 2 && IS_MATCHING_ENTRY(pSet, iEntry - 2, pvHint))
+        iEntry = iEntry - 2;
+    else if (iEntry >= 3 && IS_MATCHING_ENTRY(pSet, iEntry - 3, pvHint))
+        iEntry = iEntry - 3;
+    else if (iEntry >= 4 && IS_MATCHING_ENTRY(pSet, iEntry - 4, pvHint))
+        iEntry = iEntry - 4;
+    else if (iEntry >= 5 && IS_MATCHING_ENTRY(pSet, iEntry - 5, pvHint))
+        iEntry = iEntry - 5;
+    else if (iEntry >= 6 && IS_MATCHING_ENTRY(pSet, iEntry - 6, pvHint))
+        iEntry = iEntry - 6;
+    else if (iEntry >= 7 && IS_MATCHING_ENTRY(pSet, iEntry - 7, pvHint))
+        iEntry = iEntry - 7;
+    else
+    {
+        /*
+         * Loop till we find it.
+         */
+        bool fFound = false;
+        if (iEntry > 7)
+        {
+            iEntry -= 7;
+            while (iEntry-- > 0)
+                if (IS_MATCHING_ENTRY(pSet, iEntry, pvHint))
                 {
-                    pSet->aEntries[j] = pSet->aEntries[pSet->cEntries];
-                    pSet->aEntries[pSet->cEntries].iPage = UINT16_MAX;
-                    pSet->aEntries[pSet->cEntries].cRefs = 0;
+                    fFound = true;
+                    break;
                 }
-                else
-                {
-                    pSet->aEntries[j].iPage = UINT16_MAX;
-                    pSet->aEntries[j].cRefs = 0;
-                }
-            }
-            else
-            {
-                /* migrate the max number of refs from j into i and quit the inner loop. */
-                uint32_t cMigrate = UINT16_MAX - 1 - pSet->aEntries[i].cRefs;
-                Assert(pSet->aEntries[j].cRefs > cMigrate);
-                pSet->aEntries[j].cRefs -= cMigrate;
-                pSet->aEntries[i].cRefs = UINT16_MAX - 1;
-                break;
-            }
-        }
-    }
-}
-
-
-/**
- * Common worker code for PGMDynMapHCPhys, pgmR0DynMapHCPageInlined and
- * pgmR0DynMapGCPageInlined.
+        }
+        AssertMsgReturnVoid(fFound,
+                            ("pvHint=%p cEntries=%#x iSubset=%#x\n"
+                             "aEntries[0] = {%#x, %#x, %#x, %#x, %p}\n"
+                             "aEntries[1] = {%#x, %#x, %#x, %#x, %p}\n"
+                             "aEntries[2] = {%#x, %#x, %#x, %#x, %p}\n"
+                             "aEntries[3] = {%#x, %#x, %#x, %#x, %p}\n"
+                             "aEntries[4] = {%#x, %#x, %#x, %#x, %p}\n"
+                             "aEntries[5] = {%#x, %#x, %#x, %#x, %p}\n"
+                             ,
+                             pvHint, pSet->cEntries, pSet->iSubset,
+                             pSet->aEntries[0].iPage, pSet->aEntries[0].cRefs, pSet->aEntries[0].cInlinedRefs, pSet->aEntries[0].cUnrefs, pSet->aEntries[0].pvPage,
+                             pSet->aEntries[1].iPage, pSet->aEntries[1].cRefs, pSet->aEntries[1].cInlinedRefs, pSet->aEntries[1].cUnrefs, pSet->aEntries[1].pvPage,
+                             pSet->aEntries[2].iPage, pSet->aEntries[2].cRefs, pSet->aEntries[2].cInlinedRefs, pSet->aEntries[2].cUnrefs, pSet->aEntries[2].pvPage,
+                             pSet->aEntries[3].iPage, pSet->aEntries[3].cRefs, pSet->aEntries[3].cInlinedRefs, pSet->aEntries[3].cUnrefs, pSet->aEntries[3].pvPage,
+                             pSet->aEntries[4].iPage, pSet->aEntries[4].cRefs, pSet->aEntries[4].cInlinedRefs, pSet->aEntries[4].cUnrefs, pSet->aEntries[4].pvPage,
+                             pSet->aEntries[5].iPage, pSet->aEntries[5].cRefs, pSet->aEntries[5].cInlinedRefs, pSet->aEntries[5].cUnrefs, pSet->aEntries[5].pvPage));
+    }
+#undef IS_MATCHING_ENTRY
+
+    /*
+     * Update it.
+     */
+    uint32_t const  cTotalRefs = (uint32_t)pSet->aEntries[iEntry].cRefs + pSet->aEntries[iEntry].cInlinedRefs;
+    uint32_t const  cUnrefs    = pSet->aEntries[iEntry].cUnrefs;
+    LogFlow(("pgmRZDynMapUnusedHint: pvHint=%p #%u cRefs=%d cInlinedRefs=%d cUnrefs=%d (+1) cTotalRefs=%d %s(%d) %s\n",
+             pvHint, iEntry, pSet->aEntries[iEntry].cRefs, pSet->aEntries[iEntry].cInlinedRefs, cUnrefs, cTotalRefs, pszFile, iLine, pszFunction));
+    AssertReturnVoid(cTotalRefs > cUnrefs);
+
+    if (RT_LIKELY(cUnrefs < UINT16_MAX - 1))
+        pSet->aEntries[iEntry].cUnrefs++;
+    else if (pSet->aEntries[iEntry].cInlinedRefs)
+    {
+        uint32_t cSub = RT_MIN(pSet->aEntries[iEntry].cInlinedRefs, pSet->aEntries[iEntry].cUnrefs);
+        pSet->aEntries[iEntry].cInlinedRefs -= cSub;
+        pSet->aEntries[iEntry].cUnrefs      -= cSub;
+        pSet->aEntries[iEntry].cUnrefs++;
+    }
+    else
+        Log(("pgmRZDynMapUnusedHint: pvHint=%p ignored because of overflow! %s(%d) %s\n", pvHint, pszFile, iLine, pszFunction));
+}
+
+
+/**
+ * Common worker code for pgmRZDynMapHCPageInlined, pgmRZDynMapHCPageV2Inlined
+ * and pgmR0DynMapGCPageOffInlined.
  *
  * @returns VINF_SUCCESS, bails out to ring-3 on failure.
@@ -1835 +2229 @@
  * @remarks This is a very hot path.
  */
-int pgmR0DynMapHCPageCommon(PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv)
-{
-    LogFlow(("pgmR0DynMapHCPageCommon: pSet=%p HCPhys=%RHp ppv=%p\n", pSet, HCPhys, ppv));
-    AssertMsg(pSet->iCpu == RTMpCpuIdToSetIndex(RTMpCpuId()), ("%d %d(%d) efl=%#x\n", pSet->iCpu, RTMpCpuIdToSetIndex(RTMpCpuId()), RTMpCpuId(), ASMGetFlags()));
-    PVMCPU pVCpu = PGMR0DYNMAP_2_VMCPU(pSet);
+int pgmRZDynMapHCPageCommon(PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
+{
+    AssertMsg(pSet->iCpu == PGMRZDYNMAP_CUR_CPU(), ("%d %d efl=%#x\n", pSet->iCpu, PGMRZDYNMAP_CUR_CPU(), ASMGetFlags()));
+    PVMCPU pVCpu = PGMRZDYNMAP_SET_2_VMCPU(pSet);
+    STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
 
     /*
      * Map it.
      */
-    void *pvPage;
-    uint32_t const  iPage = pgmR0DynMapPage(g_pPGMR0DynMap, HCPhys, pSet->iCpu, pVCpu, &pvPage);
+    void           *pvPage;
+    PPGMRZDYNMAP    pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
+    uint32_t        iPage = pgmR0DynMapPage(pThis, HCPhys, pSet->iCpu, pVCpu, &pvPage);
     if (RT_UNLIKELY(iPage == UINT32_MAX))
     {
-        RTAssertMsg2Weak("PGMDynMapHCPage: cLoad=%u/%u cPages=%u cGuardPages=%u\n",
-                         g_pPGMR0DynMap->cLoad, g_pPGMR0DynMap->cMaxLoad, g_pPGMR0DynMap->cPages, g_pPGMR0DynMap->cGuardPages);
-        if (!g_fPGMR0DynMapTestRunning)
-            VMMRZCallRing3NoCpu(PGMR0DYNMAP_2_VM(pSet), VMMCALLRING3_VM_R0_ASSERTION, 0);
-        *ppv = NULL;
-        return VERR_PGM_DYNMAP_FAILED;
+        /*
+         * We're out of mapping space, optimize our set to try remedy the
+         * situation.  (Only works if there are unreference hints.)
+         */
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize);
+        pgmDynMapOptimizeAutoSet(pSet);
+
+        iPage = pgmR0DynMapPage(pThis, HCPhys, pSet->iCpu, pVCpu, &pvPage);
+        if (RT_UNLIKELY(iPage == UINT32_MAX))
+        {
+            RTAssertMsg2Weak("pgmRZDynMapHCPageCommon: cLoad=%u/%u cPages=%u cGuardPages=%u\n",
+                             pThis->cLoad, pThis->cMaxLoad, pThis->cPages, pThis->cGuardPages);
+            if (!g_fPGMR0DynMapTestRunning)
+                VMMRZCallRing3NoCpu(PGMRZDYNMAP_SET_2_VM(pSet), VMMCALLRING3_VM_R0_ASSERTION, 0);
+            *ppv = NULL;
+            STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
+            return VERR_PGM_DYNMAP_FAILED;
+        }
     }
 
@@ -1869 +2276 @@
     {
         unsigned iEntry = pSet->cEntries++;
-        pSet->aEntries[iEntry].cRefs  = 1;
-        pSet->aEntries[iEntry].iPage  = iPage;
-        pSet->aEntries[iEntry].pvPage = pvPage;
-        pSet->aEntries[iEntry].HCPhys = HCPhys;
+        pSet->aEntries[iEntry].cRefs        = 1;
+        pSet->aEntries[iEntry].cUnrefs      = 0;
+        pSet->aEntries[iEntry].cInlinedRefs = 0;
+        pSet->aEntries[iEntry].iPage        = iPage;
+        pSet->aEntries[iEntry].pvPage       = pvPage;
+        pSet->aEntries[iEntry].HCPhys       = HCPhys;
         pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
+        LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/0/0 iPage=%#x  [a] %s(%d) %s\n",
+                 pSet, HCPhys, iEntry, iEntry + 1, pvPage, 1, iPage, pszFile, iLine, pszFunction));
     }
     /* Any of the last 5 pages? */
     else if (   pSet->aEntries[i - 0].iPage == iPage
              && pSet->aEntries[i - 0].cRefs < UINT16_MAX - 1)
+    {
         pSet->aEntries[i - 0].cRefs++;
+        LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [0] %s(%d) %s\n", pSet, HCPhys, i - 0, pSet->cEntries, pvPage, pSet->aEntries[i - 0].cRefs, pSet->aEntries[i - 0].cInlinedRefs, pSet->aEntries[i - 0].cUnrefs, iPage, pszFile, iLine, pszFunction));
+    }
     else if (   pSet->aEntries[i - 1].iPage == iPage
              && pSet->aEntries[i - 1].cRefs < UINT16_MAX - 1)
+    {
         pSet->aEntries[i - 1].cRefs++;
+        LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [1] %s(%d) %s\n", pSet, HCPhys, i - 1, pSet->cEntries, pvPage, pSet->aEntries[i - 1].cRefs, pSet->aEntries[i - 1].cInlinedRefs, pSet->aEntries[i - 1].cUnrefs, iPage, pszFile, iLine, pszFunction));
+    }
     else if (   pSet->aEntries[i - 2].iPage == iPage
              && pSet->aEntries[i - 2].cRefs < UINT16_MAX - 1)
+    {
         pSet->aEntries[i - 2].cRefs++;
+        LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [2] %s(%d) %s\n", pSet, HCPhys, i - 2, pSet->cEntries, pvPage, pSet->aEntries[i - 2].cRefs, pSet->aEntries[i - 2].cInlinedRefs, pSet->aEntries[i - 2].cUnrefs, iPage, pszFile, iLine, pszFunction));
+    }
     else if (   pSet->aEntries[i - 3].iPage == iPage
              && pSet->aEntries[i - 3].cRefs < UINT16_MAX - 1)
+    {
         pSet->aEntries[i - 3].cRefs++;
+        LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [4] %s(%d) %s\n", pSet, HCPhys, i - 3, pSet->cEntries, pvPage, pSet->aEntries[i - 3].cRefs, pSet->aEntries[i - 3].cInlinedRefs, pSet->aEntries[i - 3].cUnrefs, iPage, pszFile, iLine, pszFunction));
+    }
     else if (   pSet->aEntries[i - 4].iPage == iPage
              && pSet->aEntries[i - 4].cRefs < UINT16_MAX - 1)
+    {
         pSet->aEntries[i - 4].cRefs++;
+        LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [4] %s(%d) %s\n", pSet, HCPhys, i - 4, pSet->cEntries, pvPage, pSet->aEntries[i - 4].cRefs, pSet->aEntries[i - 4].cInlinedRefs, pSet->aEntries[i - 4].cUnrefs, iPage, pszFile, iLine, pszFunction));
+    }
     /* Don't bother searching unless we're above a 60% load. */
     else if (RT_LIKELY(i <= (int32_t)RT_ELEMENTS(pSet->aEntries) * 60 / 100))
     {
         unsigned iEntry = pSet->cEntries++;
-        pSet->aEntries[iEntry].cRefs  = 1;
-        pSet->aEntries[iEntry].iPage  = iPage;
-        pSet->aEntries[iEntry].pvPage = pvPage;
-        pSet->aEntries[iEntry].HCPhys = HCPhys;
+        pSet->aEntries[iEntry].cRefs        = 1;
+        pSet->aEntries[iEntry].cUnrefs      = 0;
+        pSet->aEntries[iEntry].cInlinedRefs = 0;
+        pSet->aEntries[iEntry].iPage        = iPage;
+        pSet->aEntries[iEntry].pvPage       = pvPage;
+        pSet->aEntries[iEntry].HCPhys       = HCPhys;
         pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
+        LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=1/0/0 iPage=%#x [b] %s(%d) %s\n", pSet, HCPhys, iEntry, pSet->cEntries, pvPage, iPage, pszFile, iLine, pszFunction));
     }
     else
@@ -1911 +2340 @@
             {
                 pSet->aEntries[i].cRefs++;
-                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatR0DynMapSetSearchHits);
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchHits);
+                LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=%u/%u/%u iPage=%#x [c] %s(%d) %s\n", pSet, HCPhys, i, pSet->cEntries, pvPage, pSet->aEntries[i].cRefs, pSet->aEntries[i].cInlinedRefs, pSet->aEntries[i].cUnrefs, iPage, pszFile, iLine, pszFunction));
                 break;
             }
         if (i < 0)
         {
-            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatR0DynMapSetSearchMisses);
+            STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchMisses);
             if (pSet->iSubset < pSet->cEntries)
             {
-                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatR0DynMapSetSearchFlushes);
-                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatR0DynMapSetSize[(pSet->cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetSearchFlushes);
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->aStatRZDynMapSetFilledPct[(pSet->cEntries * 10 / RT_ELEMENTS(pSet->aEntries)) % 11]);
                 AssertMsg(pSet->cEntries < PGMMAPSET_MAX_FILL, ("%u\n", pSet->cEntries));
                 pgmDynMapFlushSubset(pSet);
@@ -1927 +2357 @@
             if (RT_UNLIKELY(pSet->cEntries >= RT_ELEMENTS(pSet->aEntries)))
             {
-                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatR0DynMapSetOptimize);
+                STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapSetOptimize);
                 pgmDynMapOptimizeAutoSet(pSet);
             }
@@ -1934 +2364 @@
             {
                 unsigned iEntry = pSet->cEntries++;
-                pSet->aEntries[iEntry].cRefs  = 1;
-                pSet->aEntries[iEntry].iPage  = iPage;
-                pSet->aEntries[iEntry].pvPage = pvPage;
-                pSet->aEntries[iEntry].HCPhys = HCPhys;
+                pSet->aEntries[iEntry].cRefs        = 1;
+                pSet->aEntries[iEntry].cUnrefs      = 0;
+                pSet->aEntries[iEntry].cInlinedRefs = 0;
+                pSet->aEntries[iEntry].iPage        = iPage;
+                pSet->aEntries[iEntry].pvPage       = pvPage;
+                pSet->aEntries[iEntry].HCPhys       = HCPhys;
                 pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
+                LogFlow(("pgmRZDynMapHCPageCommon: pSet=%p HCPhys=%RHp #%u/%u/%p cRefs=1/0/0 iPage=%#x [d] %s(%d) %s\n", pSet, HCPhys, iEntry, pSet->cEntries, pvPage, iPage, pszFile, iLine, pszFunction));
             }
             else
             {
                 /* We're screwed. */
-                pgmR0DynMapReleasePage(g_pPGMR0DynMap, iPage, 1);
-
-                RTAssertMsg2Weak("PGMDynMapHCPage: set is full!\n");
+                pgmRZDynMapReleasePage(pThis, iPage, 1);
+
+                RTAssertMsg2Weak("pgmRZDynMapHCPageCommon: set is full!\n");
                 if (!g_fPGMR0DynMapTestRunning)
-                    VMMRZCallRing3NoCpu(PGMR0DYNMAP_2_VM(pSet), VMMCALLRING3_VM_R0_ASSERTION, 0);
+                    VMMRZCallRing3NoCpu(PGMRZDYNMAP_SET_2_VM(pSet), VMMCALLRING3_VM_R0_ASSERTION, 0);
                 *ppv = NULL;
+                STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
                 return VERR_PGM_DYNMAP_FULL_SET;
             }
@@ -1955 +2389 @@
 
     *ppv = pvPage;
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPage, a);
     return VINF_SUCCESS;
 }
-
-
-#if 0 /* Not used in R0, should internalized the other PGMDynMapHC/GCPage too. */
-/* documented elsewhere - a bit of a mess. */
-VMMDECL(int) PGMDynMapHCPage(PVM pVM, RTHCPHYS HCPhys, void **ppv)
-{
-#ifdef VBOX_WITH_STATISTICS
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-#endif
-    /*
-     * Validate state.
-     */
-    STAM_PROFILE_START(&pVCpu->pgm.s.StatR0DynMapHCPage, a);
-    AssertPtr(ppv);
-    AssertMsg(pVM->pgm.s.pvR0DynMapUsed == g_pPGMR0DynMap,
-              ("%p != %p\n", pVM->pgm.s.pvR0DynMapUsed, g_pPGMR0DynMap));
-    AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
-    PVMCPU          pVCpu   = VMMGetCpu(pVM);
-    AssertPtr(pVCpu);
-    PPGMMAPSET      pSet    = &pVCpu->pgm.s.AutoSet;
-    AssertMsg(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries),
-              ("%#x (%u)\n", pSet->cEntries, pSet->cEntries));
-
-    /*
-     * Call common code.
-     */
-    int rc = pgmR0DynMapHCPageCommon(pSet, HCPhys, ppv);
-
-    STAM_PROFILE_STOP(&pVCpu->pgm.s.StatR0DynMapHCPage, a);
-    return rc;
-}
-#endif
 
 
@@ -2025 +2428 @@
 {
     LogRel(("pgmR0DynMapTest: ****** START ******\n"));
-    PPGMR0DYNMAP    pThis = g_pPGMR0DynMap;
     PPGMMAPSET      pSet  = &pVM->aCpus[0].pgm.s.AutoSet;
+    PPGMRZDYNMAP    pThis = PGMRZDYNMAP_SET_2_DYNMAP(pSet);
     uint32_t        i;
 
@@ -2047 +2450 @@
     LogRel(("Test #1\n"));
     ASMIntDisable();
-    PGMDynMapStartAutoSet(&pVM->aCpus[0]);
+    PGMRZDynMapStartAutoSet(&pVM->aCpus[0]);
 
     uint64_t cr3 = ASMGetCR3() & ~(uint64_t)PAGE_OFFSET_MASK;
     void    *pv  = (void *)(intptr_t)-1;
     void    *pv2 = (void *)(intptr_t)-2;
-    rc           = PGMDynMapHCPage(pVM, cr3, &pv);
-    int      rc2 = PGMDynMapHCPage(pVM, cr3, &pv2);
+    rc           = pgmRZDynMapHCPageCommon(pVM, cr3, &pv  RTLOG_COMMA_SRC_POS);
+    int      rc2 = pgmRZDynMapHCPageCommon(pVM, cr3, &pv2 RTLOG_COMMA_SRC_POS);
     ASMIntEnable();
     if (    RT_SUCCESS(rc2)
@@ -2068 +2471 @@
         LogRel(("Test #2\n"));
         ASMIntDisable();
-        PGMDynMapMigrateAutoSet(&pVM->aCpus[0]);
+        PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
         for (i = 0 ; i < UINT16_MAX*2 - 1 && RT_SUCCESS(rc) && pv2 == pv; i++)
         {
             pv2 = (void *)(intptr_t)-4;
-            rc = PGMDynMapHCPage(pVM, cr3, &pv2);
+            rc = pgmRZDynMapHCPageCommon(pVM, cr3, &pv2 RTLOG_COMMA_SRC_POS);
         }
         ASMIntEnable();
@@ -2106 +2509 @@
             LogRel(("Test #3\n"));
             ASMIntDisable();
-            PGMDynMapMigrateAutoSet(&pVM->aCpus[0]);
+            PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
             pv2 = NULL;
             for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) - 5 && RT_SUCCESS(rc) && pv2 != pv; i++)
             {
                 pv2 = (void *)(intptr_t)(-5 - i);
-                rc = PGMDynMapHCPage(pVM, cr3 + PAGE_SIZE * (i + 5), &pv2);
+                rc = pgmRZDynMapHCPageCommon(pVM, cr3 + PAGE_SIZE * (i + 5), &pv2 RTLOG_COMMA_SRC_POS);
             }
             ASMIntEnable();
@@ -2134 +2537 @@
                 LogRel(("Test #4\n"));
                 ASMIntDisable();
-                PGMDynMapMigrateAutoSet(&pVM->aCpus[0]);
+                PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
                 for (i = 0 ; i < RT_ELEMENTS(pSet->aEntries) + 2; i++)
                 {
-                    rc = PGMDynMapHCPage(pVM, cr3 - PAGE_SIZE * (i + 5), &pv2);
+                    rc = pgmRZDynMapHCPageCommon(pVM, cr3 - PAGE_SIZE * (i + 5), &pv2 RTLOG_COMMA_SRC_POS);
                     if (RT_SUCCESS(rc))
                         rc = PGMR0DynMapAssertIntegrity();
@@ -2149 +2552 @@
                     LogRel(("Test #5\n"));
                     ASMIntDisable();
-                    PGMDynMapMigrateAutoSet(&pVM->aCpus[0]);
-                    PGMDynMapReleaseAutoSet(&pVM->aCpus[0]);
-                    PGMDynMapStartAutoSet(&pVM->aCpus[0]);
+                    PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
+                    PGMRZDynMapReleaseAutoSet(&pVM->aCpus[0]);
+                    PGMRZDynMapStartAutoSet(&pVM->aCpus[0]);
                     ASMIntEnable();
 
@@ -2179 +2582 @@
         LogRel(("Test #5\n"));
         ASMIntDisable();
-        PGMDynMapMigrateAutoSet(&pVM->aCpus[0]);
+        PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
         RTHCPHYS  HCPhysPT = RTR0MemObjGetPagePhysAddr(pThis->pSegHead->ahMemObjPTs[0], 0);
-        rc  = PGMDynMapHCPage(pVM, HCPhysPT, &pv);
+        rc  = pgmRZDynMapHCPageCommon(pVM, HCPhysPT, &pv RTLOG_COMMA_SRC_POS);
         if (RT_SUCCESS(rc))
         {
@@ -2216 +2619 @@
     LogRel(("Cleanup.\n"));
     ASMIntDisable();
-    PGMDynMapMigrateAutoSet(&pVM->aCpus[0]);
-    PGMDynMapFlushAutoSet(&pVM->aCpus[0]);
-    PGMDynMapReleaseAutoSet(&pVM->aCpus[0]);
+    PGMR0DynMapMigrateAutoSet(&pVM->aCpus[0]);
+    PGMRZDynMapFlushAutoSet(&pVM->aCpus[0]);
+    PGMRZDynMapReleaseAutoSet(&pVM->aCpus[0]);
     ASMIntEnable();