Changeset 15344 in vbox

Timestamp:

Dec 12, 2008 12:13:56 AM (16 years ago)

Author:

vboxsync

Message:

#3202: Optimizations of the dynamic page mapping code (ring-0). Do lots of the stuff inline, using the set as a 2st level cache and not releasing it for each inner VT-x iteration.

Location:

trunk

Files:

• include/VBox/pgm.h (modified) (1 diff)
• include/VBox/vm.h (modified) (1 diff)
• src/VBox/VMM/PGM.cpp (modified) (2 diffs)
• src/VBox/VMM/PGMInternal.h (modified) (37 diffs)
• src/VBox/VMM/VMMAll/PGMAll.cpp (modified) (2 diffs)
• src/VBox/VMM/VMMAll/PGMAllPhys.cpp (modified) (1 diff)
• src/VBox/VMM/VMMAll/PGMAllPool.cpp (modified) (2 diffs)
• src/VBox/VMM/VMMR0/HWVMXR0.cpp (modified) (2 diffs)
• src/VBox/VMM/VMMR0/PGMR0DynMap.cpp (modified) (14 diffs)

trunk/include/VBox/pgm.h

@@ -448 +448 @@
 VMMDECL(void)       PGMDynMapStartAutoSet(PVMCPU pVCpu);
 VMMDECL(void)       PGMDynMapReleaseAutoSet(PVMCPU pVCpu);
+VMMDECL(void)       PGMDynMapFlushAutoSet(PVMCPU pVCpu);
 VMMDECL(void)       PGMDynMapMigrateAutoSet(PVMCPU pVCpu);
 

trunk/include/VBox/vm.h

@@ -130 +130 @@
         struct PGMCPU       s;
 #endif
-        char                padding[192];        /* multiple of 64 */
+        char                padding[1024];       /* multiple of 64 */
     } pgm;
 

trunk/src/VBox/VMM/PGM.cpp

@@ -1136 +1136 @@
      */
     pVM->pgm.s.offVM = RT_OFFSETOF(VM, pgm.s);
+    pVM->pgm.s.offVCpu = RT_OFFSETOF(VMCPU, pgm.s);
     pVM->pgm.s.enmShadowMode    = PGMMODE_INVALID;
     pVM->pgm.s.enmGuestMode     = PGMMODE_INVALID;
@@ -1597 +1598 @@
 
     /* R0 only: */
-    STAM_REG(pVM, &pPGM->StatR0DynMapHCPage,                STAMTYPE_PROFILE, "/PGM/R0/DynMapPage/HCPage",          STAMUNIT_OCCURENCES,     "Calls to PGMDynMapHCPage (ring-0).");
-    STAM_REG(pVM, &pPGM->StatR0DynMapHCPageSetOptimize,     STAMTYPE_COUNTER, "/PGM/R0/DynMapPage/HCPageSetOptimize", STAMUNIT_OCCURENCES,   "Calls to pgmDynMapOptimizeAutoSet.");
-    STAM_REG(pVM, &pPGM->StatR0DynMapHCPageSetSearchHits,   STAMTYPE_COUNTER, "/PGM/R0/DynMapPage/HCPageSetSearchHits", STAMUNIT_OCCURENCES, "Set search hits.");
-    STAM_REG(pVM, &pPGM->StatR0DynMapHCPageSetSearchMisses, STAMTYPE_COUNTER, "/PGM/R0/DynMapPage/HCPageSetSearchMisses", STAMUNIT_OCCURENCES, "Set search misses.");
     STAM_REG(pVM, &pPGM->StatR0DynMapMigrateInvlPg,         STAMTYPE_COUNTER, "/PGM/R0/DynMapMigrateInvlPg",        STAMUNIT_OCCURENCES,     "invlpg count in PGMDynMapMigrateAutoSet.");
+    STAM_REG(pVM, &pPGM->StatR0DynMapGCPageInl,             STAMTYPE_PROFILE, "/PGM/R0/DynMapPageGCPageInl",        STAMUNIT_TICKS_PER_CALL, "Calls to pgmR0DynMapGCPageInlined.");
+    STAM_REG(pVM, &pPGM->StatR0DynMapGCPageInlHits,         STAMTYPE_COUNTER, "/PGM/R0/DynMapPageGCPageInl/Hits",   STAMUNIT_OCCURENCES,     "Hash table lookup hits.");
+    STAM_REG(pVM, &pPGM->StatR0DynMapGCPageInlMisses,       STAMTYPE_COUNTER, "/PGM/R0/DynMapPageGCPageInl/Misses", STAMUNIT_OCCURENCES,     "Misses that falls back to code common with PGMDynMapHCPage.");
+    STAM_REG(pVM, &pPGM->StatR0DynMapGCPageInlRamHits,      STAMTYPE_COUNTER, "/PGM/R0/DynMapPageGCPageInl/RamHits",   STAMUNIT_OCCURENCES,  "1st ram range hits.");
+    STAM_REG(pVM, &pPGM->StatR0DynMapGCPageInlRamMisses,    STAMTYPE_COUNTER, "/PGM/R0/DynMapPageGCPageInl/RamMisses", STAMUNIT_OCCURENCES,  "1st ram range misses, takes slow path.");
+    STAM_REG(pVM, &pPGM->StatR0DynMapHCPageInl,             STAMTYPE_PROFILE, "/PGM/R0/DynMapPageHCPageInl",        STAMUNIT_TICKS_PER_CALL, "Calls to pgmR0DynMapHCPageInlined.");
+    STAM_REG(pVM, &pPGM->StatR0DynMapHCPageInlHits,         STAMTYPE_COUNTER, "/PGM/R0/DynMapPageHCPageInl/Hits",   STAMUNIT_OCCURENCES,     "Hash table lookup hits.");
+    STAM_REG(pVM, &pPGM->StatR0DynMapHCPageInlMisses,       STAMTYPE_COUNTER, "/PGM/R0/DynMapPageHCPageInl/Misses", STAMUNIT_OCCURENCES,     "Misses that falls back to code common with PGMDynMapHCPage.");
     STAM_REG(pVM, &pPGM->StatR0DynMapPage,                  STAMTYPE_COUNTER, "/PGM/R0/DynMapPage",                 STAMUNIT_OCCURENCES,     "Calls to pgmR0DynMapPage");
-    STAM_REG(pVM, &pPGM->StatR0DynMapPageHit0,              STAMTYPE_COUNTER, "/PGM/R0/DynMapPage/Hit0",            STAMUNIT_OCCURENCES,     "Hit at iPage+0");
-    STAM_REG(pVM, &pPGM->StatR0DynMapPageHit1,              STAMTYPE_COUNTER, "/PGM/R0/DynMapPage/Hit1",            STAMUNIT_OCCURENCES,     "Hit at iPage+1");
-    STAM_REG(pVM, &pPGM->StatR0DynMapPageHit2,              STAMTYPE_COUNTER, "/PGM/R0/DynMapPage/Hit2",            STAMUNIT_OCCURENCES,     "Hit at iPage+2");
+    STAM_REG(pVM, &pPGM->StatR0DynMapSetOptimize,           STAMTYPE_COUNTER, "/PGM/R0/DynMapPage/SetOptimize",     STAMUNIT_OCCURENCES,     "Calls to pgmDynMapOptimizeAutoSet.");
+    STAM_REG(pVM, &pPGM->StatR0DynMapSetSearchHits,         STAMTYPE_COUNTER, "/PGM/R0/DynMapPage/SetSearchHits",   STAMUNIT_OCCURENCES,     "Set search hits.");
+    STAM_REG(pVM, &pPGM->StatR0DynMapSetSearchMisses,       STAMTYPE_COUNTER, "/PGM/R0/DynMapPage/SetSearchMisses", STAMUNIT_OCCURENCES,     "Set search misses.");
+    STAM_REG(pVM, &pPGM->StatR0DynMapHCPage,                STAMTYPE_PROFILE, "/PGM/R0/DynMapPage/HCPage",          STAMUNIT_TICKS_PER_CALL, "Calls to PGMDynMapHCPage (ring-0).");
+    STAM_REG(pVM, &pPGM->StatR0DynMapPageHits0,             STAMTYPE_COUNTER, "/PGM/R0/DynMapPage/Hits0",           STAMUNIT_OCCURENCES,     "Hits at iPage+0");
+    STAM_REG(pVM, &pPGM->StatR0DynMapPageHits1,             STAMTYPE_COUNTER, "/PGM/R0/DynMapPage/Hits1",           STAMUNIT_OCCURENCES,     "Hits at iPage+1");
+    STAM_REG(pVM, &pPGM->StatR0DynMapPageHits2,             STAMTYPE_COUNTER, "/PGM/R0/DynMapPage/Hits2",           STAMUNIT_OCCURENCES,     "Hits at iPage+2");
     STAM_REG(pVM, &pPGM->StatR0DynMapPageInvlPg,            STAMTYPE_COUNTER, "/PGM/R0/DynMapPage/InvlPg",          STAMUNIT_OCCURENCES,     "invlpg count in pgmR0DynMapPageSlow.");
     STAM_REG(pVM, &pPGM->StatR0DynMapPageSlow,              STAMTYPE_COUNTER, "/PGM/R0/DynMapPage/Slow",            STAMUNIT_OCCURENCES,     "Calls to pgmR0DynMapPageSlow - subtract this from pgmR0DynMapPage to get 1st level hits.");

trunk/src/VBox/VMM/PGMInternal.h

@@ -226 +226 @@
  * @remark  There is no need to assert on the result.
  */
-#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
+#ifdef IN_RC
 # define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
      PGMDynMapHCPage(pVM, HCPhys, (void **)(ppv))
+#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
+# define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
+     pgmR0DynMapHCPageInlined(&(pVM)->pgm.s, HCPhys, (void **)(ppv))
 #else
 # define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
      MMPagePhys2PageEx(pVM, HCPhys, (void **)(ppv))
+#endif
+
+/** @def PGM_HCPHYS_2_PTR_BY_PGM
+ * Maps a HC physical page pool address to a virtual address.
+ *
+ * @returns VBox status code.
+ * @param   pPGM    The PGM instance data.
+ * @param   HCPhys  The HC physical address to map to a virtual one.
+ * @param   ppv     Where to store the virtual address. No need to cast this.
+ *
+ * @remark  In GC this uses PGMGCDynMapHCPage(), so it will consume of the
+ *          small page window employeed by that function. Be careful.
+ * @remark  There is no need to assert on the result.
+ */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+# define PGM_HCPHYS_2_PTR_BY_PGM(pPGM, HCPhys, ppv) \
+     pgmR0DynMapHCPageInlined(pPGM, HCPhys, (void **)(ppv))
+#else
+# define PGM_HCPHYS_2_PTR_BY_PGM(pPGM, HCPhys, ppv) \
+     PGM_HCPHYS_2_PTR(PGM2VM(pPGM), HCPhys, (void **)(ppv))
 #endif
 
@@ -246 +269 @@
  * @remark  There is no need to assert on the result.
  */
-#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
+#ifdef IN_RC
 # define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
      PGMDynMapGCPage(pVM, GCPhys, (void **)(ppv))
+#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
+# define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
+     pgmR0DynMapGCPageInlined(&(pVM)->pgm.s, GCPhys, (void **)(ppv))
 #else
 # define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
      PGMPhysGCPhys2R3Ptr(pVM, GCPhys, 1 /* one page only */, (PRTR3PTR)(ppv)) /** @todo this isn't asserting, use PGMRamGCPhys2HCPtr! */
+#endif
+
+/** @def PGM_GCPHYS_2_PTR_BY_PGM
+ * Maps a GC physical page address to a virtual address.
+ *
+ * @returns VBox status code.
+ * @param   pPGM    Pointer to the PGM instance data.
+ * @param   GCPhys  The GC physical address to map to a virtual one.
+ * @param   ppv     Where to store the virtual address. No need to cast this.
+ *
+ * @remark  In GC this uses PGMGCDynMapGCPage(), so it will consume of the
+ *          small page window employeed by that function. Be careful.
+ * @remark  There is no need to assert on the result.
+ */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+# define PGM_GCPHYS_2_PTR_BY_PGM(pPGM, GCPhys, ppv) \
+     pgmR0DynMapGCPageInlined(pPGM, GCPhys, (void **)(ppv))
+#else
+# define PGM_GCPHYS_2_PTR_BY_PGM(pPGM, GCPhys, ppv) \
+     PGM_GCPHYS_2_PTR(PGM2VM(pPGM), GCPhys, ppv)
 #endif
 
@@ -1253 +1299 @@
      * The max is UINT16_MAX - 1. */
     uint16_t                    cRefs;
+    /** Pointer to the page. */
+    RTR0PTR                     pvPage;
+    /** The physical address for this entry. */
+    RTHCPHYS                    HCPhys;
 } PGMMAPSETENTRY;
 /** Pointer to a mapping cache usage set entry. */
@@ -1271 +1321 @@
     /** The entries. */
     PGMMAPSETENTRY              aEntries[32];
+    /** HCPhys -> iEntry fast lookup table.
+     * Use PGMMAPSET_HASH for hashing.
+     * The entries may or may not be valid, check against cEntries. */
+    uint8_t                     aiHashTable[64];
 } PGMMAPSET;
 /** Pointer to the mapping cache set. */
@@ -1276 +1330 @@
 
 /** PGMMAPSET::cEntries value for a closed set. */
-#define PGMMAPSET_CLOSED        UINT32_C(0xdeadc0fe)
+#define PGMMAPSET_CLOSED            UINT32_C(0xdeadc0fe)
+
+/** Hash function for aiHashTable. */
+#define PGMMAPSET_HASH(HCPhys)      (((HCPhys) >> PAGE_SHIFT) & 63)
 
 
@@ -2042 +2099 @@
     /** Offset to the VM structure. */
     RTINT                           offVM;
+    /** Offset of the PGMCPU structure relative to VMCPU. */
+    int32_t                         offVCpu;
+    /** Alignment padding. */
+    int32_t                         i32Alignment;
 
     /*
@@ -2561 +2622 @@
 
     /* R0 only: */
+    STAMCOUNTER StatR0DynMapMigrateInvlPg;          /**< R0: invlpg in PGMDynMapMigrateAutoSet. */
+    STAMPROFILE StatR0DynMapGCPageInl;              /**< R0: Calls to pgmR0DynMapGCPageInlined. */
+    STAMCOUNTER StatR0DynMapGCPageInlHits;          /**< R0: Hash table lookup hits. */
+    STAMCOUNTER StatR0DynMapGCPageInlMisses;        /**< R0: Misses that falls back to code common with PGMDynMapHCPage. */
+    STAMCOUNTER StatR0DynMapGCPageInlRamHits;       /**< R0: 1st ram range hits. */
+    STAMCOUNTER StatR0DynMapGCPageInlRamMisses;     /**< R0: 1st ram range misses, takes slow path. */
+    STAMPROFILE StatR0DynMapHCPageInl;              /**< R0: Calls to pgmR0DynMapHCPageInlined. */
+    STAMCOUNTER StatR0DynMapHCPageInlHits;          /**< R0: Hash table lookup hits. */
+    STAMCOUNTER StatR0DynMapHCPageInlMisses;        /**< R0: Misses that falls back to code common with PGMDynMapHCPage. */
     STAMPROFILE StatR0DynMapHCPage;                 /**< R0: Calls to PGMDynMapHCPage. */
-    STAMCOUNTER StatR0DynMapHCPageSetOptimize;      /**< R0: Calls to pgmDynMapOptimizeAutoSet. */
-    STAMCOUNTER StatR0DynMapHCPageSetSearchHits;    /**< R0: Set search hits. */
-    STAMCOUNTER StatR0DynMapHCPageSetSearchMisses;  /**< R0: Set search misses. */
-    STAMCOUNTER StatR0DynMapMigrateInvlPg;          /**< R0: invlpg in PGMDynMapMigrateAutoSet. */
+    STAMCOUNTER StatR0DynMapSetOptimize;            /**< R0: Calls to pgmDynMapOptimizeAutoSet. */
+    STAMCOUNTER StatR0DynMapSetSearchHits;          /**< R0: Set search hits. */
+    STAMCOUNTER StatR0DynMapSetSearchMisses;        /**< R0: Set search misses. */
     STAMCOUNTER StatR0DynMapPage;                   /**< R0: Calls to pgmR0DynMapPage. */
-    STAMCOUNTER StatR0DynMapPageHit0;               /**< R0: Hit at iPage+0. */
-    STAMCOUNTER StatR0DynMapPageHit1;               /**< R0: Hit at iPage+1. */
-    STAMCOUNTER StatR0DynMapPageHit2;               /**< R0: Hit at iPage+2. */
+    STAMCOUNTER StatR0DynMapPageHits0;              /**< R0: Hits at iPage+0. */
+    STAMCOUNTER StatR0DynMapPageHits1;              /**< R0: Hits at iPage+1. */
+    STAMCOUNTER StatR0DynMapPageHits2;              /**< R0: Hits at iPage+2. */
     STAMCOUNTER StatR0DynMapPageInvlPg;             /**< R0: invlpg. */
     STAMCOUNTER StatR0DynMapPageSlow;               /**< R0: Calls to pgmR0DynMapPageSlow. */
@@ -2813 +2882 @@
 int             pgmR3PhysRamReset(PVM pVM);
 int             pgmR3PhysRomReset(PVM pVM);
-#ifndef VBOX_WITH_NEW_PHYS_CODE
+# ifndef VBOX_WITH_NEW_PHYS_CODE
 int             pgmr3PhysGrowRange(PVM pVM, RTGCPHYS GCPhys);
-#endif
+# endif
 
 int             pgmR3PoolInit(PVM pVM);
@@ -2822 +2891 @@
 
 #endif /* IN_RING3 */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+int             pgmR0DynMapHCPageCommon(PVM pVM, PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv);
+#endif
 #if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
 void           *pgmPoolMapPage(PVM pVM, PPGMPOOLPAGE pPage);
@@ -3157 +3229 @@
 }
 
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+
+/**
+ * Inlined version of the ring-0 version of PGMDynMapHCPage that
+ * optimizes access to pages already in the set.
+ *
+ * @returns See pgmR0DynMapHCPageCommon.
+ * @param   pPGM        Pointer to the PVM instance data.
+ * @param   HCPhys      The physical address of the page.
+ * @param   ppv         Where to store the mapping address.
+ */
+DECLINLINE(int) pgmR0DynMapHCPageInlined(PPGM pPGM, RTHCPHYS HCPhys, void **ppv)
+{
+    STAM_PROFILE_START(&pPGM->StatR0DynMapHCPageInl, a);
+    PPGMMAPSET  pSet    = &((PPGMCPU)((uint8_t *)VMMGetCpu(PGM2VM(pPGM)) + pPGM->offVCpu))->AutoSet; /* very pretty ;-) */
+    Assert(!(HCPhys & PAGE_OFFSET_MASK));
+    Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
+    int         rc;
+
+    unsigned    iHash   = PGMMAPSET_HASH(HCPhys);
+    unsigned    iEntry  = pSet->aiHashTable[iHash];
+    if (    iEntry < pSet->cEntries
+        &&  pSet->aEntries[iEntry].HCPhys == HCPhys)
+    {
+        *ppv = pSet->aEntries[iEntry].pvPage;
+        STAM_COUNTER_INC(&pPGM->StatR0DynMapHCPageInlHits);
+        rc = VINF_SUCCESS;
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pPGM->StatR0DynMapHCPageInlMisses);
+        rc = pgmR0DynMapHCPageCommon(PGM2VM(pPGM), pSet, HCPhys, ppv);
+    }
+
+    STAM_PROFILE_STOP(&pPGM->StatR0DynMapHCPageInl, a);
+    return rc;
+}
+
+
+/**
+ * Inlined version of the ring-0 version of PGMDynMapGCPage that optimizes
+ * access to pages already in the set.
+ *
+ * @returns See pgmR0DynMapHCPageCommon.
+ * @param   pPGM        Pointer to the PVM instance data.
+ * @param   HCPhys      The physical address of the page.
+ * @param   ppv         Where to store the mapping address.
+ */
+DECLINLINE(int) pgmR0DynMapGCPageInlined(PPGM pPGM, RTGCPHYS GCPhys, void **ppv)
+{
+    STAM_PROFILE_START(&pPGM->StatR0DynMapGCPageInl, a);
+    Assert(!(GCPhys & PAGE_OFFSET_MASK));
+
+    /*
+     * Get the ram range.
+     */
+    PPGMRAMRANGE    pRam = pPGM->CTX_SUFF(pRamRanges);
+    RTGCPHYS        off = GCPhys - pRam->GCPhys;
+    if (RT_UNLIKELY(off >= pRam->cb
+        /** @todo   || page state stuff */))
+    {
+        /* This case is not counted into StatR0DynMapGCPageInl. */
+        STAM_COUNTER_INC(&pPGM->StatR0DynMapGCPageInlRamMisses);
+        return PGMDynMapGCPage(PGM2VM(pPGM), GCPhys, ppv);
+    }
+
+    RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
+    STAM_COUNTER_INC(&pPGM->StatR0DynMapGCPageInlRamHits);
+
+    /*
+     * pgmR0DynMapHCPageInlined with out stats.
+     */
+    PPGMMAPSET  pSet    = &((PPGMCPU)((uint8_t *)VMMGetCpu(PGM2VM(pPGM)) + pPGM->offVCpu))->AutoSet; /* very pretty ;-) */
+    Assert(!(HCPhys & PAGE_OFFSET_MASK));
+    Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
+    int         rc;
+
+    unsigned    iHash   = PGMMAPSET_HASH(HCPhys);
+    unsigned    iEntry  = pSet->aiHashTable[iHash];
+    if (    iEntry < pSet->cEntries
+        &&  pSet->aEntries[iEntry].HCPhys == HCPhys)
+    {
+        *ppv = pSet->aEntries[iEntry].pvPage;
+        STAM_COUNTER_INC(&pPGM->StatR0DynMapGCPageInlHits);
+        rc = VINF_SUCCESS;
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pPGM->StatR0DynMapGCPageInlMisses);
+        rc = pgmR0DynMapHCPageCommon(PGM2VM(pPGM), pSet, HCPhys, ppv);
+    }
+
+    STAM_PROFILE_STOP(&pPGM->StatR0DynMapGCPageInl, a);
+    return rc;
+}
+
+#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
 
 #ifndef IN_RC
@@ -3393 +3562 @@
 #ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
     PCX86PD pGuestPD = 0;
-    int rc = PGMDynMapGCPage(PGM2VM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
+    int rc = pgmR0DynMapGCPageInlined(pPGM, pPGM->GCPhysCR3, (void **)&pGuestPD);
     if (RT_FAILURE(rc))
     {
@@ -3417 +3586 @@
 #ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
     PX86PD pGuestPD = 0;
-    int rc = PGMDynMapGCPage(PGM2VM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
+    int rc = pgmR0DynMapGCPageInlined(pPGM, pPGM->GCPhysCR3, (void **)&pGuestPD);
     AssertRCReturn(rc, 0);
     return &pGuestPD->a[GCPtr >> X86_PD_SHIFT];
@@ -3436 +3605 @@
 #ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
     PX86PD pGuestPD = 0;
-    int rc = PGMDynMapGCPage(PGM2VM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
+    int rc = pgmR0DynMapGCPageInlined(pPGM, pPGM->GCPhysCR3, (void **)&pGuestPD);
     AssertRCReturn(rc, 0);
     return pGuestPD;
@@ -3456 +3625 @@
 #ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
     PX86PDPT pGuestPDPT = 0;
-    int rc = PGMDynMapGCPage(PGM2VM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPDPT);
+    int rc = pgmR0DynMapGCPageInlined(pPGM, pPGM->GCPhysCR3, (void **)&pGuestPDPT);
     AssertRCReturn(rc, 0);
     return pGuestPDPT;
@@ -3479 +3648 @@
 #ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
     PX86PDPT pGuestPDPT = 0;
-    int rc = PGMDynMapGCPage(PGM2VM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPDPT);
+    int rc = pgmR0DynMapGCPageInlined(pPGM, pPGM->GCPhysCR3, (void **)&pGuestPDPT);
     AssertRCReturn(rc, 0);
     return &pGuestPDPT->a[(GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE];
@@ -3516 +3685 @@
         /* cache is out-of-sync. */
         PX86PDPAE pPD;
-        int rc = PGM_GCPHYS_2_PTR(PGM2VM(pPGM), pGuestPDPT->a[iPdPt].u & X86_PDPE_PG_MASK, &pPD);
+        int rc = PGM_GCPHYS_2_PTR_BY_PGM(pPGM, pGuestPDPT->a[iPdPt].u & X86_PDPE_PG_MASK, &pPD);
         if (RT_SUCCESS(rc))
             return pPD;
@@ -3555 +3724 @@
         /* The cache is out-of-sync. */
         PX86PDPAE pPD;
-        int rc = PGM_GCPHYS_2_PTR(PGM2VM(pPGM), pGuestPDPT->a[iPdPt].u & X86_PDPE_PG_MASK, &pPD);
+        int rc = PGM_GCPHYS_2_PTR_BY_PGM(pPGM, pGuestPDPT->a[iPdPt].u & X86_PDPE_PG_MASK, &pPD);
         if (RT_SUCCESS(rc))
             return &pPD->a[iPD];
@@ -3595 +3764 @@
             /* cache is out-of-sync. */
             PX86PDPAE pPD;
-            int rc = PGM_GCPHYS_2_PTR(PGM2VM(pPGM), pGuestPDPT->a[iPdPt].u & X86_PDPE_PG_MASK, &pPD);
+            int rc = PGM_GCPHYS_2_PTR_BY_PGM(pPGM, pGuestPDPT->a[iPdPt].u & X86_PDPE_PG_MASK, &pPD);
             if (RT_SUCCESS(rc))
                 return pPD->a[iPD];
@@ -3643 +3812 @@
         /* cache is out-of-sync. */
         PX86PDPAE pPD;
-        int rc = PGM_GCPHYS_2_PTR(PGM2VM(pPGM), pGuestPDPT->a[iPdPt].u & X86_PDPE_PG_MASK, &pPD);
+        int rc = PGM_GCPHYS_2_PTR_BY_PGM(pPGM, pGuestPDPT->a[iPdPt].u & X86_PDPE_PG_MASK, &pPD);
         if (RT_SUCCESS(rc))
         {
@@ -3667 +3836 @@
 #ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
     PX86PML4 pGuestPml4;
-    int rc = PGMDynMapGCPage(PGM2VM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
+    int rc = pgmR0DynMapGCPageInlined(pPGM, pPGM->GCPhysCR3, (void **)&pGuestPml4);
     AssertRCReturn(rc, NULL);
     return pGuestPml4;
@@ -3688 +3857 @@
 #ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
     PX86PML4 pGuestPml4;
-    int rc = PGMDynMapGCPage(PGM2VM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
+    int rc = pgmR0DynMapGCPageInlined(pPGM, pPGM->GCPhysCR3, (void **)&pGuestPml4);
     AssertRCReturn(rc, NULL);
     return &pGuestPml4->a[iPml4];
@@ -3709 +3878 @@
 #ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
     PX86PML4 pGuestPml4;
-    int rc = PGMDynMapGCPage(PGM2VM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
+    int rc = pgmR0DynMapGCPageInlined(pPGM, pPGM->GCPhysCR3, (void **)&pGuestPml4);
     if (RT_FAILURE(rc))
     {
@@ -3740 +3909 @@
     {
         PX86PDPT pPdpt;
-        int rc = PGM_GCPHYS_2_PTR(PGM2VM(pPGM), pPml4e->u & X86_PML4E_PG_MASK, &pPdpt);
+        int rc = PGM_GCPHYS_2_PTR_BY_PGM(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdpt);
         AssertRCReturn(rc, NULL);
 
@@ -3769 +3938 @@
     {
         PCX86PDPT   pPdptTemp;
-        int rc = PGM_GCPHYS_2_PTR(PGM2VM(pPGM), pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
+        int rc = PGM_GCPHYS_2_PTR_BY_PGM(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
         AssertRCReturn(rc, ZeroPde);
 
@@ -3777 +3946 @@
         {
             PCX86PDPAE pPD;
-            rc = PGM_GCPHYS_2_PTR(PGM2VM(pPGM), pPdptTemp->a[iPdPt].u & X86_PDPE_PG_MASK, &pPD);
+            rc = PGM_GCPHYS_2_PTR_BY_PGM(pPGM, pPdptTemp->a[iPdPt].u & X86_PDPE_PG_MASK, &pPD);
             AssertRCReturn(rc, ZeroPde);
 
@@ -3805 +3974 @@
     {
         PCX86PDPT   pPdptTemp;
-        int rc = PGM_GCPHYS_2_PTR(PGM2VM(pPGM), pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
+        int rc = PGM_GCPHYS_2_PTR_BY_PGM(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
         AssertRCReturn(rc, ZeroPde);
 
@@ -3812 +3981 @@
         {
             PCX86PDPAE pPD;
-            rc = PGM_GCPHYS_2_PTR(PGM2VM(pPGM), pPdptTemp->a[iPdPt].u & X86_PDPE_PG_MASK, &pPD);
+            rc = PGM_GCPHYS_2_PTR_BY_PGM(pPGM, pPdptTemp->a[iPdPt].u & X86_PDPE_PG_MASK, &pPD);
             AssertRCReturn(rc, ZeroPde);
 
@@ -3838 +4007 @@
     {
         PCX86PDPT   pPdptTemp;
-        int rc = PGM_GCPHYS_2_PTR(PGM2VM(pPGM), pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
+        int rc = PGM_GCPHYS_2_PTR_BY_PGM(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
         AssertRCReturn(rc, NULL);
 
@@ -3845 +4014 @@
         {
             PX86PDPAE pPD;
-            rc = PGM_GCPHYS_2_PTR(PGM2VM(pPGM), pPdptTemp->a[iPdPt].u & X86_PDPE_PG_MASK, &pPD);
+            rc = PGM_GCPHYS_2_PTR_BY_PGM(pPGM, pPdptTemp->a[iPdPt].u & X86_PDPE_PG_MASK, &pPD);
             AssertRCReturn(rc, NULL);
 
@@ -3875 +4044 @@
     {
         PCX86PDPT   pPdptTemp;
-        int rc = PGM_GCPHYS_2_PTR(PGM2VM(pPGM), pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
+        int rc = PGM_GCPHYS_2_PTR_BY_PGM(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
         AssertRCReturn(rc, NULL);
 
@@ -3883 +4052 @@
         {
             PX86PDPAE pPD;
-            rc = PGM_GCPHYS_2_PTR(PGM2VM(pPGM), pPdptTemp->a[iPdPt].u & X86_PDPE_PG_MASK, &pPD);
+            rc = PGM_GCPHYS_2_PTR_BY_PGM(pPGM, pPdptTemp->a[iPdPt].u & X86_PDPE_PG_MASK, &pPD);
             AssertRCReturn(rc, NULL);
 
@@ -3910 +4079 @@
     PX86PD          pShwPd;
     Assert(pPGM->HCPhysShw32BitPD != 0 && pPGM->HCPhysShw32BitPD != NIL_RTHCPHYS);
-    int rc = PGM_HCPHYS_2_PTR(PGM2VM(pPGM), pPGM->HCPhysShw32BitPD, &pShwPd);
+    int rc = PGM_HCPHYS_2_PTR_BY_PGM(pPGM, pPGM->HCPhysShw32BitPD, &pShwPd);
     AssertRCReturn(rc, NULL);
     return pShwPd;
@@ -3973 +4142 @@
     PX86PDPT pShwPdpt;
     Assert(pPGM->HCPhysShwPaePdpt != 0 && pPGM->HCPhysShwPaePdpt != NIL_RTHCPHYS);
-    int rc = PGM_HCPHYS_2_PTR(PGM2VM(pPGM), pPGM->HCPhysShwPaePdpt, &pShwPdpt);
+    int rc = PGM_HCPHYS_2_PTR_BY_PGM(pPGM, pPGM->HCPhysShwPaePdpt, &pShwPdpt);
     AssertRCReturn(rc, 0);
     return pShwPdpt;
@@ -4005 +4174 @@
 # ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
     PX86PDPAE       pPD;
-    int rc = PGM_HCPHYS_2_PTR(PGM2VM(pPGM), pPGM->aHCPhysPaePDs[iPdpt], &pPD);
+    int rc = PGM_HCPHYS_2_PTR_BY_PGM(pPGM, pPGM->aHCPhysPaePDs[iPdpt], &pPD);
     AssertRCReturn(rc, 0);
     return pPD;
@@ -4070 +4239 @@
     PX86PML4 pShwPml4;
     Assert(pPGM->HCPhysShwPaePml4 != 0 && pPGM->HCPhysShwPaePml4 != NIL_RTHCPHYS);
-    int rc = PGM_HCPHYS_2_PTR(PGM2VM(pPGM), pPGM->HCPhysShwPaePml4, &pShwPml4);
+    int rc = PGM_HCPHYS_2_PTR_BY_PGM(pPGM, pPGM->HCPhysShwPaePml4, &pShwPml4);
     AssertRCReturn(rc, 0);
     return pShwPml4;
@@ -4136 +4305 @@
     {
         PCX86PDPT   pPdptTemp;
-        int rc = PGM_GCPHYS_2_PTR(PGM2VM(pPGM), pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
+        int rc = PGM_GCPHYS_2_PTR_BY_PGM(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
         AssertRCReturn(rc, NULL);
 
@@ -4143 +4312 @@
         {
             PX86PDPAE pPD;
-            rc = PGM_GCPHYS_2_PTR(PGM2VM(pPGM), pPdptTemp->a[iPdPt].u & X86_PDPE_PG_MASK, &pPD);
+            rc = PGM_GCPHYS_2_PTR_BY_PGM(pPGM, pPdptTemp->a[iPdPt].u & X86_PDPE_PG_MASK, &pPD);
             AssertRCReturn(rc, NULL);
 

trunk/src/VBox/VMM/VMMAll/PGMAll.cpp

@@ -1850 +1850 @@
     RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT]);
     //Log(("PGMDynMapGCPage: GCPhys=%RGp HCPhys=%RHp\n", GCPhys, HCPhys));
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    return pgmR0DynMapHCPageInlined(&pVM->pgm.s, HCPhys, ppv);
+#else
     return PGMDynMapHCPage(pVM, HCPhys, ppv);
+#endif
 }
 
@@ -1887 +1891 @@
      */
     RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT]);
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    int rc = pgmR0DynMapHCPageInlined(&pVM->pgm.s, HCPhys, ppv);
+#else
     int rc = PGMDynMapHCPage(pVM, HCPhys, ppv);
+#endif
     if (RT_SUCCESS(rc))
         *ppv = (void *)((uintptr_t)*ppv | (GCPhys & PAGE_OFFSET_MASK));

trunk/src/VBox/VMM/VMMAll/PGMAllPhys.cpp

@@ -500 +500 @@
     RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
     Assert(HCPhys != pVM->pgm.s.HCPhysZeroPg);
+# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    return pgmR0DynMapHCPageInlined(&pVM->pgm.s, HCPhys, ppv);
+# else
     return PGMDynMapHCPage(pVM, HCPhys, ppv);
+# endif
 
 #else /* IN_RING3 || IN_RING0 */

trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp

@@ -102 +102 @@
         Assert(pPage->idx < pVM->pgm.s.CTX_SUFF(pPool)->cCurPages);
         void *pv;
+# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+        int rc = pgmR0DynMapHCPageInlined(&pVM->pgm.s, pPage->Core.Key, &pv);
+# else
         int rc = PGMDynMapHCPage(pVM, pPage->Core.Key, &pv);
+# endif
         AssertReleaseRC(rc);
         return pv;
@@ -159 +163 @@
     }
     void *pv;
-    int rc = PGMDynMapHCPage(pVM, HCPhys, &pv);
+    int rc = pgmR0DynMapHCPageInlined(&pVM->pgm.s, HCPhys, &pv);
     AssertReleaseRC(rc);
     return pv;

trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp

@@ -2040 +2040 @@
 #endif
 #ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-    PGMDynMapReleaseAutoSet(pVCpu);
+    PGMDynMapFlushAutoSet(pVCpu);
 #endif
 
@@ -2195 +2195 @@
     STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, v);
     STAM_STATS({ STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = true; });
-
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-    PGMDynMapStartAutoSet(pVCpu);
-#endif
 
     /* Some cases don't need a complete resync of the guest CPU state; handle them here. */

trunk/src/VBox/VMM/VMMR0/PGMR0DynMap.cpp

@@ -361 +361 @@
         while (j-- > 0)
         {
-            pSet->aEntries[j].iPage = UINT16_MAX;
-            pSet->aEntries[j].cRefs = 0;
+            pSet->aEntries[j].iPage  = UINT16_MAX;
+            pSet->aEntries[j].cRefs  = 0;
+            pSet->aEntries[j].pvPage = NULL;
+            pSet->aEntries[j].HCPhys = NIL_RTHCPHYS;
         }
         pSet->cEntries = PGMMAPSET_CLOSED;
+        memset(&pSet->aiHashTable[0], 0xff, sizeof(pSet->aiHashTable));
     }
 
@@ -456 +459 @@
                         AssertLogRelMsgFailed(("cRefs=%d iPage=%#x cPages=%u\n", cRefs, iPage, pThis->cPages));
 
-                    pSet->aEntries[j].iPage = UINT16_MAX;
-                    pSet->aEntries[j].cRefs = 0;
+                    pSet->aEntries[j].iPage  = UINT16_MAX;
+                    pSet->aEntries[j].cRefs  = 0;
+                    pSet->aEntries[j].pvPage = NULL;
+                    pSet->aEntries[j].HCPhys = NIL_RTHCPHYS;
                 }
                 pSet->cEntries = PGMMAPSET_CLOSED;
@@ -1289 +1294 @@
     PPGMR0DYNMAPENTRY   paPages = pThis->paPages;
     if (RT_LIKELY(paPages[iPage].HCPhys == HCPhys))
-        STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageHit0);
+        STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageHits0);
     else
     {
@@ -1296 +1301 @@
         {
             iPage = iPage2;
-            STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageHit1);
+            STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageHits1);
         }
         else
@@ -1304 +1309 @@
             {
                 iPage = iPage2;
-                STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageHit2);
+                STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapPageHits2);
             }
             else
@@ -1496 +1501 @@
 
 /**
+ * Worker that performs the actual flushing of the set.
+ *
+ * @param   pSet        The set to flush.
+ * @param   cEntries    The number of entries.
+ */
+DECLINLINE(void) pgmDynMapFlushAutoSetWorker(PPGMMAPSET pSet, uint32_t cEntries)
+{
+    /*
+     * Release any pages it's referencing.
+     */
+    if (    cEntries != 0
+        &&  RT_LIKELY(cEntries <= RT_ELEMENTS(pSet->aEntries)))
+    {
+        PPGMR0DYNMAP    pThis = g_pPGMR0DynMap;
+        RTSPINLOCKTMP   Tmp = RTSPINLOCKTMP_INITIALIZER;
+        RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
+
+        uint32_t i = cEntries;
+        while (i-- > 0)
+        {
+            uint32_t iPage = pSet->aEntries[i].iPage;
+            Assert(iPage < pThis->cPages);
+            int32_t  cRefs = pSet->aEntries[i].cRefs;
+            Assert(cRefs > 0);
+            pgmR0DynMapReleasePageLocked(pThis, iPage, cRefs);
+
+            pSet->aEntries[i].iPage = UINT16_MAX;
+            pSet->aEntries[i].cRefs = 0;
+        }
+
+        Assert(pThis->cLoad <= pThis->cPages - pThis->cGuardPages);
+        RTSpinlockRelease(pThis->hSpinlock, &Tmp);
+    }
+}
+
+
+/**
  * Releases the dynamic memory mappings made by PGMDynMapHCPage and associates
  * since the PGMDynMapStartAutoSet call.
  *
- * If the set is already closed, nothing will be done.
- *
  * @param   pVCpu       The shared data for the current virtual CPU.
  */
@@ -1508 +1548 @@
 
     /*
-     * Is the set open?
-     *
-     * We might be closed before VM execution and not reopened again before
-     * we leave for ring-3 or something.
-     */
-    uint32_t    i = pSet->cEntries;
-    if (i != PGMMAPSET_CLOSED)
-    {
-        /*
-         * Close the set
-         */
-        AssertMsg(i <= RT_ELEMENTS(pSet->aEntries), ("%#x (%u)\n", i, i));
-        pSet->cEntries = PGMMAPSET_CLOSED;
-
-        /*
-         * Release any pages it's referencing.
-         */
-        if (i != 0 && RT_LIKELY(i <= RT_ELEMENTS(pSet->aEntries)))
-        {
-            PPGMR0DYNMAP    pThis = g_pPGMR0DynMap;
-            RTSPINLOCKTMP   Tmp = RTSPINLOCKTMP_INITIALIZER;
-            RTSpinlockAcquire(pThis->hSpinlock, &Tmp);
-
-            while (i-- > 0)
-            {
-                uint32_t iPage = pSet->aEntries[i].iPage;
-                Assert(iPage < pThis->cPages);
-                int32_t  cRefs = pSet->aEntries[i].cRefs;
-                Assert(cRefs > 0);
-                pgmR0DynMapReleasePageLocked(pThis, iPage, cRefs);
-
-                pSet->aEntries[i].iPage = UINT16_MAX;
-                pSet->aEntries[i].cRefs = 0;
-            }
-
-            Assert(pThis->cLoad <= pThis->cPages - pThis->cGuardPages);
-            RTSpinlockRelease(pThis->hSpinlock, &Tmp);
-        }
-    }
-}
+     * Close and flush the set.
+     */
+    uint32_t    cEntries = pSet->cEntries;
+    AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
+    AssertMsg(cEntries <= RT_ELEMENTS(pSet->aEntries), ("%#x (%u)\n", cEntries, cEntries));
+    pSet->cEntries = PGMMAPSET_CLOSED;
+
+    pgmDynMapFlushAutoSetWorker(pSet, cEntries);
+}
+
+
+/**
+ * Flushes the set if it's above a certain threshold.
+ *
+ * @param   pVCpu       The shared data for the current virtual CPU.
+ */
+VMMDECL(void) PGMDynMapFlushAutoSet(PVMCPU pVCpu)
+{
+    PPGMMAPSET  pSet = &pVCpu->pgm.s.AutoSet;
+
+    /*
+     * Only flush it if it's 50% full.
+     */
+    uint32_t cEntries = pSet->cEntries;
+    AssertReturnVoid(cEntries != PGMMAPSET_CLOSED);
+    if (cEntries >= RT_ELEMENTS(pSet->aEntries) / 2)
+    {
+        AssertMsg(cEntries <= RT_ELEMENTS(pSet->aEntries), ("%#x (%u)\n", cEntries, cEntries));
+        pSet->cEntries = 0;
+
+        pgmDynMapFlushAutoSetWorker(pSet, cEntries);
+    }
+}
+
 
 
@@ -1638 +1672 @@
 
 
-/* documented elsewhere - a bit of a mess.
-   This is a VERY hot path. */
-VMMDECL(int) PGMDynMapHCPage(PVM pVM, RTHCPHYS HCPhys, void **ppv)
-{
-    /*
-     * Validate state.
-     */
-    STAM_PROFILE_START(&pVM->pgm.s.StatR0DynMapHCPage, a);
-    AssertPtr(ppv);
-    *ppv = NULL;
-    AssertMsgReturn(pVM->pgm.s.pvR0DynMapUsed == g_pPGMR0DynMap,
-                    ("%p != %p\n", pVM->pgm.s.pvR0DynMapUsed, g_pPGMR0DynMap),
-                    VERR_ACCESS_DENIED);
-    AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
-    PVMCPU          pVCpu   = VMMGetCpu(pVM);
-    PPGMMAPSET      pSet    = &pVCpu->pgm.s.AutoSet;
-    AssertPtrReturn(pVCpu, VERR_INTERNAL_ERROR);
-    AssertMsgReturn(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries),
-                    ("%#x (%u)\n", pSet->cEntries, pSet->cEntries), VERR_WRONG_ORDER);
-
+/**
+ * Common worker code for PGMDynMapHCPhys, pgmR0DynMapHCPageInlined and
+ * pgmR0DynMapGCPageInlined.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The shared VM structure (for statistics).
+ * @param   pSet        The set.
+ * @param   HCPhys      The physical address of the page.
+ * @param   ppv         Where to store the address of the mapping on success.
+ */
+int pgmR0DynMapHCPageCommon(PVM pVM, PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv)
+{
     /*
      * Map it.
      */
-    uint32_t const  iPage = pgmR0DynMapPage(g_pPGMR0DynMap, HCPhys, pVM, ppv);
+    void *pvPage;
+    uint32_t const  iPage = pgmR0DynMapPage(g_pPGMR0DynMap, HCPhys, pVM, &pvPage);
     if (RT_UNLIKELY(iPage == UINT32_MAX))
     {
-        STAM_PROFILE_STOP(&pVM->pgm.s.StatR0DynMapHCPage, a);
         static uint32_t s_cBitched = 0;
         if (++s_cBitched < 10)
             LogRel(("PGMDynMapHCPage: cLoad=%u/%u cPages=%u cGuardPages=%u\n",
                     g_pPGMR0DynMap->cLoad, g_pPGMR0DynMap->cMaxLoad, g_pPGMR0DynMap->cPages, g_pPGMR0DynMap->cGuardPages));
+        *ppv = NULL;
         return VERR_PGM_DYNMAP_FAILED;
     }
@@ -1684 +1711 @@
     if (i-- < 5)
     {
-        pSet->aEntries[pSet->cEntries].cRefs = 1;
-        pSet->aEntries[pSet->cEntries].iPage = iPage;
-        pSet->cEntries++;
+        unsigned iEntry = pSet->cEntries++;
+        pSet->aEntries[iEntry].cRefs  = 1;
+        pSet->aEntries[iEntry].iPage  = iPage;
+        pSet->aEntries[iEntry].pvPage = pvPage;
+        pSet->aEntries[iEntry].HCPhys = HCPhys;
+        pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
     }
     /* Any of the last 5 pages? */
@@ -1707 +1737 @@
     else if (RT_LIKELY(i <= (int32_t)RT_ELEMENTS(pSet->aEntries) / 4 * 3))
     {
-        pSet->aEntries[pSet->cEntries].cRefs = 1;
-        pSet->aEntries[pSet->cEntries].iPage = iPage;
-        pSet->cEntries++;
+        unsigned iEntry = pSet->cEntries++;
+        pSet->aEntries[iEntry].cRefs  = 1;
+        pSet->aEntries[iEntry].iPage  = iPage;
+        pSet->aEntries[iEntry].pvPage = pvPage;
+        pSet->aEntries[iEntry].HCPhys = HCPhys;
+        pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
     }
     else
@@ -1721 +1754 @@
             {
                 pSet->aEntries[i].cRefs++;
-                STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapHCPageSetSearchHits);
+                STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapSetSearchHits);
                 break;
             }
         if (i < 0)
         {
-            STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapHCPageSetSearchMisses);
+            STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapSetSearchMisses);
             if (RT_UNLIKELY(pSet->cEntries >= RT_ELEMENTS(pSet->aEntries)))
             {
-                STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapHCPageSetOptimize);
+                STAM_COUNTER_INC(&pVM->pgm.s.StatR0DynMapSetOptimize);
                 pgmDynMapOptimizeAutoSet(pSet);
             }
             if (RT_LIKELY(pSet->cEntries < RT_ELEMENTS(pSet->aEntries)))
             {
-                pSet->aEntries[pSet->cEntries].cRefs = 1;
-                pSet->aEntries[pSet->cEntries].iPage = iPage;
-                pSet->cEntries++;
+                unsigned iEntry = pSet->cEntries++;
+                pSet->aEntries[iEntry].cRefs  = 1;
+                pSet->aEntries[iEntry].iPage  = iPage;
+                pSet->aEntries[iEntry].pvPage = pvPage;
+                pSet->aEntries[iEntry].HCPhys = HCPhys;
+                pSet->aiHashTable[PGMMAPSET_HASH(HCPhys)] = iEntry;
             }
             else
@@ -1743 +1779 @@
                 pgmR0DynMapReleasePage(g_pPGMR0DynMap, iPage, 1);
 
-                STAM_PROFILE_STOP(&pVM->pgm.s.StatR0DynMapHCPage, a);
                 static uint32_t s_cBitched = 0;
                 if (++s_cBitched < 10)
@@ -1753 +1788 @@
     }
 
+    *ppv = pvPage;
+    return VINF_SUCCESS;
+}
+
+
+/* documented elsewhere - a bit of a mess.
+   This is a VERY hot path. */
+VMMDECL(int) PGMDynMapHCPage(PVM pVM, RTHCPHYS HCPhys, void **ppv)
+{
+    /*
+     * Validate state.
+     */
+    STAM_PROFILE_START(&pVM->pgm.s.StatR0DynMapHCPage, a);
+    AssertPtr(ppv);
+    AssertMsgReturn(pVM->pgm.s.pvR0DynMapUsed == g_pPGMR0DynMap,
+                    ("%p != %p\n", pVM->pgm.s.pvR0DynMapUsed, g_pPGMR0DynMap),
+                    VERR_ACCESS_DENIED);
+    AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
+    PVMCPU          pVCpu   = VMMGetCpu(pVM);
+    PPGMMAPSET      pSet    = &pVCpu->pgm.s.AutoSet;
+    AssertPtrReturn(pVCpu, VERR_INTERNAL_ERROR);
+    AssertMsgReturn(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries),
+                    ("%#x (%u)\n", pSet->cEntries, pSet->cEntries), VERR_WRONG_ORDER);
+
+    /*
+     * Call common code.
+     */
+    int rc = pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
+
     STAM_PROFILE_STOP(&pVM->pgm.s.StatR0DynMapHCPage, a);
-    return VINF_SUCCESS;
+    return rc;
 }
 
@@ -1979 +2043 @@
     ASMIntDisable();
     PGMDynMapMigrateAutoSet(&pVM->aCpus[0]);
+    PGMDynMapFlushAutoSet(&pVM->aCpus[0]);
     PGMDynMapReleaseAutoSet(&pVM->aCpus[0]);
     ASMIntEnable();