Changeset 36891 in vbox

Timestamp:

Apr 29, 2011 1:22:57 PM (14 years ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

71469

Message:

PGM: Put a TLB in front of the RAM ranges to speed up lookup (disabled).

Location:

trunk

Files:

• include/VBox/vmm/pgm.h (modified) (1 diff)
• src/VBox/Main/src-client/GuestImpl.cpp (modified) (1 diff)
• src/VBox/VMM/VMMAll/PGMAll.cpp (modified) (8 diffs)
• src/VBox/VMM/VMMAll/PGMAllBth.h (modified) (30 diffs)
• src/VBox/VMM/VMMAll/PGMAllGst.h (modified) (4 diffs)
• src/VBox/VMM/VMMAll/PGMAllHandler.cpp (modified) (14 diffs)
• src/VBox/VMM/VMMAll/PGMAllMap.cpp (modified) (8 diffs)
• src/VBox/VMM/VMMAll/PGMAllPhys.cpp (modified) (32 diffs)
• src/VBox/VMM/VMMAll/PGMAllPool.cpp (modified) (22 diffs)
• src/VBox/VMM/VMMAll/PGMAllShw.h (modified) (2 diffs)
• src/VBox/VMM/VMMR0/PGMR0.cpp (modified) (2 diffs)
• src/VBox/VMM/VMMR0/PGMR0SharedPage.cpp (modified) (2 diffs)
• src/VBox/VMM/VMMR3/PGM.cpp (modified) (7 diffs)
• src/VBox/VMM/VMMR3/PGMDbg.cpp (modified) (5 diffs)
• src/VBox/VMM/VMMR3/PGMHandler.cpp (modified) (3 diffs)
• src/VBox/VMM/VMMR3/PGMMap.cpp (modified) (4 diffs)
• src/VBox/VMM/VMMR3/PGMPhys.cpp (modified) (47 diffs)
• src/VBox/VMM/VMMR3/PGMPool.cpp (modified) (3 diffs)
• src/VBox/VMM/VMMR3/PGMSavedState.cpp (modified) (12 diffs)
• src/VBox/VMM/VMMR3/PGMSharedPage.cpp (modified) (2 diffs)
• src/VBox/VMM/include/PGMGstDefs.h (modified) (1 diff)
• src/VBox/VMM/include/PGMInline.h (modified) (31 diffs)
• src/VBox/VMM/include/PGMInternal.h (modified) (7 diffs)
• src/VBox/VMM/testcase/tstVMStructRC.cpp (modified) (1 diff)

trunk/include/VBox/vmm/pgm.h

@@ -465 +465 @@
 VMMR3DECL(int)      PGMR3PhysGetRange(PVM pVM, uint32_t iRange, PRTGCPHYS pGCPhysStart, PRTGCPHYS pGCPhysLast,
                                       const char **ppszDesc, bool *pfIsMmio);
-VMMR3DECL(int)      PGMR3QueryVMMMemoryStats(PVM pVM, uint64_t *puTotalAllocSize, uint64_t *puTotalFreeSize, uint64_t *puTotalBalloonSize, uint64_t *puTotalSharedSize);
-VMMR3DECL(int)      PGMR3QueryMemoryStats(PVM pVM, uint64_t *pulTotalMem, uint64_t *pulPrivateMem, uint64_t *puTotalSharedMem, uint64_t *puTotalZeroMem);
+VMMR3DECL(int)      PGMR3QueryMemoryStats(PVM pVM, uint64_t *pcbTotalMem, uint64_t *pcbPrivateMem, uint64_t *pcbSharedMem, uint64_t *pcbZeroMem);
+VMMR3DECL(int)      PGMR3QueryGlobalMemoryStats(PVM pVM, uint64_t *pcbAllocMem, uint64_t *pcbFreeMem, uint64_t *pcbBallonedMem, uint64_t *pcbSharedMem);
+
 VMMR3DECL(int)      PGMR3PhysMMIORegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb,
                                           R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3, RTR3PTR pvUserR3,

trunk/src/VBox/Main/src-client/GuestImpl.cpp

@@ -383 +383 @@
         uint64_t uFreeTotal, uAllocTotal, uBalloonedTotal, uSharedTotal;
         *aMemFreeTotal = 0;
-        int rc = PGMR3QueryVMMMemoryStats(pVM.raw(), &uAllocTotal, &uFreeTotal, &uBalloonedTotal, &uSharedTotal);
+        int rc = PGMR3QueryGlobalMemoryStats(pVM.raw(), &uAllocTotal, &uFreeTotal, &uBalloonedTotal, &uSharedTotal);
         AssertRC(rc);
         if (rc == VINF_SUCCESS)

trunk/src/VBox/VMM/VMMAll/PGMAll.cpp

@@ -734 +734 @@
      * Check for conflicts and pending CR3 monitoring updates.
      */
-    if (pgmMapAreMappingsFloating(&pVM->pgm.s))
+    if (pgmMapAreMappingsFloating(pVM))
     {
         if (    pgmGetMapping(pVM, GCPtrPage)
@@ -1444 +1444 @@
     RTGCPHYS    GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_PAGE_MASK;
     PPGMPAGE    pPage;
-    int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhysCR3, &pPage);
+    int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
     if (RT_SUCCESS(rc))
     {
@@ -1486 +1486 @@
     RTGCPHYS    GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_PAE_PAGE_MASK;
     PPGMPAGE    pPage;
-    int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhysCR3, &pPage);
+    int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
     if (RT_SUCCESS(rc))
     {
@@ -1534 +1534 @@
 
     PPGMPAGE        pPage;
-    int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
+    int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
     if (RT_SUCCESS(rc))
     {
@@ -1596 +1596 @@
     RTGCPHYS    GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_AMD64_PAGE_MASK;
     PPGMPAGE    pPage;
-    int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhysCR3, &pPage);
+    int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
     if (RT_SUCCESS(rc))
     {
@@ -1814 +1814 @@
         if (RT_LIKELY(rc == VINF_SUCCESS))
         {
-            if (pgmMapAreMappingsFloating(&pVM->pgm.s))
+            if (pgmMapAreMappingsFloating(pVM))
                 pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
         }
@@ -1823 +1823 @@
             pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
             pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_MAP_CR3;
-            if (pgmMapAreMappingsFloating(&pVM->pgm.s))
+            if (pgmMapAreMappingsFloating(pVM))
                 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_MONITOR_CR3;
         }
@@ -2362 +2362 @@
      */
     int rc;
-    PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
+    PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
     if (RT_LIKELY(pPage))
     {

trunk/src/VBox/VMM/VMMAll/PGMAllBth.h

@@ -550 +550 @@
         PPGMPAGE pPage;
 #   if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
-        rc = pgmPhysGetPageEx(&pVM->pgm.s, GstWalk.Core.GCPhys, &pPage);
+        rc = pgmPhysGetPageEx(pVM, GstWalk.Core.GCPhys, &pPage);
         if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
             return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
@@ -556 +556 @@
         rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
 #   else
-        rc = pgmPhysGetPageEx(&pVM->pgm.s, (RTGCPHYS)pvFault, &pPage);
+        rc = pgmPhysGetPageEx(pVM, (RTGCPHYS)pvFault, &pPage);
         if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
             return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
@@ -675 +675 @@
      * (BTW, it's impossible to have physical access handlers in a mapping.)
      */
-    if (pgmMapAreMappingsEnabled(&pVM->pgm.s))
+    if (pgmMapAreMappingsEnabled(pVM))
     {
         PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
@@ -687 +687 @@
                  * The first thing we check is if we've got an undetected conflict.
                  */
-                if (pgmMapAreMappingsFloating(&pVM->pgm.s))
+                if (pgmMapAreMappingsFloating(pVM))
                 {
                     unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
@@ -751 +751 @@
 #  endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
     PPGMPAGE pPage;
-    rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
+    rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
     if (RT_FAILURE(rc))
     {
@@ -1261 +1261 @@
              * Conflict - Let SyncPT deal with it to avoid duplicate code.
              */
-            Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
+            Assert(pgmMapAreMappingsEnabled(pVM));
             Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
             rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
@@ -1373 +1373 @@
         else
         {
-            Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
+            Assert(pgmMapAreMappingsEnabled(pVM));
             STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDMappings));
         }
@@ -1413 +1413 @@
         pPool->cPresent--;
 
-        PPGMPAGE pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysPage);
+        PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysPage);
         AssertRelease(pPhysPage);
         pgmTrackDerefGCPhys(pPool, pShwPage, pPhysPage, iPte);
@@ -1432 +1432 @@
      * Find the guest address.
      */
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
          pRam;
          pRam = pRam->CTX_SUFF(pNext))
@@ -1625 +1625 @@
          */
         PPGMPAGE pPage;
-        int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhysPage, &pPage);
+        int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
         if (RT_SUCCESS(rc))
         {
@@ -1983 +1983 @@
                                     ||  iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK)   /* always sync GCPtrPage */
                                     ||  !CSAMDoesPageNeedScanning(pVM, GCPtrCurPage)
-                                    ||  (   (pPage = pgmPhysGetPage(&pVM->pgm.s, pPteSrc->u & GST_PTE_PG_MASK))
+                                    ||  (   (pPage = pgmPhysGetPage(pVM, pPteSrc->u & GST_PTE_PG_MASK))
                                          && PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
                                    )
@@ -2030 +2030 @@
                 /* Find ram range. */
                 PPGMPAGE pPage;
-                int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
+                int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
                 if (RT_SUCCESS(rc))
                 {
@@ -2442 +2442 @@
                 if (SHW_PTE_IS_TRACK_DIRTY(*pPteDst))
                 {
-                    PPGMPAGE pPage  = pgmPhysGetPage(&pVM->pgm.s, GST_GET_PTE_GCPHYS(*pPteSrc));
+                    PPGMPAGE pPage  = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(*pPteSrc));
                     SHWPTE   PteDst = *pPteDst;
 
@@ -2613 +2613 @@
     if (PdeDst.u & PGM_PDFLAGS_MAPPING)
     {
-        Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
+        Assert(pgmMapAreMappingsEnabled(pVM));
 #  ifndef IN_RING3
         Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
@@ -2817 +2817 @@
                         if (    ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW | X86_PTE_US))
                             ||  !CSAMDoesPageNeedScanning(pVM, GCPtrCur)
-                            ||  (   (pPage = pgmPhysGetPage(&pVM->pgm.s, GST_GET_PTE_GCPHYS(PteSrc)))
+                            ||  (   (pPage = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc)))
                                  &&  PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
                            )
@@ -2841 +2841 @@
              *
              * We'll walk the ram range list in parallel and optimize lookups.
-             * We will only sync on shadow page table at a time.
+             * We will only sync one shadow page table at a time.
              */
             STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4M));
@@ -2888 +2888 @@
                   GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
                   GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
-            PPGMRAMRANGE    pRam   = pVM->pgm.s.CTX_SUFF(pRamRanges);
+            PPGMRAMRANGE    pRam   = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
             unsigned        iPTDst = 0;
             while (     iPTDst < RT_ELEMENTS(pPTDst->a)
                    &&   !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
             {
-                /* Advance ram range list. */
-                while (pRam && GCPhys > pRam->GCPhysLast)
-                    pRam = pRam->CTX_SUFF(pNext);
                 if (pRam && GCPhys >= pRam->GCPhys)
                 {
@@ -2970 +2967 @@
                     } while (   iPTDst < RT_ELEMENTS(pPTDst->a)
                              && GCPhys <= pRam->GCPhysLast);
+
+                    /* Advance ram range list. */
+                    while (pRam && GCPhys > pRam->GCPhysLast)
+                        pRam = pRam->CTX_SUFF(pNext);
                 }
                 else if (pRam)
@@ -3076 +3077 @@
         /* Check if we allocated a big page before for this 2 MB range. */
         PPGMPAGE pPage;
-        rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPtrPage & X86_PDE2M_PAE_PG_MASK, &pPage);
+        rc = pgmPhysGetPageEx(pVM, GCPtrPage & X86_PDE2M_PAE_PG_MASK, &pPage);
         if (RT_SUCCESS(rc))
         {
@@ -3585 +3586 @@
      * Nested / EPT - almost no work.
      */
-    Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
+    Assert(!pgmMapAreMappingsEnabled(pVM));
     return VINF_SUCCESS;
 
@@ -3593 +3594 @@
      * out the shadow parts when the guest modifies its tables.
      */
-    Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
+    Assert(!pgmMapAreMappingsEnabled(pVM));
     return VINF_SUCCESS;
 
@@ -3603 +3604 @@
      * are enabled and not fixed.
      */
-    if (pgmMapAreMappingsFloating(&pVM->pgm.s))
+    if (pgmMapAreMappingsFloating(pVM))
     {
         int rc = pgmMapResolveConflicts(pVM);
@@ -3614 +3615 @@
     }
 #  else
-    Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s));
+    Assert(!pgmMapAreMappingsEnabled(pVM));
 #  endif
     return VINF_SUCCESS;
@@ -3681 +3682 @@
     AssertRCReturn(rc, 1);
     HCPhys = NIL_RTHCPHYS;
-    rc = pgmRamGCPhys2HCPhys(&pVM->pgm.s, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
+    rc = pgmRamGCPhys2HCPhys(pVM, cr3 & GST_CR3_PAGE_MASK, &HCPhys);
     AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
 #  if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
@@ -3869 +3870 @@
                 if (PdeDst.u & PGM_PDFLAGS_MAPPING)
                 {
-                    Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
+                    Assert(pgmMapAreMappingsEnabled(pVM));
                     if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
                     {
@@ -3948 +3949 @@
                     }
 
-                    PPGMPAGE pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
+                    PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
                     if (!pPhysPage)
                     {
@@ -4062 +4063 @@
 # endif
 
-                            pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
+                            pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
                             if (!pPhysPage)
                             {
@@ -4294 +4295 @@
                             }
 # endif
-                            pPhysPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
+                            pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
                             if (!pPhysPage)
                             {
@@ -4416 +4417 @@
     RTHCPHYS    HCPhysGuestCR3;
     pgmLock(pVM);
-    PPGMPAGE    pPageCR3 = pgmPhysGetPage(&pVM->pgm.s, GCPhysCR3);
+    PPGMPAGE    pPageCR3 = pgmPhysGetPage(pVM, GCPhysCR3);
     AssertReturn(pPageCR3, VERR_INTERNAL_ERROR_2);
     HCPhysGuestCR3 = PGM_PAGE_GET_HCPHYS(pPageCR3);
@@ -4464 +4465 @@
                     RTGCPHYS    GCPhys = pGuestPDPT->a[i].u & X86_PDPE_PG_MASK;
                     pgmLock(pVM);
-                    PPGMPAGE    pPage  = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
+                    PPGMPAGE    pPage  = pgmPhysGetPage(pVM, GCPhys);
                     AssertReturn(pPage, VERR_INTERNAL_ERROR_2);
                     HCPhys = PGM_PAGE_GET_HCPHYS(pPage);

trunk/src/VBox/VMM/VMMAll/PGMAllGst.h

@@ -517 +517 @@
                         {
                             if (pCur->aPhysToVirt[iPage].Core.Key != NIL_RTGCPHYS)
-                                pgmHandlerVirtualClearPage(&pVM->pgm.s, pCur, iPage);
+                                pgmHandlerVirtualClearPage(pVM, pCur, iPage);
 #ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
                             AssertReleaseMsg(!pCur->aPhysToVirt[iPage].offNextAlias,
@@ -540 +540 @@
                         if (pCur->aPhysToVirt[iPage].Core.Key != NIL_RTGCPHYS)
                         {
-                            pgmHandlerVirtualClearPage(&pVM->pgm.s, pCur, iPage);
+                            pgmHandlerVirtualClearPage(pVM, pCur, iPage);
 #ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
                             AssertReleaseMsg(!pCur->aPhysToVirt[iPage].offNextAlias,
@@ -567 +567 @@
                     {
                         if (pCur->aPhysToVirt[iPage].Core.Key != NIL_RTGCPHYS)
-                            pgmHandlerVirtualClearPage(&pVM->pgm.s, pCur, iPage);
+                            pgmHandlerVirtualClearPage(pVM, pCur, iPage);
 #ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
                         AssertReleaseMsg(!pCur->aPhysToVirt[iPage].offNextAlias,
@@ -590 +590 @@
                 if (pCur->aPhysToVirt[iPage].Core.Key != NIL_RTGCPHYS)
                 {
-                    pgmHandlerVirtualClearPage(&pVM->pgm.s, pCur, iPage);
+                    pgmHandlerVirtualClearPage(pVM, pCur, iPage);
                     pCur->aPhysToVirt[iPage].Core.Key = NIL_RTGCPHYS;
                     pState->fTodo |= PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL;

trunk/src/VBox/VMM/VMMAll/PGMAllHandler.cpp

@@ -120 +120 @@
      * There is no apparent need to support ranges which cover more than one ram range.
      */
-    PPGMRAMRANGE    pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
-    while (pRam && GCPhys > pRam->GCPhysLast)
-        pRam = pRam->CTX_SUFF(pNext);
-    if (    !pRam
-        ||  GCPhysLast < pRam->GCPhys
-        ||  GCPhys > pRam->GCPhysLast)
+    PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhys);
+    if (   !pRam
+        || GCPhysLast < pRam->GCPhys
+        || GCPhys > pRam->GCPhysLast)
     {
 #ifdef IN_RING3
@@ -306 +304 @@
         if (GCPhysStart & PAGE_OFFSET_MASK)
         {
-            PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysStart);
+            PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhysStart);
             if (    pPage
                 &&  PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
@@ -323 +321 @@
         if (GCPhysLast & PAGE_OFFSET_MASK)
         {
-            PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysLast);
+            PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhysLast);
             if (    pPage
                 &&  PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
@@ -387 +385 @@
     {
         PPGMPAGE pPage;
-        int rc = pgmPhysGetPageWithHintEx(&pVM->pgm.s, GCPhys, &pPage, ppRamHint);
+        int rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pPage, ppRamHint);
         if (    RT_SUCCESS(rc)
             &&  PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) < uState)
@@ -480 +478 @@
      * Iterate the guest ram pages updating the state.
      */
-    RTUINT          cPages = pCur->cPages;
-    RTGCPHYS        GCPhys = pCur->Core.Key;
+    RTUINT          cPages   = pCur->cPages;
+    RTGCPHYS        GCPhys   = pCur->Core.Key;
     PPGMRAMRANGE    pRamHint = NULL;
-    PPGM            pPGM = &pVM->pgm.s;
     for (;;)
     {
         PPGMPAGE pPage;
-        int rc = pgmPhysGetPageWithHintEx(pPGM, GCPhys, &pPage, &pRamHint);
+        int rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pPage, &pRamHint);
         if (RT_SUCCESS(rc))
         {
@@ -562 +559 @@
              * There is no apparent need to support ranges which cover more than one ram range.
              */
-            PPGMRAMRANGE    pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
-            while (pRam && GCPhys > pRam->GCPhysLast)
-                pRam = pRam->CTX_SUFF(pNext);
-            if (    pRam
-                &&  GCPhys <= pRam->GCPhysLast
-                &&  GCPhysLast >= pRam->GCPhys)
+            PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhys);
+            if (   pRam
+                && GCPhys <= pRam->GCPhysLast
+                && GCPhysLast >= pRam->GCPhys)
             {
                 pCur->Core.Key      = GCPhys;
@@ -865 +860 @@
             {
                 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysHandlerReset)); /**@Todo move out of switch */
-                PPGMRAMRANGE pRam = pgmPhysGetRange(&pVM->pgm.s, GCPhys);
+                PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhys);
                 Assert(pRam);
                 Assert(pRam->GCPhys     <= pCur->Core.Key);
@@ -979 +974 @@
              */
             PPGMPAGE pPage;
-            int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhysPage, &pPage);
+            int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
             AssertReturnStmt(RT_SUCCESS_NP(rc), pgmUnlock(pVM), rc);
             if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
@@ -1062 +1057 @@
              */
             PPGMPAGE pPageRemap;
-            int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhysPageRemap, &pPageRemap);
+            int rc = pgmPhysGetPageEx(pVM, GCPhysPageRemap, &pPageRemap);
             AssertReturnStmt(RT_SUCCESS_NP(rc), pgmUnlock(pVM), rc);
             AssertMsgReturnStmt(PGM_PAGE_GET_TYPE(pPageRemap) == PGMPAGETYPE_MMIO2,
@@ -1069 +1064 @@
 
             PPGMPAGE pPage;
-            rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhysPage, &pPage);
+            rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
             AssertReturnStmt(RT_SUCCESS_NP(rc), pgmUnlock(pVM), rc);
             if (PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO)
@@ -1182 +1177 @@
              */
             PPGMPAGE pPage;
-            int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhysPage, &pPage);
+            int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
             AssertReturnStmt(RT_SUCCESS_NP(rc), pgmUnlock(pVM), rc);
             if (PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO)
@@ -1434 +1429 @@
              */
             PPGMPAGE pPage;
-            int rc = pgmPhysGetPageWithHintEx(&pVM->pgm.s, pPhys2Virt->Core.Key, &pPage, &pRamHint);
+            int rc = pgmPhysGetPageWithHintEx(pVM, pPhys2Virt->Core.Key, &pPage, &pRamHint);
             if (    RT_SUCCESS(rc)
                 &&  PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < uState)
@@ -1643 +1638 @@
             }
 
-            PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysGst);
+            PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhysGst);
             if (!pPage)
             {
@@ -1688 +1683 @@
      * Check the RAM flags against the handlers.
      */
-    for (PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges); pRam; pRam = pRam->CTX_SUFF(pNext))
-    {
-        const unsigned cPages = pRam->cb >> PAGE_SHIFT;
-        for (unsigned iPage = 0; iPage < cPages; iPage++)
+    for (PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRangesX); pRam; pRam = pRam->CTX_SUFF(pNext))
+    {
+        const uint32_t cPages = pRam->cb >> PAGE_SHIFT;
+        for (uint32_t iPage = 0; iPage < cPages; iPage++)
         {
             PGMPAGE const *pPage = &pRam->aPages[iPage];

trunk/src/VBox/VMM/VMMAll/PGMAllMap.cpp

@@ -270 +270 @@
 void pgmMapSetShadowPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iNewPDE)
 {
-    Log4(("pgmMapSetShadowPDEs new pde %x (mappings enabled %d)\n", iNewPDE, pgmMapAreMappingsEnabled(&pVM->pgm.s)));
-
-    if (    !pgmMapAreMappingsEnabled(&pVM->pgm.s)
+    Log4(("pgmMapSetShadowPDEs new pde %x (mappings enabled %d)\n", iNewPDE, pgmMapAreMappingsEnabled(pVM)));
+
+    if (    !pgmMapAreMappingsEnabled(pVM)
         ||  pVM->cCpus > 1)
         return;
@@ -354 +354 @@
                 PPGMPOOLPAGE    pPoolPagePd = pgmPoolGetPage(pPool, pShwPdpt->a[iPdPt].u & X86_PDPE_PG_MASK);
                 AssertFatal(pPoolPagePd);
-                if (!pgmPoolIsPageLocked(&pVM->pgm.s, pPoolPagePd))
+                if (!pgmPoolIsPageLocked(pPoolPagePd))
                     pgmPoolLockPage(pPool, pPoolPagePd);
 #ifdef VBOX_STRICT
@@ -424 +424 @@
 void pgmMapClearShadowPDEs(PVM pVM, PPGMPOOLPAGE pShwPageCR3, PPGMMAPPING pMap, unsigned iOldPDE, bool fDeactivateCR3)
 {
-    Log(("pgmMapClearShadowPDEs: old pde %x (cPTs=%x) (mappings enabled %d) fDeactivateCR3=%RTbool\n", iOldPDE, pMap->cPTs, pgmMapAreMappingsEnabled(&pVM->pgm.s), fDeactivateCR3));
+    Log(("pgmMapClearShadowPDEs: old pde %x (cPTs=%x) (mappings enabled %d) fDeactivateCR3=%RTbool\n", iOldPDE, pMap->cPTs, pgmMapAreMappingsEnabled(pVM), fDeactivateCR3));
 
     /*
      * Skip this if disabled or if it doesn't apply.
      */
-    if (    !pgmMapAreMappingsEnabled(&pVM->pgm.s)
+    if (    !pgmMapAreMappingsEnabled(pVM)
         ||  pVM->cCpus > 1)
         return;
@@ -526 +526 @@
                     PPGMPOOLPAGE pPoolPagePd = pgmPoolGetPage(pPool, pShwPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
                     AssertFatal(pPoolPagePd);
-                    if (pgmPoolIsPageLocked(&pVM->pgm.s, pPoolPagePd))
+                    if (pgmPoolIsPageLocked(pPoolPagePd))
                         pgmPoolUnlockPage(pPool, pPoolPagePd);
                 }
@@ -630 +630 @@
      * Can skip this if mappings are disabled.
      */
-    if (!pgmMapAreMappingsEnabled(&pVM->pgm.s))
+    if (!pgmMapAreMappingsEnabled(pVM))
         return;
 
@@ -665 +665 @@
      * Skip this if disabled or if it doesn't apply.
      */
-    if (    !pgmMapAreMappingsEnabled(&pVM->pgm.s)
+    if (    !pgmMapAreMappingsEnabled(pVM)
         ||  pVM->cCpus > 1)
         return VINF_SUCCESS;
@@ -702 +702 @@
      * Skip this if disabled or if it doesn't apply.
      */
-    if (    !pgmMapAreMappingsEnabled(&pVM->pgm.s)
+    if (    !pgmMapAreMappingsEnabled(pVM)
         ||  pVM->cCpus > 1)
         return VINF_SUCCESS;
@@ -733 +733 @@
      * Can skip this if mappings are safely fixed.
      */
-    if (!pgmMapAreMappingsFloating(&pVM->pgm.s))
+    if (!pgmMapAreMappingsFloating(pVM))
         return false;
 

trunk/src/VBox/VMM/VMMAll/PGMAllPhys.cpp

@@ -141 +141 @@
 
 #endif /* IN_RING3 */
+#ifdef PGM_USE_RAMRANGE_TLB
+
+/**
+ * Invalidates the RAM range TLBs.
+ *
+ * @param   pVM                 The VM handle.
+ */
+void pgmPhysInvalidRamRangeTlbs(PVM pVM)
+{
+    pgmLock(pVM);
+    for (uint32_t i = 0; i < PGM_RAMRANGE_TLB_ENTRIES; i++)
+    {
+        pVM->pgm.s.apRamRangesTlbR3[i] = NIL_RTR3PTR;
+        pVM->pgm.s.apRamRangesTlbR0[i] = NIL_RTR0PTR;
+        pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
+    }
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Slow worker for pgmPhysGetRange.
+ *
+ * @copydoc pgmPhysGetRange
+ */
+PPGMRAMRANGE pgmPhysGetRangeSlow(PVM pVM, RTGCPHYS GCPhys)
+{
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
+    while (GCPhys > pRam->GCPhysLast)
+    {
+        pRam = pRam->CTX_SUFF(pNext);
+        if (!pRam)
+            return NULL;
+    }
+    if (GCPhys < pRam->GCPhys)
+        return NULL;
+
+    pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
+    return pRam;
+}
+
+
+/**
+ * Slow worker for pgmPhysGetRangeAtOrAbove.
+ *
+ * @copydoc pgmPhysGetRangeAtOrAbove
+ */
+PPGMRAMRANGE pgmPhysGetRangeAtOrAboveSlow(PVM pVM, RTGCPHYS GCPhys)
+{
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
+    while (GCPhys > pRam->GCPhysLast)
+    {
+        pRam = pRam->CTX_SUFF(pNext);
+        if (!pRam)
+            return NULL;
+    }
+    pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(pRam->GCPhys)] = pRam;
+    return pRam;
+}
+
+
+/**
+ * Slow worker for pgmPhysGetPage.
+ *
+ * @copydoc pgmPhysGetPage
+ */
+PPGMPAGE pgmPhysGetPageSlow(PVM pVM, RTGCPHYS GCPhys)
+{
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
+         pRam;
+         pRam = pRam->CTX_SUFF(pNext))
+    {
+        RTGCPHYS off = GCPhys - pRam->GCPhys;
+        if (off < pRam->cb)
+        {
+            pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
+            return &pRam->aPages[off >> PAGE_SHIFT];
+        }
+    }
+    return NULL;
+}
+
+
+/**
+ * Slow worker for pgmPhysGetPageEx.
+ *
+ * @copydoc pgmPhysGetPageEx
+ */
+int pgmPhysGetPageExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
+{
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
+         pRam;
+         pRam = pRam->CTX_SUFF(pNext))
+    {
+        RTGCPHYS off = GCPhys - pRam->GCPhys;
+        if (off < pRam->cb)
+        {
+            pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
+            *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
+            return VINF_SUCCESS;
+        }
+    }
+    *ppPage = NULL;
+    return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+}
+
+
+/**
+ * Slow worker for pgmPhysGetPageAndRangeEx.
+ *
+ * @copydoc pgmPhysGetPageAndRangeEx
+ */
+int pgmPhysGetPageAndRangeExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
+{
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
+         pRam;
+         pRam = pRam->CTX_SUFF(pNext))
+    {
+        RTGCPHYS off = GCPhys - pRam->GCPhys;
+        if (off < pRam->cb)
+        {
+            pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
+            *ppRam  = pRam;
+            *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
+            return VINF_SUCCESS;
+        }
+    }
+
+    *ppRam  = NULL;
+    *ppPage = NULL;
+    return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+}
+
+#endif /* PGM_USE_RAMRANGE_TLB */
 
 /**
@@ -166 +305 @@
 VMMDECL(bool) PGMPhysIsGCPhysValid(PVM pVM, RTGCPHYS GCPhys)
 {
-    PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
+    PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
     return pPage != NULL;
 }
@@ -182 +321 @@
 VMMDECL(bool) PGMPhysIsGCPhysNormal(PVM pVM, RTGCPHYS GCPhys)
 {
-    PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
+    PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
     return pPage
         && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM;
@@ -205 +344 @@
     pgmLock(pVM);
     PPGMPAGE pPage;
-    int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
+    int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
     if (RT_SUCCESS(rc))
         *pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) | (GCPhys & PAGE_OFFSET_MASK);
@@ -392 +531 @@
         PPGMPAGE pBasePage;
 
-        int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhysBase, &pBasePage);
+        int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pBasePage);
         AssertRCReturn(rc, rc);     /* paranoia; can't happen. */
         if (PGM_PAGE_GET_PDE_TYPE(pBasePage) == PGM_PAGE_PDE_TYPE_DONTCARE)
@@ -531 +670 @@
 
     PPGMPAGE pFirstPage;
-    int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhysBase, &pFirstPage);
+    int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pFirstPage);
     if (    RT_SUCCESS(rc)
         &&  PGM_PAGE_GET_TYPE(pFirstPage) == PGMPAGETYPE_RAM)
@@ -550 +689 @@
             {
                 PPGMPAGE pSubPage;
-                rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pSubPage);
+                rc = pgmPhysGetPageEx(pVM, GCPhys, &pSubPage);
                 if  (   RT_FAILURE(rc)
                      || PGM_PAGE_GET_TYPE(pSubPage)  != PGMPAGETYPE_RAM      /* Anything other than ram implies monitoring. */
@@ -629 +768 @@
     {
         PPGMPAGE pPage;
-        int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
+        int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
         AssertRCBreak(rc);
 
@@ -845 +984 @@
         {
             /* Lookup the MMIO2 range and use pvR3 to calc the address. */
-            PPGMRAMRANGE pRam = pgmPhysGetRange(&pVM->pgm.s, GCPhys);
+            PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhys);
             AssertMsgReturn(pRam || !pRam->pvR3, ("pRam=%p pPage=%R[pgmpage]\n", pRam, pPage), VERR_INTERNAL_ERROR_2);
             *ppv = (void *)((uintptr_t)pRam->pvR3 + (uintptr_t)((GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK) - pRam->GCPhys));
@@ -1007 +1146 @@
 }
 
-
 #if !defined(IN_RC) && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
+
 /**
  * Load a guest page into the ring-3 physical TLB.
@@ -1018 +1157 @@
  * @param   GCPhys      The guest physical address in question.
  */
-int pgmPhysPageLoadIntoTlb(PPGM pPGM, RTGCPHYS GCPhys)
-{
-    Assert(PGMIsLocked(PGM2VM(pPGM)));
+int pgmPhysPageLoadIntoTlb(PVM pVM, RTGCPHYS GCPhys)
+{
+    Assert(PGMIsLocked(pVM));
 
     /*
@@ -1026 +1165 @@
      * 99.8% of requests are expected to be in the first range.
      */
-    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
-    RTGCPHYS off = GCPhys - pRam->GCPhys;
-    if (RT_UNLIKELY(off >= pRam->cb))
-    {
-        do
-        {
-            pRam = pRam->CTX_SUFF(pNext);
-            if (!pRam)
-            {
-                STAM_COUNTER_INC(&pPGM->CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbMisses));
-                return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
-            }
-            off = GCPhys - pRam->GCPhys;
-        } while (off >= pRam->cb);
-    }
-
-    return pgmPhysPageLoadIntoTlbWithPage(pPGM, &pRam->aPages[off >> PAGE_SHIFT], GCPhys);
+    PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
+    if (!pPage)
+    {
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbMisses));
+        return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+    }
+
+    return pgmPhysPageLoadIntoTlbWithPage(pVM, pPage, GCPhys);
 }
 
@@ -1053 +1183 @@
  * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
  *
- * @param   pPGM        The PGM instance pointer.
+ * @param   pVM         The VM handle.
  * @param   pPage       Pointer to the PGMPAGE structure corresponding to
  *                      GCPhys.
  * @param   GCPhys      The guest physical address in question.
  */
-int pgmPhysPageLoadIntoTlbWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys)
-{
-    Assert(PGMIsLocked(PGM2VM(pPGM)));
-    STAM_COUNTER_INC(&pPGM->CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbMisses));
+int pgmPhysPageLoadIntoTlbWithPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
+{
+    Assert(PGMIsLocked(pVM));
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbMisses));
 
     /*
@@ -1067 +1197 @@
      * Make a special case for the zero page as it is kind of special.
      */
-    PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
+    PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
     if (    !PGM_PAGE_IS_ZERO(pPage)
         &&  !PGM_PAGE_IS_BALLOONED(pPage))
@@ -1073 +1203 @@
         void *pv;
         PPGMPAGEMAP pMap;
-        int rc = pgmPhysPageMapCommon(PGM2VM(pPGM), pPage, GCPhys, &pMap, &pv);
+        int rc = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMap, &pv);
         if (RT_FAILURE(rc))
             return rc;
@@ -1082 +1212 @@
     else
     {
-        Assert(PGM_PAGE_GET_HCPHYS(pPage) == pPGM->HCPhysZeroPg);
+        Assert(PGM_PAGE_GET_HCPHYS(pPage) == pVM->pgm.s.HCPhysZeroPg);
         pTlbe->pMap = NULL;
-        pTlbe->pv = pPGM->CTXALLSUFF(pvZeroPg);
+        pTlbe->pv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
     }
 #ifdef PGM_WITH_PHYS_TLB
@@ -1098 +1228 @@
     return VINF_SUCCESS;
 }
+
 #endif /* !IN_RC && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
-
 
 /**
@@ -1149 +1279 @@
 #else
     PPGMPAGEMAPTLBE pTlbe;
-    rc = pgmPhysPageQueryTlbeWithPage(&pVM->pgm.s, pPage, GCPhys, &pTlbe);
+    rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
     if (RT_FAILURE(rc))
         return rc;
@@ -1194 +1324 @@
 #else
     PPGMPAGEMAPTLBE pTlbe;
-    int rc = pgmPhysPageQueryTlbeWithPage(&pVM->pgm.s, pPage, GCPhys, &pTlbe);
+    int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
     if (RT_FAILURE(rc))
         return rc;
@@ -1242 +1372 @@
      */
     PPGMPAGE pPage;
-    rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
+    rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
     if (RT_SUCCESS(rc))
     {
@@ -1278 +1408 @@
      */
     PPGMPAGEMAPTLBE pTlbe;
-    rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
+    rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
     if (RT_SUCCESS(rc))
     {
@@ -1292 +1422 @@
             {
                 AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
-                rc = pgmPhysPageQueryTlbeWithPage(&pVM->pgm.s, pPage, GCPhys, &pTlbe);
+                rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
             }
         }
@@ -1365 +1495 @@
      */
     PPGMPAGE pPage;
-    rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
+    rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
     if (RT_SUCCESS(rc))
     {
@@ -1400 +1530 @@
      */
     PPGMPAGEMAPTLBE pTlbe;
-    rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
+    rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
     if (RT_SUCCESS(rc))
     {
@@ -1624 +1754 @@
     PPGMRAMRANGE pRam;
     PPGMPAGE pPage;
-    int rc = pgmPhysGetPageAndRangeEx(&pVM->pgm.s, GCPhys, &pPage, &pRam);
+    int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
     if (RT_SUCCESS(rc))
         rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, (void **)pR3Ptr);
@@ -1883 +2013 @@
      * Copy loop on ram ranges.
      */
-    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+    PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
     for (;;)
     {
-        /* Find range. */
-        while (pRam && GCPhys > pRam->GCPhysLast)
-            pRam = pRam->CTX_SUFF(pNext);
         /* Inside range or not? */
         if (pRam && GCPhys >= pRam->GCPhys)
@@ -1953 +2080 @@
              * Unassigned address space.
              */
-            if (!pRam)
-                break;
-            size_t cb = pRam->GCPhys - GCPhys;
+            size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
             if (cb >= cbRead)
             {
@@ -1967 +2092 @@
             GCPhys += cb;
         }
+
+        /* Advance range if necessary. */
+        while (pRam && GCPhys > pRam->GCPhysLast)
+            pRam = pRam->CTX_SUFF(pNext);
     } /* Ram range walk */
 
@@ -2398 +2527 @@
      * Copy loop on ram ranges.
      */
-    PPGMRAMRANGE    pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+    PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
     for (;;)
     {
-        /* Find range. */
-        while (pRam && GCPhys > pRam->GCPhysLast)
-            pRam = pRam->CTX_SUFF(pNext);
         /* Inside range or not? */
         if (pRam && GCPhys >= pRam->GCPhys)
@@ -2478 +2604 @@
             GCPhys += cb;
         }
+
+        /* Advance range if necessary. */
+        while (pRam && GCPhys > pRam->GCPhysLast)
+            pRam = pRam->CTX_SUFF(pNext);
     } /* Ram range walk */
 
@@ -3618 +3748 @@
     PPGMPAGE pPage;
 
-    pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
+    pPage = pgmPhysGetPage(pVM, GCPhys);
     if (pPage)
         return (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
@@ -3624 +3754 @@
     return PGMPAGETYPE_INVALID;
 }
+

trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp

@@ -265 +265 @@
                         if ((uShw.pPDPae->a[iShw + i].u & (PGM_PDFLAGS_MAPPING | X86_PDE_P)) == (PGM_PDFLAGS_MAPPING | X86_PDE_P))
                         {
-                            Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
+                            Assert(pgmMapAreMappingsEnabled(pVM));
                             VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
                             LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw=%#x!\n", iShwPdpt, iShw+i));
@@ -292 +292 @@
                                 if ((uShw.pPDPae->a[iShw2].u & (PGM_PDFLAGS_MAPPING | X86_PDE_P)) == (PGM_PDFLAGS_MAPPING | X86_PDE_P))
                                 {
-                                    Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
+                                    Assert(pgmMapAreMappingsEnabled(pVM));
                                     VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
                                     LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw2=%#x!\n", iShwPdpt, iShw2));
@@ -371 +371 @@
                 if (uShw.pPD->a[iShw].u & PGM_PDFLAGS_MAPPING)
                 {
-                    Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
+                    Assert(pgmMapAreMappingsEnabled(pVM));
                     VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
                     STAM_COUNTER_INC(&(pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestCR3WriteConflict));
@@ -403 +403 @@
                         if (uShw.pPD->a[iShw2].u & PGM_PDFLAGS_MAPPING)
                         {
-                            Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
+                            Assert(pgmMapAreMappingsEnabled(pVM));
                             STAM_COUNTER_INC(&(pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestCR3WriteConflict));
                             VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
@@ -449 +449 @@
                 if (uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING)
                 {
-                    Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
+                    Assert(pgmMapAreMappingsEnabled(pVM));
                     VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
                     STAM_COUNTER_INC(&(pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestCR3WriteConflict));
@@ -486 +486 @@
                         &&  uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING)
                     {
-                        Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
+                        Assert(pgmMapAreMappingsEnabled(pVM));
                         VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
                         STAM_COUNTER_INC(&(pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestCR3WriteConflict));
@@ -527 +527 @@
                     if (uShw.pPDPT->a[iShw].u & PGM_PLXFLAGS_MAPPING)
                     {
-                        Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
+                        Assert(pgmMapAreMappingsEnabled(pVM));
                         STAM_COUNTER_INC(&(pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestCR3WriteConflict));
                         VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
@@ -558 +558 @@
                             if (uShw.pPDPT->a[iShw2].u & PGM_PLXFLAGS_MAPPING)
                             {
-                                Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
+                                Assert(pgmMapAreMappingsEnabled(pVM));
                                 STAM_COUNTER_INC(&(pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestCR3WriteConflict));
                                 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
@@ -1125 +1125 @@
         &&  pVCpu->pgm.s.cPoolAccessHandler == (pPage->cLastAccessHandlerCount + 1))
     {
-        Log(("Possible page reuse cMods=%d -> %d (locked=%d type=%s)\n", pPage->cModifications, pPage->cModifications * 2, pgmPoolIsPageLocked(&pVM->pgm.s, pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
+        Log(("Possible page reuse cMods=%d -> %d (locked=%d type=%s)\n", pPage->cModifications, pPage->cModifications * 2, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
         Assert(pPage->cModifications < 32000);
         pPage->cModifications           = pPage->cModifications * 2;
@@ -1138 +1138 @@
 
     if (pPage->cModifications >= cMaxModifications)
-        Log(("Mod overflow %RGv cMods=%d (locked=%d type=%s)\n", pvFault, pPage->cModifications, pgmPoolIsPageLocked(&pVM->pgm.s, pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
+        Log(("Mod overflow %RGv cMods=%d (locked=%d type=%s)\n", pvFault, pPage->cModifications, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
 
     /*
@@ -1146 +1146 @@
     bool fNotReusedNotForking = false;
     if (    (   pPage->cModifications < cMaxModifications   /** @todo #define */ /** @todo need to check that it's not mapping EIP. */ /** @todo adjust this! */
-             || pgmPoolIsPageLocked(&pVM->pgm.s, pPage)
+             || pgmPoolIsPageLocked(pPage)
             )
         &&  !(fReused = pgmPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault))
@@ -1255 +1255 @@
        )
     {
-        Assert(!pgmPoolIsPageLocked(&pVM->pgm.s, pPage));
+        Assert(!pgmPoolIsPageLocked(pPage));
         Assert(pPage->fDirty == false);
 
@@ -2023 +2023 @@
          * Call pgmPoolCacheUsed to move the page to the head of the age list.
          */
-        if (!pgmPoolIsPageLocked(&pPool->CTX_SUFF(pVM)->pgm.s, pPage))
+        if (!pgmPoolIsPageLocked(pPage))
             break;
         LogFlow(("pgmPoolCacheFreeOne: refuse CR3 mapping\n"));
@@ -3451 +3451 @@
         if (GCPhysBase != GCPhysPage)
         {
-            pLargePage = pgmPhysGetPage(&pVM->pgm.s, GCPhysBase);
+            pLargePage = pgmPhysGetPage(pVM, GCPhysBase);
             AssertFatal(pLargePage);
         }
@@ -3707 +3707 @@
 
     /* Safety precaution in case we change the paging for other modes too in the future. */
-    Assert(!pgmPoolIsPageLocked(&pPool->CTX_SUFF(pVM)->pgm.s, pPage));
+    Assert(!pgmPoolIsPageLocked(pPage));
 
 #ifdef VBOX_STRICT
@@ -4148 +4148 @@
  * Clear references to guest physical memory.
  *
- * This is the same as pgmPoolTracDerefGCPhys except that the guest physical address
- * is assumed to be correct, so the linear search can be skipped and we can assert
- * at an earlier point.
+ * This is the same as pgmPoolTracDerefGCPhysHint except that the guest
+ * physical address is assumed to be correct, so the linear search can be
+ * skipped and we can assert at an earlier point.
  *
  * @param   pPool       The pool.
@@ -4161 +4161 @@
 {
     /*
-     * Walk range list.
-     */
-    PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
-    while (pRam)
-    {
-        RTGCPHYS off = GCPhys - pRam->GCPhys;
-        if (off < pRam->cb)
-        {
-            /* does it match? */
-            const unsigned iPage = off >> PAGE_SHIFT;
-            Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
+     * Lookup the page and check if it checks out before derefing it.
+     */
+    PPGMPAGE pPhysPage = pgmPhysGetPage(pPool->CTX_SUFF(pVM), GCPhys);
+    if (pPhysPage)
+    {
+        Assert(PGM_PAGE_GET_HCPHYS(pPhysPage));
 #ifdef LOG_ENABLED
-            RTHCPHYS HCPhysPage = PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]);
-            Log2(("pgmPoolTracDerefGCPhys %RHp vs %RHp\n", HCPhysPage, HCPhys));
+        RTHCPHYS HCPhysPage = PGM_PAGE_GET_HCPHYS(pPhysPage);
+        Log2(("pgmPoolTracDerefGCPhys %RHp vs %RHp\n", HCPhysPage, HCPhys));
 #endif
-            if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
-            {
-                Assert(pPage->cPresent);
-                Assert(pPool->cPresent);
-                pPage->cPresent--;
-                pPool->cPresent--;
-                pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage], iPte);
-                return;
-            }
-            AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp; found page index %x HCPhys=%RHp\n", HCPhys, GCPhys, iPage, PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage])));
-            break;
-        }
-        pRam = pRam->CTX_SUFF(pNext);
+        if (PGM_PAGE_GET_HCPHYS(pPhysPage) == HCPhys)
+        {
+            Assert(pPage->cPresent);
+            Assert(pPool->cPresent);
+            pPage->cPresent--;
+            pPool->cPresent--;
+            pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte);
+            return;
+        }
+
+        AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp; found page has HCPhys=%RHp\n",
+                              HCPhys, GCPhys, PGM_PAGE_GET_HCPHYS(pPhysPage)));
     }
     AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp\n", HCPhys, GCPhys));
@@ -4205 +4199 @@
 void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint, uint16_t iPte)
 {
-    RTHCPHYS HCPhysExpected = 0xDEADBEEFDEADBEEFULL;
-
     Log4(("pgmPoolTracDerefGCPhysHint %RHp %RGp\n", HCPhys, GCPhysHint));
 
     /*
-     * Walk range list.
-     */
-    PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
-    while (pRam)
-    {
-        RTGCPHYS off = GCPhysHint - pRam->GCPhys;
-        if (off < pRam->cb)
-        {
-            /* does it match? */
-            const unsigned iPage = off >> PAGE_SHIFT;
-            Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
-            if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
-            {
-                Assert(pPage->cPresent);
-                Assert(pPool->cPresent);
-                pPage->cPresent--;
-                pPool->cPresent--;
-                pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage], iPte);
-                return;
-            }
-            HCPhysExpected = PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]);
-            break;
-        }
-        pRam = pRam->CTX_SUFF(pNext);
-    }
-
-    /*
-     * Damn, the hint didn't work. We'll have to do an expensive linear search.
+     * Try the hint first.
+     */
+    RTHCPHYS HCPhysHinted;
+    PPGMPAGE pPhysPage = pgmPhysGetPage(pPool->CTX_SUFF(pVM), GCPhysHint);
+    if (pPhysPage)
+    {
+        HCPhysHinted = PGM_PAGE_GET_HCPHYS(pPhysPage);
+        Assert(HCPhysHinted);
+        if (HCPhysHinted == HCPhys)
+        {
+            Assert(pPage->cPresent);
+            Assert(pPool->cPresent);
+            pPage->cPresent--;
+            pPool->cPresent--;
+            pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte);
+            return;
+        }
+    }
+    else
+        HCPhysHinted = UINT64_C(0xdeadbeefdeadbeef);
+
+    /*
+     * Damn, the hint didn't work.  We'll have to do an expensive linear search.
      */
     STAM_COUNTER_INC(&pPool->StatTrackLinearRamSearches);
-    pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
+    PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRangesX);
     while (pRam)
     {
@@ -4261 +4248 @@
     }
 
-    AssertFatalMsgFailed(("HCPhys=%RHp GCPhysHint=%RGp (Expected HCPhys with hint = %RHp)\n", HCPhys, GCPhysHint, HCPhysExpected));
+    AssertFatalMsgFailed(("HCPhys=%RHp GCPhysHint=%RGp (Hinted page has HCPhys = %RHp)\n", HCPhys, GCPhysHint, HCPhysHinted));
 }
 
@@ -4778 +4765 @@
      * Quietly reject any attempts at flushing the currently active shadow CR3 mapping
      */
-    if (pgmPoolIsPageLocked(&pVM->pgm.s, pPage))
+    if (pgmPoolIsPageLocked(pPage))
     {
         AssertMsg(   pPage->enmKind == PGMPOOLKIND_64BIT_PML4
@@ -5184 +5171 @@
 #endif
                         STAM_COUNTER_INC(&pPool->StatForceFlushPage);
-                    Assert(!pgmPoolIsPageLocked(&pVM->pgm.s, pPage));
+                    Assert(!pgmPoolIsPageLocked(pPage));
                     pgmPoolMonitorChainFlush(pPool, pPage);
                     return;
@@ -5328 +5315 @@
      * Clear all the GCPhys links and rebuild the phys ext free list.
      */
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
          pRam;
          pRam = pRam->CTX_SUFF(pNext))

trunk/src/VBox/VMM/VMMAll/PGMAllShw.h

@@ -301 +301 @@
         AssertFailed(); /* can't happen */
 # else
-        Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
+        Assert(pgmMapAreMappingsEnabled(pVM));
 
         PPGMMAPPING pMap = pgmGetMapping(pVM, (RTGCPTR)GCPtr);
@@ -462 +462 @@
                     {
                         Assert(fGstPte & X86_PTE_RW);
-                        PPGMPAGE pPage = pgmPhysGetPage(&pVCpu->CTX_SUFF(pVM)->pgm.s, GCPhys);
+                        PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
                         Assert(pPage);
                         if (pPage)

trunk/src/VBox/VMM/VMMR0/PGMR0.cpp

@@ -334 +334 @@
         if (RT_SUCCESS(rc))
         {
-            for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR0; RT_SUCCESS(rc) && pRam; pRam = pRam->pNextR0)
+            for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR0; RT_SUCCESS(rc) && pRam; pRam = pRam->pNextR0)
             {
                 PPGMPAGE    pPage  = &pRam->aPages[0];
@@ -526 +526 @@
         if (   (   pHandler->cAliasedPages
                 || pHandler->cTmpOffPages)
-            && (   (pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysFault)) == NULL
+            && (   (pPage = pgmPhysGetPage(pVM, GCPhysFault)) == NULL
                 || PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
            )

trunk/src/VBox/VMM/VMMR0/PGMR0SharedPage.cpp

@@ -61 +61 @@
         Assert((pRegions[idxRegion].GCRegionAddr & 0xfff) == 0);
 
-        RTGCPTR  GCRegion  = pRegions[idxRegion].GCRegionAddr;
+        RTGCPTR  GCRegion = pRegions[idxRegion].GCRegionAddr;
         unsigned cbRegion = pRegions[idxRegion].cbRegion & ~0xfff;
-        unsigned idxPage = 0;
+        unsigned idxPage  = 0;
 
         while (cbRegion)
@@ -75 +75 @@
                 &&  !(fFlags & X86_PTE_RW)) /* important as we make assumptions about this below! */
             {
-                PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
+                PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
                 Assert(!pPage || !PGM_PAGE_IS_BALLOONED(pPage));
                 if (    pPage

trunk/src/VBox/VMM/VMMR3/PGM.cpp

@@ -1695 +1695 @@
     PGM_REG_COUNTER(&pStats->StatPageMapTlbFlushes,             "/PGM/R3/Page/MapTlbFlushes",         "TLB flushes (all contexts).");
     PGM_REG_COUNTER(&pStats->StatPageMapTlbFlushEntry,          "/PGM/R3/Page/MapTlbFlushEntry",      "TLB entry flushes (all contexts).");
+
+    PGM_REG_COUNTER(&pStats->StatRZRamRangeTlbHits,             "/PGM/RZ/RamRange/TlbHits",           "TLB hits.");
+    PGM_REG_COUNTER(&pStats->StatRZRamRangeTlbMisses,           "/PGM/RZ/RamRange/TlbMisses",         "TLB misses.");
+    PGM_REG_COUNTER(&pStats->StatR3RamRangeTlbHits,             "/PGM/R3/RamRange/TlbHits",           "TLB hits.");
+    PGM_REG_COUNTER(&pStats->StatR3RamRangeTlbMisses,           "/PGM/R3/RamRange/TlbMisses",         "TLB misses.");
 
     PGM_REG_PROFILE(&pStats->StatRZSyncCR3HandlerVirtualUpdate, "/PGM/RZ/SyncCR3/Handlers/VirtualUpdate", "Profiling of the virtual handler updates.");
@@ -2224 +2229 @@
      * Ram ranges.
      */
-    if (pVM->pgm.s.pRamRangesR3)
+    if (pVM->pgm.s.pRamRangesXR3)
     {
         /* Update the pSelfRC pointers and relink them. */
-        for (PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
+        for (PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
             if (!(pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING))
                 pCur->pSelfRC = MMHyperCCToRC(pVM, pCur);
         pgmR3PhysRelinkRamRanges(pVM);
+
+#ifdef PGM_USE_RAMRANGE_TLB
+        /* Flush the RC TLB. */
+        for (unsigned i = 0; i < PGM_RAMRANGE_TLB_ENTRIES; i++)
+            pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
+#endif
     }
 
@@ -2607 +2618 @@
                     sizeof(RTHCPTR) * 2,      "pvHC            ");
 
-    for (PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
+    for (PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
         pHlp->pfnPrintf(pHlp,
                         "%RGp-%RGp %RHv %s\n",
@@ -2656 +2667 @@
                                 "%04X - %RGp P=%d U=%d RW=%d G=%d - BIG\n",
                                 iPD,
-                                pgmGstGet4MBPhysPage(&pVM->pgm.s, PdeSrc),
+                                pgmGstGet4MBPhysPage(pVM, PdeSrc),
                                 PdeSrc.b.u1Present, PdeSrc.b.u1User, PdeSrc.b.u1Write, PdeSrc.b.u1Global && fPGE);
             else
@@ -3590 +3601 @@
     if (!pVM)
         return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
-    if (!pVM->pgm.s.pRamRangesRC)
+    if (!pVM->pgm.s.pRamRangesXR3)
         return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Sorry, no Ram is registered.\n");
 
@@ -3598 +3609 @@
     int rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL, "From     - To (incl) pvHC\n");
     PPGMRAMRANGE pRam;
-    for (pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
+    for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
     {
         rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL,
@@ -3814 +3825 @@
 
     pgmLock(pVM);
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
-          pRam && pRam->GCPhys < GCPhysEnd && RT_SUCCESS(rc);
-          pRam = pRam->pNextR3)
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
+         pRam && pRam->GCPhys < GCPhysEnd && RT_SUCCESS(rc);
+         pRam = pRam->pNextR3)
     {
         /* fill the gap */

trunk/src/VBox/VMM/VMMR3/PGMDbg.cpp

@@ -160 +160 @@
         return VERR_INVALID_POINTER;
 
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
          pRam;
          pRam = pRam->CTX_SUFF(pNext))
@@ -622 +622 @@
      */
     pgmLock(pVM);
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
          pRam;
          pRam = pRam->CTX_SUFF(pNext))
@@ -793 +793 @@
         if (RT_SUCCESS(rc))
         {
-            PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
+            PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
             if (    pPage
                 &&  (   !PGM_PAGE_IS_ZERO(pPage)
@@ -1041 +1041 @@
     {
         pgmLock(pState->pVM);
-        PCPGMPAGE pPage = pgmPhysGetPage(&pState->pVM->pgm.s, GCPhys);
+        PCPGMPAGE pPage = pgmPhysGetPage(pState->pVM, GCPhys);
         if (pPage)
             RTStrPrintf(szPage, sizeof(szPage), "%R[pgmpage]", pPage);
@@ -1719 +1719 @@
     char szPage[80];
     pgmLock(pState->pVM);
-    PCPGMPAGE pPage = pgmPhysGetPage(&pState->pVM->pgm.s, GCPhys);
+    PCPGMPAGE pPage = pgmPhysGetPage(pState->pVM, GCPhys);
     if (pPage)
         RTStrPrintf(szPage, sizeof(szPage), " %R[pgmpage]", pPage);

trunk/src/VBox/VMM/VMMR3/PGMHandler.cpp

@@ -160 +160 @@
 static DECLCALLBACK(int) pgmR3HandlerPhysicalOneClear(PAVLROGCPHYSNODECORE pNode, void *pvUser)
 {
-    PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)pNode;
+    PPGMPHYSHANDLER pCur     = (PPGMPHYSHANDLER)pNode;
     PPGMRAMRANGE    pRamHint = NULL;
-    RTGCPHYS        GCPhys = pCur->Core.Key;
-    RTUINT          cPages = pCur->cPages;
-    PPGM            pPGM = &((PVM)pvUser)->pgm.s;
+    RTGCPHYS        GCPhys   = pCur->Core.Key;
+    RTUINT          cPages   = pCur->cPages;
+    PVM             pVM      = (PVM)pvUser;
     for (;;)
     {
         PPGMPAGE pPage;
-        int rc = pgmPhysGetPageWithHintEx(pPGM, GCPhys, &pPage, &pRamHint);
+        int rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pPage, &pRamHint);
         if (RT_SUCCESS(rc))
             PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, PGM_PAGE_HNDL_PHYS_STATE_NONE);
@@ -190 +190 @@
 static DECLCALLBACK(int) pgmR3HandlerPhysicalOneSet(PAVLROGCPHYSNODECORE pNode, void *pvUser)
 {
-    PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)pNode;
-    unsigned        uState = pgmHandlerPhysicalCalcState(pCur);
+    PPGMPHYSHANDLER pCur     = (PPGMPHYSHANDLER)pNode;
+    unsigned        uState   = pgmHandlerPhysicalCalcState(pCur);
     PPGMRAMRANGE    pRamHint = NULL;
-    RTGCPHYS        GCPhys = pCur->Core.Key;
-    RTUINT          cPages = pCur->cPages;
-    PPGM            pPGM = &((PVM)pvUser)->pgm.s;
+    RTGCPHYS        GCPhys   = pCur->Core.Key;
+    RTUINT          cPages   = pCur->cPages;
+    PVM             pVM      = (PVM)pvUser;
     for (;;)
     {
         PPGMPAGE pPage;
-        int rc = pgmPhysGetPageWithHintEx(pPGM, GCPhys, &pPage, &pRamHint);
+        int rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pPage, &pRamHint);
         if (RT_SUCCESS(rc))
             PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, uState);
@@ -471 +471 @@
          * Reset the flags and remove phys2virt nodes.
          */
-        PPGM pPGM = &pVM->pgm.s;
-        for (unsigned iPage = 0; iPage < pCur->cPages; iPage++)
+        for (uint32_t iPage = 0; iPage < pCur->cPages; iPage++)
             if (pCur->aPhysToVirt[iPage].offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE)
-                pgmHandlerVirtualClearPage(pPGM, pCur, iPage);
+                pgmHandlerVirtualClearPage(pVM, pCur, iPage);
 
         /*

trunk/src/VBox/VMM/VMMR3/PGMMap.cpp

@@ -510 +510 @@
      * Ignore the additions mapping fix call if disabled.
      */
-    if (!pgmMapAreMappingsEnabled(&pVM->pgm.s))
+    if (!pgmMapAreMappingsEnabled(pVM))
     {
         Assert(HWACCMIsEnabled(pVM));
@@ -551 +551 @@
     AssertMsgReturn(cb && !(cb & X86_PAGE_4M_OFFSET_MASK), ("cb (%#x) is 0 or not aligned on a 4MB address!\n", cb),
                     VERR_INVALID_PARAMETER);
-    AssertReturn(pgmMapAreMappingsEnabled(&pVM->pgm.s), VERR_INTERNAL_ERROR_3);
+    AssertReturn(pgmMapAreMappingsEnabled(pVM), VERR_INTERNAL_ERROR_3);
     AssertReturn(pVM->cCpus == 1, VERR_INTERNAL_ERROR_4);
 
@@ -726 +726 @@
 {
     Log(("PGMR3MappingsUnfix: fMappingsFixed=%RTbool fMappingsDisabled=%RTbool\n", pVM->pgm.s.fMappingsFixed, pVM->pgm.s.fMappingsDisabled));
-    if (   pgmMapAreMappingsEnabled(&pVM->pgm.s)
+    if (   pgmMapAreMappingsEnabled(pVM)
         && (    pVM->pgm.s.fMappingsFixed
             ||  pVM->pgm.s.fMappingsFixedRestored)
@@ -1020 +1020 @@
     pgmLock(pVM);                           /* to avoid assertions */
 
-    Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s) || PGMGetGuestMode(pVCpu) <= PGMMODE_PAE_NX);
+    Assert(!pgmMapAreMappingsEnabled(pVM) || PGMGetGuestMode(pVCpu) <= PGMMODE_PAE_NX);
 
     pgmMapSetShadowPDEs(pVM, pMap, iNewPDE);

trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp

@@ -119 +119 @@
      * Copy loop on ram ranges.
      */
-    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+    PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
     for (;;)
     {
-        /* Find range. */
-        while (pRam && GCPhys > pRam->GCPhysLast)
-            pRam = pRam->CTX_SUFF(pNext);
         /* Inside range or not? */
         if (pRam && GCPhys >= pRam->GCPhys)
@@ -186 +183 @@
              * Unassigned address space.
              */
-            if (!pRam)
-                break;
-            size_t cb = pRam->GCPhys - GCPhys;
+            size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
             if (cb >= cbRead)
             {
@@ -200 +195 @@
             GCPhys += cb;
         }
+
+        /* Advance range if necessary. */
+        while (pRam && GCPhys > pRam->GCPhysLast)
+            pRam = pRam->CTX_SUFF(pNext);
     } /* Ram range walk */
 
@@ -250 +249 @@
      * Copy loop on ram ranges, stop when we hit something difficult.
      */
-    PPGMRAMRANGE    pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+    PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
     for (;;)
     {
-        /* Find range. */
-        while (pRam && GCPhys > pRam->GCPhysLast)
-            pRam = pRam->CTX_SUFF(pNext);
         /* Inside range or not? */
         if (pRam && GCPhys >= pRam->GCPhys)
@@ -332 +328 @@
             GCPhys += cb;
         }
+
+        /* Advance range if necessary. */
+        while (pRam && GCPhys > pRam->GCPhysLast)
+            pRam = pRam->CTX_SUFF(pNext);
     } /* Ram range walk */
 
@@ -361 +361 @@
     {
         PPGMPAGEMAPTLBE pTlbe;
-        int rc2 = pgmPhysPageQueryTlbe(&pVM->pgm.s, *pGCPhys, &pTlbe);
+        int rc2 = pgmPhysPageQueryTlbe(pVM, *pGCPhys, &pTlbe);
         AssertFatalRC(rc2);
         PPGMPAGE pPage = pTlbe->pPage;
@@ -431 +431 @@
      */
     PPGMPAGEMAPTLBE pTlbe;
-    rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
+    rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
     if (RT_SUCCESS(rc))
     {
@@ -532 +532 @@
      */
     PPGMPAGEMAPTLBE pTlbe;
-    rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
+    rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
     if (RT_SUCCESS(rc))
     {
@@ -591 +591 @@
 
 #ifdef VBOX_STRICT
-    for (pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
+    for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
     {
         Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pCur->pSelfR0 == MMHyperCCToR0(pVM, pCur));
@@ -599 +599 @@
         Assert((pCur->cb         & PAGE_OFFSET_MASK) == 0);
         Assert(pCur->cb == pCur->GCPhysLast - pCur->GCPhys + 1);
-        for (PPGMRAMRANGE pCur2 = pVM->pgm.s.pRamRangesR3; pCur2; pCur2 = pCur2->pNextR3)
+        for (PPGMRAMRANGE pCur2 = pVM->pgm.s.pRamRangesXR3; pCur2; pCur2 = pCur2->pNextR3)
             Assert(   pCur2 == pCur
                    || strcmp(pCur2->pszDesc, pCur->pszDesc)); /** @todo fix MMIO ranges!! */
@@ -605 +605 @@
 #endif
 
-    pCur = pVM->pgm.s.pRamRangesR3;
+    pCur = pVM->pgm.s.pRamRangesXR3;
     if (pCur)
     {
-        pVM->pgm.s.pRamRangesR0 = pCur->pSelfR0;
-        pVM->pgm.s.pRamRangesRC = pCur->pSelfRC;
+        pVM->pgm.s.pRamRangesXR0 = pCur->pSelfR0;
+        pVM->pgm.s.pRamRangesXRC = pCur->pSelfRC;
 
         for (; pCur->pNextR3; pCur = pCur->pNextR3)
@@ -622 +622 @@
     else
     {
-        Assert(pVM->pgm.s.pRamRangesR0 == NIL_RTR0PTR);
-        Assert(pVM->pgm.s.pRamRangesRC == NIL_RTRCPTR);
+        Assert(pVM->pgm.s.pRamRangesXR0 == NIL_RTR0PTR);
+        Assert(pVM->pgm.s.pRamRangesXRC == NIL_RTRCPTR);
     }
     ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
@@ -644 +644 @@
     pgmLock(pVM);
 
-    PPGMRAMRANGE pRam = pPrev ? pPrev->pNextR3 : pVM->pgm.s.pRamRangesR3;
+    PPGMRAMRANGE pRam = pPrev ? pPrev->pNextR3 : pVM->pgm.s.pRamRangesXR3;
     pNew->pNextR3 = pRam;
     pNew->pNextR0 = pRam ? pRam->pSelfR0 : NIL_RTR0PTR;
@@ -657 +657 @@
     else
     {
-        pVM->pgm.s.pRamRangesR3 = pNew;
-        pVM->pgm.s.pRamRangesR0 = pNew->pSelfR0;
-        pVM->pgm.s.pRamRangesRC = pNew->pSelfRC;
+        pVM->pgm.s.pRamRangesXR3 = pNew;
+        pVM->pgm.s.pRamRangesXR0 = pNew->pSelfR0;
+        pVM->pgm.s.pRamRangesXRC = pNew->pSelfRC;
     }
     ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
@@ -675 +675 @@
 static void pgmR3PhysUnlinkRamRange2(PVM pVM, PPGMRAMRANGE pRam, PPGMRAMRANGE pPrev)
 {
-    Assert(pPrev ? pPrev->pNextR3 == pRam : pVM->pgm.s.pRamRangesR3 == pRam);
+    Assert(pPrev ? pPrev->pNextR3 == pRam : pVM->pgm.s.pRamRangesXR3 == pRam);
     Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pRam->pSelfR0 == MMHyperCCToR0(pVM, pRam));
     Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pRam->pSelfRC == MMHyperCCToRC(pVM, pRam));
@@ -690 +690 @@
     else
     {
-        Assert(pVM->pgm.s.pRamRangesR3 == pRam);
-        pVM->pgm.s.pRamRangesR3 = pNext;
-        pVM->pgm.s.pRamRangesR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
-        pVM->pgm.s.pRamRangesRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
+        Assert(pVM->pgm.s.pRamRangesXR3 == pRam);
+        pVM->pgm.s.pRamRangesXR3 = pNext;
+        pVM->pgm.s.pRamRangesXR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
+        pVM->pgm.s.pRamRangesXRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
     }
     ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
@@ -712 +712 @@
     /* find prev. */
     PPGMRAMRANGE pPrev = NULL;
-    PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesR3;
+    PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesXR3;
     while (pCur != pRam)
     {
@@ -809 +809 @@
         for (unsigned i = 0; i < cPages; i++)
         {
-            PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, paPhysPage[i]);
+            PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
             if (    pPage == NULL
                 ||  pPage->uTypeY != PGMPAGETYPE_RAM)
@@ -850 +850 @@
         for (unsigned i = 0; i < cPages; i++)
         {
-            PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, paPhysPage[i]);
+            PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
             AssertBreak(pPage && pPage->uTypeY == PGMPAGETYPE_RAM);
 
@@ -960 +960 @@
 
 /**
- * Rendezvous callback used by PGMR3WriteProtectRAM that write protects all physical RAM
+ * Rendezvous callback used by PGMR3WriteProtectRAM that write protects all
+ * physical RAM.
  *
  * This is only called on one of the EMTs while the other ones are waiting for
@@ -968 +969 @@
  * @param   pVM         The VM handle.
  * @param   pVCpu       The VMCPU for the EMT we're being called on. Unused.
- * @param   pvUser      User parameter
+ * @param   pvUser      User parameter, unused.
  */
 static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysWriteProtectRAMRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
 {
     int rc = VINF_SUCCESS;
+    NOREF(pvUser);
 
     pgmLock(pVM);
@@ -981 +983 @@
     /** @todo pointless to write protect the physical page pointed to by RSP. */
 
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
          pRam;
          pRam = pRam->CTX_SUFF(pNext))
     {
-        unsigned cPages = pRam->cb >> PAGE_SHIFT;
-        for (unsigned iPage = 0; iPage < cPages; iPage++)
+        uint32_t cPages = pRam->cb >> PAGE_SHIFT;
+        for (uint32_t iPage = 0; iPage < cPages; iPage++)
         {
             PPGMPAGE    pPage = &pRam->aPages[iPage];
@@ -1048 +1050 @@
 
 /**
- * Enumerate all dirty FT pages
+ * Enumerate all dirty FT pages.
  *
  * @returns VBox status code.
  * @param   pVM         The VM handle.
- * @param   pfnEnum     Enumerate callback handler
- * @param   pvUser      Enumerate callback handler parameter
+ * @param   pfnEnum     Enumerate callback handler.
+ * @param   pvUser      Enumerate callback handler parameter.
  */
 VMMR3DECL(int) PGMR3PhysEnumDirtyFTPages(PVM pVM, PFNPGMENUMDIRTYFTPAGES pfnEnum, void *pvUser)
@@ -1060 +1062 @@
 
     pgmLock(pVM);
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
          pRam;
          pRam = pRam->CTX_SUFF(pNext))
     {
-        unsigned cPages = pRam->cb >> PAGE_SHIFT;
-        for (unsigned iPage = 0; iPage < cPages; iPage++)
-        {
-            PPGMPAGE    pPage = &pRam->aPages[iPage];
+        uint32_t cPages = pRam->cb >> PAGE_SHIFT;
+        for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        {
+            PPGMPAGE    pPage       = &pRam->aPages[iPage];
             PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
 
@@ -1074 +1076 @@
             {
                 /*
-                    * A RAM page.
-                    */
+                 * A RAM page.
+                 */
                 switch (PGM_PAGE_GET_STATE(pPage))
                 {
-                case PGM_PAGE_STATE_ALLOCATED:
-                case PGM_PAGE_STATE_WRITE_MONITORED:
-                    if (    !PGM_PAGE_IS_WRITTEN_TO(pPage)  /* not very recently updated? */
-                        &&  PGM_PAGE_IS_FT_DIRTY(pPage))
-                    {
-                        unsigned       cbPageRange = PAGE_SIZE;
-                        unsigned       iPageClean  = iPage + 1;
-                        RTGCPHYS       GCPhysPage  = pRam->GCPhys + iPage * PAGE_SIZE;
-                        uint8_t       *pu8Page     = NULL;
-                        PGMPAGEMAPLOCK Lock;
-
-                        /* Find the next clean page, so we can merge adjacent dirty pages. */
-                        for (; iPageClean < cPages; iPageClean++)
+                    case PGM_PAGE_STATE_ALLOCATED:
+                    case PGM_PAGE_STATE_WRITE_MONITORED:
+                        if (   !PGM_PAGE_IS_WRITTEN_TO(pPage)  /* not very recently updated? */
+                            && PGM_PAGE_IS_FT_DIRTY(pPage))
                         {
-                            PPGMPAGE pPageNext = &pRam->aPages[iPageClean];
-                            if (    RT_UNLIKELY(PGM_PAGE_GET_TYPE(pPageNext) != PGMPAGETYPE_RAM)
-                                ||  PGM_PAGE_GET_STATE(pPageNext) != PGM_PAGE_STATE_ALLOCATED
-                                ||  PGM_PAGE_IS_WRITTEN_TO(pPageNext)
-                                ||  !PGM_PAGE_IS_FT_DIRTY(pPageNext)
-                                /* Crossing a chunk boundary? */
-                                ||  (GCPhysPage & GMM_PAGEID_IDX_MASK) != ((GCPhysPage + cbPageRange) & GMM_PAGEID_IDX_MASK)
-                                )
-                                break;
-
-                            cbPageRange += PAGE_SIZE;
+                            unsigned       cbPageRange = PAGE_SIZE;
+                            unsigned       iPageClean  = iPage + 1;
+                            RTGCPHYS       GCPhysPage  = pRam->GCPhys + iPage * PAGE_SIZE;
+                            uint8_t       *pu8Page     = NULL;
+                            PGMPAGEMAPLOCK Lock;
+
+                            /* Find the next clean page, so we can merge adjacent dirty pages. */
+                            for (; iPageClean < cPages; iPageClean++)
+                            {
+                                PPGMPAGE pPageNext = &pRam->aPages[iPageClean];
+                                if (    RT_UNLIKELY(PGM_PAGE_GET_TYPE(pPageNext) != PGMPAGETYPE_RAM)
+                                    ||  PGM_PAGE_GET_STATE(pPageNext) != PGM_PAGE_STATE_ALLOCATED
+                                    ||  PGM_PAGE_IS_WRITTEN_TO(pPageNext)
+                                    ||  !PGM_PAGE_IS_FT_DIRTY(pPageNext)
+                                    /* Crossing a chunk boundary? */
+                                    ||  (GCPhysPage & GMM_PAGEID_IDX_MASK) != ((GCPhysPage + cbPageRange) & GMM_PAGEID_IDX_MASK)
+                                    )
+                                    break;
+
+                                cbPageRange += PAGE_SIZE;
+                            }
+
+                            rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysPage, (const void **)&pu8Page, &Lock);
+                            if (RT_SUCCESS(rc))
+                            {
+                                /** @todo this is risky; the range might be changed, but little choice as the sync
+                                 *  costs a lot of time. */
+                                pgmUnlock(pVM);
+                                pfnEnum(pVM, GCPhysPage, pu8Page, cbPageRange, pvUser);
+                                pgmLock(pVM);
+                                PGMPhysReleasePageMappingLock(pVM, &Lock);
+                            }
+
+                            for (iPage; iPage < iPageClean; iPage++)
+                                PGM_PAGE_CLEAR_FT_DIRTY(&pRam->aPages[iPage]);
+
+                            iPage = iPageClean - 1;
                         }
-
-                        rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysPage, (const void **)&pu8Page, &Lock);
-                        if (RT_SUCCESS(rc))
-                        {
-                            /** @todo this is risky; the range might be changed, but little choice as the sync costs a lot of time */
-                            pgmUnlock(pVM);
-                            pfnEnum(pVM, GCPhysPage, pu8Page, cbPageRange, pvUser);
-                            pgmLock(pVM);
-                            PGMPhysReleasePageMappingLock(pVM, &Lock);
-                        }
-
-                        for (iPage; iPage < iPageClean; iPage++)
-                            PGM_PAGE_CLEAR_FT_DIRTY(&pRam->aPages[iPage]);
-
-                        iPage = iPageClean - 1;
-                    }
-                    break;
+                        break;
                 }
             }
@@ -1142 +1145 @@
     pgmLock(pVM);
     uint32_t cRamRanges = 0;
-    for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRanges); pCur; pCur = pCur->CTX_SUFF(pNext))
+    for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext))
         cRamRanges++;
     pgmUnlock(pVM);
@@ -1168 +1171 @@
     pgmLock(pVM);
     uint32_t iCurRange = 0;
-    for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRanges); pCur; pCur = pCur->CTX_SUFF(pNext), iCurRange++)
+    for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext), iCurRange++)
         if (iCurRange == iRange)
         {
@@ -1191 +1194 @@
  * @returns VBox status code.
  * @param   pVM                 The VM handle.
- * @param   puTotalAllocSize    Pointer to total allocated  memory inside VMMR0 (in bytes)
- * @param   puTotalFreeSize     Pointer to total free (allocated but not used yet) memory inside VMMR0 (in bytes)
- * @param   puTotalBalloonSize  Pointer to total ballooned memory inside VMMR0 (in bytes)
- * @param   puTotalSharedSize   Pointer to total shared memory inside VMMR0 (in bytes)
- */
-VMMR3DECL(int) PGMR3QueryVMMMemoryStats(PVM pVM, uint64_t *puTotalAllocSize, uint64_t *puTotalFreeSize, uint64_t *puTotalBalloonSize, uint64_t *puTotalSharedSize)
-{
-    int rc;
-
-    uint64_t cAllocPages = 0, cFreePages = 0, cBalloonPages = 0, cSharedPages = 0;
-    rc = GMMR3QueryHypervisorMemoryStats(pVM, &cAllocPages, &cFreePages, &cBalloonPages, &cSharedPages);
+ * @param   pcbAllocMem         Where to return the amount of memory allocated
+ *                              by VMs.
+ * @param   pcbFreeMem          Where to return the amount of memory that is
+ *                              allocated from the host but not currently used
+ *                              by any VMs.
+ * @param   pcbBallonedMem      Where to return the sum of memory that is
+ *                              currently ballooned by the VMs.
+ * @param   pcbSharedMem        Where to return the amount of memory that is
+ *                              currently shared.
+ */
+VMMR3DECL(int) PGMR3QueryGlobalMemoryStats(PVM pVM, uint64_t *pcbAllocMem, uint64_t *pcbFreeMem,
+                                           uint64_t *pcbBallonedMem, uint64_t *pcbSharedMem)
+{
+    uint64_t cAllocPages   = 0;
+    uint64_t cFreePages    = 0;
+    uint64_t cBalloonPages = 0;
+    uint64_t cSharedPages  = 0;
+    int rc = GMMR3QueryHypervisorMemoryStats(pVM, &cAllocPages, &cFreePages, &cBalloonPages, &cSharedPages);
     AssertRCReturn(rc, rc);
 
-    if (puTotalAllocSize)
-        *puTotalAllocSize = cAllocPages * _4K;
-
-    if (puTotalFreeSize)
-        *puTotalFreeSize = cFreePages * _4K;
-
-    if (puTotalBalloonSize)
-        *puTotalBalloonSize = cBalloonPages * _4K;
-
-    if (puTotalSharedSize)
-        *puTotalSharedSize = cSharedPages * _4K;
-
-    Log(("PGMR3QueryVMMMemoryStats: all=%x free=%x ballooned=%x shared=%x\n", cAllocPages, cFreePages, cBalloonPages, cSharedPages));
+    if (pcbAllocMem)
+        *pcbAllocMem    = cAllocPages * _4K;
+
+    if (pcbFreeMem)
+        *pcbFreeMem     = cFreePages * _4K;
+
+    if (pcbBallonedMem)
+        *pcbBallonedMem = cBalloonPages * _4K;
+
+    if (pcbSharedMem)
+        *pcbSharedMem   = cSharedPages * _4K;
+
+    Log(("PGMR3QueryVMMMemoryStats: all=%llx free=%llx ballooned=%llx shared=%llx\n",
+         cAllocPages, cFreePages, cBalloonPages, cSharedPages));
     return VINF_SUCCESS;
 }
 
-/**
- * Query memory stats for the VM
+
+/**
+ * Query memory stats for the VM.
  *
  * @returns VBox status code.
  * @param   pVM                 The VM handle.
- * @param   puTotalAllocSize    Pointer to total allocated  memory inside the VM (in bytes)
- * @param   puTotalFreeSize     Pointer to total free (allocated but not used yet) memory inside the VM (in bytes)
- * @param   puTotalBalloonSize  Pointer to total ballooned memory inside the VM (in bytes)
- * @param   puTotalSharedSize   Pointer to total shared memory inside the VM (in bytes)
- */
-VMMR3DECL(int) PGMR3QueryMemoryStats(PVM pVM, uint64_t *pulTotalMem, uint64_t *pulPrivateMem, uint64_t *puTotalSharedMem, uint64_t *puTotalZeroMem)
-{
-    if (pulTotalMem)
-        *pulTotalMem = (uint64_t)pVM->pgm.s.cAllPages * _4K;
-
-    if (pulPrivateMem)
-        *pulPrivateMem = (uint64_t)pVM->pgm.s.cPrivatePages * _4K;
-
-    if (puTotalSharedMem)
-        *puTotalSharedMem = (uint64_t)pVM->pgm.s.cReusedSharedPages * _4K;
-
-    if (puTotalZeroMem)
-        *puTotalZeroMem = (uint64_t)pVM->pgm.s.cZeroPages * _4K;
+ * @param   pcbTotalMem         Where to return total amount memory the VM may
+ *                              possibly use.
+ * @param   pcbPrivateMem       Where to return the amount of private memory
+ *                              currently allocated.
+ * @param   pcbSharedMem        Where to return the amount of actually shared
+ *                              memory currently used by the VM.
+ * @param   pcbZeroMem          Where to return the amount of memory backed by
+ *                              zero pages.
+ *
+ * @remarks The total mem is normally larger than the sum of the three
+ *          components.  There are two reasons for this, first the amount of
+ *          shared memory is what we're sure is shared instead of what could
+ *          possibly be shared with someone.  Secondly, because the total may
+ *          include some pure MMIO pages that doesn't go into any of the three
+ *          sub-counts.
+ *
+ * @todo Why do we return reused shared pages instead of anything that could
+ *       potentially be shared?  Doesn't this mean the first VM gets a much
+ *       lower number of shared pages?
+ */
+VMMR3DECL(int) PGMR3QueryMemoryStats(PVM pVM, uint64_t *pcbTotalMem, uint64_t *pcbPrivateMem,
+                                     uint64_t *pcbSharedMem, uint64_t *pcbZeroMem)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    if (pcbTotalMem)
+        *pcbTotalMem    = (uint64_t)pVM->pgm.s.cAllPages            * PAGE_SIZE;
+
+    if (pcbPrivateMem)
+        *pcbPrivateMem  = (uint64_t)pVM->pgm.s.cPrivatePages        * PAGE_SIZE;
+
+    if (pcbSharedMem)
+        *pcbSharedMem   = (uint64_t)pVM->pgm.s.cReusedSharedPages   * PAGE_SIZE;
+
+    if (pcbZeroMem)
+        *pcbZeroMem     = (uint64_t)pVM->pgm.s.cZeroPages           * PAGE_SIZE;
 
     Log(("PGMR3QueryMemoryStats: all=%x private=%x reused=%x zero=%x\n", pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cReusedSharedPages, pVM->pgm.s.cZeroPages));
     return VINF_SUCCESS;
 }
+
 
 /**
@@ -1308 +1339 @@
         case PGMRELOCATECALL_SUGGEST:
             return true;
+
         case PGMRELOCATECALL_RELOCATE:
         {
-            /* Update myself and then relink all the ranges. */
+            /*
+             * Update myself, then relink all the ranges and flush the RC TLB.
+             */
             pgmLock(pVM);
+
             pRam->pSelfRC = (RTRCPTR)(GCPtrNew + PAGE_SIZE);
+
             pgmR3PhysRelinkRamRanges(pVM);
+#ifdef PGM_USE_RAMRANGE_TLB
+            for (unsigned i = 0; i < PGM_RAMRANGE_TLB_ENTRIES; i++)
+                pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
+#endif
+
             pgmUnlock(pVM);
             return true;
@@ -1440 +1481 @@
      */
     PPGMRAMRANGE    pPrev = NULL;
-    PPGMRAMRANGE    pRam = pVM->pgm.s.pRamRangesR3;
+    PPGMRAMRANGE    pRam = pVM->pgm.s.pRamRangesXR3;
     while (pRam && GCPhysLast >= pRam->GCPhys)
     {
@@ -1563 +1604 @@
     uint64_t NanoTS = RTTimeNanoTS();
     pgmLock(pVM);
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
     {
         PPGMPAGE    pPage  = &pRam->aPages[0];
@@ -1647 +1688 @@
      * Walk the ram ranges.
      */
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
     {
         uint32_t iPage = pRam->cb >> PAGE_SHIFT;
@@ -1800 +1841 @@
      * Walk the ram ranges.
      */
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
     {
         uint32_t iPage = pRam->cb >> PAGE_SHIFT;
@@ -1886 +1927 @@
     bool fRamExists = false;
     PPGMRAMRANGE pRamPrev = NULL;
-    PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
+    PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
     while (pRam && GCPhysLast >= pRam->GCPhys)
     {
@@ -2028 +2069 @@
     VM_ASSERT_EMT(pVM);
 
+/** @todo this needs to own the PGM lock! */
     /*
      * First deregister the handler, then check if we should remove the ram range.
@@ -2036 +2078 @@
         RTGCPHYS        GCPhysLast  = GCPhys + (cb - 1);
         PPGMRAMRANGE    pRamPrev    = NULL;
-        PPGMRAMRANGE    pRam        = pVM->pgm.s.pRamRangesR3;
+        PPGMRAMRANGE    pRam        = pVM->pgm.s.pRamRangesXR3;
         while (pRam && GCPhysLast >= pRam->GCPhys)
         {
@@ -2122 +2164 @@
 
     PGMPhysInvalidatePageMapTLB(pVM);
+#ifdef PGM_USE_RAMRANGE_TLB
+    pgmPhysInvalidRamRangeTlbs(pVM);
+#endif
     return rc;
 }
@@ -2428 +2473 @@
     bool fRamExists = false;
     PPGMRAMRANGE pRamPrev = NULL;
-    PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
+    PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
     while (pRam && GCPhysLast >= pRam->GCPhys)
     {
@@ -2589 +2634 @@
     {
         /* Restore the RAM pages we've replaced. */
-        PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
+        PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
         while (pRam->GCPhys > pCur->RamRange.GCPhysLast)
             pRam = pRam->pNextR3;
@@ -2630 +2675 @@
 
     PGMPhysInvalidatePageMapTLB(pVM);
+#ifdef PGM_USE_RAMRANGE_TLB
+    pgmPhysInvalidRamRangeTlbs(pVM);
+#endif
     pgmUnlock(pVM);
 
@@ -2830 +2878 @@
     bool            fRamExists = false;
     PPGMRAMRANGE    pRamPrev = NULL;
-    PPGMRAMRANGE    pRam = pVM->pgm.s.pRamRangesR3;
+    PPGMRAMRANGE    pRam = pVM->pgm.s.pRamRangesXR3;
     while (pRam && GCPhysLast >= pRam->GCPhys)
     {
@@ -3202 +3250 @@
                 if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
                 {
-                    pShadowPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
+                    pShadowPage = pgmPhysGetPage(pVM, GCPhys);
                     AssertLogRelReturn(pShadowPage, VERR_INTERNAL_ERROR);
                 }
@@ -3413 +3461 @@
 
                     /* flush references to the page. */
-                    PPGMPAGE pRamPage = pgmPhysGetPage(&pVM->pgm.s, pRom->GCPhys + (iPage << PAGE_SHIFT));
+                    PPGMPAGE pRamPage = pgmPhysGetPage(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT));
                     int rc2 = pgmPoolTrackUpdateGCPhys(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT), pRamPage,
                                                        true /*fFlushPTEs*/, &fFlushTLB);
@@ -3904 +3952 @@
 
                 PPGMPAGE pPage;
-                rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
+                rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
                 AssertRC(rc);
 
@@ -4086 +4134 @@
                 if (idPage != NIL_GMM_PAGEID)
                 {
-                    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
+                    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
                          pRam;
                          pRam = pRam->pNextR3)
@@ -4251 +4299 @@
     PPGMRAMRANGE pRam;
     PPGMPAGE pPage;
-    int rc = pgmPhysGetPageAndRangeEx(&pVM->pgm.s, GCPhys, &pPage, &pRam);
+    int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
     if (RT_SUCCESS(rc))
     {
@@ -4305 +4353 @@
             /* Get a ring-3 mapping of the address. */
             PPGMPAGER3MAPTLBE pTlbe;
-            rc2 = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
+            rc2 = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
             AssertLogRelRCReturn(rc2, rc2);
             *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));

trunk/src/VBox/VMM/VMMR3/PGMPool.cpp

@@ -594 +594 @@
     }
     else if (    (   pPage->cModifications < 96 /* it's cheaper here. */
-                  || pgmPoolIsPageLocked(&pVM->pgm.s, pPage)
+                  || pgmPoolIsPageLocked(pPage)
                   )
              &&  cbBuf <= 4)
@@ -643 +643 @@
     unsigned cModifiedPages = 0; NOREF(cModifiedPages);
     unsigned cLeft = pPool->cUsedPages;
-    unsigned iPage = pPool->cCurPages;
+    uint32_t iPage = pPool->cCurPages;
     while (--iPage >= PGMPOOL_IDX_FIRST)
     {
@@ -799 +799 @@
      * Clear all the GCPhys links and rebuild the phys ext free list.
      */
-    for (PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
+    for (PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRangesX);
          pRam;
          pRam = pRam->CTX_SUFF(pNext))

trunk/src/VBox/VMM/VMMR3/PGMSavedState.cpp

@@ -223 +223 @@
                     RTGCPHYS GCPhys = pRom->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
                     PPGMPAGE pPage;
-                    int rc = pgmPhysGetPageWithHintEx(&pVM->pgm.s, GCPhys, &pPage, &pRamHint);
+                    int rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pPage, &pRamHint);
                     AssertLogRelMsgRC(rc, ("%Rrc GCPhys=%RGp\n", rc, GCPhys));
                     if (RT_SUCCESS(rc))
@@ -419 +419 @@
             PPGMPAGE pPage;
             if (PGMROMPROT_IS_ROM(enmProt))
-                pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
+                pPage = pgmPhysGetPage(pVM, GCPhys);
             else
                 pPage = &pRom->aPages[iPage].Virgin;
@@ -513 +513 @@
                     PGMROMPROT  enmProt = pRomPage->enmProt;
                     RTGCPHYS    GCPhys  = pRom->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
-                    PPGMPAGE    pPage   = PGMROMPROT_IS_ROM(enmProt) ? &pRomPage->Shadow : pgmPhysGetPage(&pVM->pgm.s, GCPhys);
+                    PPGMPAGE    pPage   = PGMROMPROT_IS_ROM(enmProt) ? &pRomPage->Shadow : pgmPhysGetPage(pVM, GCPhys);
                     bool        fZero   = PGM_PAGE_IS_ZERO(pPage) || PGM_PAGE_IS_BALLOONED(pPage); Assert(!PGM_PAGE_IS_BALLOONED(pPage)); /* Shouldn't be ballooned. */
                     int         rc      = VINF_SUCCESS;
@@ -1048 +1048 @@
     do
     {
-        for (pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
+        for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
         {
             if (   !pCur->paLSPages
@@ -1304 +1304 @@
     {
         uint32_t const  idRamRangesGen = pVM->pgm.s.idRamRangesGen;
-        for (pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
+        for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
         {
             if (    pCur->GCPhysLast > GCPhysCur
@@ -1531 +1531 @@
     {
         uint32_t const  idRamRangesGen = pVM->pgm.s.idRamRangesGen;
-        for (pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
+        for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
         {
             if (   pCur->GCPhysLast > GCPhysCur
@@ -1746 +1746 @@
     do
     {
-        for (pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
+        for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
         {
             if (pCur->paLSPages)
@@ -2334 +2334 @@
      */
     uint32_t i = 0;
-    for (PPGMRAMRANGE pRam = pPGM->pRamRangesR3; ; pRam = pRam->pNextR3, i++)
+    for (PPGMRAMRANGE pRam = pPGM->pRamRangesXR3; ; pRam = pRam->pNextR3, i++)
     {
         /* Check the sequence number / separator. */
@@ -2641 +2641 @@
 
                 PPGMPAGE pPage;
-                rc = pgmPhysGetPageWithHintEx(&pVM->pgm.s, GCPhys, &pPage, &pRamHint);
+                rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pPage, &pRamHint);
                 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc %RGp\n", rc, GCPhys), rc);
 
@@ -2858 +2858 @@
                 if (!pRealPage)
                 {
-                    rc = pgmPhysGetPageWithHintEx(&pVM->pgm.s, GCPhys, &pRealPage, &pRamHint);
+                    rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pRealPage, &pRamHint);
                     AssertLogRelMsgRCReturn(rc, ("rc=%Rrc %RGp\n", rc, GCPhys), rc);
                 }
@@ -3176 +3176 @@
             pVM->pgm.s.fMappingsFixedRestored = false;
             if (   pVM->pgm.s.fMappingsFixed
-                && pgmMapAreMappingsEnabled(&pVM->pgm.s))
+                && pgmMapAreMappingsEnabled(pVM))
             {
                 RTGCPTR     GCPtrFixed    = pVM->pgm.s.GCPtrMappingFixed;
@@ -3213 +3213 @@
              * when restoring other components like PATM.
              */
-            if (pgmMapAreMappingsFloating(&pVM->pgm.s))
+            if (pgmMapAreMappingsFloating(pVM))
             {
                 PVMCPU pVCpu = &pVM->aCpus[0];

trunk/src/VBox/VMM/VMMR3/PGMSharedPage.cpp

@@ -262 +262 @@
     case VINF_SUCCESS:
     {
-        PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
+        PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
         if (pPage)
         {
@@ -316 +316 @@
     pgmLock(pVM);
 
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
     {
         PPGMPAGE    pPage  = &pRam->aPages[0];

trunk/src/VBox/VMM/include/PGMGstDefs.h

@@ -140 +140 @@
 # define GST_GET_PTE_GCPHYS(Pte)                ((Pte).u & GST_PDE_PG_MASK)
 # define GST_GET_PDE_GCPHYS(Pde)                ((Pde).u & GST_PDE_PG_MASK)
-# define GST_GET_BIG_PDE_GCPHYS(pVM, Pde)       pgmGstGet4MBPhysPage(&(pVM)->pgm.s, Pde)
+# define GST_GET_BIG_PDE_GCPHYS(pVM, Pde)       pgmGstGet4MBPhysPage((pVM), Pde)
 # define GST_GET_PDE_SHW_FLAGS(pVCpu, Pde)      ((Pde).u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A))
 # define GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, Pde) \

trunk/src/VBox/VMM/include/PGMInline.h

@@ -46 +46 @@
  */
 
-/** @todo Split out all the inline stuff into a separate file.  Then we can
- *        include it later when VM and VMCPU are defined and so avoid all that
- *        &pVM->pgm.s and &pVCpu->pgm.s stuff.  It also chops ~1600 lines off
- *        this file and will make it somewhat easier to navigate... */
-
 /**
  * Gets the PGMRAMRANGE structure for a guest page.
@@ -57 +52 @@
  * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
  *
- * @param   pPGM        PGM handle.
+ * @param   pVM         The VM handle.
  * @param   GCPhys      The GC physical address.
  */
-DECLINLINE(PPGMRAMRANGE) pgmPhysGetRange(PPGM pPGM, RTGCPHYS GCPhys)
-{
+DECLINLINE(PPGMRAMRANGE) pgmPhysGetRange(PVM pVM, RTGCPHYS GCPhys)
+{
+#ifdef PGM_USE_RAMRANGE_TLB
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
+    if (!pRam || GCPhys - pRam->GCPhys >= pRam->cb)
+        pRam = pgmPhysGetRangeSlow(pVM, GCPhys);
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbHits));
+    return pRam;
+
+#else
     /*
      * Optimize for the first range.
      */
-    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
     RTGCPHYS off = GCPhys - pRam->GCPhys;
     if (RT_UNLIKELY(off >= pRam->cb))
@@ -78 +81 @@
     }
     return pRam;
-}
+#endif
+}
+
+
+/**
+ * Gets the PGMRAMRANGE structure for a guest page, if unassigned get the ram
+ * range above it.
+ *
+ * @returns Pointer to the RAM range on success.
+ * @returns NULL if the address is located after the last range.
+ *
+ * @param   pVM         The VM handle.
+ * @param   GCPhys      The GC physical address.
+ */
+DECLINLINE(PPGMRAMRANGE) pgmPhysGetRangeAtOrAbove(PVM pVM, RTGCPHYS GCPhys)
+{
+#ifdef PGM_USE_RAMRANGE_TLB
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
+    if (   !pRam
+        || (GCPhys - pRam->GCPhys) >= pRam->cb)
+        return pgmPhysGetRangeAtOrAboveSlow(pVM, GCPhys);
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbHits));
+    return pRam;
+
+#else
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
+    while (pRam && GCPhys > pRam->GCPhysLast)
+        pRam = pRam->CTX_SUFF(pNext);
+    return pRam;
+#endif
+}
+
 
 
@@ -87 +121 @@
  * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
  *
- * @param   pPGM        PGM handle.
+ * @param   pVM         The VM handle.
  * @param   GCPhys      The GC physical address.
  */
-DECLINLINE(PPGMPAGE) pgmPhysGetPage(PPGM pPGM, RTGCPHYS GCPhys)
-{
+DECLINLINE(PPGMPAGE) pgmPhysGetPage(PVM pVM, RTGCPHYS GCPhys)
+{
+#ifdef PGM_USE_RAMRANGE_TLB
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
+    RTGCPHYS off;
+    if (   !pRam
+        || (off = GCPhys - pRam->GCPhys) >= pRam->cb)
+        return pgmPhysGetPageSlow(pVM, GCPhys);
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbHits));
+    return &pRam->aPages[off >> PAGE_SHIFT];
+
+#else
     /*
      * Optimize for the first range.
      */
-    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
     RTGCPHYS off = GCPhys - pRam->GCPhys;
     if (RT_UNLIKELY(off >= pRam->cb))
@@ -108 +152 @@
     }
     return &pRam->aPages[off >> PAGE_SHIFT];
+#endif
 }
 
@@ -120 +165 @@
  * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
  *
- * @param   pPGM        PGM handle.
+ * @param   pVM         The VM handle.
  * @param   GCPhys      The GC physical address.
  * @param   ppPage      Where to store the page pointer on success.
  */
-DECLINLINE(int) pgmPhysGetPageEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
-{
+DECLINLINE(int) pgmPhysGetPageEx(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
+{
+#ifdef PGM_USE_RAMRANGE_TLB
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
+    RTGCPHYS off;
+    if (   !pRam
+        || (off = GCPhys - pRam->GCPhys) >= pRam->cb)
+        return pgmPhysGetPageExSlow(pVM, GCPhys, ppPage);
+    *ppPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbHits));
+    return VINF_SUCCESS;
+
+#else
     /*
      * Optimize for the first range.
      */
-    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
     RTGCPHYS off = GCPhys - pRam->GCPhys;
     if (RT_UNLIKELY(off >= pRam->cb))
@@ -146 +202 @@
     *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
     return VINF_SUCCESS;
+#endif
 }
 
@@ -160 +217 @@
  * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
  *
- * @param   pPGM        PGM handle.
+ * @param   pVM         The VM handle.
  * @param   GCPhys      The GC physical address.
  * @param   ppPage      Where to store the page pointer on success.
@@ -166 +223 @@
  *                      The caller initializes this to NULL before the call.
  */
-DECLINLINE(int) pgmPhysGetPageWithHintEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRamHint)
+DECLINLINE(int) pgmPhysGetPageWithHintEx(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRamHint)
 {
     RTGCPHYS off;
@@ -173 +230 @@
         ||  RT_UNLIKELY((off = GCPhys - pRam->GCPhys) >= pRam->cb))
     {
-        pRam = pPGM->CTX_SUFF(pRamRanges);
+#ifdef PGM_USE_RAMRANGE_TLB
+        pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
+        if (   !pRam
+            || (off = GCPhys - pRam->GCPhys) >= pRam->cb)
+            return pgmPhysGetPageAndRangeExSlow(pVM, GCPhys, ppPage, ppRamHint);
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbHits));
+
+#else
+        pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
         off = GCPhys - pRam->GCPhys;
         if (RT_UNLIKELY(off >= pRam->cb))
@@ -188 +253 @@
             } while (off >= pRam->cb);
         }
+#endif
         *ppRamHint = pRam;
     }
@@ -201 +267 @@
  * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
  *
- * @param   pPGM        PGM handle.
+ * @param   pVM         The VM handle.
  * @param   GCPhys      The GC physical address.
- * @param   ppRam       Where to store the pointer to the PGMRAMRANGE.
- */
-DECLINLINE(PPGMPAGE) pgmPhysGetPageAndRange(PPGM pPGM, RTGCPHYS GCPhys, PPGMRAMRANGE *ppRam)
-{
+ * @param   ppPage      Where to store the pointer to the PGMPAGE structure.
+ * @param   ppRam       Where to store the pointer to the PGMRAMRANGE structure.
+ */
+DECLINLINE(int) pgmPhysGetPageAndRangeEx(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
+{
+#ifdef PGM_USE_RAMRANGE_TLB
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
+    RTGCPHYS off;
+    if (   !pRam
+        || (off = GCPhys - pRam->GCPhys) >= pRam->cb)
+        return pgmPhysGetPageAndRangeExSlow(pVM, GCPhys, ppPage, ppRam);
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbHits));
+
+#else
     /*
      * Optimize for the first range.
      */
-    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
-    RTGCPHYS off = GCPhys - pRam->GCPhys;
-    if (RT_UNLIKELY(off >= pRam->cb))
-    {
-        do
-        {
-            pRam = pRam->CTX_SUFF(pNext);
-            if (RT_UNLIKELY(!pRam))
-                return NULL;
-            off = GCPhys - pRam->GCPhys;
-        } while (off >= pRam->cb);
-    }
-    *ppRam = pRam;
-    return &pRam->aPages[off >> PAGE_SHIFT];
-}
-
-
-/**
- * Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
- *
- * @returns Pointer to the page on success.
- * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
- *
- * @param   pPGM        PGM handle.
- * @param   GCPhys      The GC physical address.
- * @param   ppPage      Where to store the pointer to the PGMPAGE structure.
- * @param   ppRam       Where to store the pointer to the PGMRAMRANGE structure.
- */
-DECLINLINE(int) pgmPhysGetPageAndRangeEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
-{
-    /*
-     * Optimize for the first range.
-     */
-    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
     RTGCPHYS off = GCPhys - pRam->GCPhys;
     if (RT_UNLIKELY(off >= pRam->cb))
@@ -259 +302 @@
         } while (off >= pRam->cb);
     }
+#endif
     *ppRam = pRam;
     *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
@@ -269 +313 @@
  *
  * @returns VBox status.
- * @param   pPGM        PGM handle.
+ * @param   pVM         The VM handle.
  * @param   GCPhys      The GC physical address.
  * @param   pHCPhys     Where to store the corresponding HC physical address.
@@ -276 +320 @@
  *              Avoid when writing new code!
  */
-DECLINLINE(int) pgmRamGCPhys2HCPhys(PPGM pPGM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
+DECLINLINE(int) pgmRamGCPhys2HCPhys(PVM pVM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
 {
     PPGMPAGE pPage;
-    int rc = pgmPhysGetPageEx(pPGM, GCPhys, &pPage);
+    int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
     if (RT_FAILURE(rc))
         return rc;
@@ -344 +388 @@
      * Get the ram range.
      */
-    PPGMRAMRANGE    pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+#ifdef PGM_USE_RAMRANGE_TLB
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
+    RTGCPHYS off;
+    if (   !pRam
+        || (off = GCPhys - pRam->GCPhys) >= pRam->cb
+        /** @todo   || page state stuff */
+       )
+#else
+    PPGMRAMRANGE    pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
     RTGCPHYS        off  = GCPhys - pRam->GCPhys;
     if (RT_UNLIKELY(off >= pRam->cb
         /** @todo   || page state stuff */))
+#endif
     {
         /* This case is not counted into StatRZDynMapGCPageInl. */
@@ -418 +471 @@
      */
     PVM             pVM  = pVCpu->CTX_SUFF(pVM);
-    PPGMRAMRANGE    pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+#ifdef PGM_USE_RAMRANGE_TLB
+    PPGMRAMRANGE    pRam = pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)];
+    RTGCPHYS        off;
+    if (   !pRam
+        || (off = GCPhys - pRam->GCPhys) >= pRam->cb
+        /** @todo   || page state stuff */
+       )
+#else
+    PPGMRAMRANGE    pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
     RTGCPHYS        off  = GCPhys - pRam->GCPhys;
     if (RT_UNLIKELY(off >= pRam->cb
         /** @todo   || page state stuff */))
+#endif
     {
         /* This case is not counted into StatRZDynMapGCPageInl. */
@@ -513 +575 @@
  * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
  *
- * @param   pPGM        The PGM instance handle.
+ * @param   pVM         The VM handle.
  * @param   GCPhys      The address of the guest page.
  * @param   ppTlbe      Where to store the pointer to the TLB entry.
  */
-DECLINLINE(int) pgmPhysPageQueryTlbe(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
+DECLINLINE(int) pgmPhysPageQueryTlbe(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
 {
     int rc;
-    PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
+    PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
     if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
     {
-        STAM_COUNTER_INC(&pPGM->CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbHits));
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbHits));
         rc = VINF_SUCCESS;
     }
     else
-        rc = pgmPhysPageLoadIntoTlb(pPGM, GCPhys);
+        rc = pgmPhysPageLoadIntoTlb(pVM, GCPhys);
     *ppTlbe = pTlbe;
     return rc;
@@ -541 +603 @@
  * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
  *
- * @param   pPGM        The PGM instance handle.
+ * @param   pVM         The VM handle.
  * @param   pPage       Pointer to the PGMPAGE structure corresponding to
  *                      GCPhys.
@@ -547 +609 @@
  * @param   ppTlbe      Where to store the pointer to the TLB entry.
  */
-DECLINLINE(int) pgmPhysPageQueryTlbeWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
+DECLINLINE(int) pgmPhysPageQueryTlbeWithPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
 {
     int rc;
-    PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
+    PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
     if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
     {
-        STAM_COUNTER_INC(&pPGM->CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbHits));
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbHits));
         rc = VINF_SUCCESS;
     }
     else
-        rc = pgmPhysPageLoadIntoTlbWithPage(pPGM, pPage, GCPhys);
+        rc = pgmPhysPageLoadIntoTlbWithPage(pVM, pPage, GCPhys);
     *ppTlbe = pTlbe;
     return rc;
@@ -567 +629 @@
 /**
  * Enables write monitoring for an allocated page.
- *  
- * The caller is responsible for updating the shadow page tables. 
- *  
+ *
+ * The caller is responsible for updating the shadow page tables.
+ *
  * @param   pVM         The VM handle.
- * @param   pPage       The page to write monitor. 
+ * @param   pPage       The page to write monitor.
  * @param   GCPhysPage  The address of the page.
  */
@@ -585 +647 @@
     if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
     {
-        PPGMPAGE pFirstPage = pgmPhysGetPage(&pVM->pgm.s, GCPhysPage & X86_PDE2M_PAE_PG_MASK);
+        PPGMPAGE pFirstPage = pgmPhysGetPage(pVM, GCPhysPage & X86_PDE2M_PAE_PG_MASK);
         AssertFatal(pFirstPage);
         if (PGM_PAGE_GET_PDE_TYPE(pFirstPage) == PGM_PAGE_PDE_TYPE_PDE)
@@ -638 +700 @@
  *
  * @returns guest physical address
- * @param   pPGM        Pointer to the PGM instance data.
+ * @param   pVM         The VM handle.
  * @param   Pde         Guest Pde
  */
-DECLINLINE(RTGCPHYS) pgmGstGet4MBPhysPage(PPGM pPGM, X86PDE Pde)
+DECLINLINE(RTGCPHYS) pgmGstGet4MBPhysPage(PVM pVM, X86PDE Pde)
 {
     RTGCPHYS GCPhys = Pde.u & X86_PDE4M_PG_MASK;
     GCPhys |= (RTGCPHYS)Pde.b.u8PageNoHigh << 32;
 
-    return GCPhys & pPGM->GCPhys4MBPSEMask;
+    return GCPhys & pVM->pgm.s.GCPhys4MBPSEMask;
 }
 
@@ -1307 +1369 @@
 
 /**
- * Clears one physical page of a virtual handler
- *
- * @param   pPGM    Pointer to the PGM instance.
- * @param   pCur    Virtual handler structure
- * @param   iPage   Physical page index
+ * Clears one physical page of a virtual handler.
+ *
+ * @param   pVM         The VM handle.
+ * @param   pCur        Virtual handler structure.
+ * @param   iPage       Physical page index.
  *
  * @remark  Only used when PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL is being set, so no
  *          need to care about other handlers in the same page.
  */
-DECLINLINE(void) pgmHandlerVirtualClearPage(PPGM pPGM, PPGMVIRTHANDLER pCur, unsigned iPage)
+DECLINLINE(void) pgmHandlerVirtualClearPage(PVM pVM, PPGMVIRTHANDLER pCur, unsigned iPage)
 {
     const PPGMPHYS2VIRTHANDLER pPhys2Virt = &pCur->aPhysToVirt[iPage];
@@ -1331 +1393 @@
     {
         /* We're the head of the alias chain. */
-        PPGMPHYS2VIRTHANDLER pRemove = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysRemove(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key); NOREF(pRemove);
+        PPGMPHYS2VIRTHANDLER pRemove = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysRemove(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key); NOREF(pRemove);
 #ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
         AssertReleaseMsg(pRemove != NULL,
@@ -1352 +1414 @@
 #endif
             pNext->offNextAlias |= PGMPHYS2VIRTHANDLER_IS_HEAD;
-            bool fRc = RTAvlroGCPhysInsert(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, &pNext->Core);
+            bool fRc = RTAvlroGCPhysInsert(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysToVirtHandlers, &pNext->Core);
             AssertRelease(fRc);
         }
@@ -1359 +1421 @@
     {
         /* Locate the previous node in the alias chain. */
-        PPGMPHYS2VIRTHANDLER pPrev = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGet(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key);
+        PPGMPHYS2VIRTHANDLER pPrev = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key);
 #ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
         AssertReleaseMsg(pPrev != pPhys2Virt,
@@ -1405 +1467 @@
      * Clear the ram flags for this page.
      */
-    PPGMPAGE pPage = pgmPhysGetPage(pPGM, pPhys2Virt->Core.Key);
+    PPGMPAGE pPage = pgmPhysGetPage(pVM, pPhys2Virt->Core.Key);
     AssertReturnVoid(pPage);
     PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, PGM_PAGE_HNDL_VIRT_STATE_NONE);
@@ -1521 +1583 @@
  * @param   pPage       PGM pool page
  */
-DECLINLINE(bool) pgmPoolIsPageLocked(PPGM pPGM, PPGMPOOLPAGE pPage)
+DECLINLINE(bool) pgmPoolIsPageLocked(PPGMPOOLPAGE pPage)
 {
     if (pPage->cLocked)
@@ -1540 +1602 @@
  * @param   pVM         VM handle.
  */
-DECL_FORCE_INLINE(bool) pgmMapAreMappingsEnabled(PPGM pPGM)
+DECL_FORCE_INLINE(bool) pgmMapAreMappingsEnabled(PVM pVM)
 {
 #ifdef PGM_WITHOUT_MAPPINGS
     /* There are no mappings in VT-x and AMD-V mode. */
-    Assert(pPGM->fMappingsDisabled);
+    Assert(pVM->pgm.s.fMappingsDisabled);
     return false;
 #else
-    return !pPGM->fMappingsDisabled;
+    return !pVM->pgm.s.fMappingsDisabled;
 #endif
 }
@@ -1558 +1620 @@
  * @param   pVM         The VM handle.
  */
-DECL_FORCE_INLINE(bool) pgmMapAreMappingsFloating(PPGM pPGM)
+DECL_FORCE_INLINE(bool) pgmMapAreMappingsFloating(PVM pVM)
 {
 #ifdef PGM_WITHOUT_MAPPINGS
     /* There are no mappings in VT-x and AMD-V mode. */
-    Assert(pPGM->fMappingsDisabled);
+    Assert(pVM->pgm.s.fMappingsDisabled);
     return false;
 #else
-    return !pPGM->fMappingsDisabled
-        && !pPGM->fMappingsFixed;
+    return !pVM->pgm.s.fMappingsDisabled
+        && !pVM->pgm.s.fMappingsFixed;
 #endif
 }

trunk/src/VBox/VMM/include/PGMInternal.h

@@ -1326 +1326 @@
 #define PGM_RAM_RANGE_IS_AD_HOC(pRam) \
     (!!( (pRam)->fFlags & (PGM_RAM_RANGE_FLAGS_AD_HOC_ROM | PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO | PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2) ) )
+
+/* enable the tlbs. */
+//#define PGM_USE_RAMRANGE_TLB
+/** The number of entries in the RAM range TLBs (there is one for each
+ *  context).  Must be a power of two. */
+#define PGM_RAMRANGE_TLB_ENTRIES            8
+
+/**
+ * Calculates the RAM range TLB index for the physical address.
+ *
+ * @returns RAM range TLB index.
+ * @param   GCPhys      The guest physical address.
+ */
+#define PGM_RAMRANGE_TLB_IDX(a_GCPhys)      ( ((a_GCPhys) >> 20) & (PGM_RAMRANGE_TLB_ENTRIES - 1) )
+
 
 
@@ -2792 +2807 @@
     STAMCOUNTER StatR3PageMapTlbHits;               /**< R3: Ring-3/0 page mapper TLB hits. */
     STAMCOUNTER StatR3PageMapTlbMisses;             /**< R3: Ring-3/0 page mapper TLB misses. */
+    STAMCOUNTER StatRZRamRangeTlbHits;              /**< RC/R0: RAM range TLB hits. */
+    STAMCOUNTER StatRZRamRangeTlbMisses;            /**< RC/R0: RAM range TLB misses. */
+    STAMCOUNTER StatR3RamRangeTlbHits;              /**< R3: RAM range TLB hits. */
+    STAMCOUNTER StatR3RamRangeTlbMisses;            /**< R3: RAM range TLB misses. */
     STAMPROFILE StatRZSyncCR3HandlerVirtualReset;   /**< RC/R0: Profiling of the virtual handler resets. */
     STAMPROFILE StatRZSyncCR3HandlerVirtualUpdate;  /**< RC/R0: Profiling of the virtual handler updates. */
@@ -2946 +2965 @@
     /** Pointer to the list of RAM ranges (Phys GC -> Phys HC conversion) - for R3.
      * This is sorted by physical address and contains no overlapping ranges. */
-    R3PTRTYPE(PPGMRAMRANGE)         pRamRangesR3;
+    R3PTRTYPE(PPGMRAMRANGE)         pRamRangesXR3;
+#ifdef PGM_USE_RAMRANGE_TLB
+    /** Ram range TLB for R3. */
+    R3PTRTYPE(PPGMRAMRANGE)         apRamRangesTlbR3[PGM_RAMRANGE_TLB_ENTRIES];
+#endif
     /** PGM offset based trees - R3 Ptr. */
     R3PTRTYPE(PPGMTREES)            pTreesR3;
@@ -2967 +2990 @@
     /*RTR3PTR                         R3PtrAlignment0;*/
 
-
-    /** R0 pointer corresponding to PGM::pRamRangesR3. */
-    R0PTRTYPE(PPGMRAMRANGE)         pRamRangesR0;
+    /** R0 pointer corresponding to PGM::pRamRangesXR3. */
+    R0PTRTYPE(PPGMRAMRANGE)         pRamRangesXR0;
+#ifdef PGM_USE_RAMRANGE_TLB
+    /** Ram range TLB for R0. */
+    R0PTRTYPE(PPGMRAMRANGE)         apRamRangesTlbR0[PGM_RAMRANGE_TLB_ENTRIES];
+#endif
     /** PGM offset based trees - R0 Ptr. */
     R0PTRTYPE(PPGMTREES)            pTreesR0;
@@ -2984 +3010 @@
 
 
-    /** RC pointer corresponding to PGM::pRamRangesR3. */
-    RCPTRTYPE(PPGMRAMRANGE)         pRamRangesRC;
+    /** RC pointer corresponding to PGM::pRamRangesXR3. */
+    RCPTRTYPE(PPGMRAMRANGE)         pRamRangesXRC;
+#ifdef PGM_USE_RAMRANGE_TLB
+    /** Ram range TLB for RC. */
+    RCPTRTYPE(PPGMRAMRANGE)         apRamRangesTlbRC[PGM_RAMRANGE_TLB_ENTRIES];
+#endif
     /** PGM offset based trees - RC Ptr. */
     RCPTRTYPE(PPGMTREES)            pTreesRC;
@@ -3771 +3801 @@
 int             pgmPhysAllocLargePage(PVM pVM, RTGCPHYS GCPhys);
 int             pgmPhysRecheckLargePage(PVM pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage);
-int             pgmPhysPageLoadIntoTlb(PPGM pPGM, RTGCPHYS GCPhys);
-int             pgmPhysPageLoadIntoTlbWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys);
+int             pgmPhysPageLoadIntoTlb(PVM pVM, RTGCPHYS GCPhys);
+int             pgmPhysPageLoadIntoTlbWithPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
 void            pgmPhysPageMakeWriteMonitoredWritable(PVM pVM, PPGMPAGE pPage);
 int             pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
@@ -3784 +3814 @@
 VMMDECL(int)    pgmPhysRomWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
 int             pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys);
+void            pgmPhysInvalidRamRangeTlbs(PVM pVM);
+PPGMRAMRANGE    pgmPhysGetRangeSlow(PVM pVM, RTGCPHYS GCPhys);
+PPGMRAMRANGE    pgmPhysGetRangeAtOrAboveSlow(PVM pVM, RTGCPHYS GCPhys);
+PPGMPAGE        pgmPhysGetPageSlow(PVM pVM, RTGCPHYS GCPhys);
+int             pgmPhysGetPageExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage);
+int             pgmPhysGetPageAndRangeExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam);
 
 #ifdef IN_RING3

trunk/src/VBox/VMM/testcase/tstVMStructRC.cpp

@@ -564 +564 @@
     GEN_CHECK_OFF(PGM, enmHostMode);
     GEN_CHECK_OFF(PGM, GCPhys4MBPSEMask);
-    GEN_CHECK_OFF(PGM, pRamRangesR3);
-    GEN_CHECK_OFF(PGM, pRamRangesR0);
-    GEN_CHECK_OFF(PGM, pRamRangesRC);
+    GEN_CHECK_OFF(PGM, pRamRangesXR3);
+    GEN_CHECK_OFF(PGM, pRamRangesXR0);
+    GEN_CHECK_OFF(PGM, pRamRangesXRC);
     GEN_CHECK_OFF(PGM, pRomRangesR3);
     GEN_CHECK_OFF(PGM, pRomRangesR0);