Changeset 82555 in vbox for trunk/src

Timestamp:

Dec 11, 2019 11:56:54 PM (5 years ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

135429

Message:

PGMPool,MM: Use ring-0 mapping while in ring-0, so let the page pool do its own allocations rather than going through MMPage*. The MMPage* code is mostly code, but we still need it for a dummy page allocation. I'll address this tomorrow. bugref:9528

Location:

trunk/src/VBox/VMM

Files:

• Makefile.kmk (modified) (2 diffs)
• VMMAll/PGMAllPool.cpp (modified) (3 diffs)
• VMMR0/GVMMR0.cpp (modified) (2 diffs)
• VMMR0/PGMR0.cpp (modified) (2 diffs)
• VMMR0/PGMR0Pool.cpp (added)
• VMMR0/VMMR0.cpp (modified) (3 diffs)
• VMMR3/MM.cpp (modified) (3 diffs)
• VMMR3/MMPagePool.cpp (modified) (2 diffs)
• VMMR3/PGMDbg.cpp (modified) (1 diff)
• VMMR3/PGMPool.cpp (modified) (3 diffs)
• VMMR3/VMM.cpp (modified) (1 diff)
• include/PGMInternal.h (modified) (7 diffs)

trunk/src/VBox/VMM/Makefile.kmk

@@ -490 +490 @@
         VMMR0/PDMR0Driver.cpp \
         VMMR0/PGMR0.cpp \
+        VMMR0/PGMR0Pool.cpp \
         VMMR0/PGMR0SharedPage.cpp \
         VMMR0/VMMR0.cpp \
@@ -514 +515 @@
         VMMAll/MMAll.cpp \
         VMMAll/MMAllHyper.cpp \
-        VMMAll/MMAllPagePool.cpp \
         VMMAll/NEMAll.cpp \
         VMMAll/PDMAll.cpp \

trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp

@@ -4974 +4974 @@
         STAM_PROFILE_ADV_SUSPEND(&pPool->StatAlloc, a);
 #ifdef IN_RING3
-        int rc = PGMR3PoolGrow(pVM);
+        int rc = PGMR3PoolGrow(pVM, VMMGetCpu(pVM));
 #else
         int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_POOL_GROW, 0);
@@ -5189 +5189 @@
     PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
     return (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
+}
+
+
+/**
+ * Internal worker for PGM_HCPHYS_2_PTR.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   HCPhys      The HC physical address of the shadow page.
+ * @param   ppv         Where to return the address.
+ */
+int pgmPoolHCPhys2Ptr(PVM pVM, RTHCPHYS HCPhys, void **ppv)
+{
+    PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pVM->pgm.s.CTX_SUFF(pPool)->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
+    AssertMsgReturn(pPage && pPage->enmKind != PGMPOOLKIND_FREE,
+                    ("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0),
+                    VERR_PGM_POOL_GET_PAGE_FAILED);
+    *ppv = (uint8_t *)pPage->CTX_SUFF(pvPage) + (HCPhys & PAGE_OFFSET_MASK);
+    return VINF_SUCCESS;
 }
 
@@ -5333 +5352 @@
         PPGMPOOLPAGE pPage = &pPool->aPages[i];
 
-        Assert(pPage->Core.Key == MMPage2Phys(pVM, pPage->pvPageR3));
         if (pPage->fMonitored)
             pgmPoolMonitorFlush(pPool, pPage);

trunk/src/VBox/VMM/VMMR0/GVMMR0.cpp

@@ -902 +902 @@
                         RT_BZERO(pGVM, cPages << PAGE_SHIFT);
                         gvmmR0InitPerVMData(pGVM, iHandle, cCpus, pSession);
+                        pGVM->gvmm.s.VMMemObj  = hVMMemObj;
                         GMMR0InitPerVMData(pGVM);
+                        rc = PGMR0InitPerVMData(pGVM);
                         PDMR0InitPerVMData(pGVM);
                         IOMR0InitPerVMData(pGVM);
-                        pGVM->gvmm.s.VMMemObj  = hVMMemObj;
-
-                        /*
-                         * Allocate page array.
-                         * This currently have to be made available to ring-3, but this is should change eventually.
-                         */
-                        rc = RTR0MemObjAllocPage(&pGVM->gvmm.s.VMPagesMemObj, cPages * sizeof(SUPPAGE), false /* fExecutable */);
                         if (RT_SUCCESS(rc))
                         {
-                            PSUPPAGE paPages = (PSUPPAGE)RTR0MemObjAddress(pGVM->gvmm.s.VMPagesMemObj); AssertPtr(paPages);
-                            for (uint32_t iPage = 0; iPage < cPages; iPage++)
-                            {
-                                paPages[iPage].uReserved = 0;
-                                paPages[iPage].Phys = RTR0MemObjGetPagePhysAddr(pGVM->gvmm.s.VMMemObj, iPage);
-                                Assert(paPages[iPage].Phys != NIL_RTHCPHYS);
-                            }
-
                             /*
-                             * Map the page array, VM and VMCPU structures into ring-3.
+                             * Allocate page array.
+                             * This currently have to be made available to ring-3, but this is should change eventually.
                              */
-                            AssertCompileSizeAlignment(VM, PAGE_SIZE);
-                            rc = RTR0MemObjMapUserEx(&pGVM->gvmm.s.VMMapObj, pGVM->gvmm.s.VMMemObj, (RTR3PTR)-1, 0,
-                                                     RTMEM_PROT_READ | RTMEM_PROT_WRITE, NIL_RTR0PROCESS,
-                                                     0 /*offSub*/, sizeof(VM));
-                            for (VMCPUID i = 0; i < cCpus && RT_SUCCESS(rc); i++)
-                            {
-                                AssertCompileSizeAlignment(VMCPU, PAGE_SIZE);
-                                rc = RTR0MemObjMapUserEx(&pGVM->aCpus[i].gvmm.s.VMCpuMapObj, pGVM->gvmm.s.VMMemObj,
-                                                         (RTR3PTR)-1, 0, RTMEM_PROT_READ | RTMEM_PROT_WRITE, NIL_RTR0PROCESS,
-                                                         RT_UOFFSETOF_DYN(GVM, aCpus[i]), sizeof(VMCPU));
-                            }
-                            if (RT_SUCCESS(rc))
-                                rc = RTR0MemObjMapUser(&pGVM->gvmm.s.VMPagesMapObj, pGVM->gvmm.s.VMPagesMemObj, (RTR3PTR)-1,
-                                                       0 /* uAlignment */, RTMEM_PROT_READ | RTMEM_PROT_WRITE,
-                                                       NIL_RTR0PROCESS);
+                            rc = RTR0MemObjAllocPage(&pGVM->gvmm.s.VMPagesMemObj, cPages * sizeof(SUPPAGE), false /* fExecutable */);
                             if (RT_SUCCESS(rc))
                             {
+                                PSUPPAGE paPages = (PSUPPAGE)RTR0MemObjAddress(pGVM->gvmm.s.VMPagesMemObj); AssertPtr(paPages);
+                                for (uint32_t iPage = 0; iPage < cPages; iPage++)
+                                {
+                                    paPages[iPage].uReserved = 0;
+                                    paPages[iPage].Phys = RTR0MemObjGetPagePhysAddr(pGVM->gvmm.s.VMMemObj, iPage);
+                                    Assert(paPages[iPage].Phys != NIL_RTHCPHYS);
+                                }
+
                                 /*
-                                 * Initialize all the VM pointers.
+                                 * Map the page array, VM and VMCPU structures into ring-3.
                                  */
-                                PVMR3 pVMR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMMapObj);
-                                AssertPtr((void *)pVMR3);
-
-                                for (VMCPUID i = 0; i < cCpus; i++)
+                                AssertCompileSizeAlignment(VM, PAGE_SIZE);
+                                rc = RTR0MemObjMapUserEx(&pGVM->gvmm.s.VMMapObj, pGVM->gvmm.s.VMMemObj, (RTR3PTR)-1, 0,
+                                                         RTMEM_PROT_READ | RTMEM_PROT_WRITE, NIL_RTR0PROCESS,
+                                                         0 /*offSub*/, sizeof(VM));
+                                for (VMCPUID i = 0; i < cCpus && RT_SUCCESS(rc); i++)
                                 {
-                                    pGVM->aCpus[i].pVMR0 = pGVM;
-                                    pGVM->aCpus[i].pVMR3 = pVMR3;
-                                    pGVM->apCpusR3[i] = RTR0MemObjAddressR3(pGVM->aCpus[i].gvmm.s.VMCpuMapObj);
-                                    pGVM->aCpus[i].pVCpuR3 = pGVM->apCpusR3[i];
-                                    pGVM->apCpusR0[i] = &pGVM->aCpus[i];
-                                    AssertPtr((void *)pGVM->apCpusR3[i]);
+                                    AssertCompileSizeAlignment(VMCPU, PAGE_SIZE);
+                                    rc = RTR0MemObjMapUserEx(&pGVM->aCpus[i].gvmm.s.VMCpuMapObj, pGVM->gvmm.s.VMMemObj,
+                                                             (RTR3PTR)-1, 0, RTMEM_PROT_READ | RTMEM_PROT_WRITE, NIL_RTR0PROCESS,
+                                                             RT_UOFFSETOF_DYN(GVM, aCpus[i]), sizeof(VMCPU));
                                 }
-
-                                pGVM->paVMPagesR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMPagesMapObj);
-                                AssertPtr((void *)pGVM->paVMPagesR3);
-
-                                /*
-                                 * Complete the handle - take the UsedLock sem just to be careful.
-                                 */
-                                rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
-                                AssertRC(rc);
-
-                                pHandle->pGVM                   = pGVM;
-                                pHandle->hEMT0                  = hEMT0;
-                                pHandle->ProcId                 = ProcId;
-                                pGVM->pVMR3                     = pVMR3;
-                                pGVM->pVMR3Unsafe               = pVMR3;
-                                pGVM->aCpus[0].hEMT             = hEMT0;
-                                pGVM->aCpus[0].hNativeThreadR0  = hEMT0;
-                                pGVMM->cEMTs += cCpus;
-
-                                /* Associate it with the session and create the context hook for EMT0. */
-                                rc = SUPR0SetSessionVM(pSession, pGVM, pGVM);
+                                if (RT_SUCCESS(rc))
+                                    rc = RTR0MemObjMapUser(&pGVM->gvmm.s.VMPagesMapObj, pGVM->gvmm.s.VMPagesMemObj, (RTR3PTR)-1,
+                                                           0 /* uAlignment */, RTMEM_PROT_READ | RTMEM_PROT_WRITE,
+                                                           NIL_RTR0PROCESS);
                                 if (RT_SUCCESS(rc))
                                 {
-                                    rc = VMMR0ThreadCtxHookCreateForEmt(&pGVM->aCpus[0]);
+                                    /*
+                                     * Initialize all the VM pointers.
+                                     */
+                                    PVMR3 pVMR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMMapObj);
+                                    AssertPtr((void *)pVMR3);
+
+                                    for (VMCPUID i = 0; i < cCpus; i++)
+                                    {
+                                        pGVM->aCpus[i].pVMR0 = pGVM;
+                                        pGVM->aCpus[i].pVMR3 = pVMR3;
+                                        pGVM->apCpusR3[i] = RTR0MemObjAddressR3(pGVM->aCpus[i].gvmm.s.VMCpuMapObj);
+                                        pGVM->aCpus[i].pVCpuR3 = pGVM->apCpusR3[i];
+                                        pGVM->apCpusR0[i] = &pGVM->aCpus[i];
+                                        AssertPtr((void *)pGVM->apCpusR3[i]);
+                                    }
+
+                                    pGVM->paVMPagesR3 = RTR0MemObjAddressR3(pGVM->gvmm.s.VMPagesMapObj);
+                                    AssertPtr((void *)pGVM->paVMPagesR3);
+
+                                    /*
+                                     * Complete the handle - take the UsedLock sem just to be careful.
+                                     */
+                                    rc = GVMMR0_USED_EXCLUSIVE_LOCK(pGVMM);
+                                    AssertRC(rc);
+
+                                    pHandle->pGVM                   = pGVM;
+                                    pHandle->hEMT0                  = hEMT0;
+                                    pHandle->ProcId                 = ProcId;
+                                    pGVM->pVMR3                     = pVMR3;
+                                    pGVM->pVMR3Unsafe               = pVMR3;
+                                    pGVM->aCpus[0].hEMT             = hEMT0;
+                                    pGVM->aCpus[0].hNativeThreadR0  = hEMT0;
+                                    pGVMM->cEMTs += cCpus;
+
+                                    /* Associate it with the session and create the context hook for EMT0. */
+                                    rc = SUPR0SetSessionVM(pSession, pGVM, pGVM);
                                     if (RT_SUCCESS(rc))
                                     {
-                                        /*
-                                         * Done!
-                                         */
-                                        VBOXVMM_R0_GVMM_VM_CREATED(pGVM, pGVM, ProcId, (void *)hEMT0, cCpus);
-
-                                        GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
-                                        gvmmR0CreateDestroyUnlock(pGVMM);
-
-                                        CPUMR0RegisterVCpuThread(&pGVM->aCpus[0]);
-
-                                        *ppGVM = pGVM;
-                                        Log(("GVMMR0CreateVM: pVMR3=%p pGVM=%p hGVM=%d\n", pVMR3, pGVM, iHandle));
-                                        return VINF_SUCCESS;
+                                        rc = VMMR0ThreadCtxHookCreateForEmt(&pGVM->aCpus[0]);
+                                        if (RT_SUCCESS(rc))
+                                        {
+                                            /*
+                                             * Done!
+                                             */
+                                            VBOXVMM_R0_GVMM_VM_CREATED(pGVM, pGVM, ProcId, (void *)hEMT0, cCpus);
+
+                                            GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
+                                            gvmmR0CreateDestroyUnlock(pGVMM);
+
+                                            CPUMR0RegisterVCpuThread(&pGVM->aCpus[0]);
+
+                                            *ppGVM = pGVM;
+                                            Log(("GVMMR0CreateVM: pVMR3=%p pGVM=%p hGVM=%d\n", pVMR3, pGVM, iHandle));
+                                            return VINF_SUCCESS;
+                                        }
+
+                                        SUPR0SetSessionVM(pSession, NULL, NULL);
                                     }
-
-                                    SUPR0SetSessionVM(pSession, NULL, NULL);
+                                    GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
                                 }
-                                GVMMR0_USED_EXCLUSIVE_UNLOCK(pGVMM);
-                            }
-
-                            /* Cleanup mappings. */
-                            if (pGVM->gvmm.s.VMMapObj != NIL_RTR0MEMOBJ)
-                            {
-                                RTR0MemObjFree(pGVM->gvmm.s.VMMapObj, false /* fFreeMappings */);
-                                pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
-                            }
-                            for (VMCPUID i = 0; i < cCpus; i++)
-                                if (pGVM->aCpus[i].gvmm.s.VMCpuMapObj != NIL_RTR0MEMOBJ)
+
+                                /* Cleanup mappings. */
+                                if (pGVM->gvmm.s.VMMapObj != NIL_RTR0MEMOBJ)
                                 {
-                                    RTR0MemObjFree(pGVM->aCpus[i].gvmm.s.VMCpuMapObj, false /* fFreeMappings */);
-                                    pGVM->aCpus[i].gvmm.s.VMCpuMapObj = NIL_RTR0MEMOBJ;
+                                    RTR0MemObjFree(pGVM->gvmm.s.VMMapObj, false /* fFreeMappings */);
+                                    pGVM->gvmm.s.VMMapObj = NIL_RTR0MEMOBJ;
                                 }
-                            if (pGVM->gvmm.s.VMPagesMapObj != NIL_RTR0MEMOBJ)
-                            {
-                                RTR0MemObjFree(pGVM->gvmm.s.VMPagesMapObj, false /* fFreeMappings */);
-                                pGVM->gvmm.s.VMPagesMapObj = NIL_RTR0MEMOBJ;
+                                for (VMCPUID i = 0; i < cCpus; i++)
+                                    if (pGVM->aCpus[i].gvmm.s.VMCpuMapObj != NIL_RTR0MEMOBJ)
+                                    {
+                                        RTR0MemObjFree(pGVM->aCpus[i].gvmm.s.VMCpuMapObj, false /* fFreeMappings */);
+                                        pGVM->aCpus[i].gvmm.s.VMCpuMapObj = NIL_RTR0MEMOBJ;
+                                    }
+                                if (pGVM->gvmm.s.VMPagesMapObj != NIL_RTR0MEMOBJ)
+                                {
+                                    RTR0MemObjFree(pGVM->gvmm.s.VMPagesMapObj, false /* fFreeMappings */);
+                                    pGVM->gvmm.s.VMPagesMapObj = NIL_RTR0MEMOBJ;
+                                }
                             }
                         }
                     }
-
                 }
                 /* else: The user wasn't permitted to create this VM. */
@@ -1297 +1299 @@
     PDMR0CleanupVM(pGVM);
     IOMR0CleanupVM(pGVM);
+    PGMR0CleanupVM(pGVM);
 
     AssertCompile(NIL_RTTHREADCTXHOOK == (RTTHREADCTXHOOK)0); /* Depends on zero initialized memory working for NIL at the moment. */

trunk/src/VBox/VMM/VMMR0/PGMR0.cpp

@@ -33 +33 @@
 #include <iprt/assert.h>
 #include <iprt/mem.h>
+#include <iprt/memobj.h>
 
 
@@ -54 +55 @@
 #include "PGMR0Bth.h"
 #undef PGM_BTH_NAME
+
+
+/**
+ * Initializes the per-VM data for the PGM.
+ *
+ * This is called from under the GVMM lock, so it should only initialize the
+ * data so PGMR0CleanupVM and others will work smoothly.
+ *
+ * @returns VBox status code.
+ * @param   pGVM    Pointer to the global VM structure.
+ */
+VMMR0_INT_DECL(int) PGMR0InitPerVMData(PGVM pGVM)
+{
+    AssertCompile(sizeof(pGVM->pgm.s) <= sizeof(pGVM->pgm.padding));
+    AssertCompile(sizeof(pGVM->pgmr0.s) <= sizeof(pGVM->pgmr0.padding));
+
+    AssertCompile(RT_ELEMENTS(pGVM->pgmr0.s.ahPoolMemObjs) == RT_ELEMENTS(pGVM->pgmr0.s.ahPoolMapObjs));
+    for (uint32_t i = 0; i < RT_ELEMENTS(pGVM->pgmr0.s.ahPoolMemObjs); i++)
+    {
+        pGVM->pgmr0.s.ahPoolMemObjs[i] = NIL_RTR0MEMOBJ;
+        pGVM->pgmr0.s.ahPoolMapObjs[i] = NIL_RTR0MEMOBJ;
+    }
+    return RTCritSectInit(&pGVM->pgmr0.s.PoolGrowCritSect);
+}
+
+
+/**
+ * Initalize the per-VM PGM for ring-0.
+ *
+ * @returns VBox status code.
+ * @param   pGVM    Pointer to the global VM structure.
+ */
+VMMR0_INT_DECL(int) PGMR0InitVM(PGVM pGVM)
+{
+    int rc = VINF_SUCCESS;
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    rc = PGMR0DynMapInitVM(pGVM);
+#endif
+    RT_NOREF(pGVM);
+    return rc;
+}
+
+
+/**
+ * Cleans up any loose ends before the GVM structure is destroyed.
+ */
+VMMR0_INT_DECL(void) PGMR0CleanupVM(PGVM pGVM)
+{
+    for (uint32_t i = 0; i < RT_ELEMENTS(pGVM->pgmr0.s.ahPoolMemObjs); i++)
+    {
+        if (pGVM->pgmr0.s.ahPoolMapObjs[i] != NIL_RTR0MEMOBJ)
+        {
+            int rc = RTR0MemObjFree(pGVM->pgmr0.s.ahPoolMapObjs[i], true /*fFreeMappings*/);
+            AssertRC(rc);
+            pGVM->pgmr0.s.ahPoolMapObjs[i] = NIL_RTR0MEMOBJ;
+        }
+
+        if (pGVM->pgmr0.s.ahPoolMemObjs[i] != NIL_RTR0MEMOBJ)
+        {
+            int rc = RTR0MemObjFree(pGVM->pgmr0.s.ahPoolMemObjs[i], true /*fFreeMappings*/);
+            AssertRC(rc);
+            pGVM->pgmr0.s.ahPoolMemObjs[i] = NIL_RTR0MEMOBJ;
+        }
+    }
+
+    if (RTCritSectIsInitialized(&pGVM->pgmr0.s.PoolGrowCritSect))
+        RTCritSectDelete(&pGVM->pgmr0.s.PoolGrowCritSect);
+}
 
 

trunk/src/VBox/VMM/VMMR0/VMMR0.cpp

@@ -465 +465 @@
             {
                 VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-                rc = PGMR0DynMapInitVM(pGVM);
-#endif
+                rc = PGMR0InitVM(pGVM);
                 if (RT_SUCCESS(rc))
                 {
@@ -1800 +1798 @@
             break;
 
+        case VMMR0_DO_PGM_POOL_GROW:
+            if (idCpu == NIL_VMCPUID)
+                return VERR_INVALID_CPU_ID;
+            rc = PGMR0PoolGrow(pGVM);
+            VMM_CHECK_SMAP_CHECK2(pGVM, RT_NOTHING);
+            break;
+
         /*
          * GMM wrappers.
@@ -2344 +2349 @@
 
             default:
+            case VMMR0_DO_PGM_POOL_GROW:
                 break;
         }

trunk/src/VBox/VMM/VMMR3/MM.cpp

@@ -460 +460 @@
 VMMR3DECL(int) MMR3Term(PVM pVM)
 {
+#if 0
     /*
      * Destroy the page pool. (first as it used the hyper heap)
      */
     mmR3PagePoolTerm(pVM);
+#endif
 
     /* Clean up the hypervisor heap. */
@@ -740 +742 @@
 VMMR3DECL(int) MMR3HCPhys2HCVirt(PVM pVM, RTHCPHYS HCPhys, void **ppv)
 {
+#if 0
     /*
      * Try page tables.
@@ -746 +749 @@
     if (RT_SUCCESS(rc))
         return rc;
+#endif
 
     /*

trunk/src/VBox/VMM/VMMR3/MMPagePool.cpp

@@ -379 +379 @@
 }
 
+#if 0
 
 /**
@@ -483 +484 @@
 }
 
+#endif
 
 /**

trunk/src/VBox/VMM/VMMR3/PGMDbg.cpp

@@ -1116 +1116 @@
     if (!fIsMapping)
     {
-        int rc = MMPagePhys2PageTry(pState->pVM, HCPhys, &pvPage);
-        if (RT_FAILURE(rc))
+        PPGMPOOLPAGE pPoolPage = pgmPoolQueryPageForDbg(pState->pVM->pgm.s.pPoolR3, HCPhys);
+        if (pPoolPage)
         {
             pState->pHlp->pfnPrintf(pState->pHlp, "%0*llx error! %s at HCPhys=%RHp was not found in the page pool!\n",
                                     pState->cchAddress, pState->u64Address, pszDesc, HCPhys);
-            return rc;
+            return VERR_PGM_POOL_GET_PAGE_FAILED;
         }
+        pvPage = (uint8_t *)pPoolPage->pvPageR3 + (HCPhys & PAGE_OFFSET_MASK);
     }
     else

trunk/src/VBox/VMM/VMMR3/PGMPool.cpp

@@ -170 +170 @@
     AssertLogRelMsgReturn(cMaxPages <= PGMPOOL_IDX_LAST && cMaxPages >= RT_ALIGN(PGMPOOL_IDX_FIRST, 16),
                           ("cMaxPages=%u (%#x)\n", cMaxPages, cMaxPages), VERR_INVALID_PARAMETER);
-    cMaxPages = RT_ALIGN(cMaxPages, 16);
+    AssertCompile(RT_IS_POWER_OF_TWO(PGMPOOL_CFG_MAX_GROW));
+    if (cMaxPages < PGMPOOL_IDX_LAST)
+        cMaxPages = RT_ALIGN(cMaxPages, PGMPOOL_CFG_MAX_GROW / 2);
     if (cMaxPages > PGMPOOL_IDX_LAST)
         cMaxPages = PGMPOOL_IDX_LAST;
@@ -314 +316 @@
     Assert(!pPool->aPages[NIL_PGMPOOL_IDX].fReusedFlushPending);
 
-#ifdef VBOX_WITH_STATISTICS
     /*
      * Register statistics.
      */
+    STAM_REL_REG(pVM, &pPool->StatGrow,                 STAMTYPE_PROFILE,   "/PGM/Pool/Grow",           STAMUNIT_TICKS, "Profiling PGMR0PoolGrow");
+#ifdef VBOX_WITH_STATISTICS
     STAM_REG(pVM, &pPool->cCurPages,                    STAMTYPE_U16,       "/PGM/Pool/cCurPages",      STAMUNIT_PAGES,             "Current pool size.");
     STAM_REG(pVM, &pPool->cMaxPages,                    STAMTYPE_U16,       "/PGM/Pool/cMaxPages",      STAMUNIT_PAGES,             "Max pool size.");
@@ -473 +476 @@
  * @returns VBox status code.
  * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
  */
-VMMR3DECL(int) PGMR3PoolGrow(PVM pVM)
+VMMR3_INT_DECL(int) PGMR3PoolGrow(PVM pVM, PVMCPU pVCpu)
 {
-    PPGMPOOL pPool = pVM->pgm.s.pPoolR3;
-    AssertReturn(pPool->cCurPages < pPool->cMaxPages, VERR_PGM_POOL_MAXED_OUT_ALREADY);
-
-    /* With 32-bit guests and no EPT, the CR3 limits the root pages to low
-       (below 4 GB) memory. */
-    /** @todo change the pool to handle ROOT page allocations specially when
-     *        required. */
-    bool fCanUseHighMemory = HMIsNestedPagingActive(pVM);
-
-    pgmLock(pVM);
-
-    /*
-     * How much to grow it by?
-     */
-    uint32_t cPages = pPool->cMaxPages - pPool->cCurPages;
-    cPages = RT_MIN(PGMPOOL_CFG_MAX_GROW, cPages);
-    LogFlow(("PGMR3PoolGrow: Growing the pool by %d (%#x) pages. fCanUseHighMemory=%RTbool\n", cPages, cPages, fCanUseHighMemory));
-
-    for (unsigned i = pPool->cCurPages; cPages-- > 0; i++)
-    {
-        PPGMPOOLPAGE pPage = &pPool->aPages[i];
-
-        if (fCanUseHighMemory)
-            pPage->pvPageR3 = MMR3PageAlloc(pVM);
-        else
-            pPage->pvPageR3 = MMR3PageAllocLow(pVM);
-        if (!pPage->pvPageR3)
-        {
-            Log(("We're out of memory!! i=%d fCanUseHighMemory=%RTbool\n", i, fCanUseHighMemory));
-            pgmUnlock(pVM);
-            return i ? VINF_SUCCESS : VERR_NO_PAGE_MEMORY;
-        }
-        pPage->Core.Key  = MMPage2Phys(pVM, pPage->pvPageR3);
-        AssertFatal(pPage->Core.Key < _4G || fCanUseHighMemory);
-        pPage->GCPhys    = NIL_RTGCPHYS;
-        pPage->enmKind   = PGMPOOLKIND_FREE;
-        pPage->idx       = pPage - &pPool->aPages[0];
-        LogFlow(("PGMR3PoolGrow: insert page #%#x - %RHp\n", pPage->idx, pPage->Core.Key));
-        pPage->iNext     = pPool->iFreeHead;
-        pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
-        pPage->iModifiedNext  = NIL_PGMPOOL_IDX;
-        pPage->iModifiedPrev  = NIL_PGMPOOL_IDX;
-        pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
-        pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
-        pPage->iAgeNext  = NIL_PGMPOOL_IDX;
-        pPage->iAgePrev  = NIL_PGMPOOL_IDX;
-        /* commit it */
-        bool fRc = RTAvloHCPhysInsert(&pPool->HCPhysTree, &pPage->Core); Assert(fRc); NOREF(fRc);
-        pPool->iFreeHead = i;
-        pPool->cCurPages = i + 1;
-    }
-
-    pgmUnlock(pVM);
-    Assert(pPool->cCurPages <= pPool->cMaxPages);
-    return VINF_SUCCESS;
+    /* This used to do a lot of stuff, but it has moved to ring-0 (PGMR0PoolGrow). */
+    AssertReturn(pVM->pgm.s.pPoolR3->cCurPages < pVM->pgm.s.pPoolR3->cMaxPages, VERR_PGM_POOL_MAXED_OUT_ALREADY);
+    return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_PGM_POOL_GROW, 0, NULL);
 }
 

trunk/src/VBox/VMM/VMMR3/VMM.cpp

@@ -2393 +2393 @@
         case VMMCALLRING3_PGM_POOL_GROW:
         {
-            pVCpu->vmm.s.rcCallRing3 = PGMR3PoolGrow(pVM);
+            pVCpu->vmm.s.rcCallRing3 = PGMR3PoolGrow(pVM, pVCpu);
             break;
         }

trunk/src/VBox/VMM/include/PGMInternal.h

@@ -140 +140 @@
  * The maximum number of pages to add to the pool in one go.
  */
-#define PGMPOOL_CFG_MAX_GROW            (_256K >> PAGE_SHIFT)
+#define PGMPOOL_CFG_MAX_GROW            (_2M >> PAGE_SHIFT)
 
 /** @def VBOX_STRICT_PGM_HANDLER_VIRTUAL
@@ -266 +266 @@
  *                      this.
  *
- * @remark  Use with care as we don't have so much dynamic mapping space in
- *          ring-0 on 32-bit darwin and in RC.
  * @remark  There is no need to assert on the result.
  */
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-# define PGM_HCPHYS_2_PTR(pVM, pVCpu, HCPhys, ppv) \
-     pgmRZDynMapHCPageInlined(pVCpu, HCPhys, (void **)(ppv) RTLOG_COMMA_SRC_POS)
-#else
-# define PGM_HCPHYS_2_PTR(pVM, pVCpu, HCPhys, ppv) \
-     MMPagePhys2PageEx(pVM, HCPhys, (void **)(ppv))
-#endif
+#define PGM_HCPHYS_2_PTR(pVM, pVCpu, HCPhys, ppv) pgmPoolHCPhys2Ptr(pVM, HCPhys, (void **)(ppv))
 
 /** @def PGM_GCPHYS_2_PTR_V2
@@ -2055 +2047 @@
     AVLOHCPHYSNODECORE  Core;
     /** Pointer to the R3 mapping of the page. */
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
     R3PTRTYPE(void *)   pvPageR3;
-#else
-    R3R0PTRTYPE(void *) pvPageR3;
-#endif
-#if HC_ARCH_BITS == 32 && GC_ARCH_BITS == 64
-    uint32_t            Alignment0;
-#endif
+    /** Pointer to the R0 mapping of the page. */
+    R0PTRTYPE(void *)   pvPageR0;
     /** The guest physical address. */
     RTGCPHYS            GCPhys;
@@ -2347 +2334 @@
     uint32_t                    Alignment3;         /**< Align the next member on a 64-bit boundary. */
 #endif
+    /** Profiling PGMR0PoolGrow(). */
+    STAMPROFILE                 StatGrow;
     /** The AVL tree for looking up a page by its HC physical address. */
     AVLOHCPHYSTREE              HCPhysTree;
@@ -2382 +2371 @@
 {
     AssertPtr(a_pPage);
-    AssertReleaseMsg(RT_VALID_PTR(a_pPage->pvPageR3), ("enmKind=%d idx=%#x HCPhys=%RHp GCPhys=%RGp caller=%s\n", a_pPage->enmKind, a_pPage->idx, a_pPage->Core.Key, a_pPage->GCPhys, pszCaller));
-    return a_pPage->pvPageR3;
+    AssertMsg(RT_VALID_PTR(a_pPage->CTX_SUFF(pvPage)),
+              ("enmKind=%d idx=%#x HCPhys=%RHp GCPhys=%RGp pvPageR3=%p pvPageR0=%p caller=%s\n",
+               a_pPage->enmKind, a_pPage->idx, a_pPage->Core.Key, a_pPage->GCPhys, a_pPage->pvPageR3, a_pPage->pvPageR0, pszCaller));
+    return a_pPage->CTX_SUFF(pvPage);
 }
 #else
-# define PGMPOOL_PAGE_2_PTR(pVM, a_pPage)       ((a_pPage)->pvPageR3)
+# define PGMPOOL_PAGE_2_PTR(pVM, a_pPage)       ((a_pPage)->CTX_SUFF(pvPage))
 #endif
 
@@ -3834 +3825 @@
 
 
+/**
+ * PGM GVM instance data.
+ */
+typedef struct PGMR0PERVM
+{
+    /** @name PGM Pool related stuff.
+     * @{ */
+    /** Critical section for serializing pool growth. */
+    RTCRITSECT  PoolGrowCritSect;
+    /** The memory objects for the pool pages. */
+    RTR0MEMOBJ  ahPoolMemObjs[(PGMPOOL_IDX_LAST + PGMPOOL_CFG_MAX_GROW - 1) / PGMPOOL_CFG_MAX_GROW];
+    /** The ring-3 mapping objects for the pool pages. */
+    RTR0MEMOBJ  ahPoolMapObjs[(PGMPOOL_IDX_LAST + PGMPOOL_CFG_MAX_GROW - 1) / PGMPOOL_CFG_MAX_GROW];
+    /** @} */
+} PGMR0PERVM;
+
 RT_C_DECLS_BEGIN
 
@@ -3949 +3956 @@
 PPGMPOOLPAGE    pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys);
 PPGMPOOLPAGE    pgmPoolQueryPageForDbg(PPGMPOOL pPool, RTHCPHYS HCPhys);
+int             pgmPoolHCPhys2Ptr(PVM pVM, RTHCPHYS HCPhys, void **ppv);
 int             pgmPoolSyncCR3(PVMCPUCC pVCpu);
 bool            pgmPoolIsDirtyPageSlow(PVM pVM, RTGCPHYS GCPhys);