Changeset 104108 in vbox for trunk/src

Timestamp:

Mar 28, 2024 9:50:41 PM (10 months ago)

Author:

vboxsync

Message:

VMM/IEM: Slap a header on the exec mem allocations. This simplifies freeing and will help implementing defraging and more localized pruning. Statistics. bugref:10370

Location:

trunk/src/VBox/VMM

Files:

• VMMAll/IEMAllN8veRecompiler.cpp (modified) (22 diffs)
• VMMR3/IEMR3.cpp (modified) (1 diff)

trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

@@ -159 +159 @@
 /** The chunk sub-allocation unit size as a shift factor. */
 #define IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT     7
+/** Enables adding a header to the sub-allocator allocations.
+ * This is useful for freeing up executable memory among other things.  */
+#define IEMEXECMEM_ALT_SUB_WITH_ALLOC_HEADER
+
 
 #if defined(IN_RING3) && !defined(RT_OS_WINDOWS)
@@ -344 +348 @@
      * portion corresponding to an chunk). */
     uint32_t                cBitmapElementsPerChunk;
+# ifdef VBOX_WITH_STATISTICS
+    STAMPROFILE             StatAlloc;
+# endif
 #else
     /** @name Tweaks to get 64 byte aligned allocats w/o unnecessary fragmentation.
@@ -378 +385 @@
 
 
+#ifdef IEMEXECMEM_ALT_SUB_WITH_ALLOC_HEADER
+/**
+ * Allocation header.
+ */
+typedef struct IEMEXECMEMALLOCHDR
+{
+    /** Magic value / eyecatcher (IEMEXECMEMALLOCHDR_MAGIC). */
+    uint32_t        uMagic;
+    /** The allocation chunk (for speeding up freeing). */
+    uint32_t        idxChunk;
+    /** Pointer to the translation block the allocation belongs to.
+     * This is the whole point of the header. */
+    PIEMTB          pTb;
+} IEMEXECMEMALLOCHDR;
+/** Pointer to an allocation header. */
+typedef  IEMEXECMEMALLOCHDR *PIEMEXECMEMALLOCHDR;
+/** Magic value for IEMEXECMEMALLOCHDR ('ExeM'). */
+# define IEMEXECMEMALLOCHDR_MAGIC       UINT32_C(0x4d657845)
+#endif
+
+
 static int iemExecMemAllocatorGrow(PVMCPUCC pVCpu, PIEMEXECMEMALLOCATOR pExecMemAllocator);
 
@@ -385 +413 @@
  * the heap statistics.
  */
-static void * iemExecMemAllocatorAllocTailCode(PIEMEXECMEMALLOCATOR pExecMemAllocator, void *pvRet,
-                                               uint32_t cbReq, uint32_t idxChunk)
+static void *iemExecMemAllocatorAllocTailCode(PIEMEXECMEMALLOCATOR pExecMemAllocator, void *pvRet,
+                                              uint32_t cbReq, uint32_t idxChunk)
 {
     pExecMemAllocator->cAllocations += 1;
@@ -419 +447 @@
 #ifdef IEMEXECMEM_USE_ALT_SUB_ALLOCATOR
 static void *iemExecMemAllocatorAllocInChunkInt(PIEMEXECMEMALLOCATOR pExecMemAllocator, uint64_t *pbmAlloc, uint32_t idxFirst,
-                                                uint32_t cToScan, uint32_t cReqUnits, uint32_t idxChunk)
+                                                uint32_t cToScan, uint32_t cReqUnits, uint32_t idxChunk, PIEMTB pTb)
 {
     /*
@@ -447 +475 @@
             pChunk->idxFreeHint = (uint32_t)iBit + cReqUnits;
 
+# ifdef IEMEXECMEM_ALT_SUB_WITH_ALLOC_HEADER
+            PIEMEXECMEMALLOCHDR pHdr = (PIEMEXECMEMALLOCHDR)((uint8_t *)pChunk->pvChunk
+                                                             + (   (idxFirst + (uint32_t)iBit)
+                                                                << IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT));
+            pHdr->uMagic   = IEMEXECMEMALLOCHDR_MAGIC;
+            pHdr->idxChunk = idxChunk;
+            pHdr->pTb      = pTb;
+            return iemExecMemAllocatorAllocTailCode(pExecMemAllocator, pHdr + 1,
+                                                    cReqUnits << IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT, idxChunk);
+#else
+            RT_NOREF(pTb);
             void * const pvRet  = (uint8_t *)pChunk->pvChunk
                                 + ((idxFirst + (uint32_t)iBit) << IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT);
-
             return iemExecMemAllocatorAllocTailCode(pExecMemAllocator, pvRet,
                                                     cReqUnits << IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT, idxChunk);
+#endif
         }
 
@@ -461 +500 @@
 
 
-static void *iemExecMemAllocatorAllocInChunk(PIEMEXECMEMALLOCATOR pExecMemAllocator, uint32_t idxChunk, uint32_t cbReq)
+static void *
+iemExecMemAllocatorAllocInChunk(PIEMEXECMEMALLOCATOR pExecMemAllocator, uint32_t idxChunk, uint32_t cbReq, PIEMTB pTb)
 {
 #ifdef IEMEXECMEM_USE_ALT_SUB_ALLOCATOR
@@ -467 +507 @@
      * Figure out how much to allocate.
      */
-    uint32_t const cReqUnits = (cbReq + IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE - 1) >> IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT;
+# ifdef IEMEXECMEM_ALT_SUB_WITH_ALLOC_HEADER
+    uint32_t const cReqUnits = (cbReq + sizeof(IEMEXECMEMALLOCHDR) + IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE - 1)
+# else
+    uint32_t const cReqUnits = (cbReq + IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE - 1)
+# endif
+                            >> IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT;
     if (cReqUnits <= pExecMemAllocator->aChunks[idxChunk].cFreeUnits)
     {
@@ -475 +520 @@
         {
             void *pvRet = iemExecMemAllocatorAllocInChunkInt(pExecMemAllocator, pbmAlloc, idxHint,
-                                                             pExecMemAllocator->cUnitsPerChunk - idxHint, cReqUnits, idxChunk);
+                                                             pExecMemAllocator->cUnitsPerChunk - idxHint,
+                                                             cReqUnits, idxChunk, pTb);
             if (pvRet)
                 return pvRet;
@@ -481 +527 @@
         return iemExecMemAllocatorAllocInChunkInt(pExecMemAllocator, pbmAlloc, 0,
                                                   RT_MIN(pExecMemAllocator->cUnitsPerChunk, RT_ALIGN_32(idxHint + cReqUnits, 64)),
-                                                  cReqUnits, idxChunk);
+                                                  cReqUnits, idxChunk, pTb);
     }
 #else
@@ -487 +533 @@
     if (pvRet)
         return iemExecMemAllocatorAllocTailCode(pExecMemAllocator, pvRet, cbReq, idxChunk);
+    RT_NOREF(pTb);
 #endif
     return NULL;
@@ -501 +548 @@
  *                  thread.
  * @param   cbReq   How many bytes are required.
- */
-static void *iemExecMemAllocatorAlloc(PVMCPU pVCpu, uint32_t cbReq)
+ * @param   pTb     The translation block that will be using the allocation.
+ */
+static void *iemExecMemAllocatorAlloc(PVMCPU pVCpu, uint32_t cbReq, PIEMTB pTb)
 {
     PIEMEXECMEMALLOCATOR pExecMemAllocator = pVCpu->iem.s.pExecMemAllocatorR3;
     AssertReturn(pExecMemAllocator && pExecMemAllocator->uMagic == IEMEXECMEMALLOCATOR_MAGIC, NULL);
     AssertMsgReturn(cbReq > 32 && cbReq < _512K, ("%#x\n", cbReq), NULL);
+    STAM_PROFILE_START(&pExecMemAllocator->StatAlloc, a);
 
     /*
@@ -531 +580 @@
             for (uint32_t idxChunk = idxChunkHint; idxChunk < cChunks; idxChunk++)
             {
-                void *pvRet = iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk, cbReq);
+                void *pvRet = iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk, cbReq, pTb);
                 if (pvRet)
+                {
+                    STAM_PROFILE_STOP(&pExecMemAllocator->StatAlloc, a);
                     return pvRet;
+                }
             }
             for (uint32_t idxChunk = 0; idxChunk < idxChunkHint; idxChunk++)
             {
-                void *pvRet = iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk, cbReq);
+                void *pvRet = iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk, cbReq, pTb);
                 if (pvRet)
+                {
+                    STAM_PROFILE_STOP(&pExecMemAllocator->StatAlloc, a);
                     return pvRet;
+                }
             }
         }
@@ -552 +607 @@
 
             uint32_t const idxChunk = pExecMemAllocator->cChunks - 1;
-            void *pvRet = iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk, cbReq);
+            void *pvRet = iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk, cbReq, pTb);
             if (pvRet)
+            {
+                STAM_PROFILE_STOP(&pExecMemAllocator->StatAlloc, a);
                 return pvRet;
+            }
             AssertFailed();
         }
@@ -566 +624 @@
         {
             STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatNativeExecMemInstrBufAllocFailed);
+            STAM_PROFILE_STOP(&pExecMemAllocator->StatAlloc, a);
             return NULL;
         }
     }
-
 }
 
@@ -601 +659 @@
     PIEMEXECMEMALLOCATOR pExecMemAllocator = pVCpu->iem.s.pExecMemAllocatorR3;
     Assert(pExecMemAllocator && pExecMemAllocator->uMagic == IEMEXECMEMALLOCATOR_MAGIC);
-    Assert(pv);
-#ifdef IEMEXECMEM_USE_ALT_SUB_ALLOCATOR
+    AssertPtr(pv);
+#ifndef IEMEXECMEM_USE_ALT_SUB_ALLOCATOR
+    Assert(!((uintptr_t)pv & 63));
+#else
+# ifndef IEMEXECMEM_ALT_SUB_WITH_ALLOC_HEADER
     Assert(!((uintptr_t)pv & (IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE - 1)));
-#else
-    Assert(!((uintptr_t)pv & 63));
+# else
+    PIEMEXECMEMALLOCHDR pHdr = (PIEMEXECMEMALLOCHDR)pv - 1;
+    Assert(!((uintptr_t)pHdr & (IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE - 1)));
+    AssertReturnVoid(pHdr->uMagic == IEMEXECMEMALLOCHDR_MAGIC);
+    uint32_t const idxChunk = pHdr->idxChunk;
+    AssertReturnVoid(idxChunk < pExecMemAllocator->cChunks);
+    pv = pHdr;
+# endif
 #endif
 
@@ -617 +684 @@
     /* Free it / assert sanity. */
 #if defined(VBOX_STRICT) || defined(IEMEXECMEM_USE_ALT_SUB_ALLOCATOR)
+    bool           fFound  = false;
+    uint32_t const cbChunk = pExecMemAllocator->cbChunk;
+# ifndef IEMEXECMEM_ALT_SUB_WITH_ALLOC_HEADER
     uint32_t const cChunks = pExecMemAllocator->cChunks;
-    uint32_t const cbChunk = pExecMemAllocator->cbChunk;
-    bool           fFound  = false;
     for (uint32_t idxChunk = 0; idxChunk < cChunks; idxChunk++)
+# endif
     {
         uintptr_t const offChunk = (uintptr_t)pv - (uintptr_t)pExecMemAllocator->aChunks[idxChunk].pvChunk;
@@ -636 +705 @@
                 AssertReturnVoid(ASMBitTest(pbmAlloc, idxFirst + i));
             ASMBitClearRange(pbmAlloc, idxFirst, idxFirst + cReqUnits);
-
+# ifdef IEMEXECMEM_ALT_SUB_WITH_ALLOC_HEADER
+            pHdr->uMagic    = 0;
+            pHdr->idxChunk  = 0;
+            pHdr->pTb       = NULL;
+# endif
             pExecMemAllocator->aChunks[idxChunk].cFreeUnits  += cReqUnits;
             pExecMemAllocator->aChunks[idxChunk].idxFreeHint  = idxFirst;
@@ -738 +811 @@
     unsigned const cbNeededAligned  = RT_ALIGN_32(cbNeeded, IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE);
     PIMAGE_RUNTIME_FUNCTION_ENTRY const paFunctions
-        = (PIMAGE_RUNTIME_FUNCTION_ENTRY)iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk, cbNeededAligned);
+        = (PIMAGE_RUNTIME_FUNCTION_ENTRY)iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk, cbNeededAligned, NULL);
 #  else
     unsigned const cbNeededAligned  = RT_ALIGN_32(cbNeeded + pExecMemAllocator->cbHeapBlockHdr, 64)
@@ -982 +1055 @@
 #   ifdef IEMEXECMEM_USE_ALT_SUB_ALLOCATOR
     unsigned const cbNeededAligned = RT_ALIGN_32(cbNeeded, IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SIZE);
-    GDBJITSYMFILE * const pSymFile = (GDBJITSYMFILE *)iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk, cbNeededAligned);
+    GDBJITSYMFILE * const pSymFile = (GDBJITSYMFILE *)iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk,
+                                                                                      cbNeededAligned, NULL);
 #   else
     unsigned const cbNeededAligned = RT_ALIGN_32(cbNeeded + pExecMemAllocator->cbHeapBlockHdr, 64)
@@ -1559 +1633 @@
 
     pExecMemAllocator->idxChunkHint = 0;
+
+    /*
+     * Register statistics.
+     */
+    PUVM const pUVM = pVCpu->pUVCpu->pUVM;
+    STAMR3RegisterFU(pUVM, &pExecMemAllocator->cAllocations,    STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
+                     "Current number of allocations",           "/IEM/CPU%u/re/ExecMem/cAllocations", pVCpu->idCpu);
+    STAMR3RegisterFU(pUVM, &pExecMemAllocator->cChunks,         STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                     "Currently allocated chunks",              "/IEM/CPU%u/re/ExecMem/cChunks", pVCpu->idCpu);
+    STAMR3RegisterFU(pUVM, &pExecMemAllocator->cMaxChunks,      STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                     "Maximum number of chunks",                "/IEM/CPU%u/re/ExecMem/cMaxChunks", pVCpu->idCpu);
+    STAMR3RegisterFU(pUVM, &pExecMemAllocator->cbChunk,         STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
+                     "Allocation chunk size",                   "/IEM/CPU%u/re/ExecMem/cbChunk", pVCpu->idCpu);
+    STAMR3RegisterFU(pUVM, &pExecMemAllocator->cbAllocated,     STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
+                     "Number of bytes current allocated",       "/IEM/CPU%u/re/ExecMem/cbAllocated", pVCpu->idCpu);
+    STAMR3RegisterFU(pUVM, &pExecMemAllocator->cbFree,          STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
+                     "Number of bytes current free",            "/IEM/CPU%u/re/ExecMem/cbFree", pVCpu->idCpu);
+    STAMR3RegisterFU(pUVM, &pExecMemAllocator->cbTotal,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
+                     "Total number of byte",                    "/IEM/CPU%u/re/ExecMem/cbTotal", pVCpu->idCpu);
+#ifdef VBOX_WITH_STATISTICS
+    STAMR3RegisterFU(pUVM, &pExecMemAllocator->StatAlloc,       STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL,
+                     "Profiling the allocator",                 "/IEM/CPU%u/re/ExecMem/ProfAlloc", pVCpu->idCpu);
+#endif
 
     return VINF_SUCCESS;
@@ -10703 +10800 @@
         iemTbAllocatorProcessDelayedFrees(pVCpu, pVCpu->iem.s.pTbAllocatorR3);
 
-    PIEMNATIVEINSTR const paFinalInstrBuf = (PIEMNATIVEINSTR)iemExecMemAllocatorAlloc(pVCpu, off * sizeof(IEMNATIVEINSTR));
+    PIEMNATIVEINSTR const paFinalInstrBuf = (PIEMNATIVEINSTR)iemExecMemAllocatorAlloc(pVCpu, off * sizeof(IEMNATIVEINSTR), pTb);
     AssertReturn(paFinalInstrBuf, pTb);
     memcpy(paFinalInstrBuf, pReNative->pInstrBuf, off * sizeof(paFinalInstrBuf[0]));

trunk/src/VBox/VMM/VMMR3/IEMR3.cpp

@@ -396 +396 @@
 # endif
         STAMR3RegisterF(pVM, (void *)&pTbAllocator->StatPruneNative,    STAMTYPE_PROFILE,   STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL,
-                        "Time spent freeing up native TBs when out of executable memory", "/IEM/CPU%u/re/TbPruningNative", idCpu);
+                        "Time spent freeing up native TBs when out of executable memory", "/IEM/CPU%u/re/ExecMem/TbPruningNative", idCpu);
         STAMR3RegisterF(pVM, (void *)&pTbAllocator->cAllocatedChunks,   STAMTYPE_U16,   STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
                         "Populated TB chunks",                          "/IEM/CPU%u/re/cTbChunks", idCpu);