Changeset 101088 in vbox

Timestamp:

Sep 12, 2023 10:22:20 AM (20 months ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

159057

Message:

VMM/IEM,VMM/TM: Basic TB managment and allocation rewrite. bugref:10369

Location:

trunk

Files:

• include/VBox/vmm/iem.h (modified) (1 diff)
• include/VBox/vmm/tm.h (modified) (1 diff)
• src/VBox/VMM/VMMAll/IEMAllThrdRecompiler.cpp (modified) (22 diffs)
• src/VBox/VMM/VMMAll/TMAll.cpp (modified) (6 diffs)
• src/VBox/VMM/VMMR3/EM.cpp (modified) (1 diff)
• src/VBox/VMM/VMMR3/IEMR3.cpp (modified) (3 diffs)
• src/VBox/VMM/include/IEMInternal.h (modified) (9 diffs)

trunk/include/VBox/vmm/iem.h

@@ -159 +159 @@
 VMMDECL(VBOXSTRICTRC)       IEMExecOneIgnoreLock(PVMCPUCC pVCpu);
 VMMDECL(VBOXSTRICTRC)       IEMExecLots(PVMCPUCC pVCpu, uint32_t cMaxInstructions, uint32_t cPollRate, uint32_t *pcInstructions);
-VMMDECL(VBOXSTRICTRC)       IEMExecRecompilerThreaded(PVMCC pVM, PVMCPUCC pVCpu);
+VMMDECL(VBOXSTRICTRC)       IEMExecRecompiler(PVMCC pVM, PVMCPUCC pVCpu);
 /** Statistics returned by IEMExecForExits. */
 typedef struct IEMEXECFOREXITSTATS

trunk/include/VBox/vmm/tm.h

@@ -272 +272 @@
 
 VMMDECL(bool)           TMTimerPollBool(PVMCC pVM, PVMCPUCC pVCpu);
+VMMDECL(bool)           TMTimerPollBoolWith32BitMilliTS(PVMCC pVM, PVMCPUCC pVCpu, uint32_t *pmsNow);
 VMM_INT_DECL(void)      TMTimerPollVoid(PVMCC pVM, PVMCPUCC pVCpu);
 VMM_INT_DECL(uint64_t)  TMTimerPollGIP(PVMCC pVM, PVMCPUCC pVCpu, uint64_t *pu64Delta);

trunk/src/VBox/VMM/VMMAll/IEMAllThrdRecompiler.cpp

@@ -83 +83 @@
 #include <iprt/mem.h>
 #include <iprt/string.h>
+#include <iprt/sort.h>
 #include <iprt/x86.h>
 
@@ -115 +116 @@
 *********************************************************************************************************************************/
 static VBOXSTRICTRC iemThreadedTbExec(PVMCPUCC pVCpu, PIEMTB pTb);
+static void         iemTbAllocatorFree(PVMCPUCC pVCpu, PIEMTB pTb);
 
 
@@ -438 +440 @@
 
 
+/*********************************************************************************************************************************
+*   Translation Block Cache.                                                                                                     *
+*********************************************************************************************************************************/
+
+/** @callback_method_impl{FNRTSORTCMP, Compare two TBs for pruning sorting purposes.}  */
+static DECLCALLBACK(int) iemTbCachePruneCmpTb(void const *pvElement1, void const *pvElement2, void *pvUser)
+{
+    PCIEMTB const  pTb1 = (PCIEMTB)pvElement1;
+    PCIEMTB const  pTb2 = (PCIEMTB)pvElement2;
+    uint32_t const cMsSinceUse1 = (uint32_t)(uintptr_t)pvUser - pTb1->msLastUsed;
+    uint32_t const cMsSinceUse2 = (uint32_t)(uintptr_t)pvUser - pTb2->msLastUsed;
+    if (cMsSinceUse1 != cMsSinceUse2)
+        return cMsSinceUse1 < cMsSinceUse2 ? -1 : 1;
+    if (pTb1->cUsed != pTb2->cUsed)
+        return pTb1->cUsed > pTb2->cUsed ? -1 : 1;
+    if ((pTb1->fFlags & IEMTB_F_TYPE_MASK) != (pTb2->fFlags & IEMTB_F_TYPE_MASK))
+        return (pTb1->fFlags & IEMTB_F_TYPE_MASK) == IEMTB_F_TYPE_NATIVE ? -1 : 1;
+    return 0;
+}
+
+
+DECL_NO_INLINE(static, void) iemTbCacheAddWithPruning(PVMCPUCC pVCpu, PIEMTBCACHE pTbCache, PIEMTB pTb, uint32_t idxHash)
+{
+    STAM_PROFILE_START(&pTbCache->StatPrune, a);
+
+    /*
+     * First convert the collision list to an array.
+     */
+    PIEMTB    apSortedTbs[IEMTBCACHE_PTR_MAX_COUNT];
+    uintptr_t cInserted    = 0;
+    PIEMTB    pTbCollision = IEMTBCACHE_PTR_GET_TB(pTbCache->apHash[idxHash]);
+    pTbCache->apHash[idxHash] = NULL; /* Must NULL the entry before trying to free anything. */
+
+    while (pTbCollision && cInserted < RT_ELEMENTS(apSortedTbs))
+    {
+        apSortedTbs[cInserted++] = pTbCollision;
+        pTbCollision = pTbCollision->pNext;
+    }
+
+    /* Free any excess (impossible). */
+    if (RT_LIKELY(!pTbCollision))
+        Assert(cInserted == RT_ELEMENTS(apSortedTbs));
+    else
+        do
+        {
+            PIEMTB pTbToFree = pTbCollision;
+            pTbCollision = pTbToFree->pNext;
+            iemTbAllocatorFree(pVCpu, pTbToFree);
+        } while (pTbCollision);
+
+    /*
+     * Sort it by most recently used and usage count.
+     */
+    RTSortApvShell((void **)apSortedTbs, cInserted, iemTbCachePruneCmpTb, (void *)(uintptr_t)pVCpu->iem.s.msRecompilerPollNow);
+
+    /* We keep half the list for now. Perhaps a bit aggressive... */
+    uintptr_t const cKeep = cInserted / 2;
+
+    /* First free up the TBs we don't wish to keep (before creating the new
+       list because otherwise the free code will scan the list for each one
+       without ever finding it). */
+    for (uintptr_t idx = cKeep; idx < cInserted; idx++)
+        iemTbAllocatorFree(pVCpu, apSortedTbs[idx]);
+
+    /* Chain the new TB together with the ones we like to keep of the existing
+       ones and insert this list into the hash table. */
+    pTbCollision = pTb;
+    for (uintptr_t idx = 0; idx < cKeep; idx++)
+        pTbCollision = pTbCollision->pNext = apSortedTbs[idx];
+    pTbCollision->pNext = NULL;
+
+    pTbCache->apHash[idxHash] = IEMTBCACHE_PTR_MAKE(pTb, cKeep + 1);
+
+    STAM_PROFILE_STOP(&pTbCache->StatPrune, a);
+}
+
+
+static void iemTbCacheAdd(PVMCPUCC pVCpu, PIEMTBCACHE pTbCache, PIEMTB pTb)
+{
+    uint32_t const idxHash    = IEMTBCACHE_HASH(pTbCache, pTb->fFlags, pTb->GCPhysPc);
+    PIEMTB const   pTbOldHead = pTbCache->apHash[idxHash];
+    if (!pTbOldHead)
+    {
+        pTb->pNext = NULL;
+        pTbCache->apHash[idxHash] = IEMTBCACHE_PTR_MAKE(pTb, 1);  /** @todo could make 1 implicit... */
+    }
+    else
+    {
+        STAM_REL_COUNTER_INC(&pTbCache->cCollisions);
+        uintptr_t cCollisions = IEMTBCACHE_PTR_GET_COUNT(pTbOldHead);
+        if (cCollisions < IEMTBCACHE_PTR_MAX_COUNT)
+        {
+            pTb->pNext = IEMTBCACHE_PTR_GET_TB(pTbOldHead);
+            pTbCache->apHash[idxHash] = IEMTBCACHE_PTR_MAKE(pTb, cCollisions + 1);
+        }
+        else
+            iemTbCacheAddWithPruning(pVCpu, pTbCache, pTb, idxHash);
+    }
+}
+
+
+/**
+ * Unlinks @a pTb from the hash table if found in it.
+ *
+ * @returns true if unlinked, false if not present.
+ * @param   pTbCache    The hash table.
+ * @param   pTb         The TB to remove.
+ */
+static bool iemTbCacheRemove(PIEMTBCACHE pTbCache, PIEMTB pTb)
+{
+    uint32_t const idxHash = IEMTBCACHE_HASH(pTbCache, pTb->fFlags, pTb->GCPhysPc);
+    PIEMTB         pTbHash = IEMTBCACHE_PTR_GET_TB(pTbCache->apHash[idxHash]);
+
+    /* At the head of the collision list? */
+    if (pTbHash == pTb)
+    {
+        if (!pTb->pNext)
+            pTbCache->apHash[idxHash] = NULL;
+        else
+            pTbCache->apHash[idxHash] = IEMTBCACHE_PTR_MAKE(pTb->pNext,
+                                                            IEMTBCACHE_PTR_GET_COUNT(pTbCache->apHash[idxHash]) - 1);
+        return true;
+    }
+
+    /* Search the collision list. */
+    while (pTbHash)
+    {
+        PIEMTB const pNextTb = pTbHash->pNext;
+        if (pNextTb == pTb)
+        {
+            pTbHash->pNext = pTb->pNext;
+            pTbCache->apHash[idxHash] = IEMTBCACHE_PTR_MAKE(pTbCache->apHash[idxHash],
+                                                            IEMTBCACHE_PTR_GET_COUNT(pTbCache->apHash[idxHash]) - 1);
+            return true;
+        }
+        pTbHash = pNextTb;
+    }
+    return false;
+}
+
+
+/**
+ * Looks up a TB for the given PC and flags in the cache.
+ *
+ * @returns Pointer to TB on success, NULL if not found.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling thread.
+ * @param   pTbCache        The translation block cache.
+ * @param   GCPhysPc        The PC to look up a TB for.
+ * @param   fExtraFlags     The extra flags to join with IEMCPU::fExec for
+ *                          the lookup.
+ * @thread  EMT(pVCpu)
+ */
+static PIEMTB iemTbCacheLookup(PVMCPUCC pVCpu, PIEMTBCACHE pTbCache,
+                               RTGCPHYS GCPhysPc, uint32_t fExtraFlags) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    uint32_t const fFlags  = ((pVCpu->iem.s.fExec & IEMTB_F_IEM_F_MASK) | fExtraFlags | IEMTB_F_STATE_READY) & IEMTB_F_KEY_MASK;
+    uint32_t const idxHash = IEMTBCACHE_HASH_NO_KEY_MASK(pTbCache, fFlags, GCPhysPc);
+    PIEMTB         pTb     = IEMTBCACHE_PTR_GET_TB(pTbCache->apHash[idxHash]);
+#if defined(VBOX_STRICT) || defined(LOG_ENABLED)
+    int            cLeft   = IEMTBCACHE_PTR_GET_COUNT(pTbCache->apHash[idxHash]);
+#endif
+    Log10(("TB lookup: fFlags=%#x GCPhysPc=%RGp idxHash=%#x: %p L %d\n", fFlags, GCPhysPc, idxHash, pTb, cLeft));
+    while (pTb)
+    {
+        if (pTb->GCPhysPc == GCPhysPc)
+        {
+            if ((pTb->fFlags & IEMTB_F_KEY_MASK) == fFlags)
+            {
+                if (pTb->x86.fAttr == (uint16_t)pVCpu->cpum.GstCtx.cs.Attr.u)
+                {
+                    pTb->cUsed++;
+                    pTb->msLastUsed = pVCpu->iem.s.msRecompilerPollNow;
+                    STAM_COUNTER_INC(&pTbCache->cLookupHits);
+                    AssertMsg(cLeft > 0, ("%d\n", cLeft));
+                    return pTb;
+                }
+                Log11(("TB miss: CS: %#x, wanted %#x\n", pTb->x86.fAttr, (uint16_t)pVCpu->cpum.GstCtx.cs.Attr.u));
+            }
+            else
+                Log11(("TB miss: fFlags: %#x, wanted %#x\n", pTb->fFlags, fFlags));
+        }
+        else
+            Log11(("TB miss: GCPhysPc: %#x, wanted %#x\n", pTb->GCPhysPc, GCPhysPc));
+
+        pTb = pTb->pNext;
+#ifdef VBOX_STRICT
+        cLeft--;
+#endif
+    }
+    AssertMsg(cLeft == 0, ("%d\n", cLeft));
+    STAM_REL_COUNTER_INC(&pTbCache->cLookupMisses);
+    return pTb;
+}
+
+
+/*********************************************************************************************************************************
+*   Translation Block Allocator.
+*********************************************************************************************************************************/
 /*
- * Translation block management.
- */
-
-typedef struct IEMTBCACHE
-{
-    uint32_t cHash;
-    uint32_t uHashMask;
-    PIEMTB   apHash[_1M];
-} IEMTBCACHE;
-
-static IEMTBCACHE g_TbCache = { _1M, _1M - 1, }; /**< Quick and dirty. */
-
-#define IEMTBCACHE_HASH(a_paCache, a_fTbFlags, a_GCPhysPc) \
-    ( ((uint32_t)(a_GCPhysPc) ^ (a_fTbFlags)) & (a_paCache)->uHashMask)
+ * Translation block allocationmanagement.
+ */
+
+#ifdef IEMTB_SIZE_IS_POWER_OF_TWO
+# define IEMTBALLOC_IDX_TO_CHUNK(a_pTbAllocator, a_idxTb) \
+    ((a_idxTb) >> (a_pTbAllocator)->cChunkShift)
+# define IEMTBALLOC_IDX_TO_INDEX_IN_CHUNK(a_pTbAllocator, a_idxTb, a_idxChunk) \
+    ((a_idxTb) &  (a_pTbAllocator)->fChunkMask)
+# define IEMTBALLOC_IDX_FOR_CHUNK(a_pTbAllocator, a_idxChunk) \
+    ((uint32_t)(a_idxChunk) << (a_pTbAllocator)->cChunkShift)
+#else
+# define IEMTBALLOC_IDX_TO_CHUNK(a_pTbAllocator, a_idxTb) \
+    ((a_idxTb) / (a_pTbAllocator)->cTbsPerChunk)
+# define IEMTBALLOC_IDX_TO_INDEX_IN_CHUNK(a_pTbAllocator, a_idxTb, a_idxChunk) \
+    ((a_idxTb) - (a_idxChunk) * (a_pTbAllocator)->cTbsPerChunk)
+# define IEMTBALLOC_IDX_FOR_CHUNK(a_pTbAllocator, a_idxChunk) \
+    ((uint32_t)(a_idxChunk) * (a_pTbAllocator)->cTbsPerChunk)
+#endif
+/** Makes a TB index from a chunk index and TB index within that chunk. */
+#define IEMTBALLOC_IDX_MAKE(a_pTbAllocator, a_idxChunk, a_idxInChunk) \
+    (IEMTBALLOC_IDX_FOR_CHUNK(a_pTbAllocator, a_idxChunk) + (a_idxInChunk))
+
+
+/**
+ * Initializes the TB allocator and cache for an EMT.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   cInitialTbs The initial number of translation blocks to
+ *                      preallocator.
+ * @param   cMaxTbs     The max number of translation blocks allowed.
+ * @thread  EMT
+ */
+DECLCALLBACK(int) iemTbInit(PVMCC pVM, uint32_t cInitialTbs, uint32_t cMaxTbs)
+{
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    Assert(!pVCpu->iem.s.pTbCacheR3);
+    Assert(!pVCpu->iem.s.pTbAllocatorR3);
+
+    /*
+     * Calculate the chunk size of the TB allocator.
+     * The minimum chunk size is 2MiB.
+     */
+    AssertCompile(!(sizeof(IEMTB) & IEMTBCACHE_PTR_COUNT_MASK));
+    uint32_t      cbPerChunk   = _2M;
+    uint32_t      cTbsPerChunk = _2M / sizeof(IEMTB);
+#ifdef IEMTB_SIZE_IS_POWER_OF_TWO
+    uint8_t const cTbShift     = ASMBitFirstSetU32((uint32_t)sizeof(IEMTB)) - 1;
+    uint8_t       cChunkShift  = 21 - cTbShift;
+    AssertCompile(RT_BIT_32(21) == _2M); Assert(RT_BIT_32(cChunkShift) == cTbsPerChunk);
+#endif
+    for (;;)
+    {
+        if (cMaxTbs <= cTbsPerChunk * (uint64_t)RT_ELEMENTS(pVCpu->iem.s.pTbAllocatorR3->aChunks))
+            break;
+        cbPerChunk  *= 2;
+        cTbsPerChunk = cbPerChunk / sizeof(IEMTB);
+#ifdef IEMTB_SIZE_IS_POWER_OF_TWO
+        cChunkShift += 1;
+#endif
+    }
+
+    uint32_t cMaxChunks = (cMaxTbs + cTbsPerChunk - 1) / cTbsPerChunk;
+    Assert(cMaxChunks * cTbsPerChunk >= cMaxTbs);
+    Assert(cMaxChunks <= RT_ELEMENTS(pVCpu->iem.s.pTbAllocatorR3->aChunks));
+
+    cMaxTbs = cMaxChunks * cTbsPerChunk;
+
+    /*
+     * Allocate and initalize it.
+     */
+    uint32_t const        c64BitWords   = RT_ALIGN_32(cMaxTbs, 64) / 64;
+    size_t const          cbTbAllocator = RT_UOFFSETOF_DYN(IEMTBALLOCATOR, bmAllocated[c64BitWords]);
+    PIEMTBALLOCATOR const pTbAllocator  = (PIEMTBALLOCATOR)RTMemAllocZ(cbTbAllocator);
+    if (!pTbAllocator)
+        return VMSetError(pVM, VERR_NO_MEMORY, RT_SRC_POS,
+                          "Failed to allocate %zu bytes (max %u TBs) for the TB allocator of VCpu #%u",
+                          cbTbAllocator, cMaxTbs, pVCpu->idCpu);
+    pTbAllocator->uMagic        = IEMTBALLOCATOR_MAGIC;
+    pTbAllocator->cMaxChunks    = (uint8_t)cMaxChunks;
+    pTbAllocator->cTbsPerChunk  = cTbsPerChunk;
+    pTbAllocator->cbPerChunk    = cbPerChunk;
+    pTbAllocator->cMaxTbs       = cMaxTbs;
+#ifdef IEMTB_SIZE_IS_POWER_OF_TWO
+    pTbAllocator->fChunkMask    = cTbsPerChunk - 1;
+    pTbAllocator->cChunkShift   = cChunkShift;
+    Assert(RT_BIT_32(cChunkShift) == cTbsPerChunk);
+#endif
+
+    memset(pTbAllocator->bmAllocated, 0xff, c64BitWords * sizeof(uint64_t)); /* Mark all as allocated, clear as chunks are added. */
+    pVCpu->iem.s.pTbAllocatorR3 = pTbAllocator;
+
+    /*
+     * Allocate the initial chunks.
+     */
+    for (uint32_t idxChunk = 0; ; idxChunk++)
+    {
+        PIEMTB const paTbs = pTbAllocator->aChunks[idxChunk].paTbs = (PIEMTB)RTMemPageAllocZ(cbPerChunk);
+        if (!paTbs)
+            return VMSetError(pVM, VERR_NO_MEMORY, RT_SRC_POS,
+                              "Failed to initial %zu bytes for the #%u chunk of TBs for VCpu #%u",
+                              cbPerChunk, idxChunk, pVCpu->idCpu);
+
+        for (uint32_t iTb = 0; iTb < cTbsPerChunk; iTb++)
+            paTbs[iTb].idxAllocChunk = idxChunk; /* This is not strictly necessary... */
+        ASMBitClearRange(pTbAllocator->bmAllocated, idxChunk * cTbsPerChunk, (idxChunk + 1) * cTbsPerChunk);
+        pTbAllocator->cAllocatedChunks = (uint16_t)(idxChunk + 1);
+        pTbAllocator->cTotalTbs       += cTbsPerChunk;
+
+        if (idxChunk * cTbsPerChunk >= cInitialTbs)
+            break;
+    }
+
+    /*
+     * Calculate the size of the hash table. We double the max TB count and
+     * round it up to the nearest power of two.
+     */
+    uint32_t cCacheEntries = cMaxTbs * 2;
+    if (!RT_IS_POWER_OF_TWO(cCacheEntries))
+    {
+        uint8_t const iBitTop = ASMBitFirstSetU32(cCacheEntries);
+        cCacheEntries = RT_BIT_32(iBitTop);
+        Assert(cCacheEntries >= cMaxTbs * 2);
+    }
+
+    size_t const      cbTbCache = RT_UOFFSETOF_DYN(IEMTBCACHE, apHash[cCacheEntries]);
+    PIEMTBCACHE const pTbCache  = (PIEMTBCACHE)RTMemAllocZ(cbTbCache);
+    if (!pTbCache)
+        return VMSetError(pVM, VERR_NO_MEMORY, RT_SRC_POS,
+                          "Failed to allocate %zu bytes (%u entries) for the TB cache of VCpu #%u",
+                          cbTbCache, cCacheEntries, pVCpu->idCpu);
+
+    /*
+     * Initialize it (assumes zeroed by the allocator).
+     */
+    pTbCache->uMagic    = IEMTBCACHE_MAGIC;
+    pTbCache->cHash     = cCacheEntries;
+    pTbCache->uHashMask = cCacheEntries - 1;
+    Assert(pTbCache->cHash > pTbCache->uHashMask);
+    pVCpu->iem.s.pTbCacheR3 = pTbCache;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Inner free worker.
+ */
+static void iemTbAllocatorFreeInner(PVMCPUCC pVCpu, PIEMTBALLOCATOR pTbAllocator,
+                                    PIEMTB pTb, uint32_t idxChunk, uint32_t idxInChunk)
+{
+    Assert(idxChunk < pTbAllocator->cAllocatedChunks);
+    Assert(idxInChunk < pTbAllocator->cTbsPerChunk);
+    Assert((uintptr_t)(pTb - pTbAllocator->aChunks[idxChunk].paTbs) == idxInChunk);
+    Assert(ASMBitTest(&pTbAllocator->bmAllocated, IEMTBALLOC_IDX_MAKE(pTbAllocator, idxChunk, idxInChunk)));
+
+    /*
+     * Unlink the TB from the hash table.
+     */
+    iemTbCacheRemove(pVCpu->iem.s.pTbCacheR3, pTb);
+
+    /*
+     * Free the TB itself.
+     */
+    switch (pTb->fFlags & IEMTB_F_TYPE_MASK)
+    {
+        case IEMTB_F_TYPE_THREADED:
+            pTbAllocator->cThreadedTbs -= 1;
+            RTMemFree(pTb->Thrd.paCalls);
+            break;
+        case IEMTB_F_TYPE_NATIVE:
+            pTbAllocator->cNativeTbs -= 1;
+            RTMemFree(pTb->Native.pbCode); /// @todo native: fix me
+            break;
+        default:
+            AssertFailed();
+    }
+    RTMemFree(pTb->pabOpcodes);
+
+    pTb->pNext              = NULL;
+    pTb->fFlags             = 0;
+    pTb->GCPhysPc           = UINT64_MAX;
+    pTb->Gen.uPtr           = 0;
+    pTb->Gen.uData          = 0;
+    pTb->cbOpcodes          = 0;
+    pTb->cbOpcodesAllocated = 0;
+    pTb->pabOpcodes         = NULL;
+
+    ASMBitClear(&pTbAllocator->bmAllocated, IEMTBALLOC_IDX_MAKE(pTbAllocator, idxChunk, idxInChunk));
+    Assert(pTbAllocator->cInUseTbs > 0);
+
+    pTbAllocator->cInUseTbs -= 1;
+    STAM_REL_COUNTER_INC(&pTbAllocator->StatFrees);
+}
+
+
+/**
+ * Frees the given TB.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure of the calling
+ *                  thread.
+ * @param   pTb     The translation block to free..
+ * @thread  EMT(pVCpu)
+ */
+static void iemTbAllocatorFree(PVMCPUCC pVCpu, PIEMTB pTb)
+{
+    /*
+     * Validate state.
+     */
+    PIEMTBALLOCATOR const pTbAllocator = pVCpu->iem.s.pTbAllocatorR3;
+    Assert(pTbAllocator && pTbAllocator->uMagic == IEMTBALLOCATOR_MAGIC);
+    uint8_t const idxChunk = pTb->idxAllocChunk;
+    AssertLogRelReturnVoid(idxChunk < pTbAllocator->cAllocatedChunks);
+    uintptr_t const idxInChunk = pTb - pTbAllocator->aChunks[idxChunk].paTbs;
+    AssertLogRelReturnVoid(idxInChunk < pTbAllocator->cTbsPerChunk);
+
+    /*
+     * Call inner worker.
+     */
+    iemTbAllocatorFreeInner(pVCpu, pTbAllocator, pTb, idxChunk, (uint32_t)idxInChunk);
+}
+
+
+/**
+ * Grow the translation block allocator with another chunk.
+ */
+static int iemTbAllocatorGrow(PVMCPUCC pVCpu)
+{
+    /*
+     * Validate state.
+     */
+    PIEMTBALLOCATOR const pTbAllocator = pVCpu->iem.s.pTbAllocatorR3;
+    AssertReturn(pTbAllocator, VERR_WRONG_ORDER);
+    AssertReturn(pTbAllocator->uMagic == IEMTBALLOCATOR_MAGIC, VERR_INVALID_MAGIC);
+    uint32_t const idxChunk = pTbAllocator->cAllocatedChunks;
+    AssertReturn(idxChunk < pTbAllocator->cMaxChunks, VERR_OUT_OF_RESOURCES);
+
+    /*
+     * Allocate a new chunk and add it to the allocator.
+     */
+    PIEMTB const paTbs = (PIEMTB)RTMemPageAllocZ(pTbAllocator->cbPerChunk);
+    AssertLogRelReturn(paTbs, VERR_NO_PAGE_MEMORY);
+    pTbAllocator->aChunks[idxChunk].paTbs = paTbs;
+
+    uint32_t const cTbsPerChunk = pTbAllocator->cTbsPerChunk;
+    for (uint32_t iTb = 0; iTb < cTbsPerChunk; iTb++)
+        paTbs[iTb].idxAllocChunk = idxChunk; /* This is not strictly necessary... */
+    ASMBitClearRange(pTbAllocator->bmAllocated, idxChunk * cTbsPerChunk, (idxChunk + 1) * cTbsPerChunk);
+    pTbAllocator->cAllocatedChunks = (uint16_t)(idxChunk + 1);
+    pTbAllocator->cTotalTbs       += cTbsPerChunk;
+    pTbAllocator->iStartHint       = idxChunk * cTbsPerChunk;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Allocates a TB from allocator with free block.
+ *
+ * This is common code to both the fast and slow allocator code paths.
+ */
+DECL_FORCE_INLINE(PIEMTB) iemTbAllocatorAllocCore(PIEMTBALLOCATOR const pTbAllocator, bool fThreaded)
+{
+    Assert(pTbAllocator->cInUseTbs < pTbAllocator->cTotalTbs);
+
+    int idxTb;
+    if (pTbAllocator->iStartHint < pTbAllocator->cTotalTbs)
+        idxTb = ASMBitNextClear(pTbAllocator->bmAllocated,
+                                pTbAllocator->cTotalTbs,
+                                pTbAllocator->iStartHint & ~(uint32_t)63);
+    else
+        idxTb = -1;
+    if (idxTb < 0)
+    {
+        idxTb = ASMBitFirstClear(pTbAllocator->bmAllocated, pTbAllocator->cTotalTbs);
+        AssertLogRelReturn(idxTb >= 0, NULL);
+    }
+    Assert((uint32_t)idxTb < pTbAllocator->cTotalTbs);
+    ASMBitSet(pTbAllocator->bmAllocated, idxTb);
+
+    /** @todo shift/mask optimization for power of two IEMTB sizes. */
+    uint32_t const idxChunk     = IEMTBALLOC_IDX_TO_CHUNK(pTbAllocator, idxTb);
+    uint32_t const idxTbInChunk = IEMTBALLOC_IDX_TO_INDEX_IN_CHUNK(pTbAllocator, idxTb, idxChunk);
+    PIEMTB const   pTb          = &pTbAllocator->aChunks[idxChunk].paTbs[idxTbInChunk];
+    Assert(pTb->idxAllocChunk == idxChunk);
+
+    pTbAllocator->cInUseTbs        += 1;
+    if (fThreaded)
+        pTbAllocator->cThreadedTbs += 1;
+    else
+        pTbAllocator->cNativeTbs   += 1;
+    STAM_REL_COUNTER_INC(&pTbAllocator->StatAllocs);
+    return pTb;
+}
+
+
+/**
+ * Slow path for iemTbAllocatorAlloc.
+ */
+static PIEMTB iemTbAllocatorAllocSlow(PVMCPUCC pVCpu, PIEMTBALLOCATOR const pTbAllocator, bool fThreaded)
+{
+    /*
+     * With some luck we can add another chunk.
+     */
+    if (pTbAllocator->cAllocatedChunks < pTbAllocator->cMaxChunks)
+    {
+        int rc = iemTbAllocatorGrow(pVCpu);
+        if (RT_SUCCESS(rc))
+            return iemTbAllocatorAllocCore(pTbAllocator, fThreaded);
+    }
+
+    /*
+     * We have to prune stuff. Sigh.
+     *
+     * This requires scanning for older TBs and kick them out.  Not sure how to
+     * best do this as we don't want to maintain any list of TBs ordered by last
+     * usage time. But one reasonably simple approach would be that each time we
+     * get here we continue a sequential scan of the allocation chunks,
+     * considering just a smallish number of TBs and freeing a fixed portion of
+     * them.  Say, we consider the next 128 TBs, freeing the least recently used
+     * in out of groups of 4 TBs, resulting in 32 free TBs.
+     */
+    STAM_PROFILE_START(&pTbAllocator->StatPrune, a);
+    uint32_t const msNow          = pVCpu->iem.s.msRecompilerPollNow;
+    uint32_t       cFreedTbs      = 0;
+#ifdef IEMTB_SIZE_IS_POWER_OF_TWO
+    uint32_t       idxTbPruneFrom = pTbAllocator->iPruneFrom & ~(uint32_t)3;
+#else
+    uint32_t       idxTbPruneFrom = pTbAllocator->iPruneFrom;
+#endif
+    if (idxTbPruneFrom > pTbAllocator->cMaxTbs)
+        idxTbPruneFrom = 0;
+    for (uint32_t i = 0; i < 128; i += 4, idxTbPruneFrom += 4)
+    {
+        uint32_t idxChunk   = IEMTBALLOC_IDX_TO_CHUNK(pTbAllocator, idxTbPruneFrom);
+        uint32_t idxInChunk = IEMTBALLOC_IDX_TO_INDEX_IN_CHUNK(pTbAllocator, idxTbPruneFrom, idxChunk);
+        PIEMTB   pTb        = &pTbAllocator->aChunks[idxChunk].paTbs[idxInChunk];
+        uint32_t cMsAge     = msNow - pTb->msLastUsed;
+        for (uint32_t j = 1, idxChunk2 = idxChunk, idxInChunk2 = idxInChunk + 1; j < 4; j++, idxInChunk2++)
+        {
+#ifndef IEMTB_SIZE_IS_POWER_OF_TWO
+            if (idxInChunk2 < pTbAllocator->cTbsPerChunk)
+            { /* likely */ }
+            else
+            {
+                idxInChunk2 = 0;
+                idxChunk2  += 1;
+                if (idxChunk2 >= pTbAllocator->cAllocatedChunks)
+                    idxChunk2 = 0;
+            }
+#endif
+            PIEMTB   const pTb2    = &pTbAllocator->aChunks[idxChunk2].paTbs[idxInChunk2];
+            uint32_t const cMsAge2 = msNow - pTb2->msLastUsed;
+            if (   cMsAge2 > cMsAge
+                || (cMsAge2 == cMsAge && pTb2->cUsed < pTb->cUsed)
+                || (pTb2->fFlags & IEMTB_F_STATE_MASK) == IEMTB_F_STATE_OBSOLETE) /** @todo Consider state (and clean it up)! */
+            {
+                pTb        = pTb2;
+                idxChunk   = idxChunk2;
+                idxInChunk = idxInChunk2;
+                cMsAge     = cMsAge2;
+            }
+        }
+
+        /* Free the TB if in the right state. */
+        /** @todo They shall all be freeable! Otherwise we've buggered up the
+         *        accounting.  The TB state crap needs elimnating. */
+        if (   (pTb->fFlags & IEMTB_F_STATE_MASK) == IEMTB_F_STATE_READY
+            || (pTb->fFlags & IEMTB_F_STATE_MASK) == IEMTB_F_STATE_OBSOLETE)
+        {
+            iemTbAllocatorFreeInner(pVCpu, pTbAllocator, pTb, idxChunk, idxInChunk);
+            cFreedTbs++; /* just for safety */
+        }
+    }
+    pTbAllocator->iPruneFrom = idxTbPruneFrom;
+    STAM_PROFILE_STOP(&pTbAllocator->StatPrune, a);
+
+    /*
+     * Allocate a TB from the ones we've pruned.
+     */
+    if (cFreedTbs)
+        return iemTbAllocatorAllocCore(pTbAllocator, fThreaded);
+    return NULL;
+}
+
+
+/**
+ * Allocate a translation block.
+ *
+ * @returns Pointer to block on success, NULL if we're out and is unable to
+ *          free up an existing one (very unlikely once implemented).
+ * @param   pVCpu       The cross context virtual CPU structure of the calling
+ *                      thread.
+ * @param   fThreaded   Set if threaded TB being allocated, clear if native TB.
+ *                      For statistics.
+ */
+DECL_FORCE_INLINE(PIEMTB) iemTbAllocatorAlloc(PVMCPUCC pVCpu, bool fThreaded)
+{
+    PIEMTBALLOCATOR const pTbAllocator = pVCpu->iem.s.pTbAllocatorR3;
+    Assert(pTbAllocator && pTbAllocator->uMagic == IEMTBALLOCATOR_MAGIC);
+
+    /* If the allocator is full, take slow code path.*/
+    if (RT_LIKELY(pTbAllocator->cInUseTbs < pTbAllocator->cTotalTbs))
+        return iemTbAllocatorAllocCore(pTbAllocator, fThreaded);
+    return iemTbAllocatorAllocSlow(pVCpu, pTbAllocator, fThreaded);
+}
 
 
@@ -471 +1064 @@
 static PIEMTB iemThreadedTbAlloc(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysPc, uint32_t fExtraFlags)
 {
-    PIEMTB pTb = (PIEMTB)RTMemAlloc(sizeof(IEMTB));
+    PIEMTB pTb = (PIEMTB)RTMemAllocZ(sizeof(IEMTB));
     if (pTb)
     {
@@ -486 +1079 @@
                 pTb->cbOpcodes              = 0;
                 pTb->pNext                  = NULL;
-                RTListInit(&pTb->LocalList);
+                pTb->cUsed                  = 0;
+                pTb->msLastUsed             = pVCpu->iem.s.msRecompilerPollNow;
+                pTb->idxAllocChunk          = UINT8_MAX;
                 pTb->GCPhysPc               = GCPhysPc;
                 pTb->x86.fAttr              = (uint16_t)pVCpu->cpum.GstCtx.cs.Attr.u;
@@ -502 +1097 @@
                 pTb->aGCPhysPages[1]        = NIL_RTGCPHYS;
 
-                pVCpu->iem.s.cTbAllocs++;
                 return pTb;
             }
@@ -559 +1153 @@
      * complicated later, don't worry. :-)
      */
-    PIEMTB pTb = (PIEMTB)RTMemAlloc(sizeof(IEMTB));
+    PIEMTB pTb = iemTbAllocatorAlloc(pVCpu, true /*fThreaded*/);
     if (pTb)
     {
+        uint8_t const idxAllocChunk = pTb->idxAllocChunk;
         memcpy(pTb, pTbSrc, sizeof(*pTb));
+        pTb->idxAllocChunk = idxAllocChunk;
 
         unsigned const cCalls = pTbSrc->Thrd.cCalls;
@@ -577 +1173 @@
                 pTb->cbOpcodesAllocated     = cbOpcodes;
                 pTb->pNext                  = NULL;
-                RTListInit(&pTb->LocalList);
+                pTb->cUsed                  = 0;
+                pTb->msLastUsed             = pVCpu->iem.s.msRecompilerPollNow;
                 pTb->fFlags                 = (pTbSrc->fFlags & ~IEMTB_F_STATE_MASK) | IEMTB_F_STATE_READY;
 
-                pVCpu->iem.s.cTbAllocs++;
                 return pTb;
             }
             RTMemFree(pTb->Thrd.paCalls);
         }
-        RTMemFree(pTb);
+        iemTbAllocatorFree(pVCpu, pTb);
     }
     RT_NOREF(pVM);
@@ -596 +1192 @@
  * Adds the given TB to the hash table.
  *
- * @param   pVM         The cross context virtual machine structure.
  * @param   pVCpu       The cross context virtual CPU structure of the calling
  *                      thread.
+ * @param   pTbCache    The cache to add it to.
  * @param   pTb         The translation block to add.
  */
-static void iemThreadedTbAdd(PVMCC pVM, PVMCPUCC pVCpu, PIEMTB pTb)
-{
-    uint32_t const idxHash = IEMTBCACHE_HASH(&g_TbCache, pTb->fFlags, pTb->GCPhysPc);
-    pTb->pNext = g_TbCache.apHash[idxHash];
-    g_TbCache.apHash[idxHash] = pTb;
+static void iemThreadedTbAdd(PVMCPUCC pVCpu, PIEMTBCACHE pTbCache, PIEMTB pTb)
+{
+    iemTbCacheAdd(pVCpu, pTbCache, pTb);
+
     STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->iem.s.StatTbThreadedInstr, pTb->cInstructions);
     STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->iem.s.StatTbThreadedCalls, pTb->Thrd.cCalls);
@@ -611 +1206 @@
     {
         Log12(("TB added: %p %RGp LB %#x fl=%#x idxHash=%#x cRanges=%u cInstr=%u cCalls=%u\n",
-               pTb, pTb->GCPhysPc, pTb->cbOpcodes, pTb->fFlags, idxHash, pTb->cRanges, pTb->cInstructions, pTb->Thrd.cCalls));
+               pTb, pTb->GCPhysPc, pTb->cbOpcodes, pTb->fFlags, IEMTBCACHE_HASH(pTbCache, pTb->fFlags, pTb->GCPhysPc),
+               pTb->cRanges, pTb->cInstructions, pTb->Thrd.cCalls));
         for (uint8_t idxRange = 0; idxRange < pTb->cRanges; idxRange++)
             Log12((" range#%u: offPg=%#05x offOp=%#04x LB %#04x pg#%u=%RGp\n", idxRange, pTb->aRanges[idxRange].offPhysPage,
@@ -619 +1215 @@
                    : pTb->aGCPhysPages[pTb->aRanges[idxRange].idxPhysPage - 1]));
     }
-    RT_NOREF(pVM);
-}
-
-
-/**
- * Frees the given TB.
- *
- * @param   pVM     The cross context virtual machine structure.
- * @param   pVCpu   The cross context virtual CPU structure of the calling
- *                  thread.
- * @param   pTb     The translation block to free..
- */
-static void iemThreadedTbFree(PVMCC pVM, PVMCPUCC pVCpu, PIEMTB pTb)
-{
-    RT_NOREF(pVM);
-    AssertPtr(pTb);
-
-    AssertCompile(IEMTB_F_STATE_OBSOLETE == IEMTB_F_STATE_MASK);
-    pTb->fFlags |= IEMTB_F_STATE_OBSOLETE; /* works, both bits set */
-
-    /* Unlink it from the hash table: */
-    uint32_t const idxHash = IEMTBCACHE_HASH(&g_TbCache, pTb->fFlags, pTb->GCPhysPc);
-    PIEMTB pTbCur = g_TbCache.apHash[idxHash];
-    if (pTbCur == pTb)
-        g_TbCache.apHash[idxHash] = pTb->pNext;
-    else
-        while (pTbCur)
-        {
-            PIEMTB const pNextTb = pTbCur->pNext;
-            if (pNextTb == pTb)
-            {
-                pTbCur->pNext = pTb->pNext;
-                break;
-            }
-            pTbCur = pNextTb;
-        }
-
-    /* Free it. */
-    RTMemFree(pTb->Thrd.paCalls);
-    pTb->Thrd.paCalls = NULL;
-
-    RTMemFree(pTb->pabOpcodes);
-    pTb->pabOpcodes = NULL;
-
-    RTMemFree(pTb);
-    pVCpu->iem.s.cTbFrees++;
 }
 
@@ -673 +1223 @@
 void iemThreadedTbObsolete(PVMCPUCC pVCpu, PIEMTB pTb)
 {
-    iemThreadedTbFree(pVCpu->CTX_SUFF(pVM), pVCpu, pTb);
-}
-
-
-static PIEMTB iemThreadedTbLookup(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysPc, uint32_t fExtraFlags) IEM_NOEXCEPT_MAY_LONGJMP
-{
-    uint32_t const fFlags  = (pVCpu->iem.s.fExec & IEMTB_F_IEM_F_MASK) | fExtraFlags | IEMTB_F_STATE_READY;
-    uint32_t const idxHash = IEMTBCACHE_HASH(&g_TbCache, fFlags, GCPhysPc);
-    Log10(("TB lookup: idxHash=%#x fFlags=%#x GCPhysPc=%RGp\n", idxHash, fFlags, GCPhysPc));
-    PIEMTB pTb = g_TbCache.apHash[idxHash];
-    while (pTb)
-    {
-        if (pTb->GCPhysPc == GCPhysPc)
-        {
-            if (pTb->fFlags == fFlags)
-            {
-                if (pTb->x86.fAttr == (uint16_t)pVCpu->cpum.GstCtx.cs.Attr.u)
-                {
-#ifdef VBOX_WITH_STATISTICS
-                    pVCpu->iem.s.cTbLookupHits++;
-#endif
-                    return pTb;
-                }
-                Log11(("TB miss: CS: %#x, wanted %#x\n", pTb->x86.fAttr, (uint16_t)pVCpu->cpum.GstCtx.cs.Attr.u));
-            }
-            else
-                Log11(("TB miss: fFlags: %#x, wanted %#x\n", pTb->fFlags, fFlags));
-        }
-        else
-            Log11(("TB miss: GCPhysPc: %#x, wanted %#x\n", pTb->GCPhysPc, GCPhysPc));
-
-        pTb = pTb->pNext;
-    }
-    RT_NOREF(pVM);
-    pVCpu->iem.s.cTbLookupMisses++;
-    return pTb;
+    iemTbAllocatorFree(pVCpu, pTb);
 }
 
@@ -1443 +1958 @@
  * @param   GCPhysPc    The physical address corresponding to the current
  *                      RIP+CS.BASE.
- * @param   fExtraFlags Extra translation block flags: IEMTB_F_TYPE_THREADED and
- *                      maybe IEMTB_F_RIP_CHECKS.
+ * @param   fExtraFlags Extra translation block flags: IEMTB_F_INHIBIT_SHADOW,
+ *                      IEMTB_F_INHIBIT_NMI, IEMTB_F_CS_LIM_CHECKS.
  */
 static VBOXSTRICTRC iemThreadedCompile(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysPc, uint32_t fExtraFlags) IEM_NOEXCEPT_MAY_LONGJMP
 {
+    Assert(!(fExtraFlags & IEMTB_F_TYPE_MASK));
+    fExtraFlags |= IEMTB_F_TYPE_THREADED;
+
     /*
      * Get the TB we use for the recompiling.  This is a maxed-out TB so
@@ -1544 +2062 @@
     AssertReturn(pTb, VERR_IEM_TB_ALLOC_FAILED);
 
-    iemThreadedTbAdd(pVM, pVCpu, pTb);
+    iemThreadedTbAdd(pVCpu, pVCpu->iem.s.pTbCacheR3, pTb);
 
 #ifdef IEM_COMPILE_ONLY_MODE
@@ -1574 +2092 @@
     {
         Log7(("TB obsolete: %p GCPhys=%RGp\n", pTb, pTb->GCPhysPc));
-        iemThreadedTbFree(pVCpu->pVMR3, pVCpu, pTb);
+        iemThreadedTbObsolete(pVCpu, pTb);
         return VINF_SUCCESS;
     }
@@ -1680 +2198 @@
  * Determines the extra IEMTB_F_XXX flags.
  *
- * @returns IEMTB_F_TYPE_THREADED and maybe IEMTB_F_RIP_CHECKS.
+ * @returns A mix of IEMTB_F_INHIBIT_SHADOW, IEMTB_F_INHIBIT_NMI and
+ *          IEMTB_F_CS_LIM_CHECKS (or zero).
  * @param   pVCpu   The cross context virtual CPU structure of the calling
  *                  thread.
@@ -1686 +2205 @@
 DECL_FORCE_INLINE(uint32_t) iemGetTbFlagsForCurrentPc(PVMCPUCC pVCpu)
 {
-    uint32_t fRet = IEMTB_F_TYPE_THREADED;
+    uint32_t fRet = 0;
 
     /*
@@ -1715 +2234 @@
 
 
-VMMDECL(VBOXSTRICTRC) IEMExecRecompilerThreaded(PVMCC pVM, PVMCPUCC pVCpu)
+VMMDECL(VBOXSTRICTRC) IEMExecRecompiler(PVMCC pVM, PVMCPUCC pVCpu)
 {
     /*
@@ -1735 +2254 @@
      */
     iemInitExec(pVCpu, 0 /*fExecOpts*/);
+    if (RT_LIKELY(pVCpu->iem.s.msRecompilerPollNow != 0))
+    { }
+    else
+        pVCpu->iem.s.msRecompilerPollNow = (uint32_t)(TMVirtualGetNoCheck(pVM) / RT_NS_1MS);
 
     /*
@@ -1742 +2265 @@
      * having to call setjmp for each block we're executing.
      */
+    PIEMTBCACHE const pTbCache = pVCpu->iem.s.pTbCacheR3;
     for (;;)
     {
@@ -1755 +2279 @@
                 uint32_t const fExtraFlags = iemGetTbFlagsForCurrentPc(pVCpu);
 
-                pTb = iemThreadedTbLookup(pVM, pVCpu, GCPhysPc, fExtraFlags);
+                pTb = iemTbCacheLookup(pVCpu, pTbCache, GCPhysPc, fExtraFlags);
                 if (pTb)
-                    rcStrict = iemThreadedTbExec(pVCpu, pTb);
+                {
+                    if (pTb->fFlags & IEMTB_F_TYPE_THREADED)
+                        rcStrict = iemThreadedTbExec(pVCpu, pTb);
+                    else
+                        AssertFailedStmt(rcStrict = VERR_INTERNAL_ERROR_4);
+                }
                 else
                     rcStrict = iemThreadedCompile(pVM, pVCpu, GCPhysPc, fExtraFlags);
@@ -1777 +2306 @@
                     {
                         if (RT_LIKELY(   (iIterations & cPollRate) != 0
-                                      || !TMTimerPollBool(pVM, pVCpu)))
+                                      || !TMTimerPollBoolWith32BitMilliTS(pVM, pVCpu, &pVCpu->iem.s.msRecompilerPollNow)))
                         {
 

trunk/src/VBox/VMM/VMMAll/TMAll.cpp

@@ -890 +890 @@
  * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
  * @param   pu64Delta   Where to store the delta.
+ * @param   pu64Now     Where to store the current time. Optional.
  *
  * @thread  The emulation thread.
@@ -895 +896 @@
  * @remarks GIP uses ns ticks.
  */
-DECL_FORCE_INLINE(uint64_t) tmTimerPollInternal(PVMCC pVM, PVMCPUCC pVCpu, uint64_t *pu64Delta)
+DECL_FORCE_INLINE(uint64_t) tmTimerPollInternal(PVMCC pVM, PVMCPUCC pVCpu, uint64_t *pu64Delta, uint64_t *pu64Now)
 {
     VMCPUID idCpu = pVM->tm.s.idTimerCpu;
@@ -903 +904 @@
     const uint64_t u64Now = TMVirtualGetNoCheck(pVM);
     STAM_COUNTER_INC(&pVM->tm.s.StatPoll);
+    if (pu64Now)
+        *pu64Now = u64Now;
 
     /*
@@ -1104 +1107 @@
     AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
     uint64_t off = 0;
-    tmTimerPollInternal(pVM, pVCpu, &off);
+    tmTimerPollInternal(pVM, pVCpu, &off, NULL);
+    return off == 0;
+}
+
+
+/**
+ * Set FF if we've passed the next virtual event and return virtual time as MS.
+ *
+ * This function is called before FFs are checked in the inner execution EM loops.
+ *
+ * This is used by the IEM recompiler for polling timers while also providing a
+ * free time source for recent use tracking and such.
+ *
+ * @returns true if timers are pending, false if not.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pmsNow      Where to return the current virtual time in
+ *                      milliseconds.
+ * @thread  The emulation thread.
+ */
+VMMDECL(bool) TMTimerPollBoolWith32BitMilliTS(PVMCC pVM, PVMCPUCC pVCpu, uint32_t *pmsNow)
+{
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+    uint64_t off = 0;
+    uint64_t u64Now = 0;
+    tmTimerPollInternal(pVM, pVCpu, &off, &u64Now);
+    *pmsNow = (uint32_t)(u64Now / RT_NS_1MS);
     return off == 0;
 }
@@ -1121 +1151 @@
 {
     uint64_t off;
-    tmTimerPollInternal(pVM, pVCpu, &off);
+    tmTimerPollInternal(pVM, pVCpu, &off, NULL);
 }
 
@@ -1139 +1169 @@
 VMM_INT_DECL(uint64_t) TMTimerPollGIP(PVMCC pVM, PVMCPUCC pVCpu, uint64_t *pu64Delta)
 {
-    return tmTimerPollInternal(pVM, pVCpu, pu64Delta);
+    return tmTimerPollInternal(pVM, pVCpu, pu64Delta, NULL);
 }
 

trunk/src/VBox/VMM/VMMR3/EM.cpp

@@ -1088 +1088 @@
 #ifdef VBOX_WITH_IEM_RECOMPILER
             if (pVM->em.s.fIemRecompiled)
-                rcStrict = IEMExecRecompilerThreaded(pVM, pVCpu);
+                rcStrict = IEMExecRecompiler(pVM, pVCpu);
             else
 #endif

trunk/src/VBox/VMM/VMMR3/IEMR3.cpp

@@ -95 +95 @@
 VMMR3DECL(int)      IEMR3Init(PVM pVM)
 {
-#if !defined(VBOX_VMM_TARGET_ARMV8) && !defined(VBOX_WITHOUT_CPUID_HOST_CALL)
     /*
      * Read configuration.
      */
-    PCFGMNODE pIem = CFGMR3GetChild(CFGMR3GetRoot(pVM), "IEM");
-
+#if (!defined(VBOX_VMM_TARGET_ARMV8) && !defined(VBOX_WITHOUT_CPUID_HOST_CALL)) || defined(VBOX_WITH_IEM_RECOMPILER)
+    PCFGMNODE const pIem = CFGMR3GetChild(CFGMR3GetRoot(pVM), "IEM");
+    int rc;
+#endif
+
+#if !defined(VBOX_VMM_TARGET_ARMV8) && !defined(VBOX_WITHOUT_CPUID_HOST_CALL)
     /** @cfgm{/IEM/CpuIdHostCall, boolean, false}
      * Controls whether the custom VBox specific CPUID host call interface is
      * enabled or not. */
 # ifdef DEBUG_bird
-    int rc = CFGMR3QueryBoolDef(pIem, "CpuIdHostCall", &pVM->iem.s.fCpuIdHostCall, true);
+    rc = CFGMR3QueryBoolDef(pIem, "CpuIdHostCall", &pVM->iem.s.fCpuIdHostCall, true);
 # else
-    int rc = CFGMR3QueryBoolDef(pIem, "CpuIdHostCall", &pVM->iem.s.fCpuIdHostCall, false);
+    rc = CFGMR3QueryBoolDef(pIem, "CpuIdHostCall", &pVM->iem.s.fCpuIdHostCall, false);
 # endif
     AssertLogRelRCReturn(rc, rc);
+#endif
+
+#ifdef VBOX_WITH_IEM_RECOMPILER
+    /** @cfgm{/IEM/InitialTbCount, uint32_t, 32768}
+     * Initial (minimum) number of TBs per EMT in ring-3. */
+    uint32_t cInitialTbs = 0;
+    rc = CFGMR3QueryU32Def(pIem, "InitialTbCount", &cInitialTbs, _32K);
+    AssertLogRelRCReturn(rc, rc);
+    if (cInitialTbs < _16K || cInitialTbs > _8M)
+        return VMSetError(pVM, VERR_OUT_OF_RANGE, RT_SRC_POS,
+                          "InitialTbCount value %u (%#x) is out of range (min %u, max %u)", cInitialTbs, cInitialTbs, _16K, _8M);
+
+    /** @cfgm{/IEM/MaxTbCount, uint32_t, 524288}
+     * Max number of TBs per EMT. */
+    uint32_t cMaxTbs = 0;
+    rc = CFGMR3QueryU32Def(pIem, "MaxTbCount", &cMaxTbs, _512K);
+    AssertLogRelRCReturn(rc, rc);
+    if (cMaxTbs < cInitialTbs || cMaxTbs > _8M)
+        return VMSetError(pVM, VERR_OUT_OF_RANGE, RT_SRC_POS,
+                          "MaxTbCount value %u (%#x) is out of range (min %u, max %u)", cMaxTbs, cMaxTbs, cInitialTbs, _8M);
 #endif
 
@@ -125 +148 @@
         pVCpu->iem.s.CodeTlb.uTlbRevision = pVCpu->iem.s.DataTlb.uTlbRevision = uInitialTlbRevision;
         pVCpu->iem.s.CodeTlb.uTlbPhysRev  = pVCpu->iem.s.DataTlb.uTlbPhysRev  = uInitialTlbPhysRev;
-
-#if !defined(VBOX_VMM_TARGET_ARMV8) && defined(VBOX_WITH_NESTED_HWVIRT_VMX) /* quick fix for stupid structure duplication non-sense */
-
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.cInstructions,               STAMTYPE_U32,       STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "Instructions interpreted",                     "/IEM/CPU%u/cInstructions", idCpu);
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.cLongJumps,                  STAMTYPE_U32,       STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
-                        "Number of longjmp calls",                      "/IEM/CPU%u/cLongJumps", idCpu);
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.cPotentialExits,             STAMTYPE_U32,       STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "Potential exits",                              "/IEM/CPU%u/cPotentialExits", idCpu);
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.cRetAspectNotImplemented,    STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "VERR_IEM_ASPECT_NOT_IMPLEMENTED",              "/IEM/CPU%u/cRetAspectNotImplemented", idCpu);
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.cRetInstrNotImplemented,     STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "VERR_IEM_INSTR_NOT_IMPLEMENTED",               "/IEM/CPU%u/cRetInstrNotImplemented", idCpu);
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.cRetInfStatuses,             STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "Informational statuses returned",              "/IEM/CPU%u/cRetInfStatuses", idCpu);
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.cRetErrStatuses,             STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "Error statuses returned",                      "/IEM/CPU%u/cRetErrStatuses", idCpu);
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.cbWritten,                   STAMTYPE_U32,       STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
-                        "Approx bytes written",                         "/IEM/CPU%u/cbWritten", idCpu);
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.cPendingCommit,              STAMTYPE_U32,       STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
-                        "Times RC/R0 had to postpone instruction committing to ring-3", "/IEM/CPU%u/cPendingCommit", idCpu);
-
-#ifdef VBOX_WITH_STATISTICS
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.CodeTlb.cTlbHits,            STAMTYPE_U64_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "Code TLB hits",                            "/IEM/CPU%u/CodeTlb-Hits", idCpu);
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.DataTlb.cTlbHits,            STAMTYPE_U64_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "Data TLB hits",                            "/IEM/CPU%u/DataTlb-Hits", idCpu);
-#endif
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.CodeTlb.cTlbMisses,          STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "Code TLB misses",                          "/IEM/CPU%u/CodeTlb-Misses", idCpu);
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.CodeTlb.uTlbRevision,        STAMTYPE_X64,       STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
-                        "Code TLB revision",                        "/IEM/CPU%u/CodeTlb-Revision", idCpu);
-        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.CodeTlb.uTlbPhysRev, STAMTYPE_X64,       STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
-                        "Code TLB physical revision",               "/IEM/CPU%u/CodeTlb-PhysRev", idCpu);
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.CodeTlb.cTlbSlowReadPath,    STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
-                        "Code TLB slow read path",                  "/IEM/CPU%u/CodeTlb-SlowReads", idCpu);
-
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.DataTlb.cTlbMisses,          STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "Data TLB misses",                          "/IEM/CPU%u/DataTlb-Misses", idCpu);
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.DataTlb.cTlbSafeReadPath,    STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "Data TLB safe read path",                  "/IEM/CPU%u/DataTlb-SafeReads", idCpu);
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.DataTlb.cTlbSafeWritePath,   STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "Data TLB safe write path",                 "/IEM/CPU%u/DataTlb-SafeWrites", idCpu);
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.DataTlb.uTlbRevision,        STAMTYPE_X64,       STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
-                        "Data TLB revision",                        "/IEM/CPU%u/DataTlb-Revision", idCpu);
-        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.DataTlb.uTlbPhysRev, STAMTYPE_X64,       STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
-                        "Data TLB physical revision",               "/IEM/CPU%u/DataTlb-PhysRev", idCpu);
-
-
-#ifdef VBOX_WITH_IEM_RECOMPILER
-        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.cTbExec,             STAMTYPE_U64_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
-                        "Executed translation block",                   "/IEM/CPU%u/re/cTbExec", idCpu);
-        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatTbExecBreaks,    STAMTYPE_COUNTER,   STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
-                        "Times TB execution was interrupted/broken off", "/IEM/CPU%u/re/cTbExecBreaks", idCpu);
-        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.cTbAllocs,           STAMTYPE_U64,       STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
-                        "Translation block allocations",                "/IEM/CPU%u/re/cTbAllocs", idCpu);
-        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.cTbFrees,            STAMTYPE_U64,       STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
-                        "Translation block frees",                      "/IEM/CPU%u/re/cTbFrees", idCpu);
-        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.cTbLookupHits,       STAMTYPE_U64_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
-                        "Translation block lookup hits",                "/IEM/CPU%u/re/cTbLookupHits", idCpu);
-        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.cTbLookupMisses,     STAMTYPE_U64_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
-                        "Translation block lookup misses",              "/IEM/CPU%u/re/cTbLookupMisses", idCpu);
-
-        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatTbThreadedCalls, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS_PER_TB,
-                        "Calls per threaded translation block",         "/IEM/CPU%u/re/ThrdCallsPerTb", idCpu);
-        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatTbThreadedInstr, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_INSTR_PER_TB,
-                        "Instruction per threaded translation block",   "/IEM/CPU%u/re/ThrdInstrPerTb", idCpu);
-
-        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatCheckIrqBreaks,  STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "TB breaks by CheckIrq",                        "/IEM/CPU%u/re/CheckIrqBreaks", idCpu);
-        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatCheckModeBreaks, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "TB breaks by CheckMode",                       "/IEM/CPU%u/re/CheckModeBreaks", idCpu);
-        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatCheckBranchMisses, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "Branch target misses",                         "/IEM/CPU%u/re/CheckTbJmpMisses", idCpu);
-        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatCheckNeedCsLimChecking, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "Needing CS.LIM checking TB after branch or on page crossing", "/IEM/CPU%u/re/CheckTbNeedCsLimChecking", idCpu);
-#endif
-
-        for (uint32_t i = 0; i < RT_ELEMENTS(pVCpu->iem.s.aStatXcpts); i++)
-            STAMR3RegisterF(pVM, &pVCpu->iem.s.aStatXcpts[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
-                            "", "/IEM/CPU%u/Exceptions/%02x", idCpu, i);
-        for (uint32_t i = 0; i < RT_ELEMENTS(pVCpu->iem.s.aStatInts); i++)
-            STAMR3RegisterF(pVM, &pVCpu->iem.s.aStatInts[i], STAMTYPE_U32_RESET, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
-                            "", "/IEM/CPU%u/Interrupts/%02x", idCpu, i);
-
-#if !defined(VBOX_VMM_TARGET_ARMV8) && defined(VBOX_WITH_STATISTICS) && !defined(DOXYGEN_RUNNING)
-        /* Instruction statistics: */
-# define IEM_DO_INSTR_STAT(a_Name, a_szDesc) \
-            STAMR3RegisterF(pVM, &pVCpu->iem.s.StatsRZ.a_Name, STAMTYPE_U32_RESET, STAMVISIBILITY_USED, \
-                            STAMUNIT_COUNT, a_szDesc, "/IEM/CPU%u/instr-RZ/" #a_Name, idCpu); \
-            STAMR3RegisterF(pVM, &pVCpu->iem.s.StatsR3.a_Name, STAMTYPE_U32_RESET, STAMVISIBILITY_USED, \
-                            STAMUNIT_COUNT, a_szDesc, "/IEM/CPU%u/instr-R3/" #a_Name, idCpu);
-# include "IEMInstructionStatisticsTmpl.h"
-# undef IEM_DO_INSTR_STAT
-#endif
-
-#endif /* !defined(VBOX_VMM_TARGET_ARMV8) && defined(VBOX_WITH_NESTED_HWVIRT_VMX) - quick fix for stupid structure duplication non-sense */
 
         /*
@@ -285 +211 @@
         while (iMemMap-- > 0)
             pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
+    }
+
+
+#ifdef VBOX_WITH_IEM_RECOMPILER
+    /*
+     * Initialize the TB allocator and cache (/ hash table).
+     *
+     * This is done by each EMT to try get more optimal thread/numa locality of
+     * the allocations.
+     */
+    rc = VMR3ReqCallWait(pVM, VMCPUID_ALL, (PFNRT)iemTbInit, 3, pVM, cInitialTbs, cMaxTbs);
+    AssertLogRelRCReturn(rc, rc);
+#endif
+
+    /*
+     * Register statistics.
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+#if !defined(VBOX_VMM_TARGET_ARMV8) && defined(VBOX_WITH_NESTED_HWVIRT_VMX) /* quick fix for stupid structure duplication non-sense */
+        PVMCPU pVCpu = pVM->apCpusR3[idCpu];
+
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.cInstructions,               STAMTYPE_U32,       STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Instructions interpreted",                     "/IEM/CPU%u/cInstructions", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.cLongJumps,                  STAMTYPE_U32,       STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
+                        "Number of longjmp calls",                      "/IEM/CPU%u/cLongJumps", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.cPotentialExits,             STAMTYPE_U32,       STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Potential exits",                              "/IEM/CPU%u/cPotentialExits", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.cRetAspectNotImplemented,    STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "VERR_IEM_ASPECT_NOT_IMPLEMENTED",              "/IEM/CPU%u/cRetAspectNotImplemented", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.cRetInstrNotImplemented,     STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "VERR_IEM_INSTR_NOT_IMPLEMENTED",               "/IEM/CPU%u/cRetInstrNotImplemented", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.cRetInfStatuses,             STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Informational statuses returned",              "/IEM/CPU%u/cRetInfStatuses", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.cRetErrStatuses,             STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Error statuses returned",                      "/IEM/CPU%u/cRetErrStatuses", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.cbWritten,                   STAMTYPE_U32,       STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
+                        "Approx bytes written",                         "/IEM/CPU%u/cbWritten", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.cPendingCommit,              STAMTYPE_U32,       STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
+                        "Times RC/R0 had to postpone instruction committing to ring-3", "/IEM/CPU%u/cPendingCommit", idCpu);
+
+# ifdef VBOX_WITH_STATISTICS
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.CodeTlb.cTlbHits,            STAMTYPE_U64_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Code TLB hits",                            "/IEM/CPU%u/CodeTlb-Hits", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.DataTlb.cTlbHits,            STAMTYPE_U64_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Data TLB hits",                            "/IEM/CPU%u/DataTlb-Hits", idCpu);
+# endif
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.CodeTlb.cTlbMisses,          STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Code TLB misses",                          "/IEM/CPU%u/CodeTlb-Misses", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.CodeTlb.uTlbRevision,        STAMTYPE_X64,       STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Code TLB revision",                        "/IEM/CPU%u/CodeTlb-Revision", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.CodeTlb.uTlbPhysRev, STAMTYPE_X64,       STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Code TLB physical revision",               "/IEM/CPU%u/CodeTlb-PhysRev", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.CodeTlb.cTlbSlowReadPath,    STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Code TLB slow read path",                  "/IEM/CPU%u/CodeTlb-SlowReads", idCpu);
+
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.DataTlb.cTlbMisses,          STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Data TLB misses",                          "/IEM/CPU%u/DataTlb-Misses", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.DataTlb.cTlbSafeReadPath,    STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Data TLB safe read path",                  "/IEM/CPU%u/DataTlb-SafeReads", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.DataTlb.cTlbSafeWritePath,   STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Data TLB safe write path",                 "/IEM/CPU%u/DataTlb-SafeWrites", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.DataTlb.uTlbRevision,        STAMTYPE_X64,       STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Data TLB revision",                        "/IEM/CPU%u/DataTlb-Revision", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.DataTlb.uTlbPhysRev, STAMTYPE_X64,       STAMVISIBILITY_ALWAYS, STAMUNIT_NONE,
+                        "Data TLB physical revision",               "/IEM/CPU%u/DataTlb-PhysRev", idCpu);
+
+#ifdef VBOX_WITH_IEM_RECOMPILER
+        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.cTbExec,             STAMTYPE_U64_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Executed translation block",                   "/IEM/CPU%u/re/cTbExec", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatTbExecBreaks,    STAMTYPE_COUNTER,   STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                        "Times TB execution was interrupted/broken off", "/IEM/CPU%u/re/cTbExecBreaks", idCpu);
+
+        PIEMTBALLOCATOR const pTbAllocator = pVCpu->iem.s.pTbAllocatorR3;
+        STAMR3RegisterF(pVM, (void *)&pTbAllocator->StatAllocs,         STAMTYPE_COUNTER,   STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                        "Translation block allocations",                "/IEM/CPU%u/re/cTbAllocs", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pTbAllocator->StatFrees,          STAMTYPE_COUNTER,   STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                        "Translation block frees",                      "/IEM/CPU%u/re/cTbFrees", idCpu);
+# ifdef VBOX_WITH_STATISTICS
+        STAMR3RegisterF(pVM, (void *)&pTbAllocator->StatPrune,          STAMTYPE_PROFILE,   STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL,
+                        "Time spent freeing up TBs when full at alloc", "/IEM/CPU%u/re/TbPruningAlloc", idCpu);
+# endif
+        STAMR3RegisterF(pVM, (void *)&pTbAllocator->cAllocatedChunks,   STAMTYPE_U16,   STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Populated TB chunks",                          "/IEM/CPU%u/re/cTbChunks", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pTbAllocator->cMaxChunks,         STAMTYPE_U8,    STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Max number of TB chunks",                      "/IEM/CPU%u/re/cTbChunksMax", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pTbAllocator->cTotalTbs,          STAMTYPE_U32,   STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Total number of TBs in the allocator",         "/IEM/CPU%u/re/cTbTotal", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pTbAllocator->cMaxTbs,            STAMTYPE_U32,   STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Max total number of TBs allowed",              "/IEM/CPU%u/re/cTbTotalMax", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pTbAllocator->cInUseTbs,          STAMTYPE_U32,   STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Number of currently allocated TBs",            "/IEM/CPU%u/re/cTbAllocated", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pTbAllocator->cNativeTbs,         STAMTYPE_U32,   STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Number of currently allocated native TBs",     "/IEM/CPU%u/re/cTbAllocatedNative", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pTbAllocator->cThreadedTbs,       STAMTYPE_U32,   STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Number of currently allocated threaded TBs",   "/IEM/CPU%u/re/cTbAllocatedThreaded", idCpu);
+
+        PIEMTBCACHE     const pTbCache     = pVCpu->iem.s.pTbCacheR3;
+        STAMR3RegisterF(pVM, (void *)&pTbCache->cHash,                  STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Translation block lookup table size",          "/IEM/CPU%u/re/cTbHashTab", idCpu);
+
+        STAMR3RegisterF(pVM, (void *)&pTbCache->cLookupHits,            STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                        "Translation block lookup hits",                "/IEM/CPU%u/re/cTbLookupHits", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pTbCache->cLookupMisses,          STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                        "Translation block lookup misses",              "/IEM/CPU%u/re/cTbLookupMisses", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pTbCache->cCollisions,            STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
+                        "Translation block hash table collisions",      "/IEM/CPU%u/re/cTbCollisions", idCpu);
+# ifdef VBOX_WITH_STATISTICS
+        STAMR3RegisterF(pVM, (void *)&pTbCache->StatPrune,              STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL,
+                        "Time spent shortening collision lists",        "/IEM/CPU%u/re/TbPruningCollisions", idCpu);
+# endif
+
+        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatTbThreadedCalls, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS_PER_TB,
+                        "Calls per threaded translation block",         "/IEM/CPU%u/re/ThrdCallsPerTb", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatTbThreadedInstr, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_INSTR_PER_TB,
+                        "Instruction per threaded translation block",   "/IEM/CPU%u/re/ThrdInstrPerTb", idCpu);
+
+        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatCheckIrqBreaks,  STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "TB breaks by CheckIrq",                        "/IEM/CPU%u/re/CheckIrqBreaks", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatCheckModeBreaks, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "TB breaks by CheckMode",                       "/IEM/CPU%u/re/CheckModeBreaks", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatCheckBranchMisses, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Branch target misses",                         "/IEM/CPU%u/re/CheckTbJmpMisses", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatCheckNeedCsLimChecking, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Needing CS.LIM checking TB after branch or on page crossing", "/IEM/CPU%u/re/CheckTbNeedCsLimChecking", idCpu);
+#endif
+
+        for (uint32_t i = 0; i < RT_ELEMENTS(pVCpu->iem.s.aStatXcpts); i++)
+            STAMR3RegisterF(pVM, &pVCpu->iem.s.aStatXcpts[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
+                            "", "/IEM/CPU%u/Exceptions/%02x", idCpu, i);
+        for (uint32_t i = 0; i < RT_ELEMENTS(pVCpu->iem.s.aStatInts); i++)
+            STAMR3RegisterF(pVM, &pVCpu->iem.s.aStatInts[i], STAMTYPE_U32_RESET, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
+                            "", "/IEM/CPU%u/Interrupts/%02x", idCpu, i);
+
+# if !defined(VBOX_VMM_TARGET_ARMV8) && defined(VBOX_WITH_STATISTICS) && !defined(DOXYGEN_RUNNING)
+        /* Instruction statistics: */
+#  define IEM_DO_INSTR_STAT(a_Name, a_szDesc) \
+            STAMR3RegisterF(pVM, &pVCpu->iem.s.StatsRZ.a_Name, STAMTYPE_U32_RESET, STAMVISIBILITY_USED, \
+                            STAMUNIT_COUNT, a_szDesc, "/IEM/CPU%u/instr-RZ/" #a_Name, idCpu); \
+            STAMR3RegisterF(pVM, &pVCpu->iem.s.StatsR3.a_Name, STAMTYPE_U32_RESET, STAMVISIBILITY_USED, \
+                            STAMUNIT_COUNT, a_szDesc, "/IEM/CPU%u/instr-R3/" #a_Name, idCpu);
+#  include "IEMInstructionStatisticsTmpl.h"
+#  undef IEM_DO_INSTR_STAT
+# endif
+
+#endif /* !defined(VBOX_VMM_TARGET_ARMV8) && defined(VBOX_WITH_NESTED_HWVIRT_VMX) - quick fix for stupid structure duplication non-sense */
     }
 

trunk/src/VBox/VMM/include/IEMInternal.h

@@ -712 +712 @@
  *       For the same reasons, we skip all of IEM_F_X86_CTX_MASK, with the
  *       exception of SMM (which we don't implement). */
-#define IEMTB_F_KEY_MASK                ((UINT32_C(0xffffffff) & ~(IEM_F_X86_CTX_MASK | IEM_F_X86_CPL_MASK)) | IEM_F_X86_CTX_SMM)
+#define IEMTB_F_KEY_MASK                (  (UINT32_MAX & ~(IEM_F_X86_CTX_MASK | IEM_F_X86_CPL_MASK | IEMTB_F_TYPE_MASK)) \
+                                         | IEM_F_X86_CTX_SMM)
 /** @} */
 
@@ -781 +782 @@
 /**
  * Translation block.
+ *
+ * The current plan is to just keep TBs and associated lookup hash table private
+ * to each VCpu as that simplifies TB removal greatly (no races) and generally
+ * avoids using expensive atomic primitives for updating lists and stuff.
  */
 #pragma pack(2) /* to prevent the Thrd structure from being padded unnecessarily */
@@ -786 +791 @@
 {
     /** Next block with the same hash table entry. */
-    struct IEMTB * volatile pNext;
-    /** List on the local VCPU for blocks. */
-    RTLISTNODE          LocalList;
+    struct IEMTB       *pNext;
+    /** Usage counter. */
+    uint32_t            cUsed;
+    /** The IEMCPU::msRecompilerPollNow last time it was used. */
+    uint32_t            msLastUsed;
+    /** The allocation chunk this TB belongs to. */
+    uint8_t             idxAllocChunk;
+
+    uint8_t             abUnused[3];
+    uint32_t            uUnused;
+
 
     /** @name What uniquely identifies the block.
@@ -822 +835 @@
             uint16_t            cAllocated;
         } Thrd;
+        struct
+        {
+            uint8_t            *pbCode;
+            /** Amount of code that pbCode points to. */
+            uint32_t            cbAllocated;
+        } Native;
+        /** Generic view for zeroing when freeing. */
+        struct
+        {
+            uintptr_t           uPtr;
+            uint32_t            uData;
+        } Gen;
     };
 
@@ -872 +897 @@
 AssertCompileMemberOffset(IEMTB, cbOpcodes, 52);
 AssertCompileMemberSize(IEMTB, aRanges[0], 6);
+#if 1
 AssertCompileSize(IEMTB, 128);
+# define IEMTB_SIZE_IS_POWER_OF_TWO /**< The IEMTB size is a power of two. */
+#else
+AssertCompileSize(IEMTB, 168);
+# undef  IEMTB_SIZE_IS_POWER_OF_TWO
+#endif
+
 /** Pointer to a translation block. */
 typedef IEMTB *PIEMTB;
 /** Pointer to a const translation block. */
 typedef IEMTB const *PCIEMTB;
+
+/**
+ * A chunk of memory in the TB allocator.
+ */
+typedef struct IEMTBCHUNK
+{
+    /** Pointer to the translation blocks in this chunk. */
+    PIEMTB          paTbs;
+#ifdef IN_RING0
+    /** Allocation handle. */
+    RTR0MEMOBJ      hMemObj;
+#endif
+} IEMTBCHUNK;
+
+/**
+ * A per-CPU translation block allocator.
+ *
+ * Because of how the IEMTBCACHE uses the lower 6 bits of the TB address to keep
+ * the length of the collision list, and of course also for cache line alignment
+ * reasons, the TBs must be allocated with at least 64-byte alignment.
+ * Memory is there therefore allocated using one of the page aligned allocators.
+ *
+ *
+ * To avoid wasting too much memory, it is allocated piecemeal as needed,
+ * in chunks (IEMTBCHUNK) of 2 MiB or more.  The TB has an 8-bit chunk index
+ * that enables us to quickly calculate the allocation bitmap position when
+ * freeing the translation block.
+ */
+typedef struct IEMTBALLOCATOR
+{
+    /** Magic value (IEMTBALLOCATOR_MAGIC). */
+    uint32_t        uMagic;
+
+#ifdef IEMTB_SIZE_IS_POWER_OF_TWO
+    /** Mask corresponding to cTbsPerChunk - 1. */
+    uint32_t        fChunkMask;
+    /** Shift count corresponding to cTbsPerChunk. */
+    uint8_t         cChunkShift;
+#else
+    uint32_t        uUnused;
+    uint8_t         bUnused;
+#endif
+    /** Number of chunks we're allowed to allocate. */
+    uint8_t         cMaxChunks;
+    /** Number of chunks currently populated. */
+    uint16_t        cAllocatedChunks;
+    /** Number of translation blocks per chunk. */
+    uint32_t        cTbsPerChunk;
+    /** Chunk size. */
+    uint32_t        cbPerChunk;
+
+    /** The maximum number of TBs. */
+    uint32_t        cMaxTbs;
+    /** Total number of TBs in the populated chunks.
+     * (cAllocatedChunks * cTbsPerChunk) */
+    uint32_t        cTotalTbs;
+    /** The current number of TBs in use.
+     * The number of free TBs: cAllocatedTbs - cInUseTbs; */
+    uint32_t        cInUseTbs;
+    /** Statistics: Number of the cInUseTbs that are native ones. */
+    uint32_t        cNativeTbs;
+    /** Statistics: Number of the cInUseTbs that are threaded ones. */
+    uint32_t        cThreadedTbs;
+
+    /** Where to start pruning TBs from when we're out.
+     *  See iemTbAllocatorAllocSlow for details. */
+    uint32_t        iPruneFrom;
+    /** Hint about which bit to start scanning the bitmap from. */
+    uint32_t        iStartHint;
+
+    /** Statistics: Number of TB allocation calls. */
+    STAMCOUNTER     StatAllocs;
+    /** Statistics: Number of TB free calls. */
+    STAMCOUNTER     StatFrees;
+    /** Statistics: Time spend pruning. */
+    STAMPROFILE     StatPrune;
+
+    /** Allocation chunks. */
+    IEMTBCHUNK      aChunks[256];
+
+    /** Allocation bitmap for all possible chunk chunks. */
+    RT_FLEXIBLE_ARRAY_EXTENSION
+    uint64_t        bmAllocated[RT_FLEXIBLE_ARRAY];
+} IEMTBALLOCATOR;
+/** Pointer to a TB allocator. */
+typedef struct IEMTBALLOCATOR *PIEMTBALLOCATOR;
+
+/** Magic value for the TB allocator (Emmet Harley Cohen). */
+#define IEMTBALLOCATOR_MAGIC        UINT32_C(0x19900525)
+
+
+/**
+ * A per-CPU translation block cache (hash table).
+ *
+ * The hash table is allocated once during IEM initialization and size double
+ * the max TB count, rounded up to the nearest power of two (so we can use and
+ * AND mask rather than a rest division when hashing).
+ */
+typedef struct IEMTBCACHE
+{
+    /** Magic value (IEMTBCACHE_MAGIC). */
+    uint32_t        uMagic;
+    /** Size of the hash table.  This is a power of two. */
+    uint32_t        cHash;
+    /** The mask corresponding to cHash. */
+    uint32_t        uHashMask;
+    uint32_t        uPadding;
+
+    /** @name Statistics
+     * @{ */
+    /** Number of collisions ever. */
+    STAMCOUNTER     cCollisions;
+
+    /** Statistics: Number of TB lookup misses. */
+    STAMCOUNTER     cLookupMisses;
+    /** Statistics: Number of TB lookup hits (debug only). */
+    STAMCOUNTER     cLookupHits;
+    STAMCOUNTER     auPadding2[3];
+    /** Statistics: Collision list length pruning. */
+    STAMPROFILE     StatPrune;
+    /** @} */
+
+    /** The hash table itself.
+     * @note The lower 6 bits of the pointer is used for keeping the collision
+     *       list length, so we can take action when it grows too long.
+     *       This works because TBs are allocated using a 64 byte (or
+     *       higher) alignment from page aligned chunks of memory, so the lower
+     *       6 bits of the address will always be zero.
+     *       See IEMTBCACHE_PTR_COUNT_MASK, IEMTBCACHE_PTR_MAKE and friends.
+     */
+    RT_FLEXIBLE_ARRAY_EXTENSION
+    PIEMTB          apHash[RT_FLEXIBLE_ARRAY];
+} IEMTBCACHE;
+/** Pointer to a per-CPU translation block cahce. */
+typedef IEMTBCACHE *PIEMTBCACHE;
+
+/** Magic value for IEMTBCACHE (Johnny O'Neal). */
+#define IEMTBCACHE_MAGIC            UINT32_C(0x19561010)
+
+/** The collision count mask for IEMTBCACHE::apHash entries. */
+#define IEMTBCACHE_PTR_COUNT_MASK               ((uintptr_t)0x3f)
+/** The max collision count for IEMTBCACHE::apHash entries before pruning. */
+#define IEMTBCACHE_PTR_MAX_COUNT                ((uintptr_t)0x30)
+/** Combine a TB pointer and a collision list length into a value for an
+ *  IEMTBCACHE::apHash entry. */
+#define IEMTBCACHE_PTR_MAKE(a_pTb, a_cCount)    (PIEMTB)((uintptr_t)(a_pTb) | (a_cCount))
+/** Combine a TB pointer and a collision list length into a value for an
+ *  IEMTBCACHE::apHash entry. */
+#define IEMTBCACHE_PTR_GET_TB(a_pHashEntry)     (PIEMTB)((uintptr_t)(a_pHashEntry) & ~IEMTBCACHE_PTR_COUNT_MASK)
+/** Combine a TB pointer and a collision list length into a value for an
+ *  IEMTBCACHE::apHash entry. */
+#define IEMTBCACHE_PTR_GET_COUNT(a_pHashEntry)  ((uintptr_t)(a_pHashEntry) & IEMTBCACHE_PTR_COUNT_MASK)
+
+/**
+ * Calculates the hash table slot for a TB from physical PC address and TB flags.
+ */
+#define IEMTBCACHE_HASH(a_paCache, a_fTbFlags, a_GCPhysPc) \
+    IEMTBCACHE_HASH_NO_KEY_MASK(a_paCache, (a_fTbFlags) & IEMTB_F_KEY_MASK, a_GCPhysPc)
+
+/**
+ * Calculates the hash table slot for a TB from physical PC address and TB
+ * flags, ASSUMING the caller has applied IEMTB_F_KEY_MASK to @a a_fTbFlags.
+ */
+#define IEMTBCACHE_HASH_NO_KEY_MASK(a_paCache, a_fTbFlags, a_GCPhysPc) \
+    (((uint32_t)(a_GCPhysPc) ^ (a_fTbFlags)) & (a_paCache)->uHashMask)
+
 
 /** @name IEMBRANCHED_F_XXX - Branched indicator (IEMCPU::fTbBranched).
@@ -1185 +1383 @@
      * components as needed. */
     R3PTRTYPE(PIEMTB)       pNativeCompileTbR3;
+    /** Pointer to the ring-3 TB cache for this EMT. */
+    R3PTRTYPE(PIEMTBCACHE)  pTbCacheR3;
     /** The PC (RIP) at the start of pCurTbR3/pCurTbR0.
      * The TBs are based on physical addresses, so this is needed to correleated
      * RIP to opcode bytes stored in the TB (AMD-V / VT-x). */
     uint64_t                uCurTbStartPc;
-    /** Statistics: Number of TB lookup misses. */
-    uint64_t                cTbLookupMisses;
-    /** Statistics: Number of TB lookup hits (debug only). */
-    uint64_t                cTbLookupHits;
     /** Number of TBs executed. */
     uint64_t                cTbExec;
@@ -1215 +1411 @@
     bool                    fTbCurInstrIsSti;
     /** Spaced reserved for recompiler data / alignment. */
-    bool                    afRecompilerStuff1[2];
+    bool                    afRecompilerStuff1[2+4];
+    /** The virtual sync time at the last timer poll call. */
+    uint32_t                msRecompilerPollNow;
     /** Previous GCPhysInstrBuf value - only valid if fTbCrossedPage is set.   */
     RTGCPHYS                GCPhysInstrBufPrev;
@@ -1225 +1423 @@
     /** Copy of IEMCPU::uInstrBufPc after decoding a branch instruction.  */
     uint64_t                GCVirtTbBranchSrcBuf;
+    /** Pointer to the ring-3 TB allocator for this EMT. */
+    R3PTRTYPE(PIEMTBALLOCATOR) pTbAllocatorR3;
     /* Alignment. */
-    uint64_t                auAlignment10[6];
-    /** Statistics: Number of TB allocation calls. */
-    uint64_t                cTbAllocs;
-    /** Statistics: Number of TB free calls. */
-    uint64_t                cTbFrees;
+    uint64_t                auAlignment10[7];
     /** Statistics: Times TB execution was broken off before reaching the end. */
     STAMCOUNTER             StatTbExecBreaks;
@@ -5105 +5301 @@
 extern const PFNIEMOP g_apfnIemThreadedRecompilerVecMap3[1024];
 
+DECLCALLBACK(int) iemTbInit(PVMCC pVM, uint32_t cInitialTbs, uint32_t cMaxTbs);
 void            iemThreadedTbObsolete(PVMCPUCC pVCpu, PIEMTB pTb);