VirtualBox

Changeset 105261 in vbox


Ignore:
Timestamp:
Jul 10, 2024 2:51:55 PM (5 months ago)
Author:
vboxsync
Message:

VMM/IEM: Share epilog and other tail code on a per-chunk basis (due to jump range). bugref:10677

Location:
trunk/src/VBox/VMM
Files:
8 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veExecMem.cpp

    r104876 r105261  
    226226    /** Pointer to the readable/executable view of the memory chunk. */
    227227    void                   *pvChunkRx;
     228#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     229    /** Pointer to the context structure detailing the per chunk common code. */
     230    PCIEMNATIVEPERCHUNKCTX  pCtx;
     231#endif
    228232#ifdef IN_RING3
    229233    /**
    … …  
    347351static int iemExecMemAllocatorGrow(PVMCPUCC pVCpu, PIEMEXECMEMALLOCATOR pExecMemAllocator);
    348352
     353
    349354#ifdef IEMEXECMEM_ALT_SUB_WITH_ALT_PRUNING
    350355/**
    … …  
    461466static void *
    462467iemExecMemAllocatorAllocInChunkInt(PIEMEXECMEMALLOCATOR pExecMemAllocator, uint64_t *pbmAlloc, uint32_t idxFirst,
    463                                    uint32_t cToScan, uint32_t cReqUnits, uint32_t idxChunk, PIEMTB pTb, void **ppvExec)
     468                                   uint32_t cToScan, uint32_t cReqUnits, uint32_t idxChunk, PIEMTB pTb,
     469                                   void **ppvExec, PCIEMNATIVEPERCHUNKCTX *ppChunkCtx)
    464470{
    465471    /*
    … …  
    497503            void * const pvMemRw = (uint8_t *)pChunk->pvChunkRw
    498504                                 + ((idxFirst + (uint32_t)iBit) << IEMEXECMEM_ALT_SUB_ALLOC_UNIT_SHIFT);
     505
     506            if (ppChunkCtx)
     507                *ppChunkCtx = pChunk->pCtx;
    499508
    500509            /*
    … …  
    529538
    530539
    531 static void *
     540static PIEMNATIVEINSTR
    532541iemExecMemAllocatorAllocInChunk(PIEMEXECMEMALLOCATOR pExecMemAllocator, uint32_t idxChunk, uint32_t cbReq, PIEMTB pTb,
    533                                 void **ppvExec)
     542                                PIEMNATIVEINSTR *ppaExec, PCIEMNATIVEPERCHUNKCTX *ppChunkCtx)
    534543{
    535544    /*
    … …  
    550559            void *pvRet = iemExecMemAllocatorAllocInChunkInt(pExecMemAllocator, pbmAlloc, idxHint,
    551560                                                             pExecMemAllocator->cUnitsPerChunk - idxHint,
    552                                                              cReqUnits, idxChunk, pTb, ppvExec);
     561                                                             cReqUnits, idxChunk, pTb, (void **)ppaExec, ppChunkCtx);
    553562            if (pvRet)
    554                 return pvRet;
     563                return (PIEMNATIVEINSTR)pvRet;
    555564        }
    556         return iemExecMemAllocatorAllocInChunkInt(pExecMemAllocator, pbmAlloc, 0,
    557                                                   RT_MIN(pExecMemAllocator->cUnitsPerChunk, RT_ALIGN_32(idxHint + cReqUnits, 64)),
    558                                                   cReqUnits, idxChunk, pTb, ppvExec);
     565        return (PIEMNATIVEINSTR)iemExecMemAllocatorAllocInChunkInt(pExecMemAllocator, pbmAlloc, 0,
     566                                                                   RT_MIN(pExecMemAllocator->cUnitsPerChunk,
     567                                                                          RT_ALIGN_32(idxHint + cReqUnits, 64)),
     568                                                                   cReqUnits, idxChunk, pTb, (void **)ppaExec, ppChunkCtx);
    559569    }
    560570    return NULL;
    … …  
    567577 * @returns Pointer to the readable/writeable memory, NULL if out of memory or other problem
    568578 *          encountered.
    569  * @param   pVCpu   The cross context virtual CPU structure of the calling
    570  *                  thread.
    571  * @param   cbReq   How many bytes are required.
    572  * @param   pTb     The translation block that will be using the allocation.
    573  * @param   ppvExec Where to return the pointer to executable view of the allocated memory, optional.
    574  */
    575 DECLHIDDEN(void *) iemExecMemAllocatorAlloc(PVMCPU pVCpu, uint32_t cbReq, PIEMTB pTb, void **ppvExec) RT_NOEXCEPT
     579 * @param   pVCpu       The cross context virtual CPU structure of the
     580 *                      calling thread.
     581 * @param   cbReq       How many bytes are required.
     582 * @param   pTb         The translation block that will be using the allocation.
     583 * @param   ppaExec     Where to return the pointer to executable view of
     584 *                      the allocated memory, optional.
     585 * @param   ppChunkCtx  Where to return the per chunk attached context
     586 *                      if available, optional.
     587 */
     588DECLHIDDEN(PIEMNATIVEINSTR) iemExecMemAllocatorAlloc(PVMCPU pVCpu, uint32_t cbReq, PIEMTB pTb,
     589                                                     PIEMNATIVEINSTR *ppaExec, PCIEMNATIVEPERCHUNKCTX *ppChunkCtx) RT_NOEXCEPT
    576590{
    577591    PIEMEXECMEMALLOCATOR pExecMemAllocator = pVCpu->iem.s.pExecMemAllocatorR3;
    … …  
    588602            for (uint32_t idxChunk = idxChunkHint; idxChunk < cChunks; idxChunk++)
    589603            {
    590                 void *pvRet = iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk, cbReq, pTb, ppvExec);
    591                 if (pvRet)
     604                PIEMNATIVEINSTR const pRet = iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk, cbReq, pTb,
     605                                                                             ppaExec, ppChunkCtx);
     606                if (pRet)
    592607                {
    593608                    STAM_PROFILE_STOP(&pExecMemAllocator->StatAlloc, a);
    594                     return pvRet;
     609                    return pRet;
    595610                }
    596611            }
    597612            for (uint32_t idxChunk = 0; idxChunk < idxChunkHint; idxChunk++)
    598613            {
    599                 void *pvRet = iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk, cbReq, pTb, ppvExec);
    600                 if (pvRet)
     614                PIEMNATIVEINSTR const pRet = iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk, cbReq, pTb,
     615                                                                             ppaExec, ppChunkCtx);
     616                if (pRet)
    601617                {
    602618                    STAM_PROFILE_STOP(&pExecMemAllocator->StatAlloc, a);
    603                     return pvRet;
     619                    return pRet;
    604620                }
    605621            }
    … …  
    615631
    616632            uint32_t const idxChunk = pExecMemAllocator->cChunks - 1;
    617             void *pvRet = iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk, cbReq, pTb, ppvExec);
    618             if (pvRet)
     633            PIEMNATIVEINSTR const pRet = iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk, cbReq, pTb,
     634                                                                         ppaExec, ppChunkCtx);
     635            if (pRet)
    619636            {
    620637                STAM_PROFILE_STOP(&pExecMemAllocator->StatAlloc, a);
    621                 return pvRet;
     638                return pRet;
    622639            }
    623640            AssertFailed();
    … …  
    659676    sys_icache_invalidate(pv, cb);
    660677    RT_NOREF(pVCpu);
     678
    661679#elif defined(RT_OS_LINUX)
    662680    RT_NOREF(pVCpu);
    … …  
    673691
    674692    asm volatile ("dsb ish\n\t isb\n\t" : : : "memory");
     693
    675694#else
    676695    RT_NOREF(pVCpu, pv, cb);
    … …  
    747766}
    748767
     768
     769/**
     770 * Interface used by iemNativeRecompileAttachExecMemChunkCtx and unwind info
     771 * generators.
     772 */
     773DECLHIDDEN(PIEMNATIVEINSTR)
     774iemExecMemAllocatorAllocFromChunk(PVMCPU pVCpu, uint32_t idxChunk, uint32_t cbReq, PIEMNATIVEINSTR *ppaExec)
     775{
     776    PIEMEXECMEMALLOCATOR pExecMemAllocator = pVCpu->iem.s.pExecMemAllocatorR3;
     777    AssertReturn(idxChunk < pExecMemAllocator->cChunks, NULL);
     778    Assert(cbReq < _1M);
     779    return iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk, cbReq, NULL /*pTb*/, ppaExec, NULL /*ppChunkCtx*/);
     780}
     781
     782
     783#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     784/**
     785 * For getting the per-chunk context detailing common code for a TB.
     786 *
     787 * This is for use by the disassembler.
     788 */
     789DECLHIDDEN(PCIEMNATIVEPERCHUNKCTX) iemExecMemGetTbChunkCtx(PVMCPU pVCpu, PCIEMTB pTb)
     790{
     791    PIEMEXECMEMALLOCATOR pExecMemAllocator = pVCpu->iem.s.pExecMemAllocatorR3;
     792    if ((pTb->fFlags & IEMTB_F_TYPE_MASK) == IEMTB_F_TYPE_NATIVE)
     793    {
     794        uintptr_t const uAddress = (uintptr_t)pTb->Native.paInstructions;
     795        uint32_t const  cbChunk  = pExecMemAllocator->cbChunk;
     796        uint32_t        idxChunk = pExecMemAllocator->cChunks;
     797        while (idxChunk-- > 0)
     798            if (uAddress - (uintptr_t)pExecMemAllocator->aChunks[idxChunk].pvChunkRx < cbChunk)
     799                return pExecMemAllocator->aChunks[idxChunk].pCtx;
     800    }
     801    return NULL;
     802}
     803#endif
    749804
    750805
    … …  
    814869    unsigned const cbNeeded         = sizeof(IMAGE_RUNTIME_FUNCTION_ENTRY) * cFunctionEntries + cbUnwindInfo;
    815870    PIMAGE_RUNTIME_FUNCTION_ENTRY const paFunctions
    816         = (PIMAGE_RUNTIME_FUNCTION_ENTRY)iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk, cbNeeded, NULL, NULL);
     871        = (PIMAGE_RUNTIME_FUNCTION_ENTRY)iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk, cbNeeded, NULL, NULL, NULL);
    817872    AssertReturn(paFunctions, VERR_INTERNAL_ERROR_5);
    818873    pExecMemAllocator->aChunks[idxChunk].pvUnwindInfo = paFunctions;
    … …  
    10501105     */
    10511106    GDBJITSYMFILE * const pSymFile = (GDBJITSYMFILE *)iemExecMemAllocatorAllocInChunk(pExecMemAllocator, idxChunk,
    1052                                                                                       sizeof(GDBJITSYMFILE), NULL, NULL);
     1107                                                                                      sizeof(GDBJITSYMFILE), NULL, NULL, NULL);
    10531108    AssertReturn(pSymFile, VERR_INTERNAL_ERROR_5);
    10541109    unsigned const offSymFileInChunk = (uintptr_t)pSymFile - (uintptr_t)pvChunk;
    … …  
    13931448     * protections when exeuctable memory is allocated.
    13941449     */
     1450    int               rc           = VERR_NO_EXEC_MEMORY;
    13951451    mach_port_t       hPortTask    = mach_task_self();
    13961452    mach_vm_address_t AddrChunk    = (mach_vm_address_t)pvChunk;
    13971453    mach_vm_address_t AddrRemapped = 0;
    1398     vm_prot_t ProtCur, ProtMax;
     1454    vm_prot_t         ProtCur      = 0;
     1455    vm_prot_t         ProtMax      = 0;
    13991456    kern_return_t krc = mach_vm_remap(hPortTask, &AddrRemapped, pExecMemAllocator->cbChunk, 0,
    14001457                                      VM_FLAGS_ANYWHERE | VM_FLAGS_RETURN_DATA_ADDR,
    14011458                                      hPortTask, AddrChunk, FALSE, &ProtCur, &ProtMax,
    14021459                                      VM_INHERIT_NONE);
    1403     if (krc != KERN_SUCCESS)
     1460    if (krc == KERN_SUCCESS)
     1461    {
     1462        krc = mach_vm_protect(mach_task_self(), AddrRemapped, pExecMemAllocator->cbChunk, FALSE, VM_PROT_READ | VM_PROT_EXECUTE);
     1463        if (krc == KERN_SUCCESS)
     1464            rc = VINF_SUCCESS;
     1465        else
     1466        {
     1467            AssertLogRelMsgFailed(("mach_vm_protect -> %d (%#x)\n", krc, krc));
     1468            krc = mach_vm_deallocate(hPortTask, AddrRemapped, pExecMemAllocator->cbChunk);
     1469            Assert(krc == KERN_SUCCESS);
     1470        }
     1471    }
     1472    else
     1473        AssertLogRelMsgFailed(("mach_vm_remap -> %d (%#x)\n", krc, krc));
     1474    if (RT_FAILURE(rc))
    14041475    {
    14051476        RTMemPageFree(pvChunk, pExecMemAllocator->cbChunk);
    1406         AssertLogRelFailed();
    1407         return VERR_NO_EXEC_MEMORY;
    1408     }
    1409 
    1410     krc = mach_vm_protect(mach_task_self(), AddrRemapped, pExecMemAllocator->cbChunk, FALSE, VM_PROT_READ | VM_PROT_EXECUTE);
    1411     if (krc != KERN_SUCCESS)
    1412     {
    1413         krc = mach_vm_deallocate(hPortTask, AddrRemapped, pExecMemAllocator->cbChunk);
    1414         Assert(krc == KERN_SUCCESS);
    1415         RTMemPageFree(pvChunk, pExecMemAllocator->cbChunk);
    1416         AssertLogRelFailed();
    1417         return VERR_NO_EXEC_MEMORY;
     1477        return rc;
    14181478    }
    14191479
    14201480    void *pvChunkRx = (void *)AddrRemapped;
    14211481#else
     1482# if defined(IN_RING3) || defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE)
     1483    int   rc        = VINF_SUCCESS;
     1484# endif
    14221485    void *pvChunkRx = pvChunk;
    14231486#endif
    … …  
    14451508    pExecMemAllocator->cbFree      += pExecMemAllocator->cbChunk;
    14461509
     1510    /* If there is a chunk context init callback call it. */
     1511#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     1512    pExecMemAllocator->aChunks[idxChunk].pCtx = iemNativeRecompileAttachExecMemChunkCtx(pVCpu, idxChunk);
     1513    if (pExecMemAllocator->aChunks[idxChunk].pCtx)
     1514#endif
     1515    {
    14471516#ifdef IN_RING3
    1448     /*
    1449      * Initialize the unwind information (this cannot really fail atm).
    1450      * (This sets pvUnwindInfo.)
    1451      */
    1452     int rc = iemExecMemAllocatorInitAndRegisterUnwindInfoForChunk(pVCpu, pExecMemAllocator, pvChunkRx, idxChunk);
     1517        /*
     1518         * Initialize the unwind information (this cannot really fail atm).
     1519         * (This sets pvUnwindInfo.)
     1520         */
     1521        rc = iemExecMemAllocatorInitAndRegisterUnwindInfoForChunk(pVCpu, pExecMemAllocator, pvChunkRx, idxChunk);
     1522#endif
     1523    }
     1524#if defined(IN_RING3) || defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE)
    14531525    if (RT_SUCCESS(rc))
    14541526    { /* likely */ }
    … …  
    14641536        pExecMemAllocator->aChunks[idxChunk].cFreeUnits = 0;
    14651537
    1466 #ifdef RT_OS_DARWIN
     1538# ifdef RT_OS_DARWIN
    14671539        krc = mach_vm_deallocate(mach_task_self(), (mach_vm_address_t)pExecMemAllocator->aChunks[idxChunk].pvChunkRx,
    14681540                                 pExecMemAllocator->cbChunk);
    14691541        Assert(krc == KERN_SUCCESS);
    1470 #endif
     1542# endif
    14711543
    14721544        RTMemPageFree(pvChunk, pExecMemAllocator->cbChunk);
    … …  
    14741546    }
    14751547#endif
     1548
    14761549    return VINF_SUCCESS;
    14771550}
    … …  
    15231596        }
    15241597    }
     1598#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     1599# if   defined(RT_OS_AMD64)
     1600    Assert(cbChunk <= _2G);
     1601# elif defined(RT_OS_ARM64)
     1602    if (cbChunk > _128M)
     1603        cbChunk = _128M; /* Max relative branch distance is +/-2^(25+2) = +/-0x8000000 (134 217 728). */
     1604# endif
     1605#endif
    15251606
    15261607    if (cbChunk > cbMax)

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veHlpA-arm64.S

    r104798 r105261  
    9494         */
    9595        br      x2
    96 
    97 #ifdef IEMNATIVE_WITH_RECOMPILER_EPILOGUE_SINGLETON
    98 /**
    99  * This is the common epilog for all TBs, restoring all volatile registers
    100  * and cleaning up the stack frame. This is a direct jump target and not a
    101  * real function to call using bl/blr.
    102  */
    103 ALIGNCODE(IEM_HLP_FUNCTION_ALIGNMENT)
    104 BEGINPROC_HIDDEN iemNativeTbEpilog
    105         ldp     x19, x20, [sp, #IEMNATIVE_FRAME_VAR_SIZE]!
    106         ldp     x21, x22, [sp, #0x10]
    107         ldp     x23, x24, [sp, #0x20]
    108         ldp     x25, x26, [sp, #0x30]
    109         ldp     x27, x28, [sp, #0x40]
    110         ldp     x29, x30, [sp, #0x50]
    111         add     sp, sp, #IEMNATIVE_FRAME_SAVE_REG_SIZE
    112 # ifdef RT_OS_DARWIN
    113         retab
    114 # else
    115         ret
    116 # endif
    117 #endif
    11896
    11997

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompBltIn.cpp

    r104957 r105261  
    598598        } while (0)
    599599
     600# ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     601#  define NEAR_JMP_SIZE 5
     602# else
     603#  define NEAR_JMP_SIZE 6
     604# endif
     605
    600606# define CHECK_OPCODES_CMP_JMP() /* cost: 7 bytes first time, then 2 bytes */ do { \
    601607            if (offConsolidatedJump != UINT32_MAX) \
    … …  
    608614            else \
    609615            { \
    610                 pbCodeBuf[off++] = 0x74; /* jz near +6 */ \
    611                 pbCodeBuf[off++] = 0x06 + BP_ON_OBSOLETION; \
     616                pbCodeBuf[off++] = 0x74; /* jz near +NEAR_JMP_SIZE */ \
     617                pbCodeBuf[off++] = NEAR_JMP_SIZE + BP_ON_OBSOLETION; \
    612618                offConsolidatedJump = off; \
    613619                if (BP_ON_OBSOLETION) pbCodeBuf[off++] = 0xcc; \

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

    r105036 r105261  
    113113                                                            IEMNATIVEGSTREG enmGstReg, uint32_t off);
    114114DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
     115static const char      *iemNativeGetLabelName(IEMNATIVELABELTYPE enmLabel, bool fCommonCode = false);
    115116
    116117
    … …  
    20572058    pReNative->bmLabelTypes                = 0;
    20582059    pReNative->cFixups                     = 0;
     2060#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     2061    pReNative->cTbExitFixups               = 0;
     2062#endif
    20592063#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
    20602064    pReNative->pDbgInfo->cEntries          = 0;
    … …  
    21962200
    21972201/**
     2202 * Used when done emitting the per-chunk code and for iemNativeInit bailout.
     2203 */
     2204static void iemNativeTerm(PIEMRECOMPILERSTATE pReNative)
     2205{
     2206    RTMemFree(pReNative->pInstrBuf);
     2207    RTMemFree(pReNative->paLabels);
     2208    RTMemFree(pReNative->paFixups);
     2209#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     2210    RTMemFree(pReNative->paTbExitFixups);
     2211#endif
     2212#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
     2213    RTMemFree(pReNative->pDbgInfo);
     2214#endif
     2215    RTMemFree(pReNative);
     2216}
     2217
     2218
     2219/**
    21982220 * Allocates and initializes the native recompiler state.
    21992221 *
    … …  
    22032225 * @param   pVCpu   The cross context virtual CPU structure of the calling
    22042226 *                  thread.
    2205  * @param   pTb     The TB that's about to be recompiled.
     2227 * @param   pTb     The TB that's about to be recompiled.  When this is NULL,
     2228 *                  the recompiler state is for emitting the common per-chunk
     2229 *                  code from iemNativeRecompileAttachExecMemChunkCtx.
    22062230 * @thread  EMT(pVCpu)
    22072231 */
    … …  
    22162240     * Try allocate all the buffers and stuff we need.
    22172241     */
    2218     pReNative->pInstrBuf = (PIEMNATIVEINSTR)RTMemAllocZ(_64K);
    2219     pReNative->paLabels  = (PIEMNATIVELABEL)RTMemAllocZ(sizeof(IEMNATIVELABEL) * _8K);
    2220     pReNative->paFixups  = (PIEMNATIVEFIXUP)RTMemAllocZ(sizeof(IEMNATIVEFIXUP) * _16K);
     2242    uint32_t const cFactor = pTb ? 1 : 32 /* per-chunk stuff doesn't really need anything but the code buffer */;
     2243    pReNative->pInstrBuf      = (PIEMNATIVEINSTR)RTMemAllocZ(_64K);
     2244    pReNative->paLabels       = (PIEMNATIVELABEL)RTMemAllocZ(sizeof(IEMNATIVELABEL) * _8K / cFactor);
     2245    pReNative->paFixups       = (PIEMNATIVEFIXUP)RTMemAllocZ(sizeof(IEMNATIVEFIXUP) * _16K / cFactor);
     2246#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     2247    pReNative->paTbExitFixups = (PIEMNATIVEEXITFIXUP)RTMemAllocZ(sizeof(IEMNATIVEEXITFIXUP) * _8K / cFactor);
     2248#endif
    22212249#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
    2222     pReNative->pDbgInfo  = (PIEMTBDBG)RTMemAllocZ(RT_UOFFSETOF_DYN(IEMTBDBG, aEntries[_16K]));
     2250    pReNative->pDbgInfo       = (PIEMTBDBG)RTMemAllocZ(RT_UOFFSETOF_DYN(IEMTBDBG, aEntries[_16K / cFactor]));
    22232251#endif
    22242252    if (RT_LIKELY(   pReNative->pInstrBuf
    22252253                  && pReNative->paLabels
    22262254                  && pReNative->paFixups)
     2255#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     2256        && pReNative->paTbExitFixups
     2257#endif
    22272258#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
    22282259        && pReNative->pDbgInfo
    … …  
    22332264         * Set the buffer & array sizes on success.
    22342265         */
    2235         pReNative->cInstrBufAlloc = _64K / sizeof(IEMNATIVEINSTR);
    2236         pReNative->cLabelsAlloc   = _8K;
    2237         pReNative->cFixupsAlloc   = _16K;
     2266        pReNative->cInstrBufAlloc     = _64K / sizeof(IEMNATIVEINSTR);
     2267        pReNative->cLabelsAlloc       = _8K  / cFactor;
     2268        pReNative->cFixupsAlloc       = _16K / cFactor;
     2269#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     2270        pReNative->cTbExitFixupsAlloc = _8K  / cFactor;
     2271#endif
    22382272#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
    2239         pReNative->cDbgInfoAlloc  = _16K;
     2273        pReNative->cDbgInfoAlloc      = _16K / cFactor;
    22402274#endif
    22412275
    22422276        /* Other constant stuff: */
    2243         pReNative->pVCpu          = pVCpu;
     2277        pReNative->pVCpu              = pVCpu;
    22442278
    22452279        /*
    2246          * Done, just need to save it and reinit it.
     2280         * Done, just reinit it.
    22472281         */
    2248         pVCpu->iem.s.pNativeRecompilerStateR3 = pReNative;
    22492282        return iemNativeReInit(pReNative, pTb);
    22502283    }
    … …  
    22542287     */
    22552288    AssertFailed();
    2256     RTMemFree(pReNative->pInstrBuf);
    2257     RTMemFree(pReNative->paLabels);
    2258     RTMemFree(pReNative->paFixups);
    2259 #ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
    2260     RTMemFree(pReNative->pDbgInfo);
    2261 #endif
    2262     RTMemFree(pReNative);
     2289    iemNativeTerm(pReNative);
    22632290    return NULL;
    22642291}
    … …  
    22852312{
    22862313    Assert(uData == 0 || enmType >= kIemNativeLabelType_FirstWithMultipleInstances);
     2314#if defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) && defined(RT_ARCH_AMD64)
     2315    Assert(enmType >= kIemNativeLabelType_FirstWithMultipleInstances);
     2316#endif
    22872317
    22882318    /*
    … …  
    23912421
    23922422
     2423#if !defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) || !defined(RT_ARCH_AMD64)
    23932424/**
    23942425 * Looks up a lable.
    … …  
    24172448    return UINT32_MAX;
    24182449}
     2450#endif
    24192451
    24202452
    … …  
    24702502    pReNative->cFixups = cFixups + 1;
    24712503}
     2504
     2505
     2506#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     2507/**
     2508 * Adds a fixup to the per chunk tail code.
     2509 *
     2510 * @throws  VBox status code (int) on failure.
     2511 * @param   pReNative       The native recompile state.
     2512 * @param   offWhere        The instruction offset of the fixup location.
     2513 * @param   enmExitReason   The exit reason to jump to.
     2514 */
     2515DECL_HIDDEN_THROW(void)
     2516iemNativeAddTbExitFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, IEMNATIVEEXITREASON enmExitReason)
     2517{
     2518    /*
     2519     * Make sure we've room.
     2520     */
     2521    PIEMNATIVEEXITFIXUP paTbExitFixups = pReNative->paTbExitFixups;
     2522    uint32_t const      cTbExitFixups  = pReNative->cTbExitFixups;
     2523    if (RT_LIKELY(cTbExitFixups < pReNative->cTbExitFixupsAlloc))
     2524    { /* likely */ }
     2525    else
     2526    {
     2527        uint32_t cNew = pReNative->cTbExitFixupsAlloc;
     2528        AssertStmt(cNew, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_FIXUP_IPE_1));
     2529        AssertStmt(cTbExitFixups == cNew, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_FIXUP_IPE_1));
     2530        cNew *= 2;
     2531        AssertStmt(cNew <= _128K, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_FIXUP_TOO_MANY));
     2532        paTbExitFixups = (PIEMNATIVEEXITFIXUP)RTMemRealloc(paTbExitFixups, cNew * sizeof(paTbExitFixups[0]));
     2533        AssertStmt(paTbExitFixups, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_FIXUP_OUT_OF_MEMORY));
     2534        pReNative->paTbExitFixups     = paTbExitFixups;
     2535        pReNative->cTbExitFixupsAlloc = cNew;
     2536    }
     2537
     2538    /*
     2539     * Add the fixup.
     2540     */
     2541    paTbExitFixups[cTbExitFixups].off            = offWhere;
     2542    paTbExitFixups[cTbExitFixups].enmExitReason  = (uint32_t)enmExitReason;
     2543    pReNative->cTbExitFixups = cTbExitFixups + 1;
     2544}
     2545#endif
    24722546
    24732547
    … …  
    60976171# ifdef RT_ARCH_AMD64
    60986172        /* test dword [pVCpu + offVCpu], imm32 */
    6099         PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
     6173        PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 13);
    61006174        if (fEflNeeded <= 0xff)
    61016175        {
    … …  
    61136187            pCodeBuf[off++] = RT_BYTE4(fEflNeeded);
    61146188        }
     6189
     6190        off = iemNativeEmitJccToFixedEx(pCodeBuf, off, off + 3, kIemNativeInstrCond_e);
     6191        pCodeBuf[off++] = 0xcc;
     6192
    61156193        IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    61166194
    … …  
    61586236
    61596237    /* Jump to non-zero status return path. */
    6160     off = iemNativeEmitJnzToNewLabel(pReNative, off, kIemNativeLabelType_NonZeroRetOrPassUp);
     6238    off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_NonZeroRetOrPassUp);
    61616239
    61626240    /* done. */
    … …  
    61756253    pu32CodeBuf[off++] = Armv8A64MkInstrOrr(ARMV8_A64_REG_X4, ARMV8_A64_REG_X3, ARMV8_A64_REG_X0, false /*f64Bit*/);
    61766254
    6177     uint32_t const idxLabel = iemNativeLabelCreate(pReNative, kIemNativeLabelType_NonZeroRetOrPassUp);
    6178     iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
    6179     pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(true /*fJmpIfNotZero*/, 0, ARMV8_A64_REG_X4, false /*f64Bit*/);
     6255    off = iemNativeEmitTestIfGprIsNotZeroAndTbExitEx(pReNative, pu32CodeBuf, off, ARMV8_A64_REG_X4, true /*f64Bit*/,
     6256                                                     kIemNativeExitReason_NonZeroRetOrPassUp);
    61806257
    61816258#else
    … …  
    62876364
    62886365    off = iemNativeEmitCmpGpr32WithGpr(pReNative, off, idxAddrReg, idxRegCsLim);
    6289     off = iemNativeEmitJaToNewLabel(pReNative, off, kIemNativeLabelType_RaiseGp0);
     6366    off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeExitReason_RaiseGp0);
    62906367
    62916368    iemNativeRegFreeTmp(pReNative, idxRegCsLim);
    … …  
    64956572
    64966573/**
     6574 * Worker for iemNativeEmitViaLookupDoOne and iemNativeRecompileAttachExecMemChunkCtx.
     6575 */
     6576static uint32_t
     6577iemNativeEmitCoreViaLookupDoOne(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offReturnBreak, uintptr_t pfnHelper)
     6578{
     6579    off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
     6580    off = iemNativeEmitCallImm(pReNative, off, pfnHelper);
     6581
     6582    /* Jump to ReturnBreak if the return register is NULL. */
     6583    off = iemNativeEmitTestIfGprIsZeroAndJmpToFixed(pReNative, off, IEMNATIVE_CALL_RET_GREG,
     6584                                                    true /*f64Bit*/, offReturnBreak);
     6585
     6586    /* Okay, continue executing the next TB. */
     6587    off = iemNativeEmitJmpViaGpr(pReNative, off, IEMNATIVE_CALL_RET_GREG);
     6588    return off;
     6589}
     6590
     6591#ifndef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     6592
     6593/**
    64976594 * Worker for iemNativeEmitReturnBreakViaLookup.
    64986595 */
    6499 static uint32_t iemNativeEmitViaLookupDoOne(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabelReturnBreak,
     6596static uint32_t iemNativeEmitViaLookupDoOne(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offReturnBreak,
    65006597                                            IEMNATIVELABELTYPE enmLabel, uintptr_t pfnHelper)
    65016598{
    … …  
    65046601    {
    65056602        iemNativeLabelDefine(pReNative, idxLabel, off);
    6506 
    6507         off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
    6508         off = iemNativeEmitCallImm(pReNative, off, pfnHelper);
    6509 
    6510         /* Jump to ReturnBreak if the return register is NULL. */
    6511         off = iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, IEMNATIVE_CALL_RET_GREG,
    6512                                                         true /*f64Bit*/, idxLabelReturnBreak);
    6513 
    6514         /* Okay, continue executing the next TB. */
    6515         off = iemNativeEmitJmpViaGpr(pReNative, off, IEMNATIVE_CALL_RET_GREG);
     6603        off = iemNativeEmitCoreViaLookupDoOne(pReNative, off, offReturnBreak, pfnHelper);
    65166604    }
    65176605    return off;
    65186606}
     6607
    65196608
    65206609/**
    … …  
    65236612 * (returns VINF_IEM_REEXEC_FINISH_WITH_FLAGS or jumps to the next TB).
    65246613 */
    6525 static uint32_t iemNativeEmitReturnBreakViaLookup(PIEMRECOMPILERSTATE pReNative, uint32_t off)
    6526 {
    6527     uint32_t const idxLabelReturnBreak = iemNativeLabelCreate(pReNative, kIemNativeLabelType_ReturnBreak);
     6614static uint32_t iemNativeEmitReturnBreakViaLookup(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxReturnBreakLabel)
     6615{
     6616    uint32_t const offReturnBreak = pReNative->paLabels[idxReturnBreakLabel].off;
     6617    Assert(offReturnBreak < off);
    65286618
    65296619    /*
    … …  
    65316621     * The GCPhysPc is in IEMNATIVE_CALL_ARG2_GREG for ReturnBreakViaLookupWithPc.
    65326622     */
    6533     off = iemNativeEmitViaLookupDoOne(pReNative, off, idxLabelReturnBreak, kIemNativeLabelType_ReturnBreakViaLookup,
     6623    off = iemNativeEmitViaLookupDoOne(pReNative, off, offReturnBreak, kIemNativeLabelType_ReturnBreakViaLookup,
    65346624                                      (uintptr_t)iemNativeHlpReturnBreakViaLookup<false /*a_fWithIrqCheck*/>);
    6535     off = iemNativeEmitViaLookupDoOne(pReNative, off, idxLabelReturnBreak, kIemNativeLabelType_ReturnBreakViaLookupWithIrq,
     6625    off = iemNativeEmitViaLookupDoOne(pReNative, off, offReturnBreak, kIemNativeLabelType_ReturnBreakViaLookupWithIrq,
    65366626                                      (uintptr_t)iemNativeHlpReturnBreakViaLookup<true /*a_fWithIrqCheck*/>);
    6537     off = iemNativeEmitViaLookupDoOne(pReNative, off, idxLabelReturnBreak, kIemNativeLabelType_ReturnBreakViaLookupWithTlb,
     6627    off = iemNativeEmitViaLookupDoOne(pReNative, off, offReturnBreak, kIemNativeLabelType_ReturnBreakViaLookupWithTlb,
    65386628                                      (uintptr_t)iemNativeHlpReturnBreakViaLookupWithTlb<false /*a_fWithIrqCheck*/>);
    6539     off = iemNativeEmitViaLookupDoOne(pReNative, off, idxLabelReturnBreak, kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq,
     6629    off = iemNativeEmitViaLookupDoOne(pReNative, off, offReturnBreak, kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq,
    65406630                                      (uintptr_t)iemNativeHlpReturnBreakViaLookupWithTlb<true /*a_fWithIrqCheck*/>);
    65416631    return off;
    65426632}
    65436633
    6544 
     6634#endif /* !IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE */
     6635
     6636/**
     6637 * Emits the code at the ReturnWithFlags label (returns VINF_IEM_REEXEC_FINISH_WITH_FLAGS).
     6638 */
     6639static uint32_t iemNativeEmitCoreReturnWithFlags(PIEMRECOMPILERSTATE pReNative, uint32_t off)
     6640{
     6641    /* set the return status  */
     6642    return iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_CALL_RET_GREG, VINF_IEM_REEXEC_FINISH_WITH_FLAGS);
     6643}
     6644
     6645
     6646#ifndef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
    65456647/**
    65466648 * Emits the code at the ReturnWithFlags label (returns VINF_IEM_REEXEC_FINISH_WITH_FLAGS).
    … …  
    65536655        iemNativeLabelDefine(pReNative, idxLabel, off);
    65546656        /* set the return status  */
    6555         off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_CALL_RET_GREG, VINF_IEM_REEXEC_FINISH_WITH_FLAGS);
     6657        off = iemNativeEmitCoreReturnWithFlags(pReNative, off);
    65566658        /* jump back to the return sequence. */
    65576659        off = iemNativeEmitJmpToLabel(pReNative, off, idxReturnLabel);
    … …  
    65596661    return off;
    65606662}
    6561 
    6562 
     6663#endif
     6664
     6665
     6666/**
     6667 * Emits the code at the ReturnBreakFF label (returns VINF_IEM_REEXEC_BREAK_FF).
     6668 */
     6669static uint32_t iemNativeEmitCoreReturnBreakFF(PIEMRECOMPILERSTATE pReNative, uint32_t off)
     6670{
     6671    /* set the return status */
     6672    return iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_CALL_RET_GREG, VINF_IEM_REEXEC_BREAK_FF);
     6673}
     6674
     6675
     6676#ifndef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
    65636677/**
    65646678 * Emits the code at the ReturnBreakFF label (returns VINF_IEM_REEXEC_BREAK_FF).
    … …  
    65716685        iemNativeLabelDefine(pReNative, idxLabel, off);
    65726686        /* set the return status */
    6573         off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_CALL_RET_GREG, VINF_IEM_REEXEC_BREAK_FF);
     6687        off = iemNativeEmitCoreReturnBreakFF(pReNative, off);
    65746688        /* jump back to the return sequence. */
    65756689        off = iemNativeEmitJmpToLabel(pReNative, off, idxReturnLabel);
    … …  
    65776691    return off;
    65786692}
    6579 
    6580 
     6693#endif
     6694
     6695
     6696/**
     6697 * Emits the code at the ReturnBreak label (returns VINF_IEM_REEXEC_BREAK).
     6698 */
     6699static uint32_t iemNativeEmitCoreReturnBreak(PIEMRECOMPILERSTATE pReNative, uint32_t off)
     6700{
     6701    /* set the return status */
     6702    return iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_CALL_RET_GREG, VINF_IEM_REEXEC_BREAK);
     6703}
     6704
     6705
     6706#ifndef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
    65816707/**
    65826708 * Emits the code at the ReturnBreak label (returns VINF_IEM_REEXEC_BREAK).
    … …  
    65896715        iemNativeLabelDefine(pReNative, idxLabel, off);
    65906716        /* set the return status */
    6591         off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_CALL_RET_GREG, VINF_IEM_REEXEC_BREAK);
     6717        off = iemNativeEmitCoreReturnBreak(pReNative, off);
    65926718        /* jump back to the return sequence. */
    65936719        off = iemNativeEmitJmpToLabel(pReNative, off, idxReturnLabel);
    … …  
    65956721    return off;
    65966722}
    6597 
    6598 
     6723#endif
     6724
     6725
     6726/**
     6727 * Emits the RC fiddling code for handling non-zero return code or rcPassUp.
     6728 */
     6729static uint32_t iemNativeEmitCoreRcFiddling(PIEMRECOMPILERSTATE pReNative, uint32_t off)
     6730{
     6731    /*
     6732     * Generate the rc + rcPassUp fiddling code.
     6733     */
     6734    /* iemNativeHlpExecStatusCodeFiddling(PVMCPUCC pVCpu, int rc, uint8_t idxInstr) */
     6735#ifdef RT_ARCH_AMD64
     6736# ifdef RT_OS_WINDOWS
     6737#  ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
     6738    off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_x8,  X86_GREG_xCX); /* cl = instruction number */
     6739#  endif
     6740    off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xCX, IEMNATIVE_REG_FIXED_PVMCPU);
     6741    off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xDX, X86_GREG_xAX);
     6742# else
     6743    off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xDI, IEMNATIVE_REG_FIXED_PVMCPU);
     6744    off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xSI, X86_GREG_xAX);
     6745#  ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
     6746    off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xDX, X86_GREG_xCX); /* cl = instruction number */
     6747#  endif
     6748# endif
     6749# ifndef IEMNATIVE_WITH_INSTRUCTION_COUNTING
     6750    off = iemNativeEmitLoadGpr8Imm(pReNative, off, X86_GREG_xCX, 0);
     6751# endif
     6752
     6753#else
     6754    off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, IEMNATIVE_CALL_RET_GREG);
     6755    off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
     6756    /* IEMNATIVE_CALL_ARG2_GREG is already set. */
     6757#endif
     6758
     6759    off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpExecStatusCodeFiddling);
     6760    return off;
     6761}
     6762
     6763
     6764#ifndef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
    65996765/**
    66006766 * Emits the RC fiddling code for handling non-zero return code or rcPassUp.
    … …  
    66096775    {
    66106776        iemNativeLabelDefine(pReNative, idxLabel, off);
    6611 
    6612         /* iemNativeHlpExecStatusCodeFiddling(PVMCPUCC pVCpu, int rc, uint8_t idxInstr) */
    6613 #ifdef RT_ARCH_AMD64
    6614 # ifdef RT_OS_WINDOWS
    6615 #  ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
    6616         off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_x8,  X86_GREG_xCX); /* cl = instruction number */
    6617 #  endif
    6618         off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xCX, IEMNATIVE_REG_FIXED_PVMCPU);
    6619         off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xDX, X86_GREG_xAX);
    6620 # else
    6621         off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xDI, IEMNATIVE_REG_FIXED_PVMCPU);
    6622         off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xSI, X86_GREG_xAX);
    6623 #  ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
    6624         off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xDX, X86_GREG_xCX); /* cl = instruction number */
    6625 #  endif
    6626 # endif
    6627 # ifndef IEMNATIVE_WITH_INSTRUCTION_COUNTING
    6628         off = iemNativeEmitLoadGpr8Imm(pReNative, off, X86_GREG_xCX, 0);
    6629 # endif
    6630 
    6631 #else
    6632         off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, IEMNATIVE_CALL_RET_GREG);
    6633         off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
    6634         /* IEMNATIVE_CALL_ARG2_GREG is already set. */
    6635 #endif
    6636 
    6637         off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpExecStatusCodeFiddling);
     6777        off = iemNativeEmitCoreRcFiddling(pReNative, off);
    66386778        off = iemNativeEmitJmpToLabel(pReNative, off, idxReturnLabel);
    66396779    }
    66406780    return off;
    66416781}
     6782#endif
    66426783
    66436784
    … …  
    66456786 * Emits a standard epilog.
    66466787 */
    6647 static uint32_t iemNativeEmitEpilog(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t *pidxReturnLabel)
    6648 {
    6649     *pidxReturnLabel = UINT32_MAX;
    6650 
    6651     /* Flush any pending writes before returning from the last instruction (RIP updates, etc.). */
    6652     off = iemNativeRegFlushPendingWrites(pReNative, off);
    6653 
    6654     /*
    6655      * Successful return, so clear the return register (eax, w0).
    6656      */
    6657     pReNative->Core.bmHstRegs |= RT_BIT_32(IEMNATIVE_CALL_RET_GREG); /* HACK: For IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK. */
    6658     off = iemNativeEmitGprZero(pReNative, off, IEMNATIVE_CALL_RET_GREG);
    6659 
    6660     /*
    6661      * Define label for common return point.
    6662      */
    6663     uint32_t const idxReturn = iemNativeLabelCreate(pReNative, kIemNativeLabelType_Return, off);
    6664     *pidxReturnLabel = idxReturn;
     6788static uint32_t iemNativeEmitCoreEpilog(PIEMRECOMPILERSTATE pReNative, uint32_t off)
     6789{
     6790    pReNative->Core.bmHstRegs |= RT_BIT_32(IEMNATIVE_CALL_RET_GREG); /* HACK: For IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK (return register is already set to status code). */
    66656791
    66666792    IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(pReNative, off, X86_EFL_STATUS_BITS);
    66676793
    6668 #ifdef IEMNATIVE_WITH_RECOMPILER_EPILOGUE_SINGLETON
    6669     //off = iemNativeEmitBrk(pReNative, off, 0x7777);
    6670     off = iemNativeEmitJmpImm(pReNative, off, (uintptr_t)iemNativeTbEpilog);
    6671 #else
     6794    /* HACK: For IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK (return register is already set to status code). */
     6795    pReNative->Core.bmHstRegs &= ~RT_BIT_32(IEMNATIVE_CALL_RET_GREG);
     6796
    66726797    /*
    66736798     * Restore registers and return.
    66746799     */
    6675 # ifdef RT_ARCH_AMD64
     6800#ifdef RT_ARCH_AMD64
    66766801    uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 20);
    66776802
    … …  
    66916816    pbCodeBuf[off++] = X86_OP_REX_B;            /* pop r12 */
    66926817    pbCodeBuf[off++] = 0x58 + X86_GREG_x12 - 8;
    6693 #  ifdef RT_OS_WINDOWS
     6818# ifdef RT_OS_WINDOWS
    66946819    pbCodeBuf[off++] = 0x58 + X86_GREG_xDI;     /* pop rdi */
    66956820    pbCodeBuf[off++] = 0x58 + X86_GREG_xSI;     /* pop rsi */
    6696 #  endif
     6821# endif
    66976822    pbCodeBuf[off++] = 0x58 + X86_GREG_xBX;     /* pop rbx */
    66986823    pbCodeBuf[off++] = 0xc9;                    /* leave */
    … …  
    67006825    pbCodeBuf[off++] = 0xcc;                    /* int3 poison */
    67016826
    6702 # elif RT_ARCH_ARM64
     6827#elif RT_ARCH_ARM64
    67036828    uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
    67046829
    … …  
    67276852
    67286853    /* retab / ret */
    6729 #  ifdef RT_OS_DARWIN /** @todo See todo on pacibsp in the prolog. */
     6854# ifdef RT_OS_DARWIN /** @todo See todo on pacibsp in the prolog. */
    67306855    if (1)
    67316856        pu32CodeBuf[off++] = ARMV8_A64_INSTR_RETAB;
    67326857    else
    6733 #  endif
     6858# endif
    67346859        pu32CodeBuf[off++] = ARMV8_A64_INSTR_RET;
    67356860
    6736 # else
    6737 #  error "port me"
    6738 # endif
     6861#else
     6862# error "port me"
     6863#endif
    67396864    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    6740 #endif /* IEMNATIVE_WITH_RECOMPILER_EPILOGUE_SINGLETON */
    67416865
    67426866    /* HACK: For IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK. */
    67436867    pReNative->Core.bmHstRegs &= ~RT_BIT_32(IEMNATIVE_CALL_RET_GREG);
    67446868
    6745     return iemNativeEmitRcFiddling(pReNative, off, idxReturn);
    6746 }
     6869    return off;
     6870}
     6871
     6872
     6873#ifndef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     6874/**
     6875 * Emits a standard epilog.
     6876 */
     6877static uint32_t iemNativeEmitEpilog(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t *pidxReturnLabel)
     6878{
     6879    /*
     6880     * Define label for common return point.
     6881     */
     6882    *pidxReturnLabel = UINT32_MAX;
     6883    uint32_t const idxReturn = iemNativeLabelCreate(pReNative, kIemNativeLabelType_Return, off);
     6884    *pidxReturnLabel = idxReturn;
     6885
     6886    /*
     6887     * Emit the code.
     6888     */
     6889    return iemNativeEmitCoreEpilog(pReNative, off);
     6890}
     6891#endif
    67476892
    67486893
    … …  
    87688913
    87698914
    8770 DECLHIDDEN(void) iemNativeDisassembleTb(PCIEMTB pTb, PCDBGFINFOHLP pHlp) RT_NOEXCEPT
     8915/**
     8916 * Translates a label to a name.
     8917 */
     8918static const char *iemNativeGetLabelName(IEMNATIVELABELTYPE enmLabel, bool fCommonCode /*= false*/)
     8919{
     8920    switch (enmLabel)
     8921    {
     8922#define STR_CASE_CMN(a_Label) case kIemNativeLabelType_ ## a_Label: return fCommonCode ? "Chunk_" #a_Label : #a_Label;
     8923        STR_CASE_CMN(Invalid);
     8924        STR_CASE_CMN(RaiseDe);
     8925        STR_CASE_CMN(RaiseUd);
     8926        STR_CASE_CMN(RaiseSseRelated);
     8927        STR_CASE_CMN(RaiseAvxRelated);
     8928        STR_CASE_CMN(RaiseSseAvxFpRelated);
     8929        STR_CASE_CMN(RaiseNm);
     8930        STR_CASE_CMN(RaiseGp0);
     8931        STR_CASE_CMN(RaiseMf);
     8932        STR_CASE_CMN(RaiseXf);
     8933        STR_CASE_CMN(ObsoleteTb);
     8934        STR_CASE_CMN(NeedCsLimChecking);
     8935        STR_CASE_CMN(CheckBranchMiss);
     8936        STR_CASE_CMN(Return);
     8937        STR_CASE_CMN(ReturnBreak);
     8938        STR_CASE_CMN(ReturnBreakFF);
     8939        STR_CASE_CMN(ReturnWithFlags);
     8940        STR_CASE_CMN(ReturnBreakViaLookup);
     8941        STR_CASE_CMN(ReturnBreakViaLookupWithIrq);
     8942        STR_CASE_CMN(ReturnBreakViaLookupWithTlb);
     8943        STR_CASE_CMN(ReturnBreakViaLookupWithTlbAndIrq);
     8944        STR_CASE_CMN(NonZeroRetOrPassUp);
     8945#undef STR_CASE_CMN
     8946#define STR_CASE_LBL(a_Label) case kIemNativeLabelType_ ## a_Label: return #a_Label;
     8947        STR_CASE_LBL(If);
     8948        STR_CASE_LBL(Else);
     8949        STR_CASE_LBL(Endif);
     8950        STR_CASE_LBL(CheckIrq);
     8951        STR_CASE_LBL(TlbLookup);
     8952        STR_CASE_LBL(TlbMiss);
     8953        STR_CASE_LBL(TlbDone);
     8954        case kIemNativeLabelType_End: break;
     8955    }
     8956    return NULL;
     8957}
     8958
     8959
     8960/** Info for the symbols resolver used when disassembling. */
     8961typedef struct IEMNATIVDISASMSYMCTX
     8962{
     8963    PVMCPU                  pVCpu;
     8964# ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     8965    PCIEMNATIVEPERCHUNKCTX  pCtx;
     8966# endif
     8967} IEMNATIVDISASMSYMCTX;
     8968typedef IEMNATIVDISASMSYMCTX *PIEMNATIVDISASMSYMCTX;
     8969
     8970
     8971/**
     8972 * Resolve address to symbol, if we can.
     8973 */
     8974static const char *iemNativeDisasmGetSymbol(PIEMNATIVDISASMSYMCTX pSymCtx, uintptr_t uAddress)
     8975{
     8976#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     8977    PCIEMNATIVEPERCHUNKCTX pChunkCtx = pSymCtx->pCtx;
     8978    if (pChunkCtx)
     8979        for (uint32_t i = 1; i < RT_ELEMENTS(pChunkCtx->apExitLabels); i++)
     8980            if ((PIEMNATIVEINSTR)uAddress == pChunkCtx->apExitLabels[i])
     8981                return iemNativeGetLabelName((IEMNATIVELABELTYPE)i, true /*fCommonCode*/);
     8982#endif
     8983    return NULL;
     8984}
     8985
     8986#ifndef VBOX_WITH_IEM_USING_CAPSTONE_DISASSEMBLER
     8987
     8988/**
     8989 * @callback_method_impl{FNDISGETSYMBOL}
     8990 */
     8991static DECLCALLBACK(int) iemNativeDisasmGetSymbolCb(PCDISSTATE pDis, uint32_t u32Sel, RTUINTPTR uAddress,
     8992                                                   char *pszBuf, size_t cchBuf, RTINTPTR *poff, void *pvUser)
     8993{
     8994    const char * const pszSym = iemNativeDisasmGetSymbol((PIEMNATIVDISASMSYMCTX)pvUser, uAddress);
     8995    if (pszSym)
     8996    {
     8997        *poff = 0;
     8998        return RTStrCopy(pszBuf, cchBuf, pszSym);
     8999    }
     9000    RT_NOREF(pDis, u32Sel);
     9001    return VERR_SYMBOL_NOT_FOUND;
     9002}
     9003
     9004#else  /* VBOX_WITH_IEM_USING_CAPSTONE_DISASSEMBLER */
     9005
     9006/**
     9007 * Annotates an instruction decoded by the capstone disassembler.
     9008 */
     9009static const char *
     9010iemNativeDisasmAnnotateCapstone(PIEMNATIVDISASMSYMCTX pSymCtx, cs_insn const *pInstr, char *pszBuf, size_t cchBuf)
     9011{
     9012# if defined(RT_ARCH_ARM64)
     9013    if (   (pInstr->id >= ARM64_INS_LD1 && pInstr->id < ARM64_INS_LSL)
     9014        || (pInstr->id >= ARM64_INS_ST1 && pInstr->id < ARM64_INS_SUB))
     9015    {
     9016        /* This is bit crappy, but the disassembler provides incomplete addressing details. */
     9017        AssertCompile(IEMNATIVE_REG_FIXED_PVMCPU == 28 && IEMNATIVE_REG_FIXED_PCPUMCTX == 27);
     9018        char const *psz = strchr(pInstr->op_str, '[');
     9019        if (psz && psz[1] == 'x' && psz[2] == '2' && (psz[3] == '7' || psz[3] == '8'))
     9020        {
     9021            uint32_t const offVCpu = psz[3] == '8'? 0 : RT_UOFFSETOF(VMCPU, cpum.GstCtx);
     9022            int32_t        off     = -1;
     9023            psz += 4;
     9024            if (*psz == ']')
     9025                off = 0;
     9026            else if (*psz == ',')
     9027            {
     9028                psz = RTStrStripL(psz + 1);
     9029                if (*psz == '#')
     9030                    off = RTStrToInt32(&psz[1]);
     9031                /** @todo deal with index registers and LSL as well... */
     9032            }
     9033            if (off >= 0)
     9034                return iemNativeDbgVCpuOffsetToName(offVCpu + (uint32_t)off);
     9035        }
     9036    }
     9037    else if (pInstr->id == ARM64_INS_B || pInstr->id == ARM64_INS_BL)
     9038    {
     9039        const char *pszAddr = strchr(pInstr->op_str, '#');
     9040        if (pszAddr)
     9041        {
     9042            uint64_t uAddr = RTStrToUInt64(pszAddr + 1);
     9043            if (uAddr != 0)
     9044                return iemNativeDisasmGetSymbol(pSymCtx, uAddr);
     9045        }
     9046    }
     9047# endif
     9048    RT_NOREF(pszBuf, cchBuf);
     9049    return NULL;
     9050}
     9051#endif /* VBOX_WITH_IEM_USING_CAPSTONE_DISASSEMBLER */
     9052
     9053
     9054DECLHIDDEN(void) iemNativeDisassembleTb(PVMCPU pVCpu, PCIEMTB pTb, PCDBGFINFOHLP pHlp) RT_NOEXCEPT
    87719055{
    87729056    AssertReturnVoid((pTb->fFlags & IEMTB_F_TYPE_MASK) == IEMTB_F_TYPE_NATIVE);
    … …  
    87909074                                          : (pTb->fFlags & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT ? DISCPUMODE_32BIT
    87919075                                          :                                                            DISCPUMODE_64BIT;
     9076#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     9077    IEMNATIVDISASMSYMCTX    SymCtx        = { pVCpu, iemExecMemGetTbChunkCtx(pVCpu, pTb) };
     9078#else
     9079    IEMNATIVDISASMSYMCTX    SymCtx        = { pVCpu };
     9080#endif
    87929081#if   defined(RT_ARCH_AMD64) && !defined(VBOX_WITH_IEM_USING_CAPSTONE_DISASSEMBLER)
    87939082    DISCPUMODE const        enmHstCpuMode = DISCPUMODE_64BIT;
    … …  
    89809269                        case kIemTbDbgEntryType_Label:
    89819270                        {
    8982                             const char *pszName    = "what_the_fudge";
    8983                             const char *pszComment = "";
    8984                             bool        fNumbered  = pDbgInfo->aEntries[iDbgEntry].Label.uData != 0;
    8985                             switch ((IEMNATIVELABELTYPE)pDbgInfo->aEntries[iDbgEntry].Label.enmLabel)
     9271                            const char *pszName = iemNativeGetLabelName((IEMNATIVELABELTYPE)pDbgInfo->aEntries[iDbgEntry].Label.enmLabel);
     9272                            if (pDbgInfo->aEntries[iDbgEntry].Label.enmLabel >= kIemNativeLabelType_FirstWithMultipleInstances)
    89869273                            {
    8987                                 case kIemNativeLabelType_Return:                        pszName = "Return"; break;
    8988                                 case kIemNativeLabelType_ReturnBreak:                   pszName = "ReturnBreak"; break;
    8989                                 case kIemNativeLabelType_ReturnBreakFF:                 pszName = "ReturnBreakFF"; break;
    8990                                 case kIemNativeLabelType_ReturnWithFlags:               pszName = "ReturnWithFlags"; break;
    8991                                 case kIemNativeLabelType_ReturnBreakViaLookup:          pszName = "ReturnBreakViaLookup"; break;
    8992                                 case kIemNativeLabelType_ReturnBreakViaLookupWithIrq:   pszName = "ReturnBreakViaLookupWithIrq"; break;
    8993                                 case kIemNativeLabelType_ReturnBreakViaLookupWithTlb:   pszName = "ReturnBreakViaLookupWithTlb"; break;
    8994                                 case kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq: pszName = "ReturnBreakViaLookupWithTlbAndIrq"; break;
    8995                                 case kIemNativeLabelType_NonZeroRetOrPassUp:            pszName = "NonZeroRetOrPassUp"; break;
    8996                                 case kIemNativeLabelType_RaiseDe:                       pszName = "RaiseDe"; break;
    8997                                 case kIemNativeLabelType_RaiseUd:                       pszName = "RaiseUd"; break;
    8998                                 case kIemNativeLabelType_RaiseSseRelated:               pszName = "RaiseSseRelated"; break;
    8999                                 case kIemNativeLabelType_RaiseAvxRelated:               pszName = "RaiseAvxRelated"; break;
    9000                                 case kIemNativeLabelType_RaiseSseAvxFpRelated:          pszName = "RaiseSseAvxFpRelated"; break;
    9001                                 case kIemNativeLabelType_RaiseNm:                       pszName = "RaiseNm"; break;
    9002                                 case kIemNativeLabelType_RaiseGp0:                      pszName = "RaiseGp0"; break;
    9003                                 case kIemNativeLabelType_RaiseMf:                       pszName = "RaiseMf"; break;
    9004                                 case kIemNativeLabelType_RaiseXf:                       pszName = "RaiseXf"; break;
    9005                                 case kIemNativeLabelType_ObsoleteTb:                    pszName = "ObsoleteTb"; break;
    9006                                 case kIemNativeLabelType_NeedCsLimChecking:             pszName = "NeedCsLimChecking"; break;
    9007                                 case kIemNativeLabelType_CheckBranchMiss:               pszName = "CheckBranchMiss"; break;
    9008                                 case kIemNativeLabelType_If:
    9009                                     pszName = "If";
    9010                                     fNumbered = true;
    9011                                     break;
    9012                                 case kIemNativeLabelType_Else:
    9013                                     pszName = "Else";
    9014                                     fNumbered = true;
    9015                                     pszComment = "   ; regs state restored pre-if-block";
    9016                                     break;
    9017                                 case kIemNativeLabelType_Endif:
    9018                                     pszName = "Endif";
    9019                                     fNumbered = true;
    9020                                     break;
    9021                                 case kIemNativeLabelType_CheckIrq:
    9022                                     pszName = "CheckIrq_CheckVM";
    9023                                     fNumbered = true;
    9024                                     break;
    9025                                 case kIemNativeLabelType_TlbLookup:
    9026                                     pszName = "TlbLookup";
    9027                                     fNumbered = true;
    9028                                     break;
    9029                                 case kIemNativeLabelType_TlbMiss:
    9030                                     pszName = "TlbMiss";
    9031                                     fNumbered = true;
    9032                                     break;
    9033                                 case kIemNativeLabelType_TlbDone:
    9034                                     pszName = "TlbDone";
    9035                                     fNumbered = true;
    9036                                     break;
    9037                                 case kIemNativeLabelType_Invalid:
    9038                                 case kIemNativeLabelType_End:
    9039                                     break;
     9274                                const char *pszComment = pDbgInfo->aEntries[iDbgEntry].Label.enmLabel == kIemNativeLabelType_Else
     9275                                                       ? "   ; regs state restored pre-if-block" : "";
     9276                                pHlp->pfnPrintf(pHlp, "  %s_%u:%s\n", pszName, pDbgInfo->aEntries[iDbgEntry].Label.uData, pszComment);
    90409277                            }
    9041                             if (fNumbered)
    9042                                 pHlp->pfnPrintf(pHlp, "  %s_%u:%s\n", pszName, pDbgInfo->aEntries[iDbgEntry].Label.uData, pszComment);
    90439278                            else
    90449279                                pHlp->pfnPrintf(pHlp, "  %s:\n", pszName);
    … …  
    91219356                                    DIS_FMT_FLAGS_BYTES_WIDTH_MAKE(10) | DIS_FMT_FLAGS_BYTES_LEFT
    91229357                                    | DIS_FMT_FLAGS_RELATIVE_BRANCH | DIS_FMT_FLAGS_C_HEX,
    9123                                     NULL /*pfnGetSymbol*/, NULL /*pvUser*/);
     9358                                    iemNativeDisasmGetSymbolCb, &SymCtx);
    91249359                    PCDISOPPARAM pMemOp;
    91259360                    if (DISUSE_IS_EFFECTIVE_ADDR(Dis.Param1.fUse))
    … …  
    91869421            {
    91879422                Assert(cInstrs == 1);
    9188                 const char *pszAnnotation = NULL;
    9189 #  if defined(RT_ARCH_ARM64)
    9190                 if (   (pInstr->id >= ARM64_INS_LD1 && pInstr->id < ARM64_INS_LSL)
    9191                     || (pInstr->id >= ARM64_INS_ST1 && pInstr->id < ARM64_INS_SUB))
    9192                 {
    9193                     /* This is bit crappy, but the disassembler provides incomplete addressing details. */
    9194                     AssertCompile(IEMNATIVE_REG_FIXED_PVMCPU == 28 && IEMNATIVE_REG_FIXED_PCPUMCTX == 27);
    9195                     char *psz = strchr(pInstr->op_str, '[');
    9196                     if (psz && psz[1] == 'x' && psz[2] == '2' && (psz[3] == '7' || psz[3] == '8'))
    9197                     {
    9198                         uint32_t const offVCpu = psz[3] == '8'? 0 : RT_UOFFSETOF(VMCPU, cpum.GstCtx);
    9199                         int32_t        off     = -1;
    9200                         psz += 4;
    9201                         if (*psz == ']')
    9202                             off = 0;
    9203                         else if (*psz == ',')
    9204                         {
    9205                             psz = RTStrStripL(psz + 1);
    9206                             if (*psz == '#')
    9207                                 off = RTStrToInt32(&psz[1]);
    9208                             /** @todo deal with index registers and LSL as well... */
    9209                         }
    9210                         if (off >= 0)
    9211                             pszAnnotation = iemNativeDbgVCpuOffsetToName(offVCpu + (uint32_t)off);
    9212                     }
    9213                 }
    9214 #  endif
    9215 
    9216                 size_t const cchOp = strlen(pInstr->op_str);
     9423                const char * const pszAnnotation = iemNativeDisasmAnnotateCapstone(&SymCtx, pInstr, szDisBuf, sizeof(szDisBuf));
     9424                size_t const       cchOp         = strlen(pInstr->op_str);
    92179425#  if defined(RT_ARCH_AMD64)
    92189426                if (pszAnnotation)
    … …  
    93269534                                    DIS_FMT_FLAGS_BYTES_WIDTH_MAKE(10) | DIS_FMT_FLAGS_BYTES_LEFT
    93279535                                    | DIS_FMT_FLAGS_RELATIVE_BRANCH | DIS_FMT_FLAGS_C_HEX,
    9328                                     NULL /*pfnGetSymbol*/, NULL /*pvUser*/);
     9536                                    iemNativeDisasmGetSymbolCb, &SymCtx);
    93299537#  elif defined(RT_ARCH_ARM64)
    93309538                    DISFormatArmV8Ex(&Dis, szDisBuf, sizeof(szDisBuf),
    93319539                                     DIS_FMT_FLAGS_BYTES_LEFT | DIS_FMT_FLAGS_RELATIVE_BRANCH | DIS_FMT_FLAGS_C_HEX,
    9332                                      NULL /*pfnGetSymbol*/, NULL /*pvUser*/);
     9540                                     iemNativeDisasmGetSymbolCb, &SymCtx);
    93339541#  else
    93349542#   error "Port me"
    … …  
    93569564            {
    93579565                Assert(cInstrs == 1);
     9566                const char * const pszAnnotation = iemNativeDisasmAnnotateCapstone(&SymCtx, pInstr, szDisBuf, sizeof(szDisBuf));
     9567                size_t const       cchOp         = strlen(pInstr->op_str);
    93589568#  if defined(RT_ARCH_AMD64)
    9359                 pHlp->pfnPrintf(pHlp, "    %p: %.*Rhxs %-7s %s\n",
    9360                                 pNativeCur, pInstr->size, pNativeCur, pInstr->mnemonic, pInstr->op_str);
     9569                if (pszAnnotation)
     9570                    pHlp->pfnPrintf(pHlp, "    %p: %.*Rhxs %-7s %s%*s ; %s\n",
     9571                                    pNativeCur, pInstr->size, pNativeCur, pInstr->mnemonic, pInstr->op_str,
     9572                                    cchOp < 55 ? 55 - cchOp : 0, "", pszAnnotation);
     9573                else
     9574                    pHlp->pfnPrintf(pHlp, "    %p: %.*Rhxs %-7s %s\n",
     9575                                    pNativeCur, pInstr->size, pNativeCur, pInstr->mnemonic, pInstr->op_str);
     9576
    93619577#  else
    9362                 pHlp->pfnPrintf(pHlp, "    %p: %#010RX32 %-7s %s\n",
    9363                                 pNativeCur, *pNativeCur, pInstr->mnemonic, pInstr->op_str);
     9578                if (pszAnnotation)
     9579                    pHlp->pfnPrintf(pHlp, "    %p: %#010RX32 %-7s %s%*s ; %s\n",
     9580                                    pNativeCur, *pNativeCur, pInstr->mnemonic, pInstr->op_str,
     9581                                    cchOp < 55 ? 55 - cchOp : 0, "", pszAnnotation);
     9582                else
     9583                    pHlp->pfnPrintf(pHlp, "    %p: %#010RX32 %-7s %s\n",
     9584                                    pNativeCur, *pNativeCur, pInstr->mnemonic, pInstr->op_str);
    93649585#  endif
    93659586                offNative += pInstr->size / sizeof(*pNativeCur);
    … …  
    93879608
    93889609
     9610#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     9611
     9612/** Emit alignment padding between labels / functions.   */
     9613DECL_INLINE_THROW(uint32_t)
     9614iemNativeRecompileEmitAlignmentPadding(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fAlignMask)
     9615{
     9616    if (off & fAlignMask)
     9617    {
     9618        PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, fAlignMask + 1);
     9619        while (off & fAlignMask)
     9620# if   defined(RT_ARCH_AMD64)
     9621            pCodeBuf[off++] = 0xcc;
     9622# elif defined(RT_ARCH_ARM64)
     9623            pCodeBuf[off++] = Armv8A64MkInstrBrk(0xcccc);
     9624# else
     9625#  error "port me"
     9626# endif
     9627    }
     9628    return off;
     9629}
     9630
     9631
     9632/**
     9633 * Called when a new chunk is allocate to emit common per-chunk code.
     9634 *
     9635 * Allocates a per-chunk context directly from the chunk itself and place the
     9636 * common code there.
     9637 *
     9638 * @returns Pointer to the chunk context start.
     9639 * @param   pVCpu       The cross context virtual CPU structure of the calling
     9640 *                      thread.
     9641 * @param   idxChunk    The index of the chunk being added and requiring a
     9642 *                      common code context.
     9643 */
     9644DECLHIDDEN(PCIEMNATIVEPERCHUNKCTX) iemNativeRecompileAttachExecMemChunkCtx(PVMCPU pVCpu, uint32_t idxChunk)
     9645{
     9646    /*
     9647     * Allocate a new recompiler state (since we're likely to be called while
     9648     * the default one is fully loaded already with a recompiled TB).
     9649     *
     9650     * This is a bit of overkill, but this isn't a frequently used code path.
     9651     */
     9652    PIEMRECOMPILERSTATE pReNative = iemNativeInit(pVCpu, NULL);
     9653    AssertReturn(pReNative, NULL);
     9654
     9655# if   defined(RT_ARCH_AMD64)
     9656    uint32_t const fAlignMask = 15;
     9657# elif defined(RT_ARCH_ARM64)
     9658    uint32_t const fAlignMask = 31 / 4;
     9659# else
     9660#  error "port me"
     9661# endif
     9662    uint32_t aoffLabels[kIemNativeExitReason_Max] = {0};
     9663    int      rc  = VINF_SUCCESS;
     9664    uint32_t off = 0;
     9665
     9666    IEMNATIVE_TRY_SETJMP(pReNative, rc)
     9667    {
     9668        /*
     9669         * Emit the epilog code.
     9670         */
     9671        aoffLabels[kIemNativeExitReason_Return] = off;
     9672        off = iemNativeEmitCoreEpilog(pReNative, off);
     9673
     9674        /*
     9675         * Generate special jump labels.  All of these gets a copy of the epilog code.
     9676         */
     9677        static struct
     9678        {
     9679            IEMNATIVEEXITREASON  enmExitReason;
     9680            uint32_t (*pfnEmitCore)(PIEMRECOMPILERSTATE pReNative, uint32_t off);
     9681        } const s_aSpecialWithEpilogs[] =
     9682        {
     9683            { kIemNativeExitReason_NonZeroRetOrPassUp,    iemNativeEmitCoreRcFiddling           },
     9684            { kIemNativeExitReason_ReturnBreak,           iemNativeEmitCoreReturnBreak          },
     9685            { kIemNativeExitReason_ReturnBreakFF,         iemNativeEmitCoreReturnBreakFF        },
     9686            { kIemNativeExitReason_ReturnWithFlags,       iemNativeEmitCoreReturnWithFlags      },
     9687        };
     9688        for (uint32_t i = 0; i < RT_ELEMENTS(s_aSpecialWithEpilogs); i++)
     9689        {
     9690            off = iemNativeRecompileEmitAlignmentPadding(pReNative, off, fAlignMask);
     9691            Assert(aoffLabels[s_aSpecialWithEpilogs[i].enmExitReason] == 0);
     9692            aoffLabels[s_aSpecialWithEpilogs[i].enmExitReason] = off;
     9693            off = s_aSpecialWithEpilogs[i].pfnEmitCore(pReNative, off);
     9694            off = iemNativeEmitCoreEpilog(pReNative, off);
     9695        }
     9696
     9697        /*
     9698         * Do what iemNativeEmitReturnBreakViaLookup does.
     9699         */
     9700        static struct
     9701        {
     9702            IEMNATIVEEXITREASON  enmExitReason;
     9703            uintptr_t            pfnHelper;
     9704        } const s_aViaLookup[] =
     9705        {
     9706            {   kIemNativeExitReason_ReturnBreakViaLookup,
     9707                (uintptr_t)iemNativeHlpReturnBreakViaLookup<false /*a_fWithIrqCheck*/>          },
     9708            {   kIemNativeExitReason_ReturnBreakViaLookupWithIrq,
     9709                (uintptr_t)iemNativeHlpReturnBreakViaLookup<true /*a_fWithIrqCheck*/>           },
     9710            {   kIemNativeExitReason_ReturnBreakViaLookupWithTlb,
     9711                (uintptr_t)iemNativeHlpReturnBreakViaLookupWithTlb<false /*a_fWithIrqCheck*/>   },
     9712            {   kIemNativeExitReason_ReturnBreakViaLookupWithTlbAndIrq,
     9713                (uintptr_t)iemNativeHlpReturnBreakViaLookupWithTlb<true /*a_fWithIrqCheck*/>    },
     9714        };
     9715        uint32_t const offReturnBreak = aoffLabels[kIemNativeExitReason_ReturnBreak]; Assert(offReturnBreak != 0);
     9716        for (uint32_t i = 0; i < RT_ELEMENTS(s_aViaLookup); i++)
     9717        {
     9718            off = iemNativeRecompileEmitAlignmentPadding(pReNative, off, fAlignMask);
     9719            Assert(aoffLabels[s_aViaLookup[i].enmExitReason] == 0);
     9720            aoffLabels[s_aViaLookup[i].enmExitReason] = off;
     9721            off = iemNativeEmitCoreViaLookupDoOne(pReNative, off, offReturnBreak, s_aViaLookup[i].pfnHelper);
     9722        }
     9723
     9724        /*
     9725         * Generate simple TB tail labels that just calls a help with a pVCpu
     9726         * arg and either return or longjmps/throws a non-zero status.
     9727         */
     9728        typedef IEM_DECL_NATIVE_HLP_PTR(int, PFNIEMNATIVESIMPLETAILLABELCALL,(PVMCPUCC pVCpu));
     9729        static struct
     9730        {
     9731            IEMNATIVEEXITREASON             enmExitReason;
     9732            bool                            fWithEpilog;
     9733            PFNIEMNATIVESIMPLETAILLABELCALL pfnCallback;
     9734        } const s_aSimpleTailLabels[] =
     9735        {
     9736            {   kIemNativeExitReason_RaiseDe,               false, iemNativeHlpExecRaiseDe },
     9737            {   kIemNativeExitReason_RaiseUd,               false, iemNativeHlpExecRaiseUd },
     9738            {   kIemNativeExitReason_RaiseSseRelated,       false, iemNativeHlpExecRaiseSseRelated },
     9739            {   kIemNativeExitReason_RaiseAvxRelated,       false, iemNativeHlpExecRaiseAvxRelated },
     9740            {   kIemNativeExitReason_RaiseSseAvxFpRelated,  false, iemNativeHlpExecRaiseSseAvxFpRelated },
     9741            {   kIemNativeExitReason_RaiseNm,               false, iemNativeHlpExecRaiseNm },
     9742            {   kIemNativeExitReason_RaiseGp0,              false, iemNativeHlpExecRaiseGp0 },
     9743            {   kIemNativeExitReason_RaiseMf,               false, iemNativeHlpExecRaiseMf },
     9744            {   kIemNativeExitReason_RaiseXf,               false, iemNativeHlpExecRaiseXf },
     9745            {   kIemNativeExitReason_ObsoleteTb,            true,  iemNativeHlpObsoleteTb },
     9746            {   kIemNativeExitReason_NeedCsLimChecking,     true,  iemNativeHlpNeedCsLimChecking },
     9747            {   kIemNativeExitReason_CheckBranchMiss,       true,  iemNativeHlpCheckBranchMiss },
     9748        };
     9749        for (uint32_t i = 0; i < RT_ELEMENTS(s_aSimpleTailLabels); i++)
     9750        {
     9751            off = iemNativeRecompileEmitAlignmentPadding(pReNative, off, fAlignMask);
     9752            Assert(!aoffLabels[s_aSimpleTailLabels[i].enmExitReason]);
     9753            aoffLabels[s_aSimpleTailLabels[i].enmExitReason] = off;
     9754
     9755            /* int pfnCallback(PVMCPUCC pVCpu) */
     9756            off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
     9757            off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)s_aSimpleTailLabels[i].pfnCallback);
     9758
     9759            /* jump back to the return sequence / generate a return sequence. */
     9760            if (!s_aSimpleTailLabels[i].fWithEpilog)
     9761                off = iemNativeEmitJmpToFixed(pReNative, off, aoffLabels[kIemNativeExitReason_Return]);
     9762            else
     9763                off = iemNativeEmitCoreEpilog(pReNative, off);
     9764        }
     9765
     9766
     9767# ifdef VBOX_STRICT
     9768        /* Make sure we've generate code for all labels. */
     9769        for (uint32_t i = kIemNativeExitReason_Invalid + 1; i < RT_ELEMENTS(aoffLabels); i++)
     9770            Assert(aoffLabels[i] != 0 || i == kIemNativeExitReason_Return);
     9771#endif
     9772    }
     9773    IEMNATIVE_CATCH_LONGJMP_BEGIN(pReNative, rc);
     9774    {
     9775        Log(("iemNativeRecompileAttachExecMemChunkCtx: Caught %Rrc while recompiling!\n", rc));
     9776        iemNativeTerm(pReNative);
     9777        return NULL;
     9778    }
     9779    IEMNATIVE_CATCH_LONGJMP_END(pReNative);
     9780
     9781    /*
     9782     * Allocate memory for the context (first) and the common code (last).
     9783     */
     9784    PIEMNATIVEPERCHUNKCTX pCtx;
     9785    uint32_t const        cbCtx               = RT_ALIGN_32(sizeof(*pCtx), 64);
     9786    uint32_t const        cbCode              = off * sizeof(IEMNATIVEINSTR);
     9787    PIEMNATIVEINSTR       paFinalCommonCodeRx = NULL;
     9788    pCtx = (PIEMNATIVEPERCHUNKCTX)iemExecMemAllocatorAllocFromChunk(pVCpu, idxChunk, cbCtx + cbCode, &paFinalCommonCodeRx);
     9789    AssertLogRelMsgReturn(pCtx, ("cbCtx=%#x cbCode=%#x idxChunk=%#x\n", cbCtx, cbCode, idxChunk), NULL);
     9790
     9791    /*
     9792     * Copy over the generated code.
     9793     * There should be no fixups or labels defined here.
     9794     */
     9795    paFinalCommonCodeRx = (PIEMNATIVEINSTR)((uintptr_t)paFinalCommonCodeRx + cbCtx);
     9796    memcpy((PIEMNATIVEINSTR)((uintptr_t)pCtx + cbCtx), pReNative->pInstrBuf, cbCode);
     9797
     9798    Assert(pReNative->cFixups == 0);
     9799    Assert(pReNative->cLabels == 0);
     9800
     9801    /*
     9802     * Initialize the context.
     9803     */
     9804    AssertCompile(kIemNativeExitReason_Invalid == 0);
     9805    AssertCompile(RT_ELEMENTS(pCtx->apExitLabels) == RT_ELEMENTS(aoffLabels));
     9806    pCtx->apExitLabels[kIemNativeExitReason_Invalid] = 0;
     9807    for (uint32_t i = kIemNativeExitReason_Invalid + 1; i < RT_ELEMENTS(pCtx->apExitLabels); i++)
     9808    {
     9809        Assert(aoffLabels[i] != 0 || i == kIemNativeExitReason_Return);
     9810        pCtx->apExitLabels[i] = &paFinalCommonCodeRx[aoffLabels[i]];
     9811        Log10(("    apExitLabels[%u]=%p %s\n", i, pCtx->apExitLabels[i], iemNativeGetLabelName((IEMNATIVELABELTYPE)i, true)));
     9812    }
     9813
     9814    iemExecMemAllocatorReadyForUse(pVCpu, pCtx, cbCtx + cbCode);
     9815
     9816    iemNativeTerm(pReNative);
     9817    return pCtx;
     9818}
     9819
     9820#endif /* IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE */
     9821
    93899822/**
    93909823 * Recompiles the given threaded TB into a native one.
    … …  
    94059838
    94069839    /*
    9407      * The first time thru, we allocate the recompiler state, the other times
    9408      * we just need to reset it before using it again.
     9840     * The first time thru, we allocate the recompiler state and save it,
     9841     * all the other times we'll just reuse the saved one after a quick reset.
    94099842     */
    94109843    PIEMRECOMPILERSTATE pReNative = pVCpu->iem.s.pNativeRecompilerStateR3;
    … …  
    94159848        pReNative = iemNativeInit(pVCpu, pTb);
    94169849        AssertReturn(pReNative, pTb);
     9850        pVCpu->iem.s.pNativeRecompilerStateR3 = pReNative; /* save it */
    94179851    }
    94189852
    … …  
    95329966#ifdef VBOX_WITH_STATISTICS
    95339967            off = iemNativeEmitThreadCallStats(pReNative, off, pCallEntry);
     9968#endif
     9969
     9970#if 0
     9971            if (   pTb->GCPhysPc == 0x00000000000c1240
     9972                && idxCurCall == 67)
     9973                off = iemNativeEmitBrk(pReNative, off, 0xf000);
    95349974#endif
    95359975
    … …  
    960910049#endif
    961010050
     10051        /* Flush any pending writes before returning from the last instruction (RIP updates, etc.). */
     10052        off = iemNativeRegFlushPendingWrites(pReNative, off);
     10053
     10054        /*
     10055         * Successful return, so clear the return register (eax, w0).
     10056         */
     10057        off = iemNativeEmitGprZero(pReNative, off, IEMNATIVE_CALL_RET_GREG);
     10058
     10059#ifndef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
    961110060        /*
    961210061         * Emit the epilog code.
    … …  
    961410063        uint32_t idxReturnLabel;
    961510064        off = iemNativeEmitEpilog(pReNative, off, &idxReturnLabel);
    9616 
     10065#else
     10066        /*
     10067         * Jump to the common per-chunk epilog code.
     10068         */
     10069        //off = iemNativeEmitBrk(pReNative, off, 0x1227);
     10070        off = iemNativeEmitTbExit(pReNative, off, kIemNativeExitReason_Return);
     10071#endif
     10072
     10073#ifndef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
    961710074        /*
    961810075         * Generate special jump labels.
    961910076         */
    9620         if (pReNative->bmLabelTypes & (  RT_BIT_64(kIemNativeLabelType_ReturnBreakViaLookup)
    9621                                        | RT_BIT_64(kIemNativeLabelType_ReturnBreakViaLookupWithIrq)
    9622                                        | RT_BIT_64(kIemNativeLabelType_ReturnBreakViaLookupWithTlb)
    9623                                        | RT_BIT_64(kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq) ))
    9624             off = iemNativeEmitReturnBreakViaLookup(pReNative, off); /* Must come before ReturnBreak! */
    9625 
    9626         if (pReNative->bmLabelTypes & RT_BIT_64(kIemNativeLabelType_ReturnBreak))
     10077        off = iemNativeEmitRcFiddling(pReNative, off, idxReturnLabel);
     10078
     10079        bool const fReturnBreakViaLookup = RT_BOOL(  pReNative->bmLabelTypes
     10080                                                   & (  RT_BIT_64(kIemNativeLabelType_ReturnBreakViaLookup)
     10081                                                      | RT_BIT_64(kIemNativeLabelType_ReturnBreakViaLookupWithIrq)
     10082                                                      | RT_BIT_64(kIemNativeLabelType_ReturnBreakViaLookupWithTlb)
     10083                                                      | RT_BIT_64(kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq)));
     10084        if (fReturnBreakViaLookup)
     10085        {
     10086            uint32_t const idxReturnBreakLabel = iemNativeLabelCreate(pReNative, kIemNativeLabelType_ReturnBreak);
     10087            off = iemNativeEmitReturnBreak(pReNative, off, idxReturnLabel);
     10088            off = iemNativeEmitReturnBreakViaLookup(pReNative, off, idxReturnBreakLabel);
     10089        }
     10090        else if (pReNative->bmLabelTypes & RT_BIT_64(kIemNativeLabelType_ReturnBreak))
    962710091            off = iemNativeEmitReturnBreak(pReNative, off, idxReturnLabel);
    962810092
    … …  
    968910153            } while (fTailLabels);
    969010154        }
     10155
     10156#else /* IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE */
     10157        /*
     10158         * Generate tail labels with jumps to the common per-chunk code.
     10159         */
     10160# ifndef RT_ARCH_AMD64
     10161        Assert(!(pReNative->bmLabelTypes & (RT_BIT_64(kIemNativeLabelType_Return) | RT_BIT_64(kIemNativeLabelType_Invalid))));
     10162        AssertCompile(kIemNativeLabelType_Invalid == 0);
     10163        uint64_t fTailLabels = pReNative->bmLabelTypes & (RT_BIT_64(kIemNativeLabelType_LastTbExit + 1U) - 2U);
     10164        if (fTailLabels)
     10165        {
     10166            do
     10167            {
     10168                IEMNATIVELABELTYPE const enmLabel = (IEMNATIVELABELTYPE)(ASMBitFirstSetU64(fTailLabels) - 1U);
     10169                fTailLabels &= ~RT_BIT_64(enmLabel);
     10170
     10171                uint32_t const idxLabel = iemNativeLabelFind(pReNative, enmLabel);
     10172                AssertContinue(idxLabel != UINT32_MAX);
     10173                iemNativeLabelDefine(pReNative, idxLabel, off);
     10174                off = iemNativeEmitTbExit(pReNative, off, (IEMNATIVEEXITREASON)enmLabel);
     10175            } while (fTailLabels);
     10176        }
     10177# else
     10178        Assert(!(pReNative->bmLabelTypes & (RT_BIT_64(kIemNativeLabelType_LastTbExit + 1) - 1U))); /* Should not be used! */
     10179# endif
     10180#endif /* IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE */
    969110181    }
    969210182    IEMNATIVE_CATCH_LONGJMP_BEGIN(pReNative, rc);
    … …  
    972310213        iemTbAllocatorProcessDelayedFrees(pVCpu, pVCpu->iem.s.pTbAllocatorR3);
    972410214
    9725     PIEMNATIVEINSTR paFinalInstrBufRx = NULL;
    9726     PIEMNATIVEINSTR const paFinalInstrBuf = (PIEMNATIVEINSTR)iemExecMemAllocatorAlloc(pVCpu, off * sizeof(IEMNATIVEINSTR), pTb, (void **)&paFinalInstrBufRx);
     10215    PIEMNATIVEINSTR       paFinalInstrBufRx = NULL;
     10216#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     10217    PCIEMNATIVEPERCHUNKCTX pCtx             = NULL;
     10218    PIEMNATIVEINSTR const paFinalInstrBuf   = iemExecMemAllocatorAlloc(pVCpu, off * sizeof(IEMNATIVEINSTR), pTb,
     10219                                                                       &paFinalInstrBufRx, &pCtx);
     10220
     10221#else
     10222    PIEMNATIVEINSTR const paFinalInstrBuf   = iemExecMemAllocatorAlloc(pVCpu, off * sizeof(IEMNATIVEINSTR), pTb,
     10223                                                                       &paFinalInstrBufRx, NULL);
     10224#endif
    972710225    AssertReturn(paFinalInstrBuf, pTb);
    972810226    memcpy(paFinalInstrBuf, pReNative->pInstrBuf, off * sizeof(paFinalInstrBuf[0]));
    … …  
    975410252                Assert(paFixups[i].off < off);
    975510253                int32_t const offDisp = paLabels[paFixups[i].idxLabel].off - paFixups[i].off + paFixups[i].offAddend;
    9756                 Assert(offDisp >= -262144 && offDisp < 262144);
     10254                Assert(offDisp >= -33554432 && offDisp < 33554432);
    975710255                *Ptr.pu32 = (*Ptr.pu32 & UINT32_C(0xfc000000)) | ((uint32_t)offDisp & UINT32_C(0x03ffffff));
    975810256                continue;
    … …  
    978410282        AssertFailed();
    978510283    }
     10284
     10285#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     10286    /*
     10287     * Apply TB exit fixups.
     10288     */
     10289    PIEMNATIVEEXITFIXUP const paTbExitFixups   = pReNative->paTbExitFixups;
     10290    uint32_t const            cTbExitFixups    = pReNative->cTbExitFixups;
     10291    for (uint32_t i = 0; i < cTbExitFixups; i++)
     10292    {
     10293        Assert(paTbExitFixups[i].off < off);
     10294        Assert(   paTbExitFixups[i].enmExitReason < kIemNativeExitReason_Max
     10295               && paTbExitFixups[i].enmExitReason > kIemNativeExitReason_Invalid);
     10296        RTPTRUNION const Ptr = { &paFinalInstrBuf[paTbExitFixups[i].off] };
     10297
     10298# if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
     10299        Assert(paTbExitFixups[i].off + 4 <= off);
     10300        intptr_t const offDisp = pCtx->apExitLabels[paTbExitFixups[i].enmExitReason] - &paFinalInstrBufRx[paTbExitFixups[i].off + 4];
     10301        Assert(offDisp >= INT32_MIN && offDisp <= INT32_MAX);
     10302        *Ptr.pi32 = (int32_t)offDisp;
     10303
     10304# elif defined(RT_ARCH_ARM64)
     10305        intptr_t const offDisp = pCtx->apExitLabels[paTbExitFixups[i].enmExitReason] - &paFinalInstrBufRx[paTbExitFixups[i].off];
     10306        Assert(offDisp >= -33554432 && offDisp < 33554432);
     10307        *Ptr.pu32 = (*Ptr.pu32 & UINT32_C(0xfc000000)) | ((uint32_t)offDisp & UINT32_C(0x03ffffff));
     10308
     10309# else
     10310#  error "Port me!"
     10311# endif
     10312    }
     10313#endif
    978610314
    978710315    iemExecMemAllocatorReadyForUse(pVCpu, paFinalInstrBufRx, off * sizeof(IEMNATIVEINSTR));
    … …  
    981210340    {
    981310341        Log3(("----------------------------------------- %d calls ---------------------------------------\n", cCallsOrg));
    9814         iemNativeDisassembleTb(pTb, DBGFR3InfoLogHlp());
     10342        iemNativeDisassembleTb(pVCpu, pTb, DBGFR3InfoLogHlp());
    981510343# if defined(DEBUG_bird) || defined(DEBUG_aeichner)
    981610344        RTLogFlush(NULL);

  • trunk/src/VBox/VMM/VMMR3/IEMR3.cpp

    r105177 r105261  
    14031403            case IEMTB_F_TYPE_NATIVE:
    14041404                pHlp->pfnPrintf(pHlp, "PC=%RGp fFlags=%#x on #%u: %p - native\n", GCPhysPc, fFlags, pVCpu->idCpu, pTb);
    1405                 iemNativeDisassembleTb(pTb, pHlp);
     1405                iemNativeDisassembleTb(pVCpu, pTb, pHlp);
    14061406                break;
    14071407# endif

  • trunk/src/VBox/VMM/include/IEMInternal.h

    r105253 r105261  
    63786378
    63796379/* Native recompiler public bits: */
     6380
    63806381DECLHIDDEN(PIEMTB)  iemNativeRecompile(PVMCPUCC pVCpu, PIEMTB pTb) RT_NOEXCEPT;
    6381 DECLHIDDEN(void)    iemNativeDisassembleTb(PCIEMTB pTb, PCDBGFINFOHLP pHlp) RT_NOEXCEPT;
     6382DECLHIDDEN(void)    iemNativeDisassembleTb(PVMCPU pVCpu, PCIEMTB pTb, PCDBGFINFOHLP pHlp) RT_NOEXCEPT;
    63826383int                 iemExecMemAllocatorInit(PVMCPU pVCpu, uint64_t cbMax, uint64_t cbInitial, uint32_t cbChunk) RT_NOEXCEPT;
    6383 DECLHIDDEN(void *)  iemExecMemAllocatorAlloc(PVMCPU pVCpu, uint32_t cbReq, PIEMTB pTb, void **ppvExec) RT_NOEXCEPT;
     6384DECLHIDDEN(PIEMNATIVEINSTR) iemExecMemAllocatorAlloc(PVMCPU pVCpu, uint32_t cbReq, PIEMTB pTb, PIEMNATIVEINSTR *ppaExec,
     6385                                                     struct IEMNATIVEPERCHUNKCTX const **ppChunkCtx) RT_NOEXCEPT;
     6386DECLHIDDEN(PIEMNATIVEINSTR) iemExecMemAllocatorAllocFromChunk(PVMCPU pVCpu, uint32_t idxChunk, uint32_t cbReq,
     6387                                                              PIEMNATIVEINSTR *ppaExec);
    63846388DECLHIDDEN(void)    iemExecMemAllocatorReadyForUse(PVMCPUCC pVCpu, void *pv, size_t cb) RT_NOEXCEPT;
    63856389void                iemExecMemAllocatorFree(PVMCPU pVCpu, void *pv, size_t cb) RT_NOEXCEPT;
    63866390DECLASM(DECL_NO_RETURN(void)) iemNativeTbLongJmp(void *pvFramePointer, int rc) RT_NOEXCEPT;
     6391DECLHIDDEN(struct IEMNATIVEPERCHUNKCTX const *) iemExecMemGetTbChunkCtx(PVMCPU pVCpu, PCIEMTB pTb);
     6392DECLHIDDEN(struct IEMNATIVEPERCHUNKCTX const *) iemNativeRecompileAttachExecMemChunkCtx(PVMCPU pVCpu, uint32_t idxChunk);
    63876393
    63886394#endif /* !RT_IN_ASSEMBLER - ASM-NOINC-END */

  • trunk/src/VBox/VMM/include/IEMN8veRecompiler.h

    r104802 r105261  
    5858# define IEMNATIVE_WITH_EFLAGS_SKIPPING
    5959#endif
    60 
    6160
    6261/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING
    … …  
    8281#endif
    8382
    84 /** @def IEMNATIVE_WITH_RECOMPILER_EPILOGUE_SINGLETON
    85  * Enables having only a single epilogue for native TBs. */
    86 #if defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING)
    87 # define IEMNATIVE_WITH_RECOMPILER_EPILOGUE_SINGLETON
    88 #endif
     83/** @def IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     84 * Enable this to use common epilogue and tail code for all TBs in a chunk. */
     85#if 1 || defined(DOXYGEN_RUNNING)
     86# define IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     87#endif
     88
    8989
    9090/** @name Stack Frame Layout
    … …  
    466466    kIemNativeExitReason_NeedCsLimChecking,
    467467    kIemNativeExitReason_CheckBranchMiss,
    468     kIemNativeExitReason_Return, /** @todo Eliminate (needed for the compile assertion below). */
    469468    kIemNativeExitReason_ReturnBreak,
    470469    kIemNativeExitReason_ReturnBreakFF,
    … …  
    475474    kIemNativeExitReason_ReturnWithFlags,
    476475    kIemNativeExitReason_NonZeroRetOrPassUp,
     476    kIemNativeExitReason_Return,                /**< This is a little bit special, but needs to be included here. */
     477    kIemNativeExitReason_Max
    477478} IEMNATIVEEXITREASON;
    478479
    … …  
    503504    kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
    504505    /* Manually defined labels. */
    505     kIemNativeLabelType_Return,
    506506    kIemNativeLabelType_ReturnBreak,
    507507    kIemNativeLabelType_ReturnBreakFF,
    … …  
    512512    kIemNativeLabelType_ReturnWithFlags,
    513513    kIemNativeLabelType_NonZeroRetOrPassUp,
    514     /** The last fixup for branches that can span almost the whole TB length. */
    515     kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_NonZeroRetOrPassUp,
     514    kIemNativeLabelType_Return,
     515    /** The last fixup for branches that can span almost the whole TB length.
     516     * @note Whether kIemNativeLabelType_Return needs to be one of these is
     517     *       a bit questionable, since nobody jumps to it except other tail code. */
     518    kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_Return,
     519    /** The last fixup for branches that exits the TB. */
     520    kIemNativeLabelType_LastTbExit        = kIemNativeLabelType_Return,
    516521
    517522    /*
    … …  
    531536} IEMNATIVELABELTYPE;
    532537
    533 /* Temporary kludge until all jumps to TB exit labels are converted to the new TB exiting style,
    534  * see @bugref{10677}. */
     538/** Temporary kludge until all jumps to TB exit labels are converted to the new TB exiting style,
     539 * see @bugref{10677}.
     540 * @note update bird: This won't happen, unfortunately, since we'll keep using
     541 *       the local labels on arm64 so we can avoid inverting branch conditions
     542 *       and inserting extra of unconditional branches in order to reach the
     543 *       common code.  Instead we'll have everyone jump to the same tail lable
     544 *       which then jumps to the common (per chunk) code. */
    535545#define IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(a_Reason) \
    536546    ((int)kIemNativeLabelType_ ## a_Reason == (int)kIemNativeExitReason_ ## a_Reason)
    … …  
    547557              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(NeedCsLimChecking)
    548558              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(CheckBranchMiss)
    549               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(Return)
    550559              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreak)
    551560              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreakFF)
    … …  
    555564              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreakViaLookupWithTlbAndIrq)
    556565              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnWithFlags)
    557               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(NonZeroRetOrPassUp));
     566              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(NonZeroRetOrPassUp)
     567              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(Return));
     568AssertCompile((int)kIemNativeExitReason_Max == (int)kIemNativeLabelType_LastTbExit + 1);
    558569
    559570
    … …  
    605616/** Pointer to a native code generator fixup. */
    606617typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
     618
     619#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     620
     621/** Native code generator fixup to per chunk TB tail code. */
     622typedef struct IEMNATIVEEXITFIXUP
     623{
     624    /** Code offset of the fixup location. */
     625    uint32_t    off;
     626    /** The exit reason (IEMNATIVEEXITREASON). */
     627    uint32_t    enmExitReason;
     628} IEMNATIVEEXITFIXUP;
     629/** Pointer to a native code generator TB exit fixup. */
     630typedef IEMNATIVEEXITFIXUP *PIEMNATIVEEXITFIXUP;
     631
     632/**
     633 * Per executable memory chunk context with addresses for common code.
     634 */
     635typedef struct IEMNATIVEPERCHUNKCTX
     636{
     637    /** Pointers to the exit labels */
     638    PIEMNATIVEINSTR apExitLabels[kIemNativeExitReason_Max];
     639} IEMNATIVEPERCHUNKCTX;
     640/** Pointer to per-chunk recompiler context. */
     641typedef IEMNATIVEPERCHUNKCTX *PIEMNATIVEPERCHUNKCTX;
     642/** Pointer to const per-chunk recompiler context. */
     643typedef const IEMNATIVEPERCHUNKCTX *PCIEMNATIVEPERCHUNKCTX;
     644
     645#endif /* IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE */
    607646
    608647
    … …  
    14261465    PIEMNATIVEFIXUP             paFixups;
    14271466
     1467#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     1468    /** Actual number of fixups in paTbExitFixups. */
     1469    uint32_t                    cTbExitFixups;
     1470    /** Max number of entries allowed in paTbExitFixups before reallocating it. */
     1471    uint32_t                    cTbExitFixupsAlloc;
     1472    /** Buffer used by the recompiler for recording fixups when generating code. */
     1473    PIEMNATIVEEXITFIXUP         paTbExitFixups;
     1474#endif
     1475
    14281476#ifdef IEMNATIVE_WITH_TB_DEBUG_INFO
    14291477    /** Number of debug info entries allocated for pDbgInfo. */
    … …  
    16381686DECL_HIDDEN_THROW(void)     iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
    16391687                                              IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
     1688#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     1689DECL_HIDDEN_THROW(void)     iemNativeAddTbExitFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, IEMNATIVEEXITREASON enmExitReason);
     1690#endif
    16401691DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);
    16411692
    … …  
    17791830IEM_DECL_NATIVE_HLP_PROTO(int,      iemNativeHlpNeedCsLimChecking,(PVMCPUCC pVCpu));
    17801831IEM_DECL_NATIVE_HLP_PROTO(int,      iemNativeHlpCheckBranchMiss,(PVMCPUCC pVCpu));
     1832IEM_DECL_NATIVE_HLP_PROTO(int,      iemNativeHlpExecRaiseAvxRelated,(PVMCPUCC pVCpu));
     1833IEM_DECL_NATIVE_HLP_PROTO(int,      iemNativeHlpExecRaiseSseRelated,(PVMCPUCC pVCpu));
     1834IEM_DECL_NATIVE_HLP_PROTO(int,      iemNativeHlpExecRaiseSseAvxFpRelated,(PVMCPUCC pVCpu));
    17811835
    17821836IEM_DECL_NATIVE_HLP_PROTO(uint64_t, iemNativeHlpMemFetchDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg));
    … …  
    25852639#endif
    25862640
    2587 #ifdef IEMNATIVE_WITH_RECOMPILER_EPILOGUE_SINGLETON
    2588 /** The common epilog jumped to from a TB.
    2589  * @note This is not a callable function! */
    2590 extern "C" IEM_DECL_NATIVE_HLP_DEF(int, iemNativeTbEpilog, (void));
    2591 #endif
    2592 
    25932641#endif /* !RT_IN_ASSEMBLER - ASM-NOINC-END */
    25942642

  • trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h

    r105036 r105261  
    74917491
    74927492
     7493/**
     7494 * Emits code that jumps to @a offTarget if @a iGprSrc is not zero.
     7495 *
     7496 * The operand size is given by @a f64Bit.
     7497 */
     7498DECL_FORCE_INLINE_THROW(uint32_t)
     7499iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToFixedEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off,
     7500                                                     uint8_t iGprSrc, bool f64Bit, bool fJmpIfNotZero, uint32_t offTarget)
     7501{
     7502#ifdef RT_ARCH_AMD64
     7503    /* test reg32,reg32  / test reg64,reg64 */
     7504    if (f64Bit)
     7505        pCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_R | X86_OP_REX_B);
     7506    else if (iGprSrc >= 8)
     7507        pCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
     7508    pCodeBuf[off++] = 0x85;
     7509    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
     7510
     7511    /* jnz idxLabel  */
     7512    off = iemNativeEmitJccToFixedEx(pCodeBuf, off, offTarget,
     7513                                    fJmpIfNotZero ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
     7514
     7515#elif defined(RT_ARCH_ARM64)
     7516    pCodeBuf[off] = Armv8A64MkInstrCbzCbnz(fJmpIfNotZero, (int32_t)(offTarget - off), iGprSrc, f64Bit);
     7517    off++;
     7518
     7519#else
     7520# error "Port me!"
     7521#endif
     7522    return off;
     7523}
     7524
     7525
     7526/**
     7527 * Emits code that jumps to @a offTarget if @a iGprSrc is not zero.
     7528 *
     7529 * The operand size is given by @a f64Bit.
     7530 */
     7531DECL_FORCE_INLINE_THROW(uint32_t)
     7532iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
     7533                                                   bool f64Bit, bool fJmpIfNotZero, uint32_t offTarget)
     7534{
     7535#ifdef RT_ARCH_AMD64
     7536    off = iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 3 + 6),
     7537                                                               off, iGprSrc, f64Bit, fJmpIfNotZero, offTarget);
     7538#elif defined(RT_ARCH_ARM64)
     7539    off = iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 1),
     7540                                                               off, iGprSrc, f64Bit, fJmpIfNotZero, offTarget);
     7541#else
     7542# error "Port me!"
     7543#endif
     7544    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     7545    return off;
     7546}
     7547
     7548
    74937549/* if (Grp1 == 0) Jmp idxLabel; */
    74947550
    … …  
    75307586    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);
    75317587    return iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);
     7588}
     7589
     7590
     7591/**
     7592 * Emits code that jumps to @a offTarget if @a iGprSrc is zero.
     7593 *
     7594 * The operand size is given by @a f64Bit.
     7595 */
     7596DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     7597                                                                      uint8_t iGprSrc, bool f64Bit, uint32_t offTarget)
     7598{
     7599    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToFixed(pReNative, off, iGprSrc, f64Bit, false /*fJmpIfNotZero*/, offTarget);
    75327600}
    75337601
    … …  
    79378005*********************************************************************************************************************************/
    79388006
     8007/**
     8008 * Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
     8009 */
     8010DECL_FORCE_INLINE_THROW(uint32_t)
     8011iemNativeEmitJccTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
     8012                         IEMNATIVEEXITREASON enmExitReason, IEMNATIVEINSTRCOND enmCond)
     8013{
     8014#if defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) && defined(RT_ARCH_AMD64)
     8015    /* jcc rel32 */
     8016    pCodeBuf[off++] = 0x0f;
     8017    pCodeBuf[off++] = (uint8_t)enmCond | 0x80;
     8018    iemNativeAddTbExitFixup(pReNative, off, enmExitReason);
     8019    pCodeBuf[off++] = 0x00;
     8020    pCodeBuf[off++] = 0x00;
     8021    pCodeBuf[off++] = 0x00;
     8022    pCodeBuf[off++] = 0x00;
     8023
     8024#else
     8025    /* ARM64 doesn't have the necessary jump range, so we jump via local label
     8026       just like when we keep everything local. */
     8027    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, (IEMNATIVELABELTYPE)enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     8028    off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabel, enmCond);
     8029#endif
     8030    return off;
     8031}
     8032
    79398033
    79408034/**
    … …  
    79458039                       IEMNATIVEEXITREASON enmExitReason, IEMNATIVEINSTRCOND enmCond)
    79468040{
     8041#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     8042# ifdef RT_ARCH_AMD64
     8043    PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
     8044# elif defined(RT_ARCH_ARM64)
     8045    PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
     8046# else
     8047#  error "Port me!"
     8048# endif
     8049    return iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, enmExitReason, enmCond);
     8050#else
    79478051    return iemNativeEmitJccToNewLabel(pReNative, off, (IEMNATIVELABELTYPE)enmExitReason, 0 /*uData*/, enmCond);
     8052#endif
    79488053}
    79498054
    … …  
    80148119
    80158120DECL_INLINE_THROW(uint32_t)
    8016 iemNativeEmitTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEEXITREASON enmExitReason)
    8017 {
    8018     return iemNativeEmitJmpToNewLabel(pReNative, off, (IEMNATIVELABELTYPE)enmExitReason);
    8019 }
    8020 
    8021 
    8022 DECL_INLINE_THROW(uint32_t)
    80238121iemNativeEmitTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off, IEMNATIVEEXITREASON enmExitReason)
    80248122{
     8123#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     8124# ifdef RT_ARCH_AMD64
     8125    /* jmp rel32 */
     8126    pCodeBuf[off++] = 0xe9;
     8127    iemNativeAddTbExitFixup(pReNative, off, enmExitReason);
     8128    pCodeBuf[off++] = 0xfe;
     8129    pCodeBuf[off++] = 0xff;
     8130    pCodeBuf[off++] = 0xff;
     8131    pCodeBuf[off++] = 0xff;
     8132
     8133# elif defined(RT_ARCH_ARM64)
     8134    iemNativeAddTbExitFixup(pReNative, off, enmExitReason);
     8135    pCodeBuf[off++] = Armv8A64MkInstrB(-1);
     8136
     8137# else
     8138#  error "Port me!"
     8139# endif
     8140    return off;
     8141
     8142#else
    80258143    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, (IEMNATIVELABELTYPE)enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
    80268144    return iemNativeEmitJmpToLabelEx(pReNative, pCodeBuf, off, idxLabel);
     8145#endif
     8146}
     8147
     8148
     8149DECL_INLINE_THROW(uint32_t)
     8150iemNativeEmitTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEEXITREASON enmExitReason)
     8151{
     8152#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
     8153# ifdef RT_ARCH_AMD64
     8154    PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
     8155
     8156    /* jmp rel32 */
     8157    pCodeBuf[off++] = 0xe9;
     8158    iemNativeAddTbExitFixup(pReNative, off, enmExitReason);
     8159    pCodeBuf[off++] = 0xfe;
     8160    pCodeBuf[off++] = 0xff;
     8161    pCodeBuf[off++] = 0xff;
     8162    pCodeBuf[off++] = 0xff;
     8163
     8164# elif defined(RT_ARCH_ARM64)
     8165    PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
     8166    iemNativeAddTbExitFixup(pReNative, off, enmExitReason);
     8167    pCodeBuf[off++] = Armv8A64MkInstrB(-1);
     8168
     8169# else
     8170#  error "Port me!"
     8171# endif
     8172    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     8173    return off;
     8174
     8175#else
     8176    return iemNativeEmitJmpToNewLabel(pReNative, off, (IEMNATIVELABELTYPE)enmExitReason);
     8177#endif
    80278178}
    80288179
    … …  
    81018252 * Emits code that exits the current TB with the given reason if 32-bit @a iGprSrc equals @a uImm.
    81028253 */
    8103 DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32EqualsImmAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    8104                                                                        uint8_t iGprSrc, uint32_t uImm, IEMNATIVEEXITREASON enmExitReason)
     8254DECL_INLINE_THROW(uint32_t)
     8255iemNativeEmitTestIfGpr32EqualsImmAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     8256                                           uint8_t iGprSrc, uint32_t uImm, IEMNATIVEEXITREASON enmExitReason)
    81058257{
    81068258    off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
    … …  
    81168268 * @note On ARM64 the range is only +/-8191 instructions.
    81178269 */
    8118 DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndTbExitIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    8119                                                                     uint8_t iGprSrc, uint8_t iBitNo, IEMNATIVEEXITREASON enmExitReason)
    8120 {
     8270DECL_INLINE_THROW(uint32_t)
     8271iemNativeEmitTestBitInGprAndTbExitIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     8272                                        uint8_t iGprSrc, uint8_t iBitNo, IEMNATIVEEXITREASON enmExitReason)
     8273{
     8274#if defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) && defined(RT_ARCH_AMD64)
     8275    Assert(iBitNo < 64);
     8276    uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
     8277    if (iBitNo < 8)
     8278    {
     8279        /* test Eb, imm8 */
     8280        if (iGprSrc >= 4)
     8281            pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
     8282        pbCodeBuf[off++] = 0xf6;
     8283        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
     8284        pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
     8285        off = iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kIemNativeInstrCond_ne);
     8286    }
     8287    else
     8288    {
     8289        /* bt Ev, imm8 */
     8290        if (iBitNo >= 32)
     8291            pbCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
     8292        else if (iGprSrc >= 8)
     8293            pbCodeBuf[off++] = X86_OP_REX_B;
     8294        pbCodeBuf[off++] = 0x0f;
     8295        pbCodeBuf[off++] = 0xba;
     8296        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
     8297        pbCodeBuf[off++] = iBitNo;
     8298        off = iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kIemNativeInstrCond_c);
     8299    }
     8300    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     8301    return off;
     8302
     8303#else
     8304    /* ARM64 doesn't have the necessary jump range, so we jump via local label
     8305       just like when we keep everything local. */
    81218306    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, (IEMNATIVELABELTYPE)enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
    81228307    return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /*fJmpIfSet*/);
     8308#endif
    81238309}
    81248310
    … …  
    81338319                                           uint8_t iGprSrc, bool f64Bit, IEMNATIVEEXITREASON enmExitReason)
    81348320{
     8321#if defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) && defined(RT_ARCH_AMD64)
     8322    /* test reg32,reg32  / test reg64,reg64 */
     8323    if (f64Bit)
     8324        pCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_R | X86_OP_REX_B);
     8325    else if (iGprSrc >= 8)
     8326        pCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
     8327    pCodeBuf[off++] = 0x85;
     8328    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
     8329
     8330    /* jnz idxLabel  */
     8331    return iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, enmExitReason, kIemNativeInstrCond_ne);
     8332
     8333#else
     8334    /* ARM64 doesn't have the necessary jump range, so we jump via local label
     8335       just like when we keep everything local. */
    81358336    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, (IEMNATIVELABELTYPE)enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
    81368337    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
    81378338                                                                f64Bit, true /*fJmpIfNotZero*/, idxLabel);
    8138 }
    8139 
    8140 
    8141 /**
    8142  * Emits code to exit the current TB on the given condition.
    8143  */
    8144 DECL_INLINE_THROW(uint32_t)
    8145 iemNativeEmitJccTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off, IEMNATIVEEXITREASON enmExitReason, IEMNATIVEINSTRCOND enmCond)
    8146 {
    8147     uint32_t const idxLabel = iemNativeLabelCreate(pReNative, (IEMNATIVELABELTYPE)enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
    8148 #ifdef RT_ARCH_AMD64
    8149     off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabel, enmCond);
    8150 #elif defined(RT_ARCH_ARM64)
    8151     off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabel, enmCond);
    8152 #else
    8153 # error "Port me!"
    8154 #endif
    8155     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    8156     return off;
     8339#endif
    81578340}
    81588341
    … …  
    81668349                                                                     uint8_t iGprSrc, bool f64Bit, IEMNATIVEEXITREASON enmExitReason)
    81678350{
     8351#if defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) && defined(RT_ARCH_AMD64)
     8352    off = iemNativeEmitTestIfGprIsNotZeroAndTbExitEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 3 + 6),
     8353                                                     off, iGprSrc, f64Bit, enmExitReason);
     8354    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     8355    return off;
     8356#else
    81688357    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, (IEMNATIVELABELTYPE)enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
    81698358    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, true /*fJmpIfNotZero*/, idxLabel);
     8359#endif
    81708360}
    81718361
Note: See TracChangeset for help on using the changeset viewer.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette