VirtualBox

Changeset 105673 in vbox for trunk


Ignore:
Timestamp:
Aug 14, 2024 1:57:57 PM (5 months ago)
Author:
vboxsync
Message:

VMM/IEM,TM: Do full-TB looping. Redid timer polling in the recompiler. Rewrote the Blt_CheckIrq code, eliminating a conditional. Fixed some TLB related assertions. Moved some IEMCPU members around in hope of better cache-locality. bugref:10656

Location:
trunk
Files:
15 edited

Legend:

Unmodified
Added
Removed

  • trunk/include/VBox/err.h

    r105266 r105673  
    25202520 *  handling. */
    25212521#define VINF_IEM_REEXEC_FINISH_WITH_FLAGS           (5312)
     2522/** Recompiled execution: Jump back in the same TB. */
     2523#define VINF_IEM_REEXEC_JUMP                        (5313)
    25222524
    25232525/** Recompilation: End translation block. */

  • trunk/include/VBox/vmm/tm.h

    r101088 r105673  
    272272
    273273VMMDECL(bool)           TMTimerPollBool(PVMCC pVM, PVMCPUCC pVCpu);
    274 VMMDECL(bool)           TMTimerPollBoolWith32BitMilliTS(PVMCC pVM, PVMCPUCC pVCpu, uint32_t *pmsNow);
     274VMM_INT_DECL(bool)      TMTimerPollBoolWith32BitMilliTS(PVMCC pVM, PVMCPUCC pVCpu, uint32_t *pmsNow);
     275VMM_INT_DECL(bool)      TMTimerPollBoolWithNanoTS(PVMCC pVM, PVMCPUCC pVCpu, uint64_t *pnsNow);
    275276VMM_INT_DECL(void)      TMTimerPollVoid(PVMCC pVM, PVMCPUCC pVCpu);
    276277VMM_INT_DECL(uint64_t)  TMTimerPollGIP(PVMCC pVM, PVMCPUCC pVCpu, uint64_t *pu64Delta);

  • trunk/include/VBox/vmm/vm.h

    r105227 r105673  
    417417 *
    418418 * Available VM bits:
    419  *      0, 1, 5, 6, 7, 13, 14, 15, 16, 17, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30
     419 *      5, 6, 7, 13, 14, 15, 16, 17, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30
    420420 *
    421421 *
    … …  
    427427 * @{
    428428 */
     429/* Bit 0, bit 1: Reserved and must not be reused.  The recompiler ASSUMES it
     430   can OR the local and global FFs together and keept the two
     431   VMCPU_FF_INTERRUPT_XXX flags uncorrupted.  */
    429432/** The virtual sync clock has been stopped, go to TM until it has been
    430433 *  restarted... */

  • trunk/src/VBox/VMM/VMMAll/IEMAll.cpp

    r105616 r105673  
    16031603            {
    16041604                pVCpu->iem.s.fTbCrossedPage     |= offPg == 0 || pVCpu->iem.s.fTbBranched != 0; /** @todo Spurious load effect on branch handling? */
     1605#  if 0 /* unused */
    16051606                pVCpu->iem.s.GCPhysInstrBufPrev  = pVCpu->iem.s.GCPhysInstrBuf;
    1606 
     1607#  endif
    16071608                pVCpu->iem.s.offInstrNextByte = offPg + (uint32_t)cbDst;
    16081609                pVCpu->iem.s.uInstrBufPc      = GCPtrFirst & ~(RTGCPTR)X86_PAGE_OFFSET_MASK;
    … …  
    16971698            uint32_t const offPg = (GCPtrFirst & X86_PAGE_OFFSET_MASK);
    16981699            pVCpu->iem.s.fTbCrossedPage     |= offPg == 0 || pVCpu->iem.s.fTbBranched != 0;
     1700#  if 0 /* unused */
    16991701            pVCpu->iem.s.GCPhysInstrBufPrev  = pVCpu->iem.s.GCPhysInstrBuf;
    1700 
     1702#  endif
    17011703            pVCpu->iem.s.offCurInstrStart = (int16_t)(offPg - cbInstr);
    17021704            pVCpu->iem.s.offInstrNextByte = offPg + cbInstr + cbToRead;
    … …  
    62916293                       || (   (    IEM_GET_CPL(pVCpu) != 3
    62926294                               || (fAccess & IEM_ACCESS_WHAT_SYS))
    6293                            && (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)) )
     6295                           && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)) )
    62946296                    && (   (WalkFast.fEffective & X86_PTE_US)
    62956297                        || IEM_GET_CPL(pVCpu) != 3
    … …  
    71237125        Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_ACCESSED));
    71247126        Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_DIRTY) || !(fAccess & IEM_ACCESS_TYPE_WRITE));
    7125         Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_WRITE) || !(fAccess & IEM_ACCESS_TYPE_WRITE));
     7127        Assert(   !(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_WRITE)
     7128               || !(fAccess & IEM_ACCESS_TYPE_WRITE)
     7129               || (fQPage & (PGMQPAGE_F_CR0_WP0 | PGMQPAGE_F_USER_MODE)) == PGMQPAGE_F_CR0_WP0);
    71267130        Assert(   !(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER)
    71277131               || IEM_GET_CPL(pVCpu) != 3
    … …  
    75377541        pTlbe->pbMappingR3      = NULL;
    75387542        Assert(!(pTlbe->fFlagsAndPhysRev & ((fNoWriteNoDirty & IEMTLBE_F_PT_NO_DIRTY) | IEMTLBE_F_PT_NO_ACCESSED)));
    7539         Assert(!(pTlbe->fFlagsAndPhysRev & fNoWriteNoDirty & IEMTLBE_F_PT_NO_WRITE));
     7543        Assert(   !(pTlbe->fFlagsAndPhysRev & fNoWriteNoDirty & IEMTLBE_F_PT_NO_WRITE)
     7544               || (fQPage & (PGMQPAGE_F_CR0_WP0 | PGMQPAGE_F_USER_MODE)) == PGMQPAGE_F_CR0_WP0);
    75407545        Assert(!(pTlbe->fFlagsAndPhysRev & fNoUser & IEMTLBE_F_PT_NO_USER));
    75417546

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompBltIn.cpp

    r105271 r105673  
    203203
    204204/**
     205 * Worker for the CheckIrq, CheckTimers and CheckTimersAndIrq builtins below.
     206 */
     207template<bool const a_fCheckTimers, bool const a_fCheckIrqs>
     208DECL_FORCE_INLINE(uint32_t) iemNativeRecompFunc_BltIn_CheckTimersAndIrqsCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off)
     209{
     210    uint8_t const         idxEflReg  = !a_fCheckIrqs ? UINT8_MAX
     211                                     : iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
     212                                                                       kIemNativeGstRegUse_ReadOnly);
     213    uint8_t const         idxTmpReg1 = iemNativeRegAllocTmp(pReNative, &off);
     214    uint8_t const         idxTmpReg2 = a_fCheckIrqs ? iemNativeRegAllocTmp(pReNative, &off) : UINT8_MAX;
     215    PIEMNATIVEINSTR const pCodeBuf   = iemNativeInstrBufEnsure(pReNative, off, RT_ARCH_VAL == RT_ARCH_VAL_AMD64 ? 72 : 32);
     216
     217    /*
     218     * First we decrement the timer poll counter, if so desired.
     219     */
     220    if (a_fCheckTimers)
     221    {
     222# ifdef RT_ARCH_AMD64
     223        /* dec  [rbx + cIrqChecksTillNextPoll] */
     224        pCodeBuf[off++] = 0xff;
     225        off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, 1, RT_UOFFSETOF(VMCPU, iem.s.cIrqChecksTillNextPoll));
     226
     227        /* jz   ReturnBreakFF */
     228        off = iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, kIemNativeLabelType_ReturnBreakFF, kIemNativeInstrCond_e);
     229
     230# elif defined(RT_ARCH_ARM64)
     231        AssertCompile(RTASSERT_OFFSET_OF(VMCPU, iem.s.cIrqChecksTillNextPoll) < _4K * sizeof(uint32_t));
     232        off = iemNativeEmitLoadGprFromVCpuU32Ex(pCodeBuf, off, idxTmpReg1, RT_UOFFSETOF(VMCPU, iem.s.cIrqChecksTillNextPoll));
     233        pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(idxTmpReg1, idxTmpReg1, 1, false /*f64Bit*/);
     234        off = iemNativeEmitStoreGprToVCpuU32Ex(pCodeBuf, off, idxTmpReg1, RT_UOFFSETOF(VMCPU, iem.s.cIrqChecksTillNextPoll));
     235
     236        /* cbz reg1, ReturnBreakFF */
     237        off = iemNativeEmitTestIfGprIsZeroAndTbExitEx(pReNative, pCodeBuf, off, idxTmpReg1, false /*f64Bit*/,
     238                                                      kIemNativeLabelType_ReturnBreakFF);
     239
     240# else
     241#  error "port me"
     242# endif
     243    }
     244
     245    /*
     246     * Second, check forced flags, if so desired.
     247     *
     248     * We OR them together to save a conditional.  A trick here is that the
     249     * two IRQ flags are unused in the global flags, so we can still use the
     250     * resulting value to check for suppressed interrupts.
     251     */
     252    if (a_fCheckIrqs)
     253    {
     254        /* Load VMCPU::fLocalForcedActions first and mask it.  We can simplify the
     255           masking by ASSUMING none of the unwanted flags are located above bit 30.  */
     256        uint64_t const fUnwantedCpuFFs = VMCPU_FF_PGM_SYNC_CR3
     257                                       | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
     258                                       | VMCPU_FF_TLB_FLUSH
     259                                       | VMCPU_FF_UNHALT;
     260        AssertCompile(fUnwantedCpuFFs < RT_BIT_64(31));
     261        off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, idxTmpReg1, RT_UOFFSETOF(VMCPUCC, fLocalForcedActions));
     262# if defined(RT_ARCH_AMD64)
     263        /* and reg1, ~fUnwantedCpuFFs */
     264        pCodeBuf[off++] = idxTmpReg1 >= 8 ? X86_OP_REX_B | X86_OP_REX_W : X86_OP_REX_W;
     265        pCodeBuf[off++] = 0x81;
     266        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, idxTmpReg1 & 7);
     267        *(uint32_t *)&pCodeBuf[off] = ~(uint32_t)fUnwantedCpuFFs;
     268        off += 4;
     269
     270# else
     271        off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, idxTmpReg2, ~fUnwantedCpuFFs);
     272        off = iemNativeEmitAndGprByGprEx(pCodeBuf, off, idxTmpReg1, idxTmpReg2);
     273# endif
     274
     275        /* OR in VM::fGlobalForcedActions.  We access the member via pVCpu.
     276           No need to mask anything here.  Unfortunately, it's a 32-bit
     277           variable, so we can't OR it directly on x86. */
     278        AssertCompile(VM_FF_ALL_MASK == UINT32_MAX);
     279        intptr_t const offGlobalForcedActions = (intptr_t)&pReNative->pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions
     280                                              - (intptr_t)pReNative->pVCpu;
     281        Assert((int32_t)offGlobalForcedActions == offGlobalForcedActions);
     282
     283# ifdef RT_ARCH_AMD64
     284        if (idxTmpReg2 >= 8)
     285            pCodeBuf[off++] = X86_OP_REX_R;
     286        pCodeBuf[off++] = 0x8b; /* mov */
     287        off = iemNativeEmitGprByVCpuSignedDisp(pCodeBuf, off, idxTmpReg2, (int32_t)offGlobalForcedActions);
     288
     289        /* or reg1, reg2 */
     290        off = iemNativeEmitOrGprByGprEx(pCodeBuf, off, idxTmpReg1, idxTmpReg2);
     291
     292        /* jz nothing_pending */
     293        uint32_t const offFixup1 = off;
     294        off = iemNativeEmitJccToFixedEx(pCodeBuf, off, off + 64, kIemNativeInstrCond_e);
     295
     296# elif defined(RT_ARCH_ARM64)
     297        off = iemNativeEmitGprBySignedVCpuLdStEx(pCodeBuf, off, idxTmpReg2, (int32_t)offGlobalForcedActions,
     298                                                 kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
     299        off = iemNativeEmitOrGprByGprEx(pCodeBuf, off, idxTmpReg1, idxTmpReg2);
     300
     301        /* cbz nothing_pending */
     302        uint32_t const offFixup1 = off;
     303        off = iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToFixedEx(pCodeBuf, off, idxTmpReg1, true /*f64Bit*/,
     304                                                                   false /*fJmpIfNotZero*/, off + 16);
     305# else
     306#  error "port me"
     307# endif
     308
     309        /* More than just IRQ FFs pending? */
     310        AssertCompile((VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC) == 3);
     311        /* cmp reg1, 3 */
     312        off = iemNativeEmitCmpGprWithImmEx(pCodeBuf, off, idxTmpReg1, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC);
     313        /* ja ReturnBreakFF */
     314        off = iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, kIemNativeLabelType_ReturnBreakFF, kIemNativeInstrCond_nbe);
     315
     316        /*
     317         * Okay, we've only got pending IRQ related FFs: Can we dispatch IRQs?
     318         *
     319         * ASSUME that the shadow flags are cleared when they ought to be cleared,
     320         * so we can skip the RIP check.
     321         */
     322        AssertCompile(CPUMCTX_INHIBIT_SHADOW < RT_BIT_32(31));
     323        /* reg1 = efl & (IF | INHIBIT_SHADOW) */
     324        off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, idxTmpReg1, idxEflReg, X86_EFL_IF | CPUMCTX_INHIBIT_SHADOW);
     325        /* reg1 ^= IF */
     326        off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, idxTmpReg1, X86_EFL_IF);
     327
     328# ifdef RT_ARCH_AMD64
     329        /* jz   ReturnBreakFF */
     330        off = iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, kIemNativeLabelType_ReturnBreakFF, kIemNativeInstrCond_e);
     331
     332# elif defined(RT_ARCH_ARM64)
     333        /* cbz  reg1, ReturnBreakFF */
     334        off = iemNativeEmitTestIfGprIsZeroAndTbExitEx(pReNative, pCodeBuf, off, idxTmpReg1, false /*f64Bit*/,
     335                                                      kIemNativeLabelType_ReturnBreakFF);
     336# else
     337#  error "port me"
     338# endif
     339        /*
     340         * nothing_pending:
     341         */
     342        iemNativeFixupFixedJump(pReNative, offFixup1, off);
     343    }
     344
     345    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     346
     347    /*
     348     * Cleanup.
     349     */
     350    iemNativeRegFreeTmp(pReNative, idxTmpReg1);
     351    if (a_fCheckIrqs)
     352    {
     353        iemNativeRegFreeTmp(pReNative, idxTmpReg2);
     354        iemNativeRegFreeTmp(pReNative, idxEflReg);
     355    }
     356    else
     357    {
     358        Assert(idxTmpReg2 == UINT8_MAX);
     359        Assert(idxEflReg == UINT8_MAX);
     360    }
     361
     362    return off;
     363}
     364
     365
     366/**
    205367 * Built-in function that checks for pending interrupts that can be delivered or
    206368 * forced action flags.
    … …  
    212374IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckIrq)
    213375{
    214     RT_NOREF(pCallEntry);
    215 
    216376    BODY_FLUSH_PENDING_WRITES();
    217 
    218     /* It's too convenient to use iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet below
    219        and I'm too lazy to create a 'Fixed' version of that one. */
    220     uint32_t const idxLabelVmCheck = iemNativeLabelCreate(pReNative, kIemNativeLabelType_CheckIrq,
    221                                                           UINT32_MAX, pReNative->uCheckIrqSeqNo++);
    222 
    223     /* Again, we need to load the extended EFLAGS before we actually need them
    224        in case we jump.  We couldn't use iemNativeRegAllocTmpForGuestReg if we
    225        loaded them inside the check, as the shadow state would not be correct
    226        when the code branches before the load.  Ditto PC. */
    227     uint8_t const idxEflReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
    228                                                               kIemNativeGstRegUse_ReadOnly);
    229 
    230     uint8_t const idxPcReg  = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc, kIemNativeGstRegUse_ReadOnly);
    231 
    232     uint8_t idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
    233 
    234     /*
    235      * Start by checking the local forced actions of the EMT we're on for IRQs
    236      * and other FFs that needs servicing.
    237      */
    238     /** @todo this isn't even close to the NMI and interrupt conditions in EM! */
    239     /* Load FFs in to idxTmpReg and AND with all relevant flags. */
    240     off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, fLocalForcedActions));
    241     off = iemNativeEmitAndGprByImm(pReNative, off, idxTmpReg,
    242                                    VMCPU_FF_ALL_MASK & ~(  VMCPU_FF_PGM_SYNC_CR3
    243                                                          | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
    244                                                          | VMCPU_FF_TLB_FLUSH
    245                                                          | VMCPU_FF_UNHALT ),
    246                                    true /*fSetFlags*/);
    247     /* If we end up with ZERO in idxTmpReg there is nothing to do.*/
    248     uint32_t const offFixupJumpToVmCheck1 = off;
    249     off = iemNativeEmitJzToFixed(pReNative, off, off /* ASSUME jz rel8 suffices */);
    250 
    251     /* Some relevant FFs are set, but if's only APIC or/and PIC being set,
    252        these may be supressed by EFLAGS.IF or CPUMIsInInterruptShadow. */
    253     off = iemNativeEmitAndGprByImm(pReNative, off, idxTmpReg,
    254                                    ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC), true /*fSetFlags*/);
    255     /* Return VINF_IEM_REEXEC_BREAK if other FFs are set. */
    256     off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_ReturnBreakFF);
    257 
    258     /* So, it's only interrupt releated FFs and we need to see if IRQs are being
    259        suppressed by the CPU or not. */
    260     off = iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(pReNative, off, idxEflReg, X86_EFL_IF_BIT, idxLabelVmCheck);
    261     off = iemNativeEmitTestAnyBitsInGprAndTbExitIfNoneSet(pReNative, off, idxEflReg, CPUMCTX_INHIBIT_SHADOW,
    262                                                           kIemNativeLabelType_ReturnBreakFF);
    263 
    264     /* We've got shadow flags set, so we must check that the PC they are valid
    265        for matches our current PC value. */
    266     /** @todo AMD64 can do this more efficiently w/o loading uRipInhibitInt into
    267      *        a register. */
    268     off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.uRipInhibitInt));
    269     off = iemNativeEmitTestIfGprNotEqualGprAndTbExit(pReNative, off, idxTmpReg, idxPcReg,
    270                                                      kIemNativeLabelType_ReturnBreakFF);
    271 
    272     /*
    273      * Now check the force flags of the VM.
    274      */
    275     iemNativeLabelDefine(pReNative, idxLabelVmCheck, off);
    276     iemNativeFixupFixedJump(pReNative, offFixupJumpToVmCheck1, off);
    277     off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, CTX_SUFF(pVM))); /* idxTmpReg = pVM */
    278     off = iemNativeEmitLoadGprByGprU32(pReNative, off, idxTmpReg, idxTmpReg, RT_UOFFSETOF(VMCC, fGlobalForcedActions));
    279     off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, VM_FF_ALL_MASK, true /*fSetFlags*/);
    280     off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_ReturnBreakFF);
    281 
    282     /** @todo STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatCheckIrqBreaks); */
    283 
    284     /*
    285      * We're good, no IRQs or FFs pending.
    286      */
    287     iemNativeRegFreeTmp(pReNative, idxTmpReg);
    288     iemNativeRegFreeTmp(pReNative, idxEflReg);
    289     iemNativeRegFreeTmp(pReNative, idxPcReg);
    290 
    291     /*
    292      * Note down that we've been here, so we can skip FFs + IRQ checks when
    293      * doing direct linking.
    294      */
     377    off = iemNativeRecompFunc_BltIn_CheckTimersAndIrqsCommon<false, true>(pReNative, off);
     378
     379    /* Note down that we've been here, so we can skip FFs + IRQ checks when
     380       doing direct linking. */
    295381#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
    296382    pReNative->idxLastCheckIrqCallNo = pReNative->idxCurCall;
     383    RT_NOREF(pCallEntry);
    297384#else
    298385    pReNative->idxLastCheckIrqCallNo = pCallEntry - pReNative->pTbOrg->Thrd.paCalls;
    … …  
    303390
    304391IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckIrq)
     392{
     393    IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
     394    IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(pOutgoing, fEflOther);
     395    RT_NOREF(pCallEntry);
     396}
     397
     398
     399/**
     400 * Built-in function that works the cIrqChecksTillNextPoll counter on direct TB
     401 * linking, like loop-jumps.
     402 */
     403IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckTimers)
     404{
     405    BODY_FLUSH_PENDING_WRITES();
     406    RT_NOREF(pCallEntry);
     407    return iemNativeRecompFunc_BltIn_CheckTimersAndIrqsCommon<true, false>(pReNative, off);
     408}
     409
     410IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckTimers)
     411{
     412    IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
     413    RT_NOREF(pCallEntry);
     414}
     415
     416
     417/**
     418 * Combined BltIn_CheckTimers + BltIn_CheckIrq for direct linking.
     419 */
     420IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckTimersAndIrq)
     421{
     422    BODY_FLUSH_PENDING_WRITES();
     423    RT_NOREF(pCallEntry);
     424    return iemNativeRecompFunc_BltIn_CheckTimersAndIrqsCommon<true, true>(pReNative, off);
     425}
     426
     427IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckTimersAndIrq)
    305428{
    306429    IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
    … …  
    22932416#endif
    22942417
     2418
     2419/**
     2420 * Built-in function for jumping in the call sequence.
     2421 */
     2422IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_Jump)
     2423{
     2424    PCIEMTB const  pTb         = pReNative->pTbOrg;
     2425    Assert(pCallEntry->auParams[1] == 0 && pCallEntry->auParams[2] == 0);
     2426    Assert(pCallEntry->auParams[0] < pTb->Thrd.cCalls);
     2427#if 1
     2428    RT_NOREF(pCallEntry, pTb);
     2429
     2430# ifdef VBOX_WITH_STATISTICS
     2431    /* Increment StatNativeTbExitLoopFullTb. */
     2432    uint32_t const offStat = RT_UOFFSETOF(VMCPU, iem.s.StatNativeTbExitLoopFullTb);
     2433#  ifdef RT_ARCH_AMD64
     2434    off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, UINT8_MAX, UINT8_MAX, offStat);
     2435#  else
     2436    uint8_t const idxStatsTmp1 = iemNativeRegAllocTmp(pReNative, &off);
     2437    uint8_t const idxStatsTmp2 = iemNativeRegAllocTmp(pReNative, &off);
     2438    off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, idxStatsTmp1, idxStatsTmp2, offStat);
     2439    iemNativeRegFreeTmp(pReNative, idxStatsTmp1);
     2440    iemNativeRegFreeTmp(pReNative, idxStatsTmp2);
     2441#  endif
     2442# endif
     2443# ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
     2444    /** @todo
     2445    off = iemNativeEmitAddU32CounterInVCpuEx(pReNative, off, pTb->cInstructions, RT_UOFFSETOF(VMCPUCC, iem.s.cInstructions));
     2446    */
     2447# endif
     2448
     2449    /* Jump to the start of the TB. */
     2450    uint32_t idxLabel = iemNativeLabelFind(pReNative, kIemNativeLabelType_LoopJumpTarget);
     2451    AssertStmt(idxLabel < pReNative->cLabels, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_LABEL_IPE_6)); /** @todo better status */
     2452    return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
     2453#else
     2454    RT_NOREF(pReNative, pCallEntry, pTb);
     2455    return off;
     2456#endif
     2457}
     2458
     2459IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_Jump)
     2460{
     2461    IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
     2462    RT_NOREF(pCallEntry);
     2463}
     2464

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

    r105445 r105673  
    20932093    pReNative->Core.u64ArgVars             = UINT64_MAX;
    20942094
    2095     AssertCompile(RT_ELEMENTS(pReNative->aidxUniqueLabels) == 22);
     2095    AssertCompile(RT_ELEMENTS(pReNative->aidxUniqueLabels) == 23);
    20962096    pReNative->aidxUniqueLabels[0]         = UINT32_MAX;
    20972097    pReNative->aidxUniqueLabels[1]         = UINT32_MAX;
    … …  
    21162116    pReNative->aidxUniqueLabels[20]        = UINT32_MAX;
    21172117    pReNative->aidxUniqueLabels[21]        = UINT32_MAX;
     2118    pReNative->aidxUniqueLabels[22]        = UINT32_MAX;
    21182119
    21192120    pReNative->idxLastCheckIrqCallNo       = UINT32_MAX;
    … …  
    23132314    Assert(uData == 0 || enmType >= kIemNativeLabelType_FirstWithMultipleInstances);
    23142315#if defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) && defined(RT_ARCH_AMD64)
    2315     Assert(enmType >= kIemNativeLabelType_FirstWithMultipleInstances);
     2316    Assert(enmType >= kIemNativeLabelType_LoopJumpTarget);
    23162317#endif
    23172318
    … …  
    24212422
    24222423
    2423 #if !defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) || !defined(RT_ARCH_AMD64)
    24242424/**
    24252425 * Looks up a lable.
    … …  
    24272427 * @returns Label ID if found, UINT32_MAX if not.
    24282428 */
    2429 static uint32_t iemNativeLabelFind(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
    2430                                    uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT
     2429DECLHIDDEN(uint32_t) iemNativeLabelFind(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
     2430                                        uint32_t offWhere /*= UINT32_MAX*/, uint16_t uData /*= 0*/) RT_NOEXCEPT
    24312431{
    24322432    Assert((unsigned)enmType < 64);
    … …  
    24482448    return UINT32_MAX;
    24492449}
    2450 #endif
    24512450
    24522451
    … …  
    89488947#undef STR_CASE_CMN
    89498948#define STR_CASE_LBL(a_Label) case kIemNativeLabelType_ ## a_Label: return #a_Label;
     8949        STR_CASE_LBL(LoopJumpTarget);
    89508950        STR_CASE_LBL(If);
    89518951        STR_CASE_LBL(Else);
    … …  
    99299929        uint32_t             cRecompiledCalls = 0;
    99309930#endif
    9931 #if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) || defined(VBOX_STRICT) || defined(LOG_ENABLED)
     9931#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) || defined(IEM_WITH_INTRA_TB_JUMPS) || defined(VBOX_STRICT) || defined(LOG_ENABLED)
    99329932        uint32_t             idxCurCall       = 0;
    99339933#endif
    … …  
    99399939#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
    99409940            pReNative->idxCurCall                 = idxCurCall;
     9941#endif
     9942
     9943#ifdef IEM_WITH_INTRA_TB_JUMPS
     9944            /*
     9945             * Define label for jump targets (currently only the first entry).
     9946             */
     9947            if (!(pCallEntry->fFlags & IEMTHREADEDCALLENTRY_F_JUMP_TARGET))
     9948            { /* likely */ }
     9949            else
     9950            {
     9951                iemNativeLabelCreate(pReNative, kIemNativeLabelType_LoopJumpTarget, off);
     9952                Assert(idxCurCall == 0); /** @todo when jumping elsewhere, we have to save the register state. */
     9953            }
    99419954#endif
    99429955
    … …  
    1003810051             */
    1003910052            pCallEntry++;
    10040 #if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) || defined(VBOX_STRICT) || defined(LOG_ENABLED)
     10053#if defined(IEMNATIVE_WITH_LIVENESS_ANALYSIS) || defined(IEM_WITH_INTRA_TB_JUMPS) || defined(VBOX_STRICT) || defined(LOG_ENABLED)
    1004110054            idxCurCall++;
    1004210055#endif

  • trunk/src/VBox/VMM/VMMAll/IEMAllThrdFuncsBltIn.cpp

    r105072 r105673  
    145145
    146146/**
     147 * Worker for iemThreadedFunc_BltIn_CheckIrq and
     148 * iemThreadedFunc_BltIn_CheckTimersAndIrqs that checks for pending FFs
     149 * and IRQs, and if it's only the latter whether we can dispatch them now.
     150 */
     151DECL_FORCE_INLINE(int) iemThreadedFunc_BltIn_CheckIrqCommon(PVMCPUCC pVCpu)
     152{
     153    /* Get and mask the per-CPU FFs.*/
     154    uint64_t const fCpuRaw = pVCpu->fLocalForcedActions;
     155    uint64_t       fFlags  = fCpuRaw & (VMCPU_FF_ALL_MASK & ~(  VMCPU_FF_PGM_SYNC_CR3
     156                                                              | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
     157                                                              | VMCPU_FF_TLB_FLUSH
     158                                                              | VMCPU_FF_UNHALT ));
     159
     160    /* OR in VM-wide FFs and check them together. */
     161    uint32_t const fVmRaw = pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions;
     162    fFlags |= fVmRaw;
     163    if (RT_LIKELY(!fFlags))
     164        return VINF_SUCCESS;
     165
     166    /* Since the VMCPU_FF_INTERUPT_XXX flags was once upon a time in fVm and
     167       we haven't reused the bits yet, we can still reliably check whether
     168       we're only here for reasons of pending interrupts and whether these
     169       are supressed by EFLAGS.IF=0 or interrupt shadowing. */
     170    Assert(!(fVmRaw & (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
     171    AssertCompile((VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC) == 3);
     172    if (   fFlags <= (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)
     173        && (   !pVCpu->cpum.GstCtx.rflags.Bits.u1IF
     174            || CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx)))
     175        return VINF_SUCCESS;
     176
     177    Log(("%04x:%08RX32: Pending IRQ and/or FF: fCpu=%#RX64 fVm=%#RX32 IF=%d\n",
     178         pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, fCpuRaw, fVmRaw, pVCpu->cpum.GstCtx.rflags.Bits.u1IF));
     179    STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatCheckIrqBreaks);
     180    return VINF_IEM_REEXEC_BREAK;
     181}
     182
     183
     184/**
    147185 * Built-in function that checks for pending interrupts that can be delivered or
    148186 * forced action flags.
    … …  
    152190 * a non-zero status that stops TB execution.
    153191 */
     192/** @todo add VMX / SVM variants of this. */
    154193IEM_DECL_IEMTHREADEDFUNC_DEF(iemThreadedFunc_BltIn_CheckIrq)
    155194{
    156195    RT_NOREF(uParam0, uParam1, uParam2);
    157 
    158     /*
    159      * Check for IRQs and other FFs that needs servicing.
    160      */
    161     uint64_t fCpu = pVCpu->fLocalForcedActions;
    162     fCpu &= VMCPU_FF_ALL_MASK & ~(  VMCPU_FF_PGM_SYNC_CR3
    163                                   | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
    164                                   | VMCPU_FF_TLB_FLUSH
    165                                   | VMCPU_FF_UNHALT );
    166     /** @todo this isn't even close to the NMI and interrupt conditions in EM! */
    167     if (RT_LIKELY(   (   !fCpu
    168                       || (   !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
    169                           && (   !pVCpu->cpum.GstCtx.rflags.Bits.u1IF
    170                               || CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx)) ) )
    171                   && !VM_FF_IS_ANY_SET(pVCpu->CTX_SUFF(pVM), VM_FF_ALL_MASK) ))
     196    return iemThreadedFunc_BltIn_CheckIrqCommon(pVCpu);
     197}
     198
     199
     200/**
     201 * Built-in function that works the cIrqChecksTillNextPoll counter on direct TB
     202 * linking, like loop-jumps.
     203 */
     204IEM_DECL_IEMTHREADEDFUNC_DEF(iemThreadedFunc_BltIn_CheckTimers)
     205{
     206    if (RT_LIKELY(--pVCpu->iem.s.cIrqChecksTillNextPoll > 0))
    172207        return VINF_SUCCESS;
    173208
    174     Log(("%04x:%08RX32: Pending IRQ and/or FF: fCpu=%#RX64 fVm=%#RX32 IF=%d\n",
    175          pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, fCpu,
    176          pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions & VM_FF_ALL_MASK, pVCpu->cpum.GstCtx.rflags.Bits.u1IF));
    177     STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatCheckIrqBreaks);
     209    Log(("%04x:%08RX32: Check timers\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip));
     210    STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatCheckTimersBreaks);
     211    RT_NOREF(uParam0, uParam1, uParam2);
     212    return VINF_IEM_REEXEC_BREAK;
     213}
     214
     215
     216/**
     217 * Combined BltIn_CheckTimers + BltIn_CheckIrq for direct linking.
     218 */
     219IEM_DECL_IEMTHREADEDFUNC_DEF(iemThreadedFunc_BltIn_CheckTimersAndIrq)
     220{
     221    if (RT_LIKELY(--pVCpu->iem.s.cIrqChecksTillNextPoll > 0))
     222        return iemThreadedFunc_BltIn_CheckIrqCommon(pVCpu);
     223
     224    Log(("%04x:%08RX32: Check timers\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip));
     225    STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatCheckTimersBreaks);
     226    RT_NOREF(uParam0, uParam1, uParam2);
    178227    return VINF_IEM_REEXEC_BREAK;
    179228}
    … …  
    829878}
    830879
     880
     881/**
     882 * Built-in function for jumping in the call sequence.
     883 */
     884IEM_DECL_IEMTHREADEDFUNC_DEF(iemThreadedFunc_BltIn_Jump)
     885{
     886    Assert(uParam1 == 0 && uParam2 == 0);
     887    RT_NOREF(pVCpu, uParam0, uParam1, uParam2);
     888    return VINF_IEM_REEXEC_JUMP;
     889}
     890

  • trunk/src/VBox/VMM/VMMAll/IEMAllThrdPython.py

    r105652 r105673  
    28312831        ( 'DeferToCImpl0',                                      2, True  ),
    28322832        ( 'CheckIrq',                                           0, True  ),
     2833        ( 'CheckTimers',                                        0, True  ),
     2834        ( 'CheckTimersAndIrq',                                  0, True  ),
    28332835        ( 'CheckMode',                                          1, True  ),
    28342836        ( 'CheckHwInstrBps',                                    0, False ),
    … …  
    28582860        ( 'CheckOpcodesOnNewPageLoadingTlb',                    2, True  ),
    28592861        ( 'CheckOpcodesOnNewPageLoadingTlbConsiderCsLim',       2, True  ),
     2862
     2863        ( 'Jump',                                               1, True  ),
    28602864    );
    28612865

  • trunk/src/VBox/VMM/VMMAll/IEMAllThrdRecompiler.cpp

    r105560 r105673  
    19411941        pVCpu->iem.s.fTbCrossedPage         = false;
    19421942        pVCpu->iem.s.cInstrTillIrqCheck     = !(fExtraFlags & IEMTB_F_INHIBIT_SHADOW) ? 32 : 0;
     1943        pVCpu->iem.s.idxLastCheckIrqCallNo  = UINT16_MAX;
    19431944        pVCpu->iem.s.fTbCurInstrIsSti       = false;
    19441945        /* Force RF clearing and TF checking on first instruction in the block
    … …  
    20602061    pCall->offOpcode   = 0;
    20612062    pCall->uTbLookup   = 0;
    2062     pCall->uUnused0    = 0;
     2063    pCall->fFlags      = 0;
    20632064    pCall->auParams[0] = 0;
    20642065    pCall->auParams[1] = 0;
    … …  
    20862087    pCall->offOpcode   = 0;
    20872088    pCall->uTbLookup   = 0;
    2088     pCall->uUnused0    = 0;
     2089    pCall->fFlags      = 0;
    20892090    pCall->auParams[0] = RT_MAKE_U16(pCall->idxInstr, idxCall); /* currently not used, but whatever */
    20902091    pCall->auParams[1] = 0;
    … …  
    21192120
    21202121
     2122#ifdef IEM_WITH_INTRA_TB_JUMPS
     2123/**
     2124 * Emits the necessary tail calls for a full TB loop-jump.
     2125 */
     2126static bool iemThreadedCompileFullTbJump(PVMCPUCC pVCpu, PIEMTB pTb)
     2127{
     2128    /*
     2129     * We need a timer and maybe IRQ check before jumping, so make sure
     2130     * we've got sufficient call entries left before emitting anything.
     2131     */
     2132    uint32_t idxCall = pTb->Thrd.cCalls;
     2133    if (idxCall + 1U <= pTb->Thrd.cAllocated)
     2134    {
     2135        /*
     2136         * We're good, emit the calls.
     2137         */
     2138        PIEMTHRDEDCALLENTRY pCall = &pTb->Thrd.paCalls[idxCall];
     2139        pTb->Thrd.cCalls = (uint16_t)(idxCall + 2);
     2140
     2141        /* Always check timers as we risk getting stuck in a loop otherwise.  We
     2142           combine it with an IRQ check if that's not performed in the TB already. */
     2143        pCall->enmFunction = pVCpu->iem.s.idxLastCheckIrqCallNo < idxCall
     2144                           ? kIemThreadedFunc_BltIn_CheckTimers
     2145                           : kIemThreadedFunc_BltIn_CheckTimersAndIrq;
     2146        pCall->idxInstr    = 0;
     2147        pCall->offOpcode   = 0;
     2148        pCall->cbOpcode    = 0;
     2149        pCall->uTbLookup   = 0;
     2150        pCall->fFlags      = 0;
     2151        pCall->auParams[0] = 0;
     2152        pCall->auParams[1] = 0;
     2153        pCall->auParams[2] = 0;
     2154        pCall++;
     2155
     2156        /* The jump callentry[0]. */
     2157        pCall->enmFunction = kIemThreadedFunc_BltIn_Jump;
     2158        pCall->idxInstr    = 0;
     2159        pCall->offOpcode   = 0;
     2160        pCall->cbOpcode    = 0;
     2161        pCall->uTbLookup   = 0;
     2162        pCall->fFlags      = 0;
     2163        pCall->auParams[0] = 0; /* jump target is call zero */
     2164        pCall->auParams[1] = 0;
     2165        pCall->auParams[2] = 0;
     2166
     2167        /* Mark callentry #0 as a jump target. */
     2168        pTb->Thrd.paCalls[0].fFlags |= IEMTHREADEDCALLENTRY_F_JUMP_TARGET;
     2169    }
     2170
     2171    return false;
     2172}
     2173#endif /* IEM_WITH_INTRA_TB_JUMPS */
     2174
     2175
    21212176/**
    21222177 * Called by IEM_MC2_BEGIN_EMIT_CALLS() under one of these conditions:
    … …  
    21782233    pCall->offOpcode   = offOpcode;
    21792234    pCall->uTbLookup   = 0;
    2180     pCall->uUnused0    = 0;
     2235    pCall->fFlags      = 0;
    21812236    pCall->auParams[0] = (uint32_t)cbInstr
    21822237                       | (uint32_t)(pVCpu->iem.s.fExec << 8) /* liveness: Enough of fExec for IEM_F_MODE_X86_IS_FLAT. */
    … …  
    22892344                        {
    22902345                            Log8(("%04x:%08RX64: loop detected after branch\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
    2291                             STAM_COUNTER_INC(&pVCpu->iem.s.StatTbLoopInTbDetected);
     2346#ifdef IEM_WITH_INTRA_TB_JUMPS
     2347                            /* If we're looping back to the start of the TB and the mode is still the same,
     2348                               we could emit a jump optimization.  For now we don't do page transitions
     2349                               as that implies TLB loading and such. */
     2350                            if (   idxLoopRange == 0
     2351                                && offPhysPc == pTb->aRanges[0].offPhysPage
     2352                                &&    (pVCpu->iem.s.fExec & IEMTB_F_IEM_F_MASK & IEMTB_F_KEY_MASK)
     2353                                   == (pTb->fFlags        & IEMTB_F_KEY_MASK   & ~IEMTB_F_CS_LIM_CHECKS)
     2354                                &&    (pVCpu->iem.s.fTbBranched & (  IEMBRANCHED_F_INDIRECT | IEMBRANCHED_F_FAR
     2355                                                                   | IEMBRANCHED_F_STACK | IEMBRANCHED_F_RELATIVE))
     2356                                   == IEMBRANCHED_F_RELATIVE)
     2357                            {
     2358                                STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatTbLoopFullTbDetected);
     2359                                return iemThreadedCompileFullTbJump(pVCpu, pTb);
     2360                            }
     2361#endif
     2362                            STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatTbLoopInTbDetected);
    22922363                            return false;
    22932364                        }
    … …  
    25652636    pCall->offOpcode   = 0;
    25662637    pCall->uTbLookup   = 0;
    2567     pCall->uUnused0    = 0;
     2638    pCall->fFlags      = 0;
    25682639    pCall->auParams[0] = pVCpu->iem.s.fExec;
    25692640    pCall->auParams[1] = 0;
    … …  
    25972668        /* Emit the call. */
    25982669        AssertReturn(idxCall < pTb->Thrd.cAllocated, false);
     2670        pVCpu->iem.s.idxLastCheckIrqCallNo = (uint16_t)idxCall;
     2671        pTb->Thrd.cCalls                   = (uint16_t)(idxCall + 1);
    25992672        PIEMTHRDEDCALLENTRY pCall = &pTb->Thrd.paCalls[idxCall];
    2600         pTb->Thrd.cCalls = (uint16_t)(idxCall + 1);
    26012673        pCall->enmFunction = kIemThreadedFunc_BltIn_CheckIrq;
    26022674        pCall->idxInstr    = pTb->cInstructions;
    … …  
    26042676        pCall->cbOpcode    = 0;
    26052677        pCall->uTbLookup   = 0;
    2606         pCall->uUnused0    = 0;
     2678        pCall->fFlags      = 0;
    26072679        pCall->auParams[0] = 0;
    26082680        pCall->auParams[1] = 0;
    … …  
    26512723     * Emit the call.
    26522724     */
    2653     AssertReturn(pTb->Thrd.cCalls < pTb->Thrd.cAllocated, false);
    2654     PIEMTHRDEDCALLENTRY pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls++];
     2725    uint32_t const idxCall = pTb->Thrd.cCalls;
     2726    AssertReturn(idxCall < pTb->Thrd.cAllocated, false);
     2727    pVCpu->iem.s.idxLastCheckIrqCallNo = (uint16_t)idxCall;
     2728    pTb->Thrd.cCalls                   = (uint16_t)(idxCall + 1);
     2729    PIEMTHRDEDCALLENTRY pCall = &pTb->Thrd.paCalls[idxCall];
    26552730    pCall->enmFunction = kIemThreadedFunc_BltIn_CheckIrq;
    26562731    pCall->idxInstr    = pTb->cInstructions;
    … …  
    26582733    pCall->cbOpcode    = 0;
    26592734    pCall->uTbLookup   = 0;
    2660     pCall->uUnused0    = 0;
     2735    pCall->fFlags      = 0;
    26612736    pCall->auParams[0] = 0;
    26622737    pCall->auParams[1] = 0;
    … …  
    28442919*   Recompiled Execution Core                                                                                                    *
    28452920*********************************************************************************************************************************/
     2921
     2922/**
     2923 * Helper for polling timers.
     2924 */
     2925DECLHIDDEN(int) iemPollTimers(PVMCC pVM, PVMCPUCC pVCpu) RT_NOEXCEPT
     2926{
     2927    /*
     2928     * Do the polling and calculate the time since the last time.
     2929     */
     2930    uint64_t       nsNow        = 0;
     2931    bool const     fExpired     = TMTimerPollBoolWithNanoTS(pVM, pVCpu, &nsNow);
     2932    uint64_t const cNsSinceLast = nsNow - pVCpu->iem.s.nsRecompilerPollNow;
     2933
     2934    /* Store the new timstamps.  */
     2935    pVCpu->iem.s.nsRecompilerPollNow = nsNow;
     2936    pVCpu->iem.s.msRecompilerPollNow = (uint32_t)(nsNow / RT_NS_1MS);
     2937
     2938    /*
     2939     * Set the next polling count down value.
     2940     *
     2941     * We take the previous value and adjust it according to the cNsSinceLast
     2942     * value, if it's not within reason.   This can't be too accurate since the
     2943     * CheckIrq and intra-TB-checks aren't evenly spaced, they depends highly
     2944     * on the guest code.
     2945     */
     2946/** @todo can we make this even more adaptive based on current timer config as well? */
     2947    uint32_t       cIrqChecksTillNextPoll = pVCpu->iem.s.cIrqChecksTillNextPollPrev;
     2948    uint32_t const cNsIdealPollInterval   = pVCpu->iem.s.cNsIdealPollInterval;
     2949    int64_t const  nsFromIdeal            = cNsSinceLast - cNsIdealPollInterval;
     2950    if (nsFromIdeal < 0)
     2951    {
     2952        if ((uint64_t)-nsFromIdeal > cNsIdealPollInterval / 8 && cIrqChecksTillNextPoll < _64K)
     2953        {
     2954            cIrqChecksTillNextPoll += cIrqChecksTillNextPoll / 8;
     2955            pVCpu->iem.s.cIrqChecksTillNextPollPrev = cIrqChecksTillNextPoll;
     2956        }
     2957    }
     2958    else
     2959    {
     2960        if ((uint64_t)nsFromIdeal > cNsIdealPollInterval / 8 && cIrqChecksTillNextPoll > 256)
     2961        {
     2962            cIrqChecksTillNextPoll -= cIrqChecksTillNextPoll / 8;
     2963            pVCpu->iem.s.cIrqChecksTillNextPollPrev = cIrqChecksTillNextPoll;
     2964        }
     2965    }
     2966    pVCpu->iem.s.cIrqChecksTillNextPoll = cIrqChecksTillNextPoll;
     2967
     2968    /*
     2969     * Repeat the IRQ and FF checks.
     2970     */
     2971    if (!fExpired)
     2972    {
     2973        uint32_t fCpu = pVCpu->fLocalForcedActions;
     2974        fCpu &= VMCPU_FF_ALL_MASK & ~(  VMCPU_FF_PGM_SYNC_CR3
     2975                                      | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
     2976                                      | VMCPU_FF_TLB_FLUSH
     2977                                      | VMCPU_FF_UNHALT );
     2978        if (RT_LIKELY(   (   !fCpu
     2979                          || (   !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
     2980                              && (   !pVCpu->cpum.GstCtx.rflags.Bits.u1IF
     2981                                  || CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx)) ) )
     2982                      && !VM_FF_IS_ANY_SET(pVCpu->CTX_SUFF(pVM), VM_FF_ALL_MASK) ))
     2983            return VINF_SUCCESS;
     2984    }
     2985    return VINF_IEM_REEXEC_BREAK_FF;
     2986}
     2987
    28462988
    28472989/** Helper for iemTbExec. */
    … …  
    29883130                          && pVCpu->iem.s.rcPassUp == VINF_SUCCESS /** @todo this isn't great. */))
    29893131                pCallEntry++;
     3132            else if (rcStrict == VINF_IEM_REEXEC_JUMP)
     3133            {
     3134                Assert(pVCpu->iem.s.rcPassUp == VINF_SUCCESS);
     3135                Assert(cCallsLeft == 0);
     3136                uint32_t const idxTarget = (uint32_t)pCallEntry->auParams[0];
     3137                cCallsLeft = pTb->Thrd.cCalls;
     3138                AssertBreak(idxTarget < cCallsLeft - 1);
     3139                cCallsLeft -= idxTarget;
     3140                pCallEntry  = &pTb->Thrd.paCalls[idxTarget];
     3141                AssertBreak(pCallEntry->fFlags & IEMTHREADEDCALLENTRY_F_JUMP_TARGET);
     3142            }
    29903143            else
    29913144            {
    … …  
    30453198     */
    30463199    uint64_t uPc = pVCpu->cpum.GstCtx.rip;
     3200#if 0 /* unused */
    30473201    pVCpu->iem.s.uCurTbStartPc = uPc;
     3202#endif
    30483203    Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 || !IEM_IS_64BIT_CODE(pVCpu));
    30493204    uPc += pVCpu->cpum.GstCtx.cs.u64Base;
    … …  
    31543309        IEM_TRY_SETJMP(pVCpu, rcStrict)
    31553310        {
    3156             uint32_t const cPollRate = 511; /* EM.cpp passes 4095 to IEMExecLots, so an eigth of that seems reasonable for now. */
    3157             for (uint32_t iIterations = 0; ; iIterations++)
     3311            for (;;)
    31583312            {
    31593313                /* Translate PC to physical address, we'll need this for both lookup and compilation. */
    … …  
    31773331                        return rcStrict;
    31783332#endif
    3179                     rcStrict = IEMExecLots(pVCpu, 2048, cPollRate, NULL);
     3333                    rcStrict = IEMExecLots(pVCpu, 2048, 511, NULL);
    31803334                }
    31813335                if (rcStrict == VINF_SUCCESS)
    … …  
    31833337                    Assert(pVCpu->iem.s.cActiveMappings == 0);
    31843338
     3339                    /* Note! This IRQ/FF check is repeated in iemPollTimers, iemThreadedFunc_BltIn_CheckIrq
     3340                             and emitted by iemNativeRecompFunc_BltIn_CheckIrq. */
    31853341                    uint64_t fCpu = pVCpu->fLocalForcedActions;
    31863342                    fCpu &= VMCPU_FF_ALL_MASK & ~(  VMCPU_FF_PGM_SYNC_CR3
    … …  
    31953351                                  && !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) ))
    31963352                    {
    3197                         if (RT_LIKELY(   (iIterations & cPollRate) != 0
    3198                                       || !TMTimerPollBoolWith32BitMilliTS(pVM, pVCpu, &pVCpu->iem.s.msRecompilerPollNow)))
     3353                        /* Once in a while we need to poll timers here. */
     3354                        if ((int32_t)--pVCpu->iem.s.cIrqChecksTillNextPoll > 0)
    31993355                        { /* likely */ }
    32003356                        else
    3201                             return VINF_SUCCESS;
     3357                        {
     3358                            int rc = iemPollTimers(pVM, pVCpu);
     3359                            if (rc != VINF_SUCCESS)
     3360                                return VINF_SUCCESS;
     3361                        }
    32023362                    }
    32033363                    else

  • trunk/src/VBox/VMM/VMMAll/IEMAllThrdTables.h

    r104357 r105673  
    234234        pCall->offOpcode   = offOpcodeMc2; \
    235235        pCall->uTbLookup   = 0; \
    236         pCall->uUnused0    = 0; \
     236        pCall->fFlags      = 0; \
    237237        pCall->auParams[0] = 0; \
    238238        pCall->auParams[1] = 0; \
    … …  
    250250        pCall->offOpcode   = offOpcodeMc2; \
    251251        pCall->uTbLookup   = 0; \
    252         pCall->uUnused0    = 0; \
     252        pCall->fFlags      = 0; \
    253253        pCall->auParams[0] = a_uArg0; \
    254254        pCall->auParams[1] = 0; \
    … …  
    267267        pCall->offOpcode   = offOpcodeMc2; \
    268268        pCall->uTbLookup   = 0; \
    269         pCall->uUnused0    = 0; \
     269        pCall->fFlags      = 0; \
    270270        pCall->auParams[0] = a_uArg0; \
    271271        pCall->auParams[1] = a_uArg1; \
    … …  
    285285        pCall->cbOpcode    = cbInstrMc2; \
    286286        pCall->uTbLookup   = 0; \
    287         pCall->uUnused0    = 0; \
     287        pCall->fFlags      = 0; \
    288288        pCall->auParams[0] = a_uArg0; \
    289289        pCall->auParams[1] = a_uArg1; \
    … …  
    302302        pCall->uTbLookup   = IEM_TB_LOOKUP_TAB_MAKE(pTb->cTbLookupEntries, a_fLargeTbLookup); \
    303303        pTb->cTbLookupEntries += !(a_fLargeTbLookup) ? 1 : IEM_TB_LOOKUP_TAB_LARGE_SIZE; \
    304         pCall->uUnused0    = 0; \
     304        pCall->fFlags      = 0; \
    305305        pCall->auParams[0] = 0; \
    306306        pCall->auParams[1] = 0; \
    … …  
    319319        pCall->uTbLookup   = IEM_TB_LOOKUP_TAB_MAKE(pTb->cTbLookupEntries, a_fLargeTbLookup); \
    320320        pTb->cTbLookupEntries += !(a_fLargeTbLookup) ? 1 : IEM_TB_LOOKUP_TAB_LARGE_SIZE; \
    321         pCall->uUnused0    = 0; \
     321        pCall->fFlags      = 0; \
    322322        pCall->auParams[0] = a_uArg0; \
    323323        pCall->auParams[1] = 0; \
    … …  
    337337        pCall->uTbLookup   = IEM_TB_LOOKUP_TAB_MAKE(pTb->cTbLookupEntries, a_fLargeTbLookup); \
    338338        pTb->cTbLookupEntries += !(a_fLargeTbLookup) ? 1 : IEM_TB_LOOKUP_TAB_LARGE_SIZE; \
    339         pCall->uUnused0    = 0; \
     339        pCall->fFlags      = 0; \
    340340        pCall->auParams[0] = a_uArg0; \
    341341        pCall->auParams[1] = a_uArg1; \
    … …  
    356356        pCall->uTbLookup   = IEM_TB_LOOKUP_TAB_MAKE(pTb->cTbLookupEntries, a_fLargeTbLookup); \
    357357        pTb->cTbLookupEntries += !(a_fLargeTbLookup) ? 1 : IEM_TB_LOOKUP_TAB_LARGE_SIZE; \
    358         pCall->uUnused0    = 0; \
     358        pCall->fFlags      = 0; \
    359359        pCall->auParams[0] = a_uArg0; \
    360360        pCall->auParams[1] = a_uArg1; \

  • trunk/src/VBox/VMM/VMMAll/TMAll.cpp

    r104788 r105673  
    11361136 * @thread  The emulation thread.
    11371137 */
    1138 VMMDECL(bool) TMTimerPollBoolWith32BitMilliTS(PVMCC pVM, PVMCPUCC pVCpu, uint32_t *pmsNow)
     1138VMM_INT_DECL(bool) TMTimerPollBoolWith32BitMilliTS(PVMCC pVM, PVMCPUCC pVCpu, uint32_t *pmsNow)
    11391139{
    11401140    AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
    … …  
    11431143    tmTimerPollInternal(pVM, pVCpu, &off, &u64Now);
    11441144    *pmsNow = (uint32_t)(u64Now / RT_NS_1MS);
     1145    return off == 0;
     1146}
     1147
     1148
     1149/**
     1150 * Set FF if we've passed the next virtual event and return virtual time as MS.
     1151 *
     1152 * This function is called before FFs are checked in the inner execution EM loops.
     1153 *
     1154 * This is used by the IEM recompiler for polling timers while also providing a
     1155 * free time source for recent use tracking and such.
     1156 *
     1157 * @returns true if timers are pending, false if not.
     1158 *
     1159 * @param   pVM         The cross context VM structure.
     1160 * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
     1161 * @param   pnsNow      Where to return the current virtual time in nanoseconds.
     1162 * @thread  The emulation thread.
     1163 */
     1164VMM_INT_DECL(bool) TMTimerPollBoolWithNanoTS(PVMCC pVM, PVMCPUCC pVCpu, uint64_t *pnsNow)
     1165{
     1166    AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
     1167    uint64_t off = 0;
     1168    tmTimerPollInternal(pVM, pVCpu, &off, pnsNow);
    11451169    return off == 0;
    11461170}

  • trunk/src/VBox/VMM/VMMR3/IEMR3.cpp

    r105663 r105673  
    224224        pVCpu->iem.s.DataTlb.NonGlobalLargePageRange.uFirstTag = UINT64_MAX;
    225225        pVCpu->iem.s.DataTlb.GlobalLargePageRange.uFirstTag    = UINT64_MAX;
     226#endif
     227
     228#ifndef VBOX_VMM_TARGET_ARMV8
     229        /* Poll timers every 400 us / 2500 Hz. (source: thin air) */
     230        pVCpu->iem.s.cNsIdealPollInterval       = 400U * RT_NS_1US;
     231        pVCpu->iem.s.cIrqChecksTillNextPoll     = 128;
     232        pVCpu->iem.s.cIrqChecksTillNextPollPrev = 128;
    226233#endif
    227234
    … …  
    595602                        "Times threaded TB execution was interrupted/broken off on a call without lookup entries", "/IEM/CPU%u/re/cTbExecThreadedBreaksWithoutLookup", idCpu);
    596603# endif
     604
     605        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.cIrqChecksTillNextPollPrev, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
     606                        "Timer polling interval",                       "/IEM/CPU%u/re/cIrqChecksTillNextPollPrev", idCpu);
    597607
    598608        PIEMTBALLOCATOR const pTbAllocator = pVCpu->iem.s.pTbAllocatorR3;
    … …  
    648658        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatCheckIrqBreaks,  STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
    649659                        "TB breaks by CheckIrq",                        "/IEM/CPU%u/re/CheckIrqBreaks", idCpu);
     660        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatCheckTimersBreaks, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
     661                        "TB breaks by CheckIrq",                        "/IEM/CPU%u/re/CheckTimersBreaks", idCpu);
    650662        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatCheckModeBreaks, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
    651663                        "TB breaks by CheckMode",                       "/IEM/CPU%u/re/CheckModeBreaks", idCpu);
    … …  
    654666        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatCheckNeedCsLimChecking, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
    655667                        "Needing CS.LIM checking TB after branch or on page crossing", "/IEM/CPU%u/re/CheckTbNeedCsLimChecking", idCpu);
    656 # ifdef VBOX_WITH_STATISTICS
     668
     669        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatTbLoopFullTbDetected, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
     670                        "Detected loop full TB",  "/IEM/CPU%u/re/LoopFullTbDetected", idCpu);
    657671        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatTbLoopInTbDetected, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
    658672                        "Detected loop within TB", "/IEM/CPU%u/re/LoopInTbDetected", idCpu);
    659 #endif
    660673
    661674        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatNativeExecMemInstrBufAllocFailed, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
    … …  
    883896                        "Number of times the TB finished raising a #XF exception",
    884897                        RAISE_PREFIX "RaiseXf", idCpu);
     898
     899#  ifdef VBOX_WITH_STATISTICS
     900        STAMR3RegisterF(pVM, &pVCpu->iem.s.StatNativeTbExitLoopFullTb, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
     901                        "Number of full TB loops.",
     902                        "/IEM/CPU%u/re/NativeTbExit/LoopFullTb", idCpu);
     903#  endif
    885904
    886905        STAMR3RegisterF(pVM, &pVCpu->iem.s.StatNativeTbExitDirectLinking1Irq, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,

  • trunk/src/VBox/VMM/include/IEMInternal.h

    r105664 r105673  
    9292# define IEM_WITH_THROW_CATCH
    9393#endif /*ASM-NOINC-END*/
     94
     95/** @def IEM_WITH_INTRA_TB_JUMPS
     96 * Enables loop-jumps within a TB (currently only to the first call).
     97 */
     98#if defined(DOXYGEN_RUNNING) || 1
     99# define IEM_WITH_INTRA_TB_JUMPS
     100#endif
    94101
    95102/** @def IEMNATIVE_WITH_DELAYED_PC_UPDATING
    … …  
    12451252    uint8_t     uTbLookup;
    12461253
    1247     /** Unused atm. */
    1248     uint8_t     uUnused0;
     1254    /** Flags - IEMTHREADEDCALLENTRY_F_XXX. */
     1255    uint8_t     fFlags;
    12491256
    12501257    /** Generic parameters. */
    … …  
    12701277/** Make a IEMTHRDEDCALLENTRY::uTbLookup value. */
    12711278#define IEM_TB_LOOKUP_TAB_MAKE(a_idxTable, a_fLarge)    ((a_idxTable) | ((a_fLarge) ? 0x80 : 0))
     1279
     1280
     1281/** The call entry is a jump target. */
     1282#define IEMTHREADEDCALLENTRY_F_JUMP_TARGET              UINT8_C(0x01)
     1283
    12721284
    12731285/**
    … …  
    21292141    uint8_t                 abAlignment9[42];
    21302142
    2131     /** @name Recompilation
     2143
     2144    /** @name Recompiled Exection
    21322145     * @{ */
    21332146    /** Pointer to the current translation block.
    … …  
    21562169    uint64_t                u64Unused;
    21572170#endif
    2158     /** Fixed TB used for threaded recompilation.
    2159      * This is allocated once with maxed-out sizes and re-used afterwards. */
    2160     R3PTRTYPE(PIEMTB)       pThrdCompileTbR3;
    21612171    /** Pointer to the ring-3 TB cache for this EMT. */
    21622172    R3PTRTYPE(PIEMTBCACHE)  pTbCacheR3;
    … …  
    21652175     * entry, thus it can always safely be used w/o NULL checking. */
    21662176    R3PTRTYPE(PIEMTB *)     ppTbLookupEntryR3;
     2177#if 0 /* unused */
    21672178    /** The PC (RIP) at the start of pCurTbR3/pCurTbR0.
    21682179     * The TBs are based on physical addresses, so this is needed to correleated
    21692180     * RIP to opcode bytes stored in the TB (AMD-V / VT-x). */
    21702181    uint64_t                uCurTbStartPc;
     2182#endif
     2183
    21712184    /** Number of threaded TBs executed. */
    21722185    uint64_t                cTbExecThreaded;
    21732186    /** Number of native TBs executed. */
    21742187    uint64_t                cTbExecNative;
     2188
     2189    /** The number of IRQ/FF checks till the next timer poll call. */
     2190    uint32_t                cIrqChecksTillNextPoll;
     2191    /** The virtual sync time at the last timer poll call in milliseconds. */
     2192    uint32_t                msRecompilerPollNow;
     2193    /** The virtual sync time at the last timer poll call in nanoseconds. */
     2194    uint64_t                nsRecompilerPollNow;
     2195    /** The previous cIrqChecksTillNextPoll value. */
     2196    uint32_t                cIrqChecksTillNextPollPrev;
     2197    /** The ideal nanosecond interval between two timer polls.
     2198     * @todo make this adaptive?  */
     2199    uint32_t                cNsIdealPollInterval;
     2200
     2201    /** The current instruction number in a native TB.
     2202     * This is set by code that may trigger an unexpected TB exit (throw/longjmp)
     2203     * and will be picked up by the TB execution loop. Only used when
     2204     * IEMNATIVE_WITH_INSTRUCTION_COUNTING is defined. */
     2205    uint8_t                 idxTbCurInstr;
     2206    /** @} */
     2207
     2208    /** @name Recompilation
     2209     * @{ */
    21752210    /** Whether we need to check the opcode bytes for the current instruction.
    21762211     * This is set by a previous instruction if it modified memory or similar.  */
    … …  
    21822217    /** Whether to end the current TB. */
    21832218    bool                    fEndTb;
     2219    /** Indicates that the current instruction is an STI.  This is set by the
     2220     * iemCImpl_sti code and subsequently cleared by the recompiler. */
     2221    bool                    fTbCurInstrIsSti;
     2222    /** Spaced reserved for recompiler data / alignment. */
     2223    bool                    afRecompilerStuff1[1];
    21842224    /** Number of instructions before we need emit an IRQ check call again.
    21852225     * This helps making sure we don't execute too long w/o checking for
    … …  
    21892229     * fTbCurInstrIsSti. */
    21902230    uint8_t                 cInstrTillIrqCheck;
    2191     /** Indicates that the current instruction is an STI.  This is set by the
    2192      * iemCImpl_sti code and subsequently cleared by the recompiler. */
    2193     bool                    fTbCurInstrIsSti;
     2231    /** The index of the last CheckIrq call during threaded recompilation. */
     2232    uint16_t                idxLastCheckIrqCallNo;
    21942233    /** The size of the IEMTB::pabOpcodes allocation in pThrdCompileTbR3. */
    21952234    uint16_t                cbOpcodesAllocated;
    2196     /** The current instruction number in a native TB.
    2197      * This is set by code that may trigger an unexpected TB exit (throw/longjmp)
    2198      * and will be picked up by the TB execution loop. Only used when
    2199      * IEMNATIVE_WITH_INSTRUCTION_COUNTING is defined. */
    2200     uint8_t                 idxTbCurInstr;
    2201     /** Spaced reserved for recompiler data / alignment. */
    2202     bool                    afRecompilerStuff1[3];
    2203     /** The virtual sync time at the last timer poll call. */
    2204     uint32_t                msRecompilerPollNow;
    22052235    /** The IEMTB::cUsed value when to attempt native recompilation of a TB. */
    22062236    uint32_t                uTbNativeRecompileAtUsedCount;
    … …  
    22122242     *  currently not up to date in EFLAGS. */
    22132243    uint32_t                fSkippingEFlags;
     2244    /** Spaced reserved for recompiler data / alignment. */
     2245    uint32_t                u32RecompilerStuff2;
     2246#if 0  /* unused */
    22142247    /** Previous GCPhysInstrBuf value - only valid if fTbCrossedPage is set.   */
    22152248    RTGCPHYS                GCPhysInstrBufPrev;
     2249#endif
     2250
     2251    /** Fixed TB used for threaded recompilation.
     2252     * This is allocated once with maxed-out sizes and re-used afterwards. */
     2253    R3PTRTYPE(PIEMTB)       pThrdCompileTbR3;
    22162254    /** Pointer to the ring-3 TB allocator for this EMT. */
    22172255    R3PTRTYPE(PIEMTBALLOCATOR) pTbAllocatorR3;
    … …  
    22222260    /** Dummy entry for ppTbLookupEntryR3. */
    22232261    R3PTRTYPE(PIEMTB)       pTbLookupEntryDummyR3;
     2262    /** @} */
    22242263
    22252264    /** Dummy TLB entry used for accesses to pages with databreakpoints. */
    … …  
    22302269    /** Statistics: Times BltIn_CheckIrq breaks out of the TB. */
    22312270    STAMCOUNTER             StatCheckIrqBreaks;
     2271    /** Statistics: Times BltIn_CheckTimers breaks direct linking TBs. */
     2272    STAMCOUNTER             StatCheckTimersBreaks;
    22322273    /** Statistics: Times BltIn_CheckMode breaks out of the TB. */
    22332274    STAMCOUNTER             StatCheckModeBreaks;
    … …  
    22402281    /** Statistics: Times a jump or page crossing required a TB with CS.LIM checking. */
    22412282    STAMCOUNTER             StatCheckNeedCsLimChecking;
    2242     /** Statistics: Times a loop was detected within a TB.. */
     2283    /** Statistics: Times a loop was detected within a TB. */
    22432284    STAMCOUNTER             StatTbLoopInTbDetected;
     2285    /** Statistics: Times a loop back to the start of the TB was detected. */
     2286    STAMCOUNTER             StatTbLoopFullTbDetected;
    22442287    /** Exec memory allocator statistics: Number of times allocaintg executable memory failed. */
    22452288    STAMCOUNTER             StatNativeExecMemInstrBufAllocFailed;
    … …  
    24212464    STAMCOUNTER             StatNativeTbExitObsoleteTb;
    24222465
     2466    /** Native recompiler: Number of full TB loops (jumps from end to start). */
     2467    STAMCOUNTER             StatNativeTbExitLoopFullTb;
     2468
    24232469    /** Native recompiler: Failure situations with direct linking scenario \#1.
    24242470     * Counter with StatNativeTbExitReturnBreak. Not in release builds.
    … …  
    24482494
    24492495#ifdef IEM_WITH_TLB_TRACE
    2450     uint64_t                au64Padding[2];
     2496    uint64_t                au64Padding[6];
    24512497#else
    2452     uint64_t                au64Padding[4];
    2453 #endif
    2454     /** @} */
     2498    //uint64_t                au64Padding[1];
     2499#endif
    24552500
    24562501#ifdef IEM_WITH_TLB_TRACE
    … …  
    24912536AssertCompileMemberAlignment(IEMCPU, aMemMappingLocks, 16);
    24922537AssertCompileMemberAlignment(IEMCPU, aBounceBuffers, 64);
     2538AssertCompileMemberAlignment(IEMCPU, pCurTbR3, 64);
    24932539AssertCompileMemberAlignment(IEMCPU, DataTlb, 64);
    24942540AssertCompileMemberAlignment(IEMCPU, CodeTlb, 64);
    … …  
    67426788extern const PFNIEMOP g_apfnIemThreadedRecompilerVecMap3[1024];
    67436789
     6790DECLHIDDEN(int)     iemPollTimers(PVMCC pVM, PVMCPUCC pVCpu) RT_NOEXCEPT;
     6791
    67446792DECLCALLBACK(int)   iemTbInit(PVMCC pVM, uint32_t cInitialTbs, uint32_t cMaxTbs,
    67456793                              uint64_t cbInitialExec, uint64_t cbMaxExec, uint32_t cbChunkExec);
    … …  
    67776825
    67786826IEM_DECL_IEMTHREADEDFUNC_PROTO(iemThreadedFunc_BltIn_CheckIrq);
     6827IEM_DECL_IEMTHREADEDFUNC_PROTO(iemThreadedFunc_BltIn_CheckTimers);
     6828IEM_DECL_IEMTHREADEDFUNC_PROTO(iemThreadedFunc_BltIn_CheckTimersAndIrq);
    67796829IEM_DECL_IEMTHREADEDFUNC_PROTO(iemThreadedFunc_BltIn_CheckMode);
    67806830IEM_DECL_IEMTHREADEDFUNC_PROTO(iemThreadedFunc_BltIn_CheckHwInstrBps);
    … …  
    68066856IEM_DECL_IEMTHREADEDFUNC_PROTO(iemThreadedFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb);
    68076857IEM_DECL_IEMTHREADEDFUNC_PROTO(iemThreadedFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim);
     6858
     6859IEM_DECL_IEMTHREADEDFUNC_PROTO(iemThreadedFunc_BltIn_Jump);
    68086860
    68096861bool iemThreadedCompileEmitIrqCheckBefore(PVMCPUCC pVCpu, PIEMTB pTb);

  • trunk/src/VBox/VMM/include/IEMN8veRecompiler.h

    r105490 r105673  
    491491    kIemNativeLabelType_LastTbExit        = kIemNativeLabelType_Return,
    492492
     493    /** Loop-jump target. */
     494    kIemNativeLabelType_LoopJumpTarget,
     495
    493496    /*
    494497     * Labels with data, potentially multiple instances per TB:
    … …  
    14491452    /** Condition sequence number (for generating unique labels). */
    14501453    uint16_t                    uCondSeqNo;
    1451     /** Check IRQ seqeunce number (for generating unique labels). */
     1454    /** Check IRQ sequence number (for generating unique labels). */
    14521455    uint16_t                    uCheckIrqSeqNo;
    14531456    /** TLB load sequence number (for generating unique labels). */
    … …  
    16321635                                                 uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
    16331636DECL_HIDDEN_THROW(void)     iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
     1637DECLHIDDEN(uint32_t)        iemNativeLabelFind(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
     1638                                               uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
    16341639DECL_HIDDEN_THROW(void)     iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
    16351640                                              IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);

  • trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h

    r105491 r105673  
    503503}
    504504
     505/**
     506 * Special variant of iemNativeEmitGprByVCpuDisp for accessing the VM structure.
     507 */
     508DECL_FORCE_INLINE(uint32_t)
     509iemNativeEmitGprByVCpuSignedDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offVCpu)
     510{
     511    Assert(offVCpu < 0);
     512    if (offVCpu < 128 && offVCpu >= -128)
     513    {
     514        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
     515        pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
     516    }
     517    else
     518    {
     519        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
     520        pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
     521        pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
     522        pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
     523        pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
     524    }
     525    return off;
     526}
     527
    505528#elif defined(RT_ARCH_ARM64)
    506529
    … …  
    585608}
    586609
     610
     611/**
     612 * Special variant of iemNativeEmitGprByVCpuLdStEx for accessing the VM
     613 * structure.
     614 *
     615 * @note Loads can use @a iGprReg for large offsets, stores requires a temporary
     616 *       registers (@a iGprTmp).
     617 * @note DON'T try this with prefetch.
     618 */
     619DECL_FORCE_INLINE_THROW(uint32_t)
     620iemNativeEmitGprBySignedVCpuLdStEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offVCpu,
     621                                   ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData, uint8_t iGprTmp = UINT8_MAX)
     622{
     623    Assert(offVCpu < 0);
     624    Assert((uint32_t)-offVCpu < RT_BIT_32(28)); /* we should be way out of range for problematic sign extending issues. */
     625    Assert(!((uint32_t)-offVCpu & (cbData - 1)));
     626
     627   /*
     628     * For negative offsets we need to use put the displacement in a register
     629     * as the two variants with signed immediates will either post or pre
     630     * increment the base address register.
     631     */
     632    if (!ARMV8A64INSTRLDSTTYPE_IS_STORE(enmOperation) || iGprTmp != UINT8_MAX)
     633    {
     634        uint8_t const idxIndexReg = !ARMV8A64INSTRLDSTTYPE_IS_STORE(enmOperation) ? iGprReg : IEMNATIVE_REG_FIXED_TMP0;
     635        off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, idxIndexReg, offVCpu / (int32_t)cbData);
     636        pCodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, idxIndexReg,
     637                                                    kArmv8A64InstrLdStExtend_Sxtw, cbData > 1 /*fShifted*/);
     638    }
     639    else
     640# ifdef IEM_WITH_THROW_CATCH
     641        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     642# else
     643        AssertReleaseFailedStmt(off = UINT32_MAX);
     644# endif
     645
     646    return off;
     647}
     648
     649/**
     650 * Special variant of iemNativeEmitGprByVCpuLdSt for accessing the VM structure.
     651 */
     652DECL_FORCE_INLINE_THROW(uint32_t)
     653iemNativeEmitGprBySignedVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
     654                                 int32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
     655{
     656    off = iemNativeEmitGprBySignedVCpuLdStEx(iemNativeInstrBufEnsure(pReNative, off, 2 + 1), off, iGprReg,
     657                                             offVCpu, enmOperation, cbData, IEMNATIVE_REG_FIXED_TMP0);
     658    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     659    return off;
     660}
     661
    587662#endif /* RT_ARCH_ARM64 */
    588663
    … …  
    601676        pCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
    602677    pCodeBuf[off++] = 0x8b;
    603     off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off,iGpr, offVCpu);
     678    off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, iGpr, offVCpu);
    604679
    605680#elif defined(RT_ARCH_ARM64)
    … …  
    773848#endif
    774849    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     850    return off;
     851}
     852
     853
     854/**
     855 * Emits a store of a GPR value to a 32-bit VCpu field.
     856 *
     857 * @note Limited range on ARM64.
     858 */
     859DECL_INLINE_THROW(uint32_t)
     860iemNativeEmitStoreGprToVCpuU32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
     861{
     862#ifdef RT_ARCH_AMD64
     863    /* mov mem32, reg32 */
     864    if (iGpr >= 8)
     865        pCodeBuf[off++] = X86_OP_REX_R;
     866    pCodeBuf[off++] = 0x89;
     867    off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, iGpr, offVCpu);
     868
     869#elif defined(RT_ARCH_ARM64)
     870    off = iemNativeEmitGprByVCpuLdStEx(pCodeBuf, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
     871
     872#else
     873# error "port me"
     874#endif
    775875    return off;
    776876}
    … …  
    54365536{
    54375537#if defined(RT_ARCH_AMD64)
    5438     /* and Ev, imm */
     5538    /* xor Ev, imm */
    54395539    if (iGprDst >= 8)
    54405540        pCodeBuf[off++] = X86_OP_REX_B;
    … …  
    61306230 */
    61316231DECL_INLINE_THROW(uint32_t)
     6232iemNativeEmitCmpGprWithImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft,
     6233                             uint64_t uImm, uint8_t idxTmpReg = UINT8_MAX)
     6234{
     6235#ifdef RT_ARCH_AMD64
     6236    if ((int8_t)uImm == (int64_t)uImm)
     6237    {
     6238        /* cmp Ev, Ib */
     6239        pCodeBuf[off++] = X86_OP_REX_W | (iGprLeft >= 8 ? X86_OP_REX_B : 0);
     6240        pCodeBuf[off++] = 0x83;
     6241        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
     6242        pCodeBuf[off++] = (uint8_t)uImm;
     6243        return off;
     6244    }
     6245    if ((int32_t)uImm == (int64_t)uImm)
     6246    {
     6247        /* cmp Ev, imm */
     6248        pCodeBuf[off++] = X86_OP_REX_W | (iGprLeft >= 8 ? X86_OP_REX_B : 0);
     6249        pCodeBuf[off++] = 0x81;
     6250        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
     6251        pCodeBuf[off++] = RT_BYTE1(uImm);
     6252        pCodeBuf[off++] = RT_BYTE2(uImm);
     6253        pCodeBuf[off++] = RT_BYTE3(uImm);
     6254        pCodeBuf[off++] = RT_BYTE4(uImm);
     6255        return off;
     6256    }
     6257
     6258#elif defined(RT_ARCH_ARM64)
     6259    if (uImm < _4K)
     6260    {
     6261        pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
     6262                                                      true /*64Bit*/, true /*fSetFlags*/);
     6263        return off;
     6264    }
     6265    if ((uImm & ~(uint64_t)0xfff000) == 0)
     6266    {
     6267        pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm >> 12,
     6268                                                      true /*64Bit*/, true /*fSetFlags*/, true /*fShift12*/);
     6269        return off;
     6270    }
     6271
     6272#else
     6273# error "Port me!"
     6274#endif
     6275
     6276    if (idxTmpReg != UINT8_MAX)
     6277    {
     6278        /* Use temporary register for the immediate. */
     6279        off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, idxTmpReg, uImm);
     6280        off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, iGprLeft, idxTmpReg);
     6281    }
     6282    else
     6283# ifdef IEM_WITH_THROW_CATCH
     6284        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     6285# else
     6286        AssertReleaseFailedStmt(off = UINT32_MAX);
     6287# endif
     6288
     6289    return off;
     6290}
     6291
     6292
     6293/**
     6294 * Emits a compare of a 64-bit GPR with a constant value, settings status
     6295 * flags/whatever for use with conditional instruction.
     6296 */
     6297DECL_INLINE_THROW(uint32_t)
    61326298iemNativeEmitCmpGprWithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint64_t uImm)
    61336299{
    61346300#ifdef RT_ARCH_AMD64
    6135     if (uImm <= UINT32_C(0xff))
     6301    if ((int8_t)uImm == (int64_t)uImm)
    61366302    {
    61376303        /* cmp Ev, Ib */
    … …  
    61426308        pbCodeBuf[off++] = (uint8_t)uImm;
    61436309    }
    6144     else if ((int64_t)uImm == (int32_t)uImm)
     6310    else if ((int32_t)uImm == (int64_t)uImm)
    61456311    {
    61466312        /* cmp Ev, imm */
    … …  
    69747140
    69757141#elif defined(RT_ARCH_ARM64)
     7142    int32_t const    offDisp     = offTarget - offFixup;
    69767143    uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
    69777144    if ((pu32CodeBuf[offFixup] & UINT32_C(0xff000000)) == UINT32_C(0x54000000))
    69787145    {
    69797146        /* B.COND + BC.COND */
    6980         int32_t const offDisp = offTarget - offFixup;
    69817147        Assert(offDisp >= -262144 && offDisp < 262144);
    69827148        pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xff00001f))
    69837149                              | (((uint32_t)offDisp    & UINT32_C(0x0007ffff)) << 5);
    69847150    }
    6985     else
     7151    else if ((pu32CodeBuf[offFixup] & UINT32_C(0xfc000000)) == UINT32_C(0x14000000))
    69867152    {
    69877153        /* B imm26 */
    6988         Assert((pu32CodeBuf[offFixup] & UINT32_C(0xfc000000)) == UINT32_C(0x14000000));
    6989         int32_t const offDisp = offTarget - offFixup;
    69907154        Assert(offDisp >= -33554432 && offDisp < 33554432);
    69917155        pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xfc000000))
    69927156                              | ((uint32_t)offDisp     & UINT32_C(0x03ffffff));
     7157    }
     7158    else
     7159    {
     7160        /* CBZ / CBNZ reg, imm19 */
     7161        Assert((pu32CodeBuf[offFixup] & UINT32_C(0x7e000000)) == UINT32_C(0x34000000));
     7162        Assert(offDisp >= -1048576 && offDisp < 1048576);
     7163        pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup]    & UINT32_C(0xff00001f))
     7164                              | (((uint32_t)offDisp << 5) & UINT32_C(0x00ffffe0));
     7165
    69937166    }
    69947167
    … …  
    84518624
    84528625
     8626/**
     8627 * Emits code that exits the current TB with @a enmExitReason if @a iGprSrc is zero.
     8628 *
     8629 * The operand size is given by @a f64Bit.
     8630 */
     8631DECL_FORCE_INLINE_THROW(uint32_t)
     8632iemNativeEmitTestIfGprIsZeroAndTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
     8633                                        uint8_t iGprSrc, bool f64Bit, IEMNATIVELABELTYPE enmExitReason)
     8634{
     8635    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
     8636#if defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) && defined(RT_ARCH_AMD64)
     8637    /* test reg32,reg32  / test reg64,reg64 */
     8638    if (f64Bit)
     8639        pCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_R | X86_OP_REX_B);
     8640    else if (iGprSrc >= 8)
     8641        pCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
     8642    pCodeBuf[off++] = 0x85;
     8643    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
     8644
     8645    /* jnz idxLabel  */
     8646    return iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, enmExitReason, kIemNativeInstrCond_e);
     8647
     8648#else
     8649    /* ARM64 doesn't have the necessary jump range, so we jump via local label
     8650       just like when we keep everything local. */
     8651    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     8652    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
     8653                                                                f64Bit, false /*fJmpIfNotZero*/, idxLabel);
     8654#endif
     8655}
     8656
     8657
     8658/**
     8659 * Emits code to exit the current TB with the given reason @a enmExitReason if @a iGprSrc is zero.
     8660 *
     8661 * The operand size is given by @a f64Bit.
     8662 */
     8663DECL_INLINE_THROW(uint32_t)
     8664iemNativeEmitTestIfGprIsZeroAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     8665                                      uint8_t iGprSrc, bool f64Bit, IEMNATIVELABELTYPE enmExitReason)
     8666{
     8667#if defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) && defined(RT_ARCH_AMD64)
     8668    off = iemNativeEmitTestIfGprIsZeroAndTbExitEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 3 + 6),
     8669                                                  off, iGprSrc, f64Bit, enmExitReason);
     8670    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     8671    return off;
     8672#else
     8673    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     8674    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, false /*fJmpIfNotZero*/, idxLabel);
     8675#endif
     8676}
     8677
     8678
    84538679#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    84548680/*********************************************************************************************************************************
Note: See TracChangeset for help on using the changeset viewer.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette