VirtualBox

Changeset 104797 in vbox for trunk/src/VBox


Ignore:
Timestamp:
May 28, 2024 5:50:30 AM (6 months ago)
Author:
vboxsync
Message:

VMM/IEM: Introduce special helpers for generating code to exit a TB in order to be able to experiment with different approaches more easily and convert the code emitters to make use of them, bugref:10677

Location:
trunk/src/VBox/VMM
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompBltIn.cpp

    r104506 r104797  
    215215                                                          UINT32_MAX, pReNative->uCheckIrqSeqNo++);
    216216
    217     uint32_t const idxLabelReturnBreakFF = iemNativeLabelCreate(pReNative, kIemNativeLabelType_ReturnBreakFF);
    218 
    219217    /* Again, we need to load the extended EFLAGS before we actually need them
    220218       in case we jump.  We couldn't use iemNativeRegAllocTmpForGuestReg if we
    … …  
    250248                                   ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC), true /*fSetFlags*/);
    251249    /* Return VINF_IEM_REEXEC_BREAK if other FFs are set. */
    252     off = iemNativeEmitJnzToLabel(pReNative, off, idxLabelReturnBreakFF);
     250    off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_ReturnBreakFF);
    253251
    254252    /* So, it's only interrupt releated FFs and we need to see if IRQs are being
    255253       suppressed by the CPU or not. */
    256254    off = iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(pReNative, off, idxEflReg, X86_EFL_IF_BIT, idxLabelVmCheck);
    257     off = iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(pReNative, off, idxEflReg, CPUMCTX_INHIBIT_SHADOW,
    258                                                               idxLabelReturnBreakFF);
     255    off = iemNativeEmitTestAnyBitsInGprAndTbExitIfNoneSet(pReNative, off, idxEflReg, CPUMCTX_INHIBIT_SHADOW,
     256                                                          kIemNativeExitReason_ReturnBreakFF);
    259257
    260258    /* We've got shadow flags set, so we must check that the PC they are valid
    … …  
    263261     *        a register. */
    264262    off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.uRipInhibitInt));
    265     off = iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, idxTmpReg, idxPcReg, idxLabelReturnBreakFF);
     263    off = iemNativeEmitTestIfGprNotEqualGprAndTbExit(pReNative, off, idxTmpReg, idxPcReg,
     264                                                     kIemNativeExitReason_ReturnBreakFF);
    266265
    267266    /*
    … …  
    273272    off = iemNativeEmitLoadGprByGprU32(pReNative, off, idxTmpReg, idxTmpReg, RT_UOFFSETOF(VMCC, fGlobalForcedActions));
    274273    off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, VM_FF_ALL_MASK, true /*fSetFlags*/);
    275     off = iemNativeEmitJnzToLabel(pReNative, off, idxLabelReturnBreakFF);
     274    off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_ReturnBreakFF);
    276275
    277276    /** @todo STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatCheckIrqBreaks); */
    … …  
    315314    off = iemNativeEmitLoadGprFromVCpuU32(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, iem.s.fExec));
    316315    off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, IEMTB_F_KEY_MASK);
    317     off = iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(pReNative, off, idxTmpReg, fExpectedExec & IEMTB_F_KEY_MASK,
    318                                                               kIemNativeLabelType_ReturnBreak);
     316    off = iemNativeEmitTestIfGpr32NotEqualImmAndTbExit(pReNative, off, idxTmpReg, fExpectedExec & IEMTB_F_KEY_MASK,
     317                                                       kIemNativeExitReason_ReturnBreak);
    319318    iemNativeRegFreeTmp(pReNative, idxTmpReg);
    320319
    … …  
    445444
    446445    /* 3. Jump if greater. */
    447     off = iemNativeEmitJaToNewLabel(pReNative, off, kIemNativeLabelType_RaiseGp0);
     446    off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeExitReason_RaiseGp0);
    448447
    449448    iemNativeRegFreeTmp(pReNative, idxRegCsLim);
    … …  
    533532    /* Compare the two and jump out if we're too close to the limit. */
    534533    off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegLeft, idxRegRight);
    535     off = iemNativeEmitJlToNewLabel(pReNative, off, kIemNativeLabelType_NeedCsLimChecking);
     534    off = iemNativeEmitJlTbExit(pReNative, off, kIemNativeExitReason_NeedCsLimChecking);
    536535
    537536    iemNativeRegFreeTmp(pReNative, idxRegRight);
    … …  
    566565    off = iemNativeEmitMarker(pReNative, off, 0x80000003);
    567566#endif
    568 
    569     uint32_t const      idxLabelObsoleteTb = iemNativeLabelCreate(pReNative, kIemNativeLabelType_ObsoleteTb);
    570567
    571568    /*
    … …  
    605602            else \
    606603            { \
    607                 pbCodeBuf[off++] = 0x74; /* jz near +5 */ \
    608                 pbCodeBuf[off++] = 0x05 + BP_ON_OBSOLETION; \
     604                pbCodeBuf[off++] = 0x74; /* jz near +6 */ \
     605                pbCodeBuf[off++] = 0x06 + BP_ON_OBSOLETION; \
    609606                offConsolidatedJump = off; \
    610607                if (BP_ON_OBSOLETION) pbCodeBuf[off++] = 0xcc; \
    611                 pbCodeBuf[off++] = 0xe9; /* jmp rel32 */ \
    612                 iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_Rel32, -4); \
    613                 pbCodeBuf[off++] = 0x00; \
    614                 pbCodeBuf[off++] = 0x00; \
    615                 pbCodeBuf[off++] = 0x00; \
    616                 pbCodeBuf[off++] = 0x00; \
     608                off = iemNativeEmitTbExitEx(pReNative, pCodeBuf, off, kIemNativeExitReason_ObsoleteTb); \
    617609            } \
    618610        } while (0)
    … …  
    680672        }
    681673
    682         uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5 + 14 + 54 + 8 + 6 + BP_ON_OBSOLETION /* = 87 */);
     674        uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6 + 14 + 54 + 8 + 6 + BP_ON_OBSOLETION /* = 88 */);
    683675
    684676        if (cbLeft > 8)
    … …  
    724716        uint8_t const idxRegCx = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xCX));
    725717
    726         uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5 + 10 + 5 + 5 + 3 + 4 + 3 + BP_ON_OBSOLETION /*= 35*/);
     718        uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6 + 10 + 5 + 5 + 3 + 4 + 3 + BP_ON_OBSOLETION /*= 36*/);
    727719
    728720        /** @todo profile and optimize this further.  Maybe an idea to align by
    … …  
    858850                if (fPendingJmp)
    859851                {
    860                     iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_RelImm19At5);
    861                     pu32CodeBuf[off++] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, 0);
     852                    off = iemNativeEmitJccTbExitEx(pReNative, pu32CodeBuf, off, kIemNativeExitReason_ObsoleteTb,
     853                                                   kArmv8InstrCond_Ne);
    862854                    fPendingJmp = false;
    863855                }
    … …  
    895887                                                            ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
    896888
    897                 iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_RelImm19At5);
    898                 pu32CodeBuf[off++] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, 0);
     889                off = iemNativeEmitJccTbExitEx(pReNative, pu32CodeBuf, off, kIemNativeExitReason_ObsoleteTb,
     890                                               kArmv8InstrCond_Ne);
    899891
    900892                /* Advance and loop. */
    … …  
    10291021     */
    10301022    if (fPendingJmp)
    1031     {
    1032         iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_RelImm19At5);
    1033         pu32CodeBuf[off++] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, 0);
    1034     }
    1035     RT_NOREF(pu32CodeBuf, cbLeft, offPage, pbOpcodes, offConsolidatedJump, idxLabelObsoleteTb);
     1023        off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_ObsoleteTb);
     1024
     1025    RT_NOREF(pu32CodeBuf, cbLeft, offPage, pbOpcodes, offConsolidatedJump);
    10361026
    10371027    /* max costs: memcmp-loop=54; memcmp-no-loop=41; only-src1-ptr=32 */
    … …  
    11921182    /* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal. */
    11931183    off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
    1194     off = iemNativeEmitJaToNewLabel(pReNative, off, kIemNativeLabelType_CheckBranchMiss);
     1184    off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeExitReason_CheckBranchMiss);
    11951185
    11961186    /* 4. Add iem.s.GCPhysInstrBuf and compare with GCPhysRangePageWithOffset. */
    … …  
    12231213                                                | pTb->aRanges[idxRange].offPhysPage)
    12241214                                             + offRange;
    1225     off = iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(pReNative, off, idxRegTmp, GCPhysRangePageWithOffset,
    1226                                                             kIemNativeLabelType_CheckBranchMiss);
     1215    off = iemNativeEmitTestIfGprNotEqualImmAndTbExit(pReNative, off, idxRegTmp, GCPhysRangePageWithOffset,
     1216                                                     kIemNativeExitReason_CheckBranchMiss);
    12271217
    12281218    iemNativeRegFreeTmp(pReNative, idxRegTmp);
    … …  
    13331323     */
    13341324    RTGCPHYS const GCPhysNewPage = iemTbGetRangePhysPageAddr(pTb, idxRange);
    1335     off = iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(pReNative, off, idxRegGCPhys, GCPhysNewPage,
    1336                                                             kIemNativeLabelType_ObsoleteTb);
     1325    off = iemNativeEmitTestIfGprNotEqualImmAndTbExit(pReNative, off, idxRegGCPhys, GCPhysNewPage,
     1326                                                     kIemNativeExitReason_ObsoleteTb);
    13371327
    13381328    iemNativeRegFreeTmp(pReNative, idxRegGCPhys);
    … …  
    13711361     * Define labels and allocate the register for holding the GCPhys of the new page.
    13721362     */
    1373     uint32_t const idxLabelCheckBranchMiss   = iemNativeLabelCreate(pReNative, kIemNativeLabelType_CheckBranchMiss);
    13741363    uint16_t const uTlbSeqNo                 = pReNative->uTlbSeqNo++;
    13751364    RTGCPHYS const GCPhysRangePageWithOffset = iemTbGetRangePhysPageAddr(pTb, idxRange)
    … …  
    15461535    off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegTmp2, GCPhysRangePageWithOffset);
    15471536    off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegTmp, idxRegTmp2);
    1548     off = iemNativeEmitJnzToLabel(pReNative, off, idxLabelCheckBranchMiss);
     1537    off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_CheckBranchMiss);
    15491538    uint32_t const offFixedJumpToEnd = off;
    15501539    off = iemNativeEmitJmpToFixed(pReNative, off, off + 512 /* force rel32 */);
    … …  
    15581547
    15591548    /* Check that we haven't been here before. */
    1560     off = iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(pReNative, off, idxRegTmp2,  false /*f64Bit*/, idxLabelCheckBranchMiss);
     1549    off = iemNativeEmitTestIfGprIsNotZeroAndTbExit(pReNative, off, idxRegTmp2,  false /*f64Bit*/,
     1550                                                   kIemNativeExitReason_CheckBranchMiss);
    15611551
    15621552    /* Jump to the TLB lookup code. */

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompFuncs.h

    r104468 r104797  
    332332                                                              kIemNativeGstRegUse_ForUpdate, false /*fNoVolatileRegs*/,
    333333                                                              true /*fSkipLivenessAssert*/);
    334     off = iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(pReNative, off, idxEflReg,
    335                                                              X86_EFL_TF | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK,
    336                                                              iemNativeLabelCreate(pReNative, kIemNativeLabelType_ReturnWithFlags));
     334    off = iemNativeEmitTestAnyBitsInGprAndTbExitIfAnySet(pReNative, off, idxEflReg,
     335                                                         X86_EFL_TF | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK,
     336                                                         kIemNativeExitReason_ReturnWithFlags);
    337337    off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxEflReg, ~(uint32_t)(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW));
    338338    off = iemNativeEmitStoreGprToVCpuU32(pReNative, off, idxEflReg, RT_UOFFSETOF(VMCPU, cpum.GstCtx.eflags));
    … …  
    394394                                             RT_UOFFSETOF(VMCPU, iem.s.ppTbLookupEntryR3));
    395395
    396         return iemNativeEmitJmpToNewLabel(pReNative, off, kIemNativeLabelType_ReturnBreak);
     396        return iemNativeEmitTbExit(pReNative, off, kIemNativeExitReason_ReturnBreak);
    397397
    398398#else
    … …  
    437437
    438438                if (pReNative->idxLastCheckIrqCallNo != UINT32_MAX)
    439                     return iemNativeEmitJmpToNewLabel(pReNative, off, kIemNativeLabelType_ReturnBreakViaLookup);
    440                 return iemNativeEmitJmpToNewLabel(pReNative, off, kIemNativeLabelType_ReturnBreakViaLookupWithIrq);
     439                    return iemNativeEmitTbExit(pReNative, off, kIemNativeExitReason_ReturnBreakViaLookup);
     440                return iemNativeEmitTbExit(pReNative, off, kIemNativeExitReason_ReturnBreakViaLookupWithIrq);
    441441            }
    442442        }
    443443        if (pReNative->idxLastCheckIrqCallNo != UINT32_MAX)
    444             return iemNativeEmitJmpToNewLabel(pReNative, off, kIemNativeLabelType_ReturnBreakViaLookupWithTlb);
    445         return iemNativeEmitJmpToNewLabel(pReNative, off, kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq);
     444            return iemNativeEmitTbExit(pReNative, off, kIemNativeExitReason_ReturnBreakViaLookupWithTlb);
     445        return iemNativeEmitTbExit(pReNative, off, kIemNativeExitReason_ReturnBreakViaLookupWithTlbAndIrq);
    446446#endif
    447447    }
    … …  
    19541954        /* Allocate a temporary CR0 register. */
    19551955        uint8_t const idxCr0Reg       = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Cr0, kIemNativeGstRegUse_ReadOnly);
    1956         uint8_t const idxLabelRaiseNm = iemNativeLabelCreate(pReNative, kIemNativeLabelType_RaiseNm);
    19571956
    19581957        /*
    … …  
    19611960         */
    19621961        /* Test and jump. */
    1963         off = iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(pReNative, off, idxCr0Reg, X86_CR0_EM | X86_CR0_TS, idxLabelRaiseNm);
     1962        off = iemNativeEmitTestAnyBitsInGprAndTbExitIfAnySet(pReNative, off, idxCr0Reg, X86_CR0_EM | X86_CR0_TS, kIemNativeExitReason_RaiseNm);
    19641963
    19651964        /* Free but don't flush the CR0 register. */
    … …  
    20192018        off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxCr0Reg, X86_CR0_MP | X86_CR0_TS);
    20202019        /* Test and jump. */
    2021         off = iemNativeEmitTestIfGpr32EqualsImmAndJmpToNewLabel(pReNative, off, idxCr0Reg, X86_CR0_MP | X86_CR0_TS, kIemNativeLabelType_RaiseNm);
     2020        off = iemNativeEmitTestIfGpr32EqualsImmAndTbExit(pReNative, off, idxCr0Reg, X86_CR0_MP | X86_CR0_TS, kIemNativeExitReason_RaiseNm);
    20222021
    20232022        /* Free the CR0 register. */
    … …  
    20642063    /* Allocate a temporary FSW register. */
    20652064    uint8_t const idxFpuFswReg    = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_FpuFsw, kIemNativeGstRegUse_ReadOnly);
    2066     uint8_t const idxLabelRaiseMf = iemNativeLabelCreate(pReNative, kIemNativeLabelType_RaiseMf);
    20672065
    20682066    /*
    … …  
    20712069     */
    20722070    /* Test and jump. */
    2073     off = iemNativeEmitTestBitInGprAndJmpToLabelIfSet(pReNative, off, idxFpuFswReg, X86_FSW_ES_BIT, idxLabelRaiseMf);
     2071    off = iemNativeEmitTestBitInGprAndTbExitIfSet(pReNative, off, idxFpuFswReg, X86_FSW_ES_BIT, kIemNativeExitReason_RaiseMf);
    20742072
    20752073    /* Free but don't flush the FSW register. */
    … …  
    21142112
    21152113        /* Allocate a temporary CR0 and CR4 register. */
    2116         uint8_t const idxLabelRaiseSseRelated = iemNativeLabelCreate(pReNative, kIemNativeLabelType_RaiseSseRelated);
    21172114        uint8_t const idxCr0Reg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Cr0);
    21182115        uint8_t const idxCr4Reg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Cr4);
    … …  
    21362133        off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off,   idxTmpReg, X86_CR0_EM | X86_CR0_TS | X86_CR4_OSFXSR);
    21372134        off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off,   idxTmpReg, X86_CR4_OSFXSR);
    2138         off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelRaiseSseRelated, kIemNativeInstrCond_ne);
     2135        off = iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, kIemNativeExitReason_RaiseSseRelated, kIemNativeInstrCond_ne);
    21392136
    21402137#elif defined(RT_ARCH_ARM64)
    … …  
    21532150        pCodeBuf[off++] = Armv8A64MkInstrEorImm(idxTmpReg, idxTmpReg, 0, 0, false /*f64Bit*/);
    21542151        /* -> idxTmpReg[0]=~OSFXSR; idxTmpReg[2]=EM; idxTmpReg[3]=TS; (the rest is zero) */
    2155         off = iemNativeEmitTestIfGprIsNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, idxTmpReg, false /*f64Bit*/,
    2156                                                              idxLabelRaiseSseRelated);
     2152        off = iemNativeEmitTestIfGprIsNotZeroAndTbExitEx(pReNative, pCodeBuf, off, idxTmpReg, false /*f64Bit*/,
     2153                                                         kIemNativeExitReason_RaiseSseRelated);
    21572154
    21582155#else
    … …  
    22102207
    22112208        /* Allocate a temporary CR0, CR4 and XCR0 register. */
    2212         uint8_t const idxLabelRaiseAvxRelated = iemNativeLabelCreate(pReNative, kIemNativeLabelType_RaiseAvxRelated);
    22132209        uint8_t const idxCr0Reg  = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Cr0);
    22142210        uint8_t const idxCr4Reg  = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Cr4);
    … …  
    22432239        off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off,                idxTmpReg, ((XSAVE_C_YMM | XSAVE_C_SSE) << 2) | 2);
    22442240        /* -> idxTmpReg[0]=CR0.TS idxTmpReg[1]=~CR4.OSXSAVE; idxTmpReg[2]=0; idxTmpReg[3]=~SSE; idxTmpReg[4]=~YMM; */
    2245         off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelRaiseAvxRelated, kIemNativeInstrCond_ne);
     2241        off = iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, kIemNativeExitReason_RaiseAvxRelated, kIemNativeInstrCond_ne);
    22462242
    22472243#elif defined(RT_ARCH_ARM64)
    … …  
    22612257        pCodeBuf[off++] = Armv8A64MkInstrBfxil(idxTmpReg, idxCr0Reg, X86_CR0_TS_BIT, 1, false /*f64Bit*/);
    22622258        /* -> idxTmpReg[0]=CR0.TS; idxTmpReg[1]=~CR4.OSXSAVE; idxTmpReg[2]=~SSE; idxTmpReg[3]=~YMM; (the rest is zero) */
    2263         off = iemNativeEmitTestIfGprIsNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, idxTmpReg, false /*f64Bit*/,
    2264                                                              idxLabelRaiseAvxRelated);
     2259        off = iemNativeEmitTestIfGprIsNotZeroAndTbExitEx(pReNative, pCodeBuf, off, idxTmpReg, false /*f64Bit*/,
     2260                                                         kIemNativeExitReason_RaiseAvxRelated);
    22652261
    22662262#else
    … …  
    23032299#endif
    23042300
    2305     uint8_t const idxLabelRaiseSseAvxFpRelated = iemNativeLabelCreate(pReNative, kIemNativeLabelType_RaiseSseAvxFpRelated);
    23062301    uint8_t const idxRegMxCsr = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_MxCsr, kIemNativeGstRegUse_ReadOnly);
    23072302    uint8_t const idxRegTmp   = iemNativeRegAllocTmp(pReNative, &off);
    … …  
    23172312    /* tmp &= mxcsr */
    23182313    off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, idxRegTmp, idxRegMxCsr);
    2319     off = iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(pReNative, off, idxRegTmp, X86_MXCSR_XCPT_FLAGS,
    2320                                                              idxLabelRaiseSseAvxFpRelated);
     2314    off = iemNativeEmitTestAnyBitsInGprAndTbExitIfAnySet(pReNative, off, idxRegTmp, X86_MXCSR_XCPT_FLAGS,
     2315                                                         kIemNativeExitReason_RaiseSseAvxFpRelated);
    23212316
    23222317    /* Free but don't flush the MXCSR register. */
    … …  
    23542349#endif
    23552350
    2356     uint8_t const idxLabelRaiseDe = iemNativeLabelCreate(pReNative, kIemNativeLabelType_RaiseDe);
    2357 
    23582351    /* raise \#DE exception unconditionally. */
    2359     off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelRaiseDe);
    2360 
    2361     return off;
     2352    return iemNativeEmitTbExit(pReNative, off, kIemNativeExitReason_RaiseDe);
    23622353}
    23632354
    … …  
    23932384#endif
    23942385
    2395     uint8_t const idxLabelRaiseGp0 = iemNativeLabelCreate(pReNative, kIemNativeLabelType_RaiseGp0);
    2396     uint8_t const idxVarReg        = iemNativeVarRegisterAcquire(pReNative, idxVarEffAddr, &off);
    2397 
    2398     off = iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(pReNative, off, idxVarReg, cbAlign - 1, idxLabelRaiseGp0);
     2386    uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxVarEffAddr, &off);
     2387
     2388    off = iemNativeEmitTestAnyBitsInGprAndTbExitIfAnySet(pReNative, off, idxVarReg, cbAlign - 1,
     2389                                                         kIemNativeExitReason_RaiseGp0);
    23992390
    24002391    iemNativeVarRegisterRelease(pReNative, idxVarEffAddr);

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

    r104524 r104797  
    62216221    off = iemNativeEmitAddGpr32Imm(pReNative, off, iTmpReg, (int32_t)0x8000);
    62226222    off = iemNativeEmitShiftGprRight(pReNative, off, iTmpReg, 16);
    6223     off = iemNativeEmitJnzToNewLabel(pReNative, off, kIemNativeLabelType_RaiseGp0);
     6223    off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_RaiseGp0);
    62246224
    62256225    iemNativeRegFreeTmp(pReNative, iTmpReg);
    … …  
    62406240    off = iemNativeEmitAddTwoGprs(pReNative, off, iTmpReg, idxAddrReg);
    62416241    off = iemNativeEmitCmpArm64(pReNative, off, ARMV8_A64_REG_XZR, iTmpReg, true /*f64Bit*/, 48 /*cShift*/, kArmv8A64InstrShift_Lsr);
    6242     off = iemNativeEmitJnzToNewLabel(pReNative, off, kIemNativeLabelType_RaiseGp0);
     6242    off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_RaiseGp0);
    62436243
    62446244    iemNativeRegFreeTmp(pReNative, iTmpReg);

  • trunk/src/VBox/VMM/include/IEMN8veRecompiler.h

    r104468 r104797  
    445445
    446446
     447/** TB exit reasons. */
     448typedef enum
     449{
     450    kIemNativeExitReason_Invalid = 0,
     451    kIemNativeExitReason_RaiseDe,                /**< Raise (throw) X86_XCPT_DE (00h). */
     452    kIemNativeExitReason_RaiseUd,                /**< Raise (throw) X86_XCPT_UD (06h). */
     453    kIemNativeExitReason_RaiseSseRelated,        /**< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. */
     454    kIemNativeExitReason_RaiseAvxRelated,        /**< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. */
     455    kIemNativeExitReason_RaiseSseAvxFpRelated,   /**< Raise (throw) X86_XCPT_UD or X86_XCPT_XF according to c4. */
     456    kIemNativeExitReason_RaiseNm,                /**< Raise (throw) X86_XCPT_NM (07h). */
     457    kIemNativeExitReason_RaiseGp0,               /**< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. */
     458    kIemNativeExitReason_RaiseMf,                /**< Raise (throw) X86_XCPT_MF (10h). */
     459    kIemNativeExitReason_RaiseXf,                /**< Raise (throw) X86_XCPT_XF (13h). */
     460    kIemNativeExitReason_ObsoleteTb,
     461    kIemNativeExitReason_NeedCsLimChecking,
     462    kIemNativeExitReason_CheckBranchMiss,
     463    kIemNativeExitReason_Return, /** @todo Eliminate (needed for the compile assertion below). */
     464    kIemNativeExitReason_ReturnBreak,
     465    kIemNativeExitReason_ReturnBreakFF,
     466    kIemNativeExitReason_ReturnBreakViaLookup,
     467    kIemNativeExitReason_ReturnBreakViaLookupWithIrq,
     468    kIemNativeExitReason_ReturnBreakViaLookupWithTlb,
     469    kIemNativeExitReason_ReturnBreakViaLookupWithTlbAndIrq,
     470    kIemNativeExitReason_ReturnWithFlags,
     471    kIemNativeExitReason_NonZeroRetOrPassUp,
     472} IEMNATIVEEXITREASON;
     473
     474
    447475/** Native code generator label types. */
    448476typedef enum
    … …  
    498526} IEMNATIVELABELTYPE;
    499527
     528/* Temporary kludge until all jumps to TB exit labels are converted to the new TB exiting style,
     529 * see @bugref{10677}. */
     530#define IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(a_Reason) \
     531    ((int)kIemNativeLabelType_ ## a_Reason == (int)kIemNativeExitReason_ ## a_Reason)
     532AssertCompile(   IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseDe)
     533              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseUd)
     534              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseSseRelated)
     535              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseAvxRelated)
     536              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseSseAvxFpRelated)
     537              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseNm)
     538              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseGp0)
     539              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseMf)
     540              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseXf)
     541              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ObsoleteTb)
     542              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(NeedCsLimChecking)
     543              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(CheckBranchMiss)
     544              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(Return)
     545              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreak)
     546              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreakFF)
     547              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreakViaLookup)
     548              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreakViaLookupWithIrq)
     549              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreakViaLookupWithTlb)
     550              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreakViaLookupWithTlbAndIrq)
     551              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnWithFlags)
     552              && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(NonZeroRetOrPassUp));
     553
     554
    500555/** Native code generator label definition. */
    501556typedef struct IEMNATIVELABEL

  • trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h

    r104468 r104797  
    77907790
    77917791
     7792/*********************************************************************************************************************************
     7793*   TB exiting helpers.                                                                                                          *
     7794*********************************************************************************************************************************/
     7795
     7796
     7797/**
     7798 * Emits a Jcc rel32 / B.cc imm19 to the epilog.
     7799 */
     7800DECL_INLINE_THROW(uint32_t)
     7801iemNativeEmitJccTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     7802                       IEMNATIVEEXITREASON enmExitReason, IEMNATIVEINSTRCOND enmCond)
     7803{
     7804    return iemNativeEmitJccToNewLabel(pReNative, off, (IEMNATIVELABELTYPE)enmExitReason, 0 /*uData*/, enmCond);
     7805}
     7806
     7807
     7808/**
     7809 * Emits a JNZ/JNE rel32 / B.NE imm19 to the TB exit routine with the given reason.
     7810 */
     7811DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     7812                                                   IEMNATIVEEXITREASON enmExitReason)
     7813{
     7814#ifdef RT_ARCH_AMD64
     7815    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kIemNativeInstrCond_ne);
     7816#elif defined(RT_ARCH_ARM64)
     7817    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kArmv8InstrCond_Ne);
     7818#else
     7819# error "Port me!"
     7820#endif
     7821}
     7822
     7823
     7824/**
     7825 * Emits a JZ/JE rel32 / B.EQ imm19 to the TB exit routine with the given reason.
     7826 */
     7827DECL_INLINE_THROW(uint32_t) iemNativeEmitJzTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     7828                                                   IEMNATIVEEXITREASON enmExitReason)
     7829{
     7830#ifdef RT_ARCH_AMD64
     7831    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kIemNativeInstrCond_e);
     7832#elif defined(RT_ARCH_ARM64)
     7833    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kArmv8InstrCond_Eq);
     7834#else
     7835# error "Port me!"
     7836#endif
     7837}
     7838
     7839
     7840/**
     7841 * Emits a JA/JNBE rel32 / B.HI imm19 to the TB exit.
     7842 */
     7843DECL_INLINE_THROW(uint32_t) iemNativeEmitJaTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     7844                                                  IEMNATIVEEXITREASON enmExitReason)
     7845{
     7846#ifdef RT_ARCH_AMD64
     7847    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kIemNativeInstrCond_nbe);
     7848#elif defined(RT_ARCH_ARM64)
     7849    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kArmv8InstrCond_Hi);
     7850#else
     7851# error "Port me!"
     7852#endif
     7853}
     7854
     7855
     7856/**
     7857 * Emits a JL/JNGE rel32 / B.LT imm19 to the TB exit with the given reason.
     7858 */
     7859DECL_INLINE_THROW(uint32_t) iemNativeEmitJlTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     7860                                                  IEMNATIVEEXITREASON enmExitReason)
     7861{
     7862#ifdef RT_ARCH_AMD64
     7863    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kIemNativeInstrCond_l);
     7864#elif defined(RT_ARCH_ARM64)
     7865    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kArmv8InstrCond_Lt);
     7866#else
     7867# error "Port me!"
     7868#endif
     7869}
     7870
     7871
     7872DECL_INLINE_THROW(uint32_t)
     7873iemNativeEmitTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEEXITREASON enmExitReason)
     7874{
     7875    return iemNativeEmitJmpToNewLabel(pReNative, off, (IEMNATIVELABELTYPE)enmExitReason);
     7876}
     7877
     7878
     7879DECL_INLINE_THROW(uint32_t)
     7880iemNativeEmitTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off, IEMNATIVEEXITREASON enmExitReason)
     7881{
     7882    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, (IEMNATIVELABELTYPE)enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     7883    return iemNativeEmitJmpToLabelEx(pReNative, pCodeBuf, off, idxLabel);
     7884}
     7885
     7886
     7887/**
     7888 * Emits a jump to the TB exit with @a enmExitReason on the condition _any_ of the bits in @a fBits
     7889 * are set in @a iGprSrc.
     7890 */
     7891DECL_INLINE_THROW(uint32_t)
     7892iemNativeEmitTestAnyBitsInGprAndTbExitIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     7893                                               uint8_t iGprSrc, uint64_t fBits, IEMNATIVEEXITREASON enmExitReason)
     7894{
     7895    Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
     7896
     7897    off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
     7898    return iemNativeEmitJnzTbExit(pReNative, off, enmExitReason);
     7899}
     7900
     7901
     7902/**
     7903 * Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
     7904 * are set in @a iGprSrc.
     7905 */
     7906DECL_INLINE_THROW(uint32_t)
     7907iemNativeEmitTestAnyBitsInGprAndTbExitIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     7908                                                uint8_t iGprSrc, uint64_t fBits, IEMNATIVEEXITREASON enmExitReason)
     7909{
     7910    Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
     7911
     7912    off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
     7913    return iemNativeEmitJzTbExit(pReNative, off, enmExitReason);
     7914}
     7915
     7916
     7917/**
     7918 * Emits code that exits the TB with the given reason if @a iGprLeft and @a iGprRight
     7919 * differs.
     7920 */
     7921DECL_INLINE_THROW(uint32_t)
     7922iemNativeEmitTestIfGprNotEqualGprAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     7923                                           uint8_t iGprLeft, uint8_t iGprRight, IEMNATIVEEXITREASON enmExitReason)
     7924{
     7925    off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);
     7926    off = iemNativeEmitJnzTbExit(pReNative, off, enmExitReason);
     7927    return off;
     7928}
     7929
     7930
     7931/**
     7932 * Emits code that jumps to the given label if 32-bit @a iGprSrc differs from
     7933 * @a uImm.
     7934 */
     7935DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32NotEqualImmAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     7936                                                                         uint8_t iGprSrc, uint32_t uImm, IEMNATIVEEXITREASON enmExitReason)
     7937{
     7938    off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
     7939    off = iemNativeEmitJnzTbExit(pReNative, off, enmExitReason);
     7940    return off;
     7941}
     7942
     7943
     7944/**
     7945 * Emits code that exits the current TB if @a iGprSrc differs from @a uImm.
     7946 */
     7947DECL_INLINE_THROW(uint32_t)
     7948iemNativeEmitTestIfGprNotEqualImmAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     7949                                           uint8_t iGprSrc, uint64_t uImm, IEMNATIVEEXITREASON enmExitReason)
     7950{
     7951    off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);
     7952    off = iemNativeEmitJnzTbExit(pReNative, off, enmExitReason);
     7953    return off;
     7954}
     7955
     7956
     7957/**
     7958 * Emits code that exits the current TB with the given reason if 32-bit @a iGprSrc equals @a uImm.
     7959 */
     7960DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32EqualsImmAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     7961                                                                       uint8_t iGprSrc, uint32_t uImm, IEMNATIVEEXITREASON enmExitReason)
     7962{
     7963    off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
     7964    off = iemNativeEmitJzTbExit(pReNative, off, enmExitReason);
     7965    return off;
     7966}
     7967
     7968
     7969/**
     7970 * Emits code to exit the current TB with the reason @a enmExitReason on the condition that bit @a iBitNo _is_ _set_ in
     7971 * @a iGprSrc.
     7972 *
     7973 * @note On ARM64 the range is only +/-8191 instructions.
     7974 */
     7975DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndTbExitIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     7976                                                                    uint8_t iGprSrc, uint8_t iBitNo, IEMNATIVEEXITREASON enmExitReason)
     7977{
     7978    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, (IEMNATIVELABELTYPE)enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     7979    return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /*fJmpIfSet*/);
     7980}
     7981
     7982
     7983/**
     7984 * Emits code that exits the current TB with @a enmExitReason if @a iGprSrc is not zero.
     7985 *
     7986 * The operand size is given by @a f64Bit.
     7987 */
     7988DECL_FORCE_INLINE_THROW(uint32_t)
     7989iemNativeEmitTestIfGprIsNotZeroAndTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
     7990                                           uint8_t iGprSrc, bool f64Bit, IEMNATIVEEXITREASON enmExitReason)
     7991{
     7992    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, (IEMNATIVELABELTYPE)enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     7993    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
     7994                                                                f64Bit, true /*fJmpIfNotZero*/, idxLabel);
     7995}
     7996
     7997
     7998/**
     7999 * Emits code to exit the current TB on the given condition.
     8000 */
     8001DECL_INLINE_THROW(uint32_t)
     8002iemNativeEmitJccTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off, IEMNATIVEEXITREASON enmExitReason, IEMNATIVEINSTRCOND enmCond)
     8003{
     8004    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, (IEMNATIVELABELTYPE)enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     8005#ifdef RT_ARCH_AMD64
     8006    off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabel, enmCond);
     8007#elif defined(RT_ARCH_ARM64)
     8008    off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabel, enmCond);
     8009#else
     8010# error "Port me!"
     8011#endif
     8012    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     8013    return off;
     8014}
     8015
     8016
     8017/**
     8018 * Emits code to exit the current TB with the given reason @a enmExitReason if @a iGprSrc is not zero.
     8019 *
     8020 * The operand size is given by @a f64Bit.
     8021 */
     8022DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsNotZeroAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     8023                                                                     uint8_t iGprSrc, bool f64Bit, IEMNATIVEEXITREASON enmExitReason)
     8024{
     8025    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, (IEMNATIVELABELTYPE)enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     8026    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, true /*fJmpIfNotZero*/, idxLabel);
     8027}
     8028
     8029
    77928030#ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
     8031/*********************************************************************************************************************************
     8032*   SIMD helpers.                                                                                                                *
     8033*********************************************************************************************************************************/
     8034
     8035
    77938036/**
    77948037 * Emits code to load the variable address into an argument GPR.
Note: See TracChangeset for help on using the changeset viewer.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette