VirtualBox

Ignore:
Timestamp:
Jun 20, 2024 2:07:04 PM (9 months ago)
Author:
vboxsync
svn:sync-xref-src-repo-rev:
163594
Message:

VMM/IEM: Relax alignment restrictions in native code TLB lookup, avoid the fallback/tlbmiss code path for most accesses as long as they're within the same page. bugref:10687

File:
1 edited

Legend:

Unmodified
Added
Removed

  • TabularUnified trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h

    r104798 r104984  
    632632}
    633633
    634 
    635634/**
    636635 * Emits a 32-bit GPR load of a VCpu value.
    … …  
    638637 */
    639638DECL_INLINE_THROW(uint32_t)
     639iemNativeEmitLoadGprFromVCpuU32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
     640{
     641#ifdef RT_ARCH_AMD64
     642    /* mov reg32, mem32 */
     643    if (iGpr >= 8)
     644        pCodeBuf[off++] = X86_OP_REX_R;
     645    pCodeBuf[off++] = 0x8b;
     646    off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, iGpr, offVCpu);
     647
     648#elif defined(RT_ARCH_ARM64)
     649    off = iemNativeEmitGprByVCpuLdStEx(pCodeBuf, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
     650
     651#else
     652# error "port me"
     653#endif
     654    return off;
     655}
     656
     657
     658/**
     659 * Emits a 32-bit GPR load of a VCpu value.
     660 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
     661 */
     662DECL_INLINE_THROW(uint32_t)
    640663iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
    641664{
    642665#ifdef RT_ARCH_AMD64
    643     /* mov reg32, mem32 */
    644     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
    645     if (iGpr >= 8)
    646         pbCodeBuf[off++] = X86_OP_REX_R;
    647     pbCodeBuf[off++] = 0x8b;
    648     off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
     666    off = iemNativeEmitLoadGprFromVCpuU32Ex(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGpr, offVCpu);
    649667    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    650668
    … …  
    69526970 */
    69536971DECL_INLINE_THROW(uint32_t)
    6954 iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
    6955                                            uint8_t iBitNo, uint32_t idxLabel, bool fJmpIfSet)
     6972iemNativeEmitTestBitInGprAndJmpToFixedIfCcEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t iBitNo,
     6973                                             uint32_t offTarget, uint32_t *poffFixup, bool fJmpIfSet)
    69566974{
    69576975    Assert(iBitNo < 64);
    69586976#ifdef RT_ARCH_AMD64
    6959     uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
    69606977    if (iBitNo < 8)
    69616978    {
    69626979        /* test Eb, imm8 */
    69636980        if (iGprSrc >= 4)
    6964             pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
    6965         pbCodeBuf[off++] = 0xf6;
    6966         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
    6967         pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
    6968         off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
     6981            pCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
     6982        pCodeBuf[off++] = 0xf6;
     6983        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
     6984        pCodeBuf[off++] = (uint8_t)1 << iBitNo;
     6985        if (poffFixup)
     6986            *poffFixup = off;
     6987        off = iemNativeEmitJccToFixedEx(pCodeBuf, off, offTarget, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
    69696988    }
    69706989    else
    … …  
    69726991        /* bt Ev, imm8 */
    69736992        if (iBitNo >= 32)
    6974             pbCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
     6993            pCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
    69756994        else if (iGprSrc >= 8)
    6976             pbCodeBuf[off++] = X86_OP_REX_B;
    6977         pbCodeBuf[off++] = 0x0f;
    6978         pbCodeBuf[off++] = 0xba;
    6979         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
    6980         pbCodeBuf[off++] = iBitNo;
    6981         off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
     6995            pCodeBuf[off++] = X86_OP_REX_B;
     6996        pCodeBuf[off++] = 0x0f;
     6997        pCodeBuf[off++] = 0xba;
     6998        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
     6999        pCodeBuf[off++] = iBitNo;
     7000        if (poffFixup)
     7001            *poffFixup = off;
     7002        off = iemNativeEmitJccToFixedEx(pCodeBuf, off, offTarget, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
     7003    }
     7004
     7005#elif defined(RT_ARCH_ARM64)
     7006    /* Just use the TBNZ instruction here. */
     7007    if (poffFixup)
     7008        *poffFixup = off;
     7009    pCodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, off - offTarget, iGprSrc, iBitNo);
     7010
     7011#else
     7012# error "Port me!"
     7013#endif
     7014    return off;
     7015}
     7016
     7017
     7018/**
     7019 * Emits a jump to @a idxTarget on the condition that bit @a iBitNo _is_ _set_
     7020 * in @a iGprSrc.
     7021 */
     7022DECL_INLINE_THROW(uint32_t)
     7023iemNativeEmitTestBitInGprAndJmpToFixedIfSetEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t iBitNo,
     7024                                              uint32_t offTarget, uint32_t *poffFixup)
     7025{
     7026    return iemNativeEmitTestBitInGprAndJmpToFixedIfCcEx(pCodeBuf, off, iGprSrc, iBitNo, offTarget, poffFixup, true /*fJmpIfSet*/);
     7027}
     7028
     7029
     7030/**
     7031 * Emits a jump to @a idxTarget on the condition that bit @a iBitNo _is_ _not_
     7032 * _set_ in @a iGprSrc.
     7033 */
     7034DECL_INLINE_THROW(uint32_t)
     7035iemNativeEmitTestBitInGprAndJmpToLabelIfNotSetEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t iBitNo,
     7036                                                 uint32_t offTarget, uint32_t *poffFixup)
     7037{
     7038    return iemNativeEmitTestBitInGprAndJmpToFixedIfCcEx(pCodeBuf, off, iGprSrc, iBitNo, offTarget, poffFixup, false /*fJmpIfSet*/);
     7039}
     7040
     7041
     7042
     7043/**
     7044 * Internal helper, don't call directly.
     7045 */
     7046DECL_INLINE_THROW(uint32_t)
     7047iemNativeEmitTestBitInGprAndJmpToLabelIfCcEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
     7048                                             uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel, bool fJmpIfSet)
     7049{
     7050    Assert(iBitNo < 64);
     7051#ifdef RT_ARCH_AMD64
     7052    if (iBitNo < 8)
     7053    {
     7054        /* test Eb, imm8 */
     7055        if (iGprSrc >= 4)
     7056            pCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
     7057        pCodeBuf[off++] = 0xf6;
     7058        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
     7059        pCodeBuf[off++] = (uint8_t)1 << iBitNo;
     7060        off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabel,
     7061                                        fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
     7062    }
     7063    else
     7064    {
     7065        /* bt Ev, imm8 */
     7066        if (iBitNo >= 32)
     7067            pCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
     7068        else if (iGprSrc >= 8)
     7069            pCodeBuf[off++] = X86_OP_REX_B;
     7070        pCodeBuf[off++] = 0x0f;
     7071        pCodeBuf[off++] = 0xba;
     7072        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
     7073        pCodeBuf[off++] = iBitNo;
     7074        off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabel,
     7075                                        fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);
    69827076    }
    69837077
    69847078#elif defined(RT_ARCH_ARM64)
    69857079    /* Use the TBNZ instruction here. */
    6986     uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
    69877080    if (pReNative->paLabels[idxLabel].enmType > kIemNativeLabelType_LastWholeTbBranch)
    69887081    {
    … …  
    69937086        {
    69947087            iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5);
    6995             pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
     7088            pCodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);
    69967089        }
    69977090        //else
    … …  
    69997092        //    RT_BREAKPOINT();
    70007093        //    Assert(off - offLabel <= 0x1fffU);
    7001         //    pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, offLabel - off, iGprSrc, iBitNo);
     7094        //    pCodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, offLabel - off, iGprSrc, iBitNo);
    70027095        //
    70037096        //}
    … …  
    70067099    {
    70077100        Assert(Armv8A64ConvertImmRImmS2Mask64(0x40, (64U - iBitNo) & 63U) == RT_BIT_64(iBitNo));
    7008         pu32CodeBuf[off++] = Armv8A64MkInstrTstImm(iGprSrc, 0x40, (64U - iBitNo) & 63U);
     7101        pCodeBuf[off++] = Armv8A64MkInstrTstImm(iGprSrc, 0x40, (64U - iBitNo) & 63U);
    70097102        iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
    7010         pu32CodeBuf[off++] = Armv8A64MkInstrBCond(fJmpIfSet ? kArmv8InstrCond_Ne : kArmv8InstrCond_Eq, 0);
     7103        pCodeBuf[off++] = Armv8A64MkInstrBCond(fJmpIfSet ? kArmv8InstrCond_Ne : kArmv8InstrCond_Eq, 0);
    70117104    }
    70127105
    … …  
    70147107# error "Port me!"
    70157108#endif
    7016     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    70177109    return off;
    70187110}
    … …  
    70227114 * Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in
    70237115 * @a iGprSrc.
    7024  *
    7025  * @note On ARM64 the range is only +/-8191 instructions.
     7116 */
     7117DECL_INLINE_THROW(uint32_t)
     7118iemNativeEmitTestBitInGprAndJmpToLabelIfSetEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
     7119                                              uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
     7120{
     7121    return iemNativeEmitTestBitInGprAndJmpToLabelIfCcEx(pReNative, pCodeBuf, off, iGprSrc, iBitNo, idxLabel, true /*fJmpIfSet*/);
     7122}
     7123
     7124
     7125/**
     7126 * Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_
     7127 * _set_ in @a iGprSrc.
     7128 */
     7129DECL_INLINE_THROW(uint32_t)
     7130iemNativeEmitTestBitInGprAndJmpToLabelIfNotSetEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
     7131                                                 uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)
     7132{
     7133    return iemNativeEmitTestBitInGprAndJmpToLabelIfCcEx(pReNative, pCodeBuf, off, iGprSrc, iBitNo, idxLabel, false /*fJmpIfSet*/);
     7134}
     7135
     7136
     7137/**
     7138 * Internal helper, don't call directly.
     7139 */
     7140DECL_INLINE_THROW(uint32_t)
     7141iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
     7142                                           uint8_t iBitNo, uint32_t idxLabel, bool fJmpIfSet)
     7143{
     7144#ifdef RT_ARCH_AMD64
     7145    off = iemNativeEmitTestBitInGprAndJmpToLabelIfCcEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 5+6), off,
     7146                                                       iGprSrc, iBitNo, idxLabel, fJmpIfSet);
     7147#elif defined(RT_ARCH_ARM64)
     7148    off = iemNativeEmitTestBitInGprAndJmpToLabelIfCcEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 2), off,
     7149                                                       iGprSrc, iBitNo, idxLabel, fJmpIfSet);
     7150#else
     7151# error "Port me!"
     7152#endif
     7153    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     7154    return off;
     7155}
     7156
     7157
     7158/**
     7159 * Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in
     7160 * @a iGprSrc.
    70267161 */
    70277162DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    … …  
    70357170 * Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_
    70367171 * _set_ in @a iGprSrc.
    7037  *
    7038  * @note On ARM64 the range is only +/-8191 instructions.
    70397172 */
    70407173DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
Note: See TracChangeset for help on using the changeset viewer.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette