VirtualBox

Changeset 102850 in vbox for trunk/src/VBox/VMM/include


Ignore:
Timestamp:
Jan 12, 2024 12:47:47 AM (15 months ago)
Author:
vboxsync
svn:sync-xref-src-repo-rev:
161051
Message:

VMM/IEM: Implemented the first of two code TLB lookups. bugref:10371

Location:
trunk/src/VBox/VMM/include
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/VBox/VMM/include/IEMInternal.h

    r102841 r102850  
    17451745    /** Native recompiled execution: Code TLB misses for new page. */
    17461746    STAMCOUNTER             StatNativeCodeTlbMissesNewPage;
    1747     uint64_t                au64Padding[6];
     1747    /** Native recompiled execution: Code TLB hits for new page. */
     1748    STAMCOUNTER             StatNativeCodeTlbHitsForNewPage;
     1749    /** Native recompiled execution: Code TLB misses for new page with offset. */
     1750    STAMCOUNTER             StatNativeCodeTlbMissesNewPageWithOffset;
     1751    /** Native recompiled execution: Code TLB hits for new page with offset. */
     1752    STAMCOUNTER             StatNativeCodeTlbHitsForNewPageWithOffset;
     1753    uint64_t                au64Padding[3];
    17481754    /** @} */
    17491755

  • trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h

    r102803 r102850  
    710710 * Emits a store of a GPR value to a 64-bit VCpu field.
    711711 */
     712DECL_FORCE_INLINE_THROW(uint32_t)
     713iemNativeEmitStoreGprToVCpuU64Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint32_t offVCpu,
     714                                 uint8_t iGprTmp = UINT8_MAX)
     715{
     716#ifdef RT_ARCH_AMD64
     717    /* mov mem64, reg64 */
     718    if (iGpr < 8)
     719        pCodeBuf[off++] = X86_OP_REX_W;
     720    else
     721        pCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
     722    pCodeBuf[off++] = 0x89;
     723    off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, iGpr, offVCpu);
     724    RT_NOREF(iGprTmp);
     725
     726#elif defined(RT_ARCH_ARM64)
     727    off = iemNativeEmitGprByVCpuLdStEx(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t), iGprTmp);
     728
     729#else
     730# error "port me"
     731#endif
     732    return off;
     733}
     734
     735
     736/**
     737 * Emits a store of a GPR value to a 64-bit VCpu field.
     738 */
    712739DECL_INLINE_THROW(uint32_t)
    713740iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
    714741{
    715742#ifdef RT_ARCH_AMD64
    716     /* mov mem64, reg64 */
    717     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
    718     if (iGpr < 8)
    719         pbCodeBuf[off++] = X86_OP_REX_W;
    720     else
    721         pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
    722     pbCodeBuf[off++] = 0x89;
    723     off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
    724     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    725 
    726 #elif defined(RT_ARCH_ARM64)
    727     off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
    728 
    729 #else
    730 # error "port me"
    731 #endif
     743    off = iemNativeEmitStoreGprToVCpuU64Ex(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGpr, offVCpu);
     744#elif defined(RT_ARCH_ARM64)
     745    off = iemNativeEmitStoreGprToVCpuU64Ex(iemNativeInstrBufEnsure(pReNative, off, 5), off, iGpr, offVCpu,
     746                                           IEMNATIVE_REG_FIXED_TMP0);
     747#else
     748# error "port me"
     749#endif
     750    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    732751    return off;
    733752}
    … …  
    802821#elif defined(RT_ARCH_ARM64)
    803822    off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
     823
     824#else
     825# error "port me"
     826#endif
     827    return off;
     828}
     829
     830
     831/**
     832 * Emits a store of an immediate value to a 16-bit VCpu field.
     833 *
     834 * @note ARM64: A idxTmp1 is always required! The idxTmp2 depends on whehter the
     835 *       offset can be encoded as an immediate or not.  The @a offVCpu immediate
     836 *       range is 0..8190 bytes from VMCPU and the same from CPUMCPU.
     837 */
     838DECL_FORCE_INLINE_THROW(uint32_t)
     839iemNativeEmitStoreImmToVCpuU16Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint16_t uImm, uint32_t offVCpu,
     840                                 uint8_t idxTmp1 = UINT8_MAX, uint8_t idxTmp2 = UINT8_MAX)
     841{
     842#ifdef RT_ARCH_AMD64
     843    /* mov mem16, imm16 */
     844    pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
     845    pCodeBuf[off++] = 0xc7;
     846    off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, 0, offVCpu);
     847    pCodeBuf[off++] = RT_BYTE1(uImm);
     848    pCodeBuf[off++] = RT_BYTE2(uImm);
     849    RT_NOREF(idxTmp1, idxTmp2);
     850
     851#elif defined(RT_ARCH_ARM64)
     852    if (idxTmp1 != UINT8_MAX)
     853    {
     854        pCodeBuf[off++] = Armv8A64MkInstrMovZ(idxTmp1, uImm);
     855        off = iemNativeEmitGprByVCpuLdStEx(pCodeBuf, off, idxTmp1, offVCpu, kArmv8A64InstrLdStType_St_Half,
     856                                           sizeof(uint16_t), idxTmp2);
     857    }
     858    else
     859# ifdef IEM_WITH_THROW_CATCH
     860        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     861# else
     862        AssertReleaseFailedStmt(off = UINT32_MAX);
     863# endif
    804864
    805865#else
    … …  
    38573917 *       and ARM64 hosts.
    38583918 */
    3859 DECL_INLINE_THROW(uint32_t)
    3860 iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
     3919DECL_FORCE_INLINE(uint32_t)
     3920iemNativeEmitAndGprByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
    38613921{
    38623922#if defined(RT_ARCH_AMD64)
    38633923    /* and Gv, Ev */
    3864     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
    3865     pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
    3866     pbCodeBuf[off++] = 0x23;
    3867     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
     3924    pCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
     3925    pCodeBuf[off++] = 0x23;
     3926    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
    38683927    RT_NOREF(fSetFlags);
    38693928
    38703929#elif defined(RT_ARCH_ARM64)
    3871     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    38723930    if (!fSetFlags)
    3873         pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
    3874     else
    3875         pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
    3876 
     3931        pCodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
     3932    else
     3933        pCodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
     3934
     3935#else
     3936# error "Port me"
     3937#endif
     3938    return off;
     3939}
     3940
     3941
     3942/**
     3943 * Emits code for AND'ing two 64-bit GPRs.
     3944 *
     3945 * @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
     3946 *       and ARM64 hosts.
     3947 */
     3948DECL_INLINE_THROW(uint32_t)
     3949iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
     3950{
     3951#if defined(RT_ARCH_AMD64)
     3952    off = iemNativeEmitAndGprByGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc, fSetFlags);
     3953#elif defined(RT_ARCH_ARM64)
     3954    off = iemNativeEmitAndGprByGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc, fSetFlags);
    38773955#else
    38783956# error "Port me"
    … …  
    40874165}
    40884166
     4167
     4168/**
     4169 * Emits code for AND'ing an 64-bit GPRs with a constant.
     4170 *
     4171 * @note For ARM64 any complicated immediates w/o a AND/ANDS compatible
     4172 *       encoding will assert / throw exception if @a iGprDst and @a iGprSrc are
     4173 *       the same.
     4174 */
     4175DECL_FORCE_INLINE_THROW(uint32_t)
     4176iemNativeEmitGprEqGprAndImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, uint64_t uImm,
     4177                              bool fSetFlags = false)
     4178{
     4179#if defined(RT_ARCH_AMD64)
     4180    off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprDst, uImm);
     4181    off = iemNativeEmitAndGprByGprEx(pCodeBuf, off, iGprDst, iGprSrc);
     4182    RT_NOREF(fSetFlags);
     4183
     4184#elif defined(RT_ARCH_ARM64)
     4185    uint32_t uImmR     = 0;
     4186    uint32_t uImmNandS = 0;
     4187    if (Armv8A64ConvertMask64ToImmRImmS(uImm, &uImmNandS, &uImmR))
     4188    {
     4189        if (!fSetFlags)
     4190            pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, uImmNandS, uImmR);
     4191        else
     4192            pCodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprSrc, uImmNandS, uImmR);
     4193    }
     4194    else if (iGprDst != iGprSrc)
     4195    {
     4196        off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprDst, uImm);
     4197        off = iemNativeEmitAndGprByGprEx(pCodeBuf, off, iGprDst, iGprSrc, fSetFlags);
     4198    }
     4199    else
     4200# ifdef IEM_WITH_THROW_CATCH
     4201        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     4202# else
     4203        AssertReleaseFailedStmt(off = UINT32_MAX);
     4204# endif
     4205
     4206#else
     4207# error "Port me"
     4208#endif
     4209    return off;
     4210}
    40894211
    40904212/**

  • trunk/src/VBox/VMM/include/IEMN8veRecompilerTlbLookup.h

    r102847 r102850  
    160160    }
    161161
    162     void freeRegsAndReleaseVars(PIEMRECOMPILERSTATE a_pReNative, uint8_t idxVarGCPtrMem = UINT8_MAX) const
    163     {
    164         if (idxRegPtr != UINT8_MAX)
    165         {
    166             if (idxRegPtrHlp == UINT8_MAX)
     162    /* Alternative constructor for the code TLB lookups where we implictly use RIP
     163       variable, only a register derived from the guest RSP. */
     164    IEMNATIVEEMITTLBSTATE(PIEMRECOMPILERSTATE a_pReNative, bool a_fFlat, uint32_t *a_poff)
     165#ifdef IEMNATIVE_WITH_TLB_LOOKUP
     166        :           fSkip(false)
     167#else
     168        :           fSkip(true)
     169#endif
     170        ,    idxRegPtrHlp(UINT8_MAX)
     171        ,       idxRegPtr(iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, kIemNativeGstReg_Pc))
     172        ,   idxRegSegBase(a_fFlat || fSkip
     173                          ? UINT8_MAX
     174                          : iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS)))
     175        ,  idxRegSegLimit(/*a_fFlat || fSkip
     176                          ? UINT8_MAX
     177                          : iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS))*/
     178                          UINT8_MAX)
     179        , idxRegSegAttrib(UINT8_MAX)
     180        ,         idxReg1(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
     181        ,         idxReg2(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
     182#if defined(RT_ARCH_ARM64)
     183        ,         idxReg3(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
     184#endif
     185        ,         uAbsPtr(UINT64_MAX)
     186
     187    {
     188    }
     189
     190    void freeRegsAndReleaseVars(PIEMRECOMPILERSTATE a_pReNative, uint8_t idxVarGCPtrMem = UINT8_MAX, bool fIsCode = false) const
     191    {
     192        if (!fIsCode)
     193        {
     194            if (idxRegPtr != UINT8_MAX)
    167195            {
    168                 if (idxVarGCPtrMem != UINT8_MAX)
    169                     iemNativeVarRegisterRelease(a_pReNative, idxVarGCPtrMem);
     196                if (idxRegPtrHlp == UINT8_MAX)
     197                {
     198                    if (idxVarGCPtrMem != UINT8_MAX)
     199                        iemNativeVarRegisterRelease(a_pReNative, idxVarGCPtrMem);
     200                }
     201                else
     202                {
     203                    Assert(idxRegPtrHlp == idxRegPtr);
     204                    iemNativeRegFreeTmpImm(a_pReNative, idxRegPtrHlp);
     205                }
    170206            }
    171207            else
    172             {
    173                 Assert(idxRegPtrHlp == idxRegPtr);
    174                 iemNativeRegFreeTmpImm(a_pReNative, idxRegPtrHlp);
    175             }
    176         }
    177         else
     208                Assert(idxRegPtrHlp == UINT8_MAX);
     209        }
     210        else
     211        {
     212            Assert(idxVarGCPtrMem == UINT8_MAX);
    178213            Assert(idxRegPtrHlp == UINT8_MAX);
     214            iemNativeRegFreeTmp(a_pReNative, idxRegPtr); /* RIP */
     215        }
    179216        if (idxRegSegBase != UINT8_MAX)
    180217            iemNativeRegFreeTmp(a_pReNative, idxRegSegBase);
    181218        if (idxRegSegLimit != UINT8_MAX)
    182         {
    183219            iemNativeRegFreeTmp(a_pReNative, idxRegSegLimit);
     220        if (idxRegSegAttrib != UINT8_MAX)
    184221            iemNativeRegFreeTmp(a_pReNative, idxRegSegAttrib);
    185         }
    186         else
    187             Assert(idxRegSegAttrib == UINT8_MAX);
    188222#if defined(RT_ARCH_ARM64)
    189223        iemNativeRegFreeTmp(a_pReNative, idxReg3);
    … …  
    207241
    208242    /** This is only for avoid assertions. */
    209     uint32_t getActiveRegsWithShadows() const
     243    uint32_t getActiveRegsWithShadows(bool fCode = false) const
    210244    {
    211245#ifdef VBOX_STRICT
    212246        if (!fSkip)
    213             return RT_BIT_32(idxRegSegBase) | RT_BIT_32(idxRegSegLimit) | RT_BIT_32(idxRegSegAttrib);
     247            return (idxRegSegBase   != UINT8_MAX ? RT_BIT_32(idxRegSegBase)   : 0)
     248                 | (idxRegSegLimit  != UINT8_MAX ? RT_BIT_32(idxRegSegLimit)  : 0)
     249                 | (idxRegSegAttrib != UINT8_MAX ? RT_BIT_32(idxRegSegAttrib) : 0)
     250                 | (fCode                        ? RT_BIT_32(idxRegPtr)       : 0);
    214251#endif
    215252        return 0;
    … …  
    229266{
    230267    Assert(!pTlbState->fSkip);
     268    uint32_t const   offVCpuTlb = a_fDataTlb ? RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb) : RT_UOFFSETOF(VMCPUCC, iem.s.CodeTlb);
    231269# if defined(RT_ARCH_AMD64)
    232     uint8_t * const  pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 512);
     270    uint8_t * const  pCodeBuf   = iemNativeInstrBufEnsure(pReNative, off, 512);
    233271# elif defined(RT_ARCH_ARM64)
    234     uint32_t * const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
     272    uint32_t * const pCodeBuf   = iemNativeInstrBufEnsure(pReNative, off, 64);
    235273# endif
    236274
    … …  
    296334     *
    297335     * 1a. Check segment limit and attributes if non-flat 32-bit code.  This is complicated.
    298      */
    299     if (iSegReg != UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT)
     336     *
     337     *     This can be skipped for code TLB lookups because limit is checked by jmp, call,
     338     *     ret, and iret prior to making it.  It is also checked by the helpers prior to
     339     *     doing TLB loading.
     340     */
     341    if (a_fDataTlb && iSegReg != UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT)
    300342    {
    301343        /* Check that we've got a segment loaded and that it allows the access.
    … …  
    539581    pCodeBuf[off++] = pTlbState->idxReg1 < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R;
    540582    pCodeBuf[off++] = 0x0b; /* OR r64,r/m64 */
    541     off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, pTlbState->idxReg1, RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbRevision));
     583    off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, pTlbState->idxReg1, offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbRevision));
    542584# else
    543     off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg3, RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbRevision));
     585    off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg3, offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbRevision));
    544586    off = iemNativeEmitOrGprByGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg3);
    545587# endif
    … …  
    548590     * 3b. Calc pTlbe.
    549591     */
     592    uint32_t const offTlbEntries = offVCpuTlb + RT_UOFFSETOF(IEMTLB, aEntries);
    550593# if defined(RT_ARCH_AMD64)
    551594    /* movzx reg2, byte reg1 */
    … …  
    560603    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, pTlbState->idxReg2 & 7, 4 /*SIB*/);
    561604    pCodeBuf[off++] = X86_SIB_MAKE(IEMNATIVE_REG_FIXED_PVMCPU & 7, pTlbState->idxReg2 & 7, 0);
    562     pCodeBuf[off++] = RT_BYTE1(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
    563     pCodeBuf[off++] = RT_BYTE2(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
    564     pCodeBuf[off++] = RT_BYTE3(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
    565     pCodeBuf[off++] = RT_BYTE4(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
     605    pCodeBuf[off++] = RT_BYTE1(offTlbEntries);
     606    pCodeBuf[off++] = RT_BYTE2(offTlbEntries);
     607    pCodeBuf[off++] = RT_BYTE3(offTlbEntries);
     608    pCodeBuf[off++] = RT_BYTE4(offTlbEntries);
    566609
    567610# elif defined(RT_ARCH_ARM64)
    … …  
    569612    pCodeBuf[off++] = Armv8A64MkInstrUbfiz(pTlbState->idxReg2, pTlbState->idxReg1, 5, 8);
    570613    /* reg2 += offsetof(VMCPUCC, iem.s.DataTlb.aEntries) */
    571     off = iemNativeEmitAddGprImmEx(pCodeBuf, off, pTlbState->idxReg2, RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.aEntries),
    572                                    pTlbState->idxReg3 /*iGprTmp*/);
     614    off = iemNativeEmitAddGprImmEx(pCodeBuf, off, pTlbState->idxReg2, offTlbEntries, pTlbState->idxReg3 /*iGprTmp*/);
    573615    /* reg2 += pVCpu */
    574616    off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, pTlbState->idxReg2, IEMNATIVE_REG_FIXED_PVMCPU);
    … …  
    602644    uint64_t       fTlbe   = IEMTLBE_F_PHYS_REV | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_UNASSIGNED | IEMTLBE_F_PT_NO_ACCESSED
    603645                           | fNoUser;
     646    if (fAccess & IEM_ACCESS_TYPE_EXEC)
     647        fTlbe |= IEMTLBE_F_PT_NO_EXEC /*| IEMTLBE_F_PG_NO_READ?*/;
    604648    if (fAccess & IEM_ACCESS_TYPE_READ)
    605649        fTlbe |= IEMTLBE_F_PG_NO_READ;
    … …  
    618662    pCodeBuf[off++] = 0x3b;
    619663    off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1, IEMNATIVE_REG_FIXED_PVMCPU,
    620                                     RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbPhysRev));
     664                                    offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbPhysRev));
    621665# elif defined(RT_ARCH_ARM64)
    622666    off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, pTlbState->idxReg3, pTlbState->idxReg2,
    623667                                         RT_UOFFSETOF(IEMTLBENTRY, fFlagsAndPhysRev));
    624668    pCodeBuf[off++] = Armv8A64MkInstrAnd(pTlbState->idxReg1, pTlbState->idxReg1, pTlbState->idxReg3);
    625     off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg3, RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbPhysRev));
     669    off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg3, offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbPhysRev));
    626670    off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg3);
    627671# else
    … …  
    634678     * 5. Check that pbMappingR3 isn't NULL (paranoia) and calculate the
    635679     *    resulting pointer.
     680     *
     681     *    For code TLB lookups we have some more work to do here to set various
     682     *    IEMCPU members and we return a GCPhys address rather than a host pointer.
    636683     */
    637684    /* mov  reg1, [reg2->pbMappingR3] */
    … …  
    643690                                                      true /*f64Bit*/, idxLabelTlbMiss);
    644691
    645     if (idxRegFlatPtr == idxRegMemResult) /* See step 1b. */
    646     {
    647         /* and result, 0xfff */
    648         off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
     692    if (a_fDataTlb)
     693    {
     694        if (idxRegFlatPtr == idxRegMemResult) /* See step 1b. */
     695        {
     696            /* and result, 0xfff */
     697            off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
     698        }
     699        else
     700        {
     701            Assert(idxRegFlatPtr == pTlbState->idxRegPtr);
     702            /* result = regflat & 0xfff */
     703            off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, idxRegMemResult, idxRegFlatPtr, GUEST_PAGE_OFFSET_MASK);
     704        }
     705
     706        /* add result, reg1 */
     707        off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, idxRegMemResult, pTlbState->idxReg1);
    649708    }
    650709    else
    651710    {
    652         Assert(idxRegFlatPtr == pTlbState->idxRegPtr);
    653         /* result = regflat & 0xfff */
    654         off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, idxRegMemResult, idxRegFlatPtr, GUEST_PAGE_OFFSET_MASK);
    655     }
    656     /* add result, reg1 */
    657     off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, idxRegMemResult, pTlbState->idxReg1);
     711        /*
     712         * Code TLB use a la iemOpcodeFetchBytesJmp - keep reg2 pointing to the TLBE.
     713         *
     714         * Note. We do not need to set offCurInstrStart or offInstrNextByte.
     715         */
     716# ifdef RT_ARCH_AMD64
     717        uint8_t const idxReg3 = UINT8_MAX;
     718# else
     719        uint8_t const idxReg3 = pTlbState->idxReg3;
     720# endif
     721        /* Set pbInstrBuf first since we've got it loaded already. */
     722        off = iemNativeEmitStoreGprToVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg1,
     723                                               RT_UOFFSETOF(VMCPUCC, iem.s.pbInstrBuf), idxReg3);
     724        /* Set uInstrBufPc to (FlatPC & ~GUEST_PAGE_OFFSET_MASK). */
     725        off = iemNativeEmitGprEqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1, idxRegFlatPtr, ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK);
     726        off = iemNativeEmitStoreGprToVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg1,
     727                                               RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc), idxReg3);
     728        /* Set cbInstrBufTotal to GUEST_PAGE_SIZE. */ /** @todo this is a simplifications. Calc right size using CS.LIM and EIP? */
     729        off = iemNativeEmitStoreImmToVCpuU16Ex(pCodeBuf, off, GUEST_PAGE_SIZE, RT_UOFFSETOF(VMCPUCC, iem.s.cbInstrBufTotal),
     730                                               pTlbState->idxReg1, idxReg3);
     731        /* Now set GCPhysInstrBuf last as we'll be returning it in idxRegMemResult. */
     732        off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, pTlbState->idxReg1,
     733                                             pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, GCPhys));
     734        off = iemNativeEmitStoreGprToVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg1,
     735                                               RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf), idxReg3);
     736        /* Set idxRegMemResult. */
     737        if (idxRegFlatPtr == idxRegMemResult) /* See step 1b. */
     738            off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
     739        else
     740            off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, idxRegMemResult, idxRegFlatPtr, GUEST_PAGE_OFFSET_MASK);
     741        off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, idxRegMemResult, pTlbState->idxReg1);
     742    }
    658743
    659744# if 0
    … …  
    665750     */
    666751#  ifdef RT_ARCH_AMD64
    667     /* push     seg | (cbMem << 8) | (fAccess << 16) */
    668     pCodeBuf[off++] = 0x68;
    669     pCodeBuf[off++] = iSegReg;
    670     pCodeBuf[off++] = cbMem;
    671     pCodeBuf[off++] = RT_BYTE1(fAccess);
    672     pCodeBuf[off++] = RT_BYTE2(fAccess);
    673     /* push     pTlbState->idxRegPtr / immediate address. */
    674     if (pTlbState->idxRegPtr != UINT8_MAX)
    675     {
    676         if (pTlbState->idxRegPtr >= 8)
     752    if (a_fDataTlb)
     753    {
     754        /* push     seg | (cbMem << 8) | (fAccess << 16) */
     755        pCodeBuf[off++] = 0x68;
     756        pCodeBuf[off++] = iSegReg;
     757        pCodeBuf[off++] = cbMem;
     758        pCodeBuf[off++] = RT_BYTE1(fAccess);
     759        pCodeBuf[off++] = RT_BYTE2(fAccess);
     760        /* push     pTlbState->idxRegPtr / immediate address. */
     761        if (pTlbState->idxRegPtr != UINT8_MAX)
     762        {
     763            if (pTlbState->idxRegPtr >= 8)
     764                pCodeBuf[off++] = X86_OP_REX_B;
     765            pCodeBuf[off++] = 0x50 + (pTlbState->idxRegPtr & 7);
     766        }
     767        else
     768        {
     769            off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->uAbsPtr);
     770            if (pTlbState->idxReg1 >= 8)
     771                pCodeBuf[off++] = X86_OP_REX_B;
     772            pCodeBuf[off++] = 0x50 + (pTlbState->idxReg1 & 7);
     773        }
     774        /* push     idxRegMemResult */
     775        if (idxRegMemResult >= 8)
    677776            pCodeBuf[off++] = X86_OP_REX_B;
    678         pCodeBuf[off++] = 0x50 + (pTlbState->idxRegPtr & 7);
    679     }
    680     else
    681     {
    682         off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->uAbsPtr);
    683         if (pTlbState->idxReg1 >= 8)
    684             pCodeBuf[off++] = X86_OP_REX_B;
    685         pCodeBuf[off++] = 0x50 + (pTlbState->idxReg1 & 7);
    686     }
    687     /* push     idxRegMemResult */
    688     if (idxRegMemResult >= 8)
    689         pCodeBuf[off++] = X86_OP_REX_B;
    690     pCodeBuf[off++] = 0x50 + (idxRegMemResult & 7);
    691     /* push     pVCpu */
    692     pCodeBuf[off++] = 0x50 + IEMNATIVE_REG_FIXED_PVMCPU;
    693     /* mov      reg1, helper */
    694     off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, pTlbState->idxReg1, (uintptr_t)iemNativeHlpAsmSafeWrapCheckTlbLookup);
    695     /* call     [reg1] */
    696     pCodeBuf[off++] = X86_OP_REX_W | (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_B);
    697     pCodeBuf[off++] = 0xff;
    698     pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, pTlbState->idxReg1 & 7);
    699     /* The stack is cleaned up by helper function. */
     777        pCodeBuf[off++] = 0x50 + (idxRegMemResult & 7);
     778        /* push     pVCpu */
     779        pCodeBuf[off++] = 0x50 + IEMNATIVE_REG_FIXED_PVMCPU;
     780        /* mov      reg1, helper */
     781        off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, pTlbState->idxReg1, (uintptr_t)iemNativeHlpAsmSafeWrapCheckTlbLookup);
     782        /* call     [reg1] */
     783        pCodeBuf[off++] = X86_OP_REX_W | (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_B);
     784        pCodeBuf[off++] = 0xff;
     785        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, pTlbState->idxReg1 & 7);
     786        /* The stack is cleaned up by helper function. */
     787    }
    700788
    701789#  else
Note: See TracChangeset for help on using the changeset viewer.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette