VirtualBox

Changeset 102756 in vbox for trunk/src/VBox/VMM/VMMAll


Ignore:
Timestamp:
Jan 3, 2024 7:45:42 PM (15 months ago)
Author:
vboxsync
svn:sync-xref-src-repo-rev:
160934
Message:

VMM/IEM: Rough TLB lookup code for fetches and stores. bugref:10371

Location:
trunk/src/VBox/VMM/VMMAll
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompBltIn.cpp

    r102755 r102756  
    242242    iemNativeFixupFixedJump(pReNative, offFixupJumpToVmCheck1, off);
    243243    off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, CTX_SUFF(pVM))); /* idxTmpReg = pVM */
    244     off = iemNativeEmitLoadGpr32ByGpr(pReNative, off, idxTmpReg, idxTmpReg, RT_UOFFSETOF(VMCC, fGlobalForcedActions));
     244    off = iemNativeEmitLoadGprByGprU32(pReNative, off, idxTmpReg, idxTmpReg, RT_UOFFSETOF(VMCC, fGlobalForcedActions));
    245245    off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, VM_FF_ALL_MASK, true /*fSetFlags*/);
    246246    off = iemNativeEmitJnzToLabel(pReNative, off, idxLabelReturnBreak);

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

    r102742 r102756  
    1002110021
    1002210022    IEMNATIVEEMITTLBSTATE(PIEMRECOMPILERSTATE a_pReNative, uint32_t *a_poff, uint8_t a_idxVarGCPtrMem,
    10023                           uint8_t a_iSegReg, uint8_t a_cbMem)
     10023                          uint8_t a_iSegReg, uint8_t a_cbMem, uint8_t a_offDisp = 0)
    1002410024#ifdef IEMNATIVE_WITH_TLB_LOOKUP
    1002510025        /* 32-bit and 64-bit wraparound will require special handling, so skip these for absolute addresses. */
    1002610026        :           fSkip(   a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind == kIemNativeVarKind_Immediate
    1002710027                          &&   (  (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT
    10028                                 ? (uint64_t)(UINT32_MAX - a_cbMem)
    10029                                 : (uint64_t)(UINT64_MAX - a_cbMem))
     10028                                ? (uint64_t)(UINT32_MAX - a_cbMem - a_offDisp)
     10029                                : (uint64_t)(UINT64_MAX - a_cbMem - a_offDisp))
    1003010030                             < a_pReNative->Core.aVars[a_idxVarGCPtrMem].u.uValue)
    1003110031#else
    … …  
    1004710047                          ? UINT8_MAX
    1004810048                          : iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_BASE(a_iSegReg)))
    10049         ,  idxRegSegLimit((a_iSegReg == UINT8_MAX && (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT) || fSkip
     10049        ,  idxRegSegLimit((a_iSegReg == UINT8_MAX || (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT) || fSkip
    1005010050                          ? UINT8_MAX
    1005110051                          : iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg)))
    10052         , idxRegSegAttrib((a_iSegReg == UINT8_MAX && (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT) || fSkip
     10052        , idxRegSegAttrib((a_iSegReg == UINT8_MAX || (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT) || fSkip
    1005310053                          ? UINT8_MAX
    1005410054                          : iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg)))
    … …  
    1017810178                       uint8_t offDisp = 0)
    1017910179{
    10180     RT_NOREF(offDisp);
    1018110180    Assert(!pTlbState->fSkip);
    1018210181# if defined(RT_ARCH_AMD64)
    … …  
    1019610195    {
    1019710196off = iemNativeEmitBrkEx(pCodeBuf, off, 1); /** @todo this needs testing */
    10198         /* cmp  seglim, regptr */   /** @todo r=bird: why 64-bit compare again? */
    10199         if (pTlbState->idxRegPtr != UINT8_MAX)
    10200             off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit, pTlbState->idxRegPtr);
     10197        /* cmp  seglim, regptr */
     10198        if (pTlbState->idxRegPtr != UINT8_MAX && offDisp == 0)
     10199            off = iemNativeEmitCmpGpr32WithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit, pTlbState->idxRegPtr);
     10200        else if (pTlbState->idxRegPtr == UINT8_MAX)
     10201            off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxRegSegLimit,
     10202                                                 (uint32_t)(pTlbState->uAbsPtr + offDisp));
     10203        else if (cbMem == 1)
     10204            off = iemNativeEmitCmpGpr32WithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit, pTlbState->idxReg2);
    1020110205        else
    10202             off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxRegSegLimit, (uint32_t)pTlbState->uAbsPtr);
     10206        {   /* use idxRegMemResult to calc the displaced address. */
     10207            off = iemNativeEmitGpr32EqGprPlusImmEx(pCodeBuf, off, idxRegMemResult, pTlbState->idxRegPtr, offDisp);
     10208            off = iemNativeEmitCmpGpr32WithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit, idxRegMemResult);
     10209        }
    1020310210        /* ja  tlbmiss */
    1020410211        off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
     10212
    1020510213        /* reg1 = segattr & X86DESCATTR_D (0x4000) */
    1020610214        off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxRegSegAttrib, X86DESCATTR_D);
    … …  
    1021510223        if (pTlbState->idxRegPtr != UINT8_MAX)
    1021610224            off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxReg1,
    10217                                                cbMem > 1 ? pTlbState->idxReg2 : pTlbState->idxRegPtr);
     10225                                               cbMem > 1 || offDisp != 0 ? pTlbState->idxReg2 : pTlbState->idxRegPtr);
    1021810226        else
    10219             off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, (uint32_t)(pTlbState->uAbsPtr + cbMem - 1));
     10227            off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1,
     10228                                                 (uint32_t)(pTlbState->uAbsPtr + offDisp + cbMem - 1)); /* fSkip=true on overflow. */
    1022010229        /* jbe  tlbmiss */
    1022110230        off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_be);
    … …  
    1024010249    if (iSegReg != UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT)
    1024110250    {
    10242         /* If we're accessing more than one byte, put the last address we'll be
    10243            accessing in idxReg2 (64-bit). */
    10244         if (cbMem > 1 && pTlbState->idxRegPtr != UINT8_MAX)
    10245         {
    10246             /* reg2 = regptr + cbMem - 1 ; 64-bit result so we can fend of wraparounds/overflows. */
    10247             off = iemNativeEmitGprEqGprPlusImmEx(pCodeBuf, off, pTlbState->idxReg2,/*=*/ pTlbState->idxRegPtr,/*+*/ cbMem - 1);
    10248         }
    10249 
    1025010251        /* Check that we've got a segment loaded and that it allows the access.
    1025110252           For write access this means a writable data segment.
    … …  
    1029610297        }
    1029710298
     10299        /* If we're accessing more than one byte or if we're working with a non-zero offDisp,
     10300           put the last address we'll be accessing in idxReg2 (64-bit). */
     10301        if ((cbMem > 1 || offDisp != 0) && pTlbState->idxRegPtr != UINT8_MAX)
     10302        {
     10303            if (!offDisp)
     10304                /* reg2 = regptr + cbMem - 1; 64-bit result so we can fend of wraparounds/overflows. */
     10305                off = iemNativeEmitGprEqGprPlusImmEx(pCodeBuf, off, pTlbState->idxReg2,/*=*/ pTlbState->idxRegPtr,/*+*/ cbMem - 1);
     10306            else
     10307            {
     10308                /* reg2 = (uint32_t)(regptr + offDisp) + cbMem - 1;. */
     10309                off = iemNativeEmitGpr32EqGprPlusImmEx(pCodeBuf, off,
     10310                                                       pTlbState->idxReg2,/*=*/ pTlbState->idxRegPtr,/*+*/ + offDisp);
     10311                off = iemNativeEmitAddGprImmEx(pCodeBuf, off, pTlbState->idxReg2, cbMem - 1);
     10312            }
     10313        }
     10314
    1029810315        /*
    1029910316         * Check the limit.  If this is a write access, we know that it's a
    … …  
    1032310340           cmp  seglim, regptr/reg2/imm */
    1032410341        if (pTlbState->idxRegPtr != UINT8_MAX)
    10325             off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit, cbMem > 1 ? pTlbState->idxReg2 : pTlbState->idxRegPtr);
     10342            off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit,
     10343                                               cbMem > 1 || offDisp != 0 ? pTlbState->idxReg2 : pTlbState->idxRegPtr);
    1032610344        else
    10327             off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxRegSegLimit, (uint32_t)pTlbState->uAbsPtr);
     10345            off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxRegSegLimit,
     10346                                                 (uint32_t)pTlbState->uAbsPtr + offDisp + cbMem - 1U); /* fSkip=true on overflow. */
    1032810347        /* jbe  tlbmiss */
    1032910348        off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_be);
    … …  
    1033310352    }
    1033410353
    10335     /* 1b. Add the segment base. We use idxRegMemResult for the ptr register if this step is
    10336            required or if the address is a constant (simplicity). */
    10337     uint8_t const idxRegFlatPtr = iSegReg != UINT8_MAX || pTlbState->idxRegPtr == UINT8_MAX
     10354    /* 1b. Add the segment base.  We use idxRegMemResult for the ptr register if
     10355           this step is required or if the address is a constant (simplicity) or
     10356           if offDisp is non-zero. */
     10357    uint8_t const idxRegFlatPtr = iSegReg != UINT8_MAX || pTlbState->idxRegPtr == UINT8_MAX || offDisp != 0
    1033810358                                ? idxRegMemResult : pTlbState->idxRegPtr;
    1033910359    if (iSegReg != UINT8_MAX)
    1034010360    {
     10361        Assert(idxRegFlatPtr != pTlbState->idxRegPtr);
    1034110362        /* regflat = segbase + regptr/imm */
    1034210363        if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
    … …  
    1034410365            Assert(iSegReg >= X86_SREG_FS);
    1034510366            if (pTlbState->idxRegPtr != UINT8_MAX)
     10367            {
    1034610368                off = iemNativeEmitGprEqGprPlusGprEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase, pTlbState->idxRegPtr);
     10369                if (offDisp != 0)
     10370                    off = iemNativeEmitAddGprImmEx(pCodeBuf, off, idxRegFlatPtr, offDisp);
     10371            }
    1034710372            else
    10348                 off = iemNativeEmitGprEqGprPlusImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase, pTlbState->uAbsPtr);
     10373                off = iemNativeEmitGprEqGprPlusImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase,
     10374                                                     pTlbState->uAbsPtr + offDisp);
    1034910375        }
    1035010376        else if (pTlbState->idxRegPtr != UINT8_MAX)
     10377        {
    1035110378            off = iemNativeEmitGpr32EqGprPlusGprEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase, pTlbState->idxRegPtr);
     10379            if (offDisp != 0)
     10380                off = iemNativeEmitAddGpr32ImmEx(pCodeBuf, off, idxRegFlatPtr, offDisp);
     10381        }
    1035210382        else
    1035310383            off = iemNativeEmitGpr32EqGprPlusImmEx(pCodeBuf, off, idxRegFlatPtr,
    10354                                                    pTlbState->idxRegSegBase, (uint32_t)pTlbState->uAbsPtr);
     10384                                                   pTlbState->idxRegSegBase, (uint32_t)pTlbState->uAbsPtr + offDisp);
    1035510385    }
    1035610386    else if (pTlbState->idxRegPtr == UINT8_MAX)
    10357         off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->uAbsPtr);
     10387    {
     10388        if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
     10389            off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->uAbsPtr + offDisp);
     10390        else
     10391            off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, idxRegFlatPtr, (uint32_t)pTlbState->uAbsPtr + offDisp);
     10392    }
     10393    else if (offDisp != 0)
     10394    {
     10395        Assert(idxRegFlatPtr != pTlbState->idxRegPtr);
     10396        if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
     10397            off = iemNativeEmitGprEqGprPlusImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegPtr, offDisp);
     10398        else
     10399            off = iemNativeEmitGpr32EqGprPlusImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegPtr, offDisp);
     10400    }
     10401    else
     10402        Assert(idxRegFlatPtr == pTlbState->idxRegPtr);
    1035810403
    1035910404    /*
    … …  
    1049110536    off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, uTag));
    1049210537# elif defined(RT_ARCH_ARM64)
    10493     off = iemNativeEmitLoadGprByGprEx(pCodeBuf, off, pTlbState->idxReg3, pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, uTag));
     10538    off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, pTlbState->idxReg3, pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, uTag));
    1049410539    off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg3);
    1049510540# else
    … …  
    1052510570                                    RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbPhysRev));
    1052610571# elif defined(RT_ARCH_ARM64)
    10527     off = iemNativeEmitLoadGprByGprEx(pCodeBuf, off, pTlbState->idxReg3, pTlbState->idxReg2,
    10528                                       RT_UOFFSETOF(IEMTLBENTRY, fFlagsAndPhysRev));
     10572    off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, pTlbState->idxReg3, pTlbState->idxReg2,
     10573                                         RT_UOFFSETOF(IEMTLBENTRY, fFlagsAndPhysRev));
    1052910574    pCodeBuf[off++] = Armv8A64MkInstrAnd(pTlbState->idxReg1, pTlbState->idxReg1, pTlbState->idxReg3);
    1053010575    off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg3, RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbPhysRev));
    … …  
    1054110586     */
    1054210587    /* mov  reg1, [reg2->pbMappingR3] */
    10543     off = iemNativeEmitLoadGprByGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2,
    10544                                       RT_UOFFSETOF(IEMTLBENTRY, pbMappingR3));
     10588    off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2,
     10589                                         RT_UOFFSETOF(IEMTLBENTRY, pbMappingR3));
    1054510590    /* if (!reg1) goto tlbmiss; */
    1054610591    /** @todo eliminate the need for this test? */
    … …  
    1077310818     * register if we can).
    1077410819     */
    10775     uint16_t const uTlbSeqNo        = pReNative->uTlbSeqNo++;
    10776     uint32_t const idxLabelTlbMiss  = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, UINT32_MAX, uTlbSeqNo);
    10777     uint32_t const idxLabelTlbDone  = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
    10778     uint8_t  const idxRegValueFetch = enmOp == kIemNativeEmitMemOp_Store ? UINT8_MAX /* special case value storing below */
    10779                                     : !(pReNative->Core.bmHstRegs & RT_BIT_32(IEMNATIVE_CALL_RET_GREG))
    10780                                     ? iemNativeVarRegisterSetAndAcquire(pReNative, idxVarValue, IEMNATIVE_CALL_RET_GREG, &off)
    10781                                     : iemNativeVarRegisterAcquire(pReNative, idxVarValue, &off);
    10782 
    10783     /*
    10784      * First we try to go via the TLB.
    10785      */
    10786 //pReNative->pInstrBuf[off++] = 0xcc;
    10787     /** @todo later. */
    10788     RT_NOREF(fAlignMask, cbMem);
    10789 
    10790     /*
     10820    uint16_t const uTlbSeqNo         = pReNative->uTlbSeqNo++;
     10821    uint8_t  const idxRegValueFetch  = enmOp == kIemNativeEmitMemOp_Store ? UINT8_MAX
     10822                                     : !(pReNative->Core.bmHstRegs & RT_BIT_32(IEMNATIVE_CALL_RET_GREG))
     10823                                     ? iemNativeVarRegisterSetAndAcquire(pReNative, idxVarValue, IEMNATIVE_CALL_RET_GREG, &off)
     10824                                     : iemNativeVarRegisterAcquire(pReNative, idxVarValue, &off);
     10825    IEMNATIVEEMITTLBSTATE const TlbState(pReNative, &off, idxVarGCPtrMem, iSegReg, cbMem, offDisp);
     10826    uint8_t  const idxRegValueStore  =    !TlbState.fSkip
     10827                                       && enmOp == kIemNativeEmitMemOp_Store
     10828                                       && pReNative->Core.aVars[idxVarValue].enmKind != kIemNativeVarKind_Immediate
     10829                                     ? iemNativeVarRegisterAcquire(pReNative, idxVarValue, &off)
     10830                                     : UINT8_MAX;
     10831    uint32_t const idxRegMemResult   = !TlbState.fSkip ? iemNativeRegAllocTmp(pReNative, &off) : UINT8_MAX;
     10832    uint32_t const idxLabelTlbLookup = !TlbState.fSkip
     10833                                     ? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
     10834                                     : UINT32_MAX;
     10835
     10836    /*
     10837     * Jump to the TLB lookup code.
     10838     */
     10839    if (!TlbState.fSkip)
     10840        off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
     10841
     10842    /*
     10843     * TlbMiss:
     10844     *
    1079110845     * Call helper to do the fetching.
    1079210846     * We flush all guest register shadow copies here.
    1079310847     */
    10794     iemNativeLabelDefine(pReNative, idxLabelTlbMiss, off);
     10848    uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
    1079510849
    1079610850#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
    … …  
    1080010854#endif
    1080110855
    10802     uint8_t idxRegArgValue;
    10803     if (iSegReg == UINT8_MAX)
    10804         idxRegArgValue = IEMNATIVE_CALL_ARG2_GREG;
    10805     else
    10806     {
    10807         /* IEMNATIVE_CALL_ARG2_GREG = iSegReg */
    10808         AssertStmt(iSegReg < 6, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_BAD_SEG_REG_NO));
    10809         off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG2_GREG, iSegReg);
    10810 
    10811         idxRegArgValue = IEMNATIVE_CALL_ARG3_GREG;
    10812     }
    10813 
    1081410856    /* IEMNATIVE_CALL_ARG2/3_GREG = uValue (idxVarValue) - if store */
    1081510857    if (enmOp == kIemNativeEmitMemOp_Store)
    1081610858    {
     10859        uint8_t const idxRegArgValue = iSegReg == UINT8_MAX ? IEMNATIVE_CALL_ARG2_GREG : IEMNATIVE_CALL_ARG3_GREG;
    1081710860        if (pReNative->Core.aVars[idxVarValue].enmKind == kIemNativeVarKind_Immediate)
    1081810861            off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegArgValue, pReNative->Core.aVars[idxVarValue].u.uValue);
    … …  
    1086010903    }
    1086110904
     10905    if (iSegReg != UINT8_MAX)
     10906    {
     10907        /* IEMNATIVE_CALL_ARG2_GREG = iSegReg */
     10908        AssertStmt(iSegReg < 6, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_BAD_SEG_REG_NO));
     10909        off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG2_GREG, iSegReg);
     10910    }
     10911
    1086210912    /* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
    1086310913    off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
    … …  
    1087410924        if (idxRegValueFetch != IEMNATIVE_CALL_RET_GREG)
    1087510925            off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegValueFetch, IEMNATIVE_CALL_RET_GREG);
     10926    }
     10927
     10928#ifdef IEMNATIVE_WITH_TLB_LOOKUP
     10929    if (!TlbState.fSkip)
     10930    {
     10931        /* end of tlbsmiss - Jump to the done label. */
     10932        uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
     10933        off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
     10934
     10935        /*
     10936         * TlbLookup:
     10937         */
     10938        off = iemNativeEmitTlbLookup(pReNative, off, &TlbState, iSegReg, cbMem, fAlignMask,
     10939                                     enmOp == kIemNativeEmitMemOp_Store ? IEM_ACCESS_TYPE_WRITE : IEM_ACCESS_TYPE_READ,
     10940                                     idxLabelTlbLookup, idxLabelTlbMiss, idxRegMemResult, offDisp);
     10941
     10942        /*
     10943         * Emit code to do the actual storing / fetching.
     10944         */
     10945        PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
     10946        switch (enmOp)
     10947        {
     10948            case kIemNativeEmitMemOp_Store:
     10949                if (pReNative->Core.aVars[idxVarValue].enmKind != kIemNativeVarKind_Immediate)
     10950                {
     10951                    switch (cbMem)
     10952                    {
     10953                        case 1:
     10954                            off = iemNativeEmitStoreGpr8ByGprEx(pCodeBuf, off, idxRegValueStore, idxRegMemResult);
     10955                            break;
     10956                        case 2:
     10957                            off = iemNativeEmitStoreGpr16ByGprEx(pCodeBuf, off, idxRegValueStore, idxRegMemResult);
     10958                            break;
     10959                        case 4:
     10960                            off = iemNativeEmitStoreGpr32ByGprEx(pCodeBuf, off, idxRegValueStore, idxRegMemResult);
     10961                            break;
     10962                        case 8:
     10963                            off = iemNativeEmitStoreGpr64ByGprEx(pCodeBuf, off, idxRegValueStore, idxRegMemResult);
     10964                            break;
     10965                        default:
     10966                            AssertFailed();
     10967                    }
     10968                }
     10969                else
     10970                {
     10971                    switch (cbMem)
     10972                    {
     10973                        case 1:
     10974                            off = iemNativeEmitStoreImm8ByGprEx(pCodeBuf, off,
     10975                                                                (uint8_t)pReNative->Core.aVars[idxVarValue].u.uValue,
     10976                                                                idxRegMemResult, TlbState.idxReg1);
     10977                            break;
     10978                        case 2:
     10979                            off = iemNativeEmitStoreImm16ByGprEx(pCodeBuf, off,
     10980                                                                 (uint16_t)pReNative->Core.aVars[idxVarValue].u.uValue,
     10981                                                                 idxRegMemResult, TlbState.idxReg1);
     10982                            break;
     10983                        case 4:
     10984                            off = iemNativeEmitStoreImm32ByGprEx(pCodeBuf, off,
     10985                                                                 (uint32_t)pReNative->Core.aVars[idxVarValue].u.uValue,
     10986                                                                 idxRegMemResult, TlbState.idxReg1);
     10987                            break;
     10988                        case 8:
     10989                            off = iemNativeEmitStoreImm64ByGprEx(pCodeBuf, off, pReNative->Core.aVars[idxVarValue].u.uValue,
     10990                                                                 idxRegMemResult, TlbState.idxReg1);
     10991                            break;
     10992                        default:
     10993                            AssertFailed();
     10994                    }
     10995                }
     10996                break;
     10997
     10998            case kIemNativeEmitMemOp_Fetch:
     10999            case kIemNativeEmitMemOp_Fetch_Zx_U16:
     11000            case kIemNativeEmitMemOp_Fetch_Zx_U32:
     11001            case kIemNativeEmitMemOp_Fetch_Zx_U64:
     11002                switch (cbMem)
     11003                {
     11004                    case 1:
     11005                        off = iemNativeEmitLoadGprByGprU8Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
     11006                        break;
     11007                    case 2:
     11008                        off = iemNativeEmitLoadGprByGprU16Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
     11009                        break;
     11010                    case 4:
     11011                        off = iemNativeEmitLoadGprByGprU32Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
     11012                        break;
     11013                    case 8:
     11014                        off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
     11015                        break;
     11016                    default:
     11017                        AssertFailed();
     11018                }
     11019                break;
     11020
     11021            case kIemNativeEmitMemOp_Fetch_Sx_U16:
     11022                Assert(cbMem == 1);
     11023                off = iemNativeEmitLoadGprByGprU16SignExtendedFromS8Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
     11024                break;
     11025
     11026            case kIemNativeEmitMemOp_Fetch_Sx_U32:
     11027                Assert(cbMem == 1 || cbMem == 2);
     11028                if (cbMem == 1)
     11029                    off = iemNativeEmitLoadGprByGprU32SignExtendedFromS8Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
     11030                else
     11031                    off = iemNativeEmitLoadGprByGprU32SignExtendedFromS16Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
     11032                break;
     11033
     11034            case kIemNativeEmitMemOp_Fetch_Sx_U64:
     11035                switch (cbMem)
     11036                {
     11037                    case 1:
     11038                        off = iemNativeEmitLoadGprByGprU64SignExtendedFromS8Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
     11039                        break;
     11040                    case 2:
     11041                        off = iemNativeEmitLoadGprByGprU64SignExtendedFromS16Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
     11042                        break;
     11043                    case 4:
     11044                        off = iemNativeEmitLoadGprByGprU64SignExtendedFromS32Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
     11045                        break;
     11046                    default:
     11047                        AssertFailed();
     11048                }
     11049                break;
     11050
     11051            default:
     11052                AssertFailed();
     11053        }
     11054
     11055        iemNativeRegFreeTmp(pReNative, idxRegMemResult);
     11056
     11057        /*
     11058         * TlbDone:
     11059         */
     11060        iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
     11061
     11062        TlbState.freeRegsAndReleaseVars(pReNative, idxVarGCPtrMem);
     11063
     11064# ifndef IEMNATIVE_WITH_FREE_AND_FLUSH_VOLATILE_REGS_AT_TLB_LOOKUP
     11065        /* Temp Hack: Flush all guest shadows in volatile registers in case of TLB miss. */
     11066        iemNativeRegFlushGuestShadowsByHostMask(pReNative, IEMNATIVE_CALL_VOLATILE_GREG_MASK);
     11067# endif
     11068    }
     11069#else
     11070    RT_NOREF(fAccess, fAlignMask, idxLabelTlbMiss);
     11071#endif
     11072
     11073    if (idxRegValueFetch != UINT8_MAX || idxRegValueStore != UINT8_MAX)
    1087611074        iemNativeVarRegisterRelease(pReNative, idxVarValue);
    10877     }
    10878 
    10879     iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
    10880 
    1088111075    return off;
    1088211076}
    … …  
    1178211976    /*
    1178311977     * Define labels and allocate the result register (trying for the return
    11784      * register if we can - which we of course can, given the above call).
    11785      */
    11786     uint16_t const uTlbSeqNo        = pReNative->uTlbSeqNo++;
    11787     uint8_t  const idxRegMemResult  = !(pReNative->Core.bmHstRegs & RT_BIT_32(IEMNATIVE_CALL_RET_GREG))
    11788                                     ? iemNativeVarRegisterSetAndAcquire(pReNative, idxVarMem, IEMNATIVE_CALL_RET_GREG, &off)
    11789                                     : iemNativeVarRegisterAcquire(pReNative, idxVarMem, &off);
    11790 
     11978     * register if we can).
     11979     */
     11980    uint16_t const uTlbSeqNo         = pReNative->uTlbSeqNo++;
     11981    uint8_t  const idxRegMemResult   = !(pReNative->Core.bmHstRegs & RT_BIT_32(IEMNATIVE_CALL_RET_GREG))
     11982                                     ? iemNativeVarRegisterSetAndAcquire(pReNative, idxVarMem, IEMNATIVE_CALL_RET_GREG, &off)
     11983                                     : iemNativeVarRegisterAcquire(pReNative, idxVarMem, &off);
    1179111984    IEMNATIVEEMITTLBSTATE const TlbState(pReNative, &off, idxVarGCPtrMem, iSegReg, cbMem);
    11792 
    1179311985    uint32_t const idxLabelTlbLookup = !TlbState.fSkip
    1179411986                                     ? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
    … …  
    1180211994
    1180311995    /*
    11804      * tlbmiss:
     11996     * TlbMiss:
    1180511997     *
    1180611998     * Call helper to do the fetching.
    … …  
    1187012062
    1187112063        /*
    11872          * tlblookup:
     12064         * TlbLookup:
    1187312065         */
    1187412066        off = iemNativeEmitTlbLookup(pReNative, off, &TlbState, iSegReg, cbMem, fAlignMask, fAccess,
    1187512067                                     idxLabelTlbLookup, idxLabelTlbMiss, idxRegMemResult);
    1187612068
     12069        /* [idxVarUnmapInfo] = 0; */
     12070        off = iemNativeEmitLoadGprImm32(pReNative, off, idxRegUnmapInfo, 0);
     12071
    1187712072        /*
    11878          * Lookup tail code, specific to the MC when the above is moved into a separate function.
    11879          */
    11880         /* [idxVarUnmapInfo] = 0 - allocate register for it. There must be free ones now, so no spilling required. */
    11881         off = iemNativeEmitLoadGprImm32(pReNative, off, idxRegUnmapInfo, 0);
    11882 
    11883         /*
    11884          * tlbdone:
     12073         * TlbDone:
    1188512074         */
    1188612075        iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
Note: See TracChangeset for help on using the changeset viewer.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette