VirtualBox

Changeset 102724 in vbox for trunk/src


Ignore:
Timestamp:
Dec 28, 2023 9:15:52 PM (13 months ago)
Author:
vboxsync
Message:

VMM/IEM: Moved the TLB lookup code emitting into a separate function and straighten the code path a little by starting with a jump and moving the check_expand_down stuff in after tlbmiss. bugref:10371

Location:
trunk/src/VBox/VMM
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

    r102720 r102724  
    97369736
    97379737
     9738/*********************************************************************************************************************************
     9739*   TLB Lookup.                                                                                                                  *
     9740*********************************************************************************************************************************/
     9741
     9742#if defined(RT_ARCH_AMD64) && 1
     9743# define IEMNATIVE_WITH_TLB_LOOKUP
     9744#endif
     9745
     9746
     9747/**
     9748 * This must be instantiate *before* branching off to the lookup code,
     9749 * so that register spilling and whatnot happens for everyone.
     9750 */
     9751typedef struct IEMNATIVEEMITTLBSTATE
     9752{
     9753    uint64_t const  uAbsPtr;
     9754    bool const      fSkip;
     9755    uint8_t const   idxRegPtr;
     9756    uint8_t const   idxRegSegBase;
     9757    uint8_t const   idxRegSegLimit;
     9758    uint8_t const   idxRegSegAttrib;
     9759    uint8_t const   idxReg1;
     9760    uint8_t const   idxReg2;
     9761
     9762    IEMNATIVEEMITTLBSTATE(PIEMRECOMPILERSTATE a_pReNative, uint32_t *a_poff, uint8_t a_idxVarGCPtrMem,
     9763                          uint8_t a_iSegReg, uint8_t a_cbMem)
     9764        :         uAbsPtr(  a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate
     9765                          ? UINT64_MAX
     9766                          : a_pReNative->Core.aVars[a_idxVarGCPtrMem].u.uValue)
     9767#ifdef IEMNATIVE_WITH_TLB_LOOKUP
     9768        /* 32-bit and 64-bit wraparound will require special handling, so skip these for absolute addresses. */
     9769        ,           fSkip(   a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind == kIemNativeVarKind_Immediate
     9770                          &&   (  (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT
     9771                                ? (uint64_t)(UINT32_MAX - a_cbMem)
     9772                                : (uint64_t)(UINT64_MAX - a_cbMem))
     9773                             < a_pReNative->Core.aVars[a_idxVarGCPtrMem].u.uValue)
     9774#else
     9775        ,           fSkip(true)
     9776#endif
     9777        ,       idxRegPtr(a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate
     9778                          ? iemNativeVarRegisterAcquire(a_pReNative, a_idxVarGCPtrMem, a_poff,
     9779                                                        true /*fInitialized*/, IEMNATIVE_CALL_ARG2_GREG)
     9780                          : UINT8_MAX)
     9781        ,   idxRegSegBase(a_iSegReg == UINT8_MAX || fSkip
     9782                          ? UINT8_MAX
     9783                          : iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_BASE(a_iSegReg)))
     9784        ,  idxRegSegLimit((a_iSegReg == UINT8_MAX && (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT) || fSkip
     9785                          ? UINT8_MAX
     9786                          : iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg)))
     9787        , idxRegSegAttrib((a_iSegReg == UINT8_MAX && (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT) || fSkip
     9788                          ? UINT8_MAX
     9789                          : iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg)))
     9790        ,         idxReg1(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
     9791        ,         idxReg2(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
     9792
     9793    {
     9794        RT_NOREF_PV(a_cbMem);
     9795    }
     9796
     9797    void freeRegsAndReleaseVars(PIEMRECOMPILERSTATE a_pReNative, uint8_t idxVarGCPtrMem) const
     9798    {
     9799        if (idxRegPtr != UINT8_MAX)
     9800            iemNativeVarRegisterRelease(a_pReNative, idxVarGCPtrMem);
     9801        if (idxRegSegBase != UINT8_MAX)
     9802            iemNativeRegFreeTmp(a_pReNative, idxRegSegBase);
     9803        if (idxRegSegLimit != UINT8_MAX)
     9804        {
     9805            iemNativeRegFreeTmp(a_pReNative, idxRegSegLimit);
     9806            iemNativeRegFreeTmp(a_pReNative, idxRegSegAttrib);
     9807        }
     9808        else
     9809            Assert(idxRegSegAttrib == UINT8_MAX);
     9810        iemNativeRegFreeTmp(a_pReNative, idxReg2);
     9811        iemNativeRegFreeTmp(a_pReNative, idxReg1);
     9812    }
     9813} IEMNATIVEEMITTLBSTATE;
     9814
     9815
     9816#ifdef IEMNATIVE_WITH_TLB_LOOKUP
     9817DECL_INLINE_THROW(uint32_t)
     9818iemNativeEmitTlbLookup(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEEMITTLBSTATE const * const pTlbState,
     9819                       uint8_t iSegReg, uint8_t cbMem, uint8_t fAlignMask, uint32_t fAccess,
     9820                       uint32_t idxLabelTlbLookup, uint32_t idxLabelTlbMiss, uint8_t idxRegMemResult,
     9821                       uint8_t offDisp = 0)
     9822{
     9823    RT_NOREF(offDisp);
     9824    Assert(!pTlbState->fSkip);
     9825# if defined(RT_ARCH_AMD64)
     9826    uint8_t * const  pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 512);
     9827# elif defined(RT_ARCH_ARM64)
     9828    uint32_t * const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
     9829# endif
     9830
     9831    /*
     9832     * The expand down check isn't use all that much, so we emit here to keep
     9833     * the lookup straighter.
     9834     */
     9835    /* check_expand_down: ; complicted! */
     9836    uint32_t const offCheckExpandDown = off;
     9837    uint32_t       offFixupLimitDone  = 0;
     9838    if (iSegReg != UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT)
     9839    {
     9840off = iemNativeEmitBrkEx(pCodeBuf, off, 1); /** @todo this needs testing */
     9841        /* cmp  seglim, regptr */
     9842        if (pTlbState->idxRegPtr != UINT8_MAX)
     9843            off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit, pTlbState->idxRegPtr);
     9844        else
     9845            off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxRegSegLimit, (uint32_t)pTlbState->uAbsPtr);
     9846        /* ja  tlbmiss */
     9847        off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
     9848        /* mov  reg1, X86DESCATTR_D (0x4000) */
     9849        off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, X86DESCATTR_D);
     9850        /* and  reg1, segattr */
     9851        off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxRegSegAttrib);
     9852        /* xor  reg1, X86DESCATTR_D */
     9853        off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1, X86DESCATTR_D);
     9854        /* shl  reg1, 2 (16 - 14) */
     9855        AssertCompile((X86DESCATTR_D << 2) == UINT32_C(0x10000));
     9856        off = iemNativeEmitShiftGpr32LeftEx(pCodeBuf, off, pTlbState->idxReg1, 2);
     9857        /* dec  reg1 (=> 0xffff if D=0; 0xffffffff if D=1) */
     9858        off = iemNativeEmitSubGpr32ImmEx(pCodeBuf, off, pTlbState->idxReg1, 1);
     9859        /* cmp  reg1, reg2 (64-bit) / imm (32-bit) */
     9860        if (pTlbState->idxRegPtr != UINT8_MAX)
     9861            off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2);
     9862        else
     9863            off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, (uint32_t)(pTlbState->uAbsPtr + cbMem - 1));
     9864        /* jbe  tlbmiss */
     9865        off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_be);
     9866        /* jmp  limitdone */
     9867        offFixupLimitDone = off;
     9868        off = iemNativeEmitJmpToFixedEx(pCodeBuf, off, off /* ASSUME short jump suffices */);
     9869    }
     9870
     9871    /*
     9872     * tlblookup:
     9873     */
     9874    iemNativeLabelDefine(pReNative, idxLabelTlbLookup, off);
     9875
     9876    /*
     9877     * 1. Segmentation.
     9878     *
     9879     * 1a. Check segment limit and attributes if non-flat 32-bit code.  This is complicated.
     9880     */
     9881    if (iSegReg != UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT)
     9882    {
     9883        /* If we're accessing more than one byte, put the last address we'll be
     9884           accessing in idxReg2 (64-bit). */
     9885        if (cbMem > 1 && pTlbState->idxRegPtr != UINT8_MAX)
     9886        {
     9887# if 1
     9888            Assert(cbMem - 1 <= 127);
     9889            /* mov reg2, regptr */
     9890            off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, pTlbState->idxReg2, pTlbState->idxRegPtr);
     9891            /* add reg2, cbMem-1 */
     9892            off = iemNativeEmitAddGpr32Imm8Ex(pCodeBuf, off, pTlbState->idxReg2, cbMem - 1);
     9893# else
     9894            /* mov reg2, cbMem-1 */
     9895            off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, pTlbState->idxReg2, cbMem - 1);
     9896            /* add reg2, regptr */
     9897            off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, pTlbState->idxReg2, pTlbState->idxRegPtr);
     9898# endif
     9899        }
     9900
     9901        /* Check that we've got a segment loaded and that it allows the access.
     9902           For write access this means a writable data segment.
     9903           For read-only accesses this means a readable code segment or any data segment. */
     9904        if (fAccess & IEM_ACCESS_TYPE_WRITE)
     9905        {
     9906            uint32_t const fMustBe1 = X86DESCATTR_P        | X86DESCATTR_DT    | X86_SEL_TYPE_WRITE;
     9907            uint32_t const fMustBe0 = X86DESCATTR_UNUSABLE | X86_SEL_TYPE_CODE;
     9908            /* mov reg1, must1|must0 */
     9909            off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, pTlbState->idxReg1, fMustBe1 | fMustBe0);
     9910            /* and reg1, segattrs */
     9911            off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxRegSegAttrib);
     9912            /* cmp reg1, must1 */
     9913            off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, fMustBe1);
     9914            /* jne tlbmiss */
     9915            off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
     9916        }
     9917        else
     9918        {
     9919            /*  U  | !P |!DT |!CD | RW |
     9920                16 |  8 |  4 |  3 |  1 |
     9921              -------------------------------
     9922                0  |  0 |  0 |  0 |  0 | execute-only code segment. - must be excluded
     9923                0  |  0 |  0 |  0 |  1 | execute-read code segment.
     9924                0  |  0 |  0 |  1 |  0 | read-only data segment.
     9925                0  |  0 |  0 |  1 |  1 | read-write data segment.   - last valid combination
     9926            */
     9927            /* mov reg1, relevant attributes  */
     9928            off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, pTlbState->idxReg1,
     9929                                                X86DESCATTR_UNUSABLE | X86DESCATTR_P | X86DESCATTR_DT
     9930                                              | X86_SEL_TYPE_CODE    | X86_SEL_TYPE_WRITE);
     9931            /* and reg1, segattrs */
     9932            off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxRegSegAttrib);
     9933            /* xor reg1, X86DESCATTR_P | X86DESCATTR_DT | X86_SEL_TYPE_CODE ; place C=1 RW=0 at the bottom & limit the range.
     9934                                            ; EO-code=0,  ER-code=2, RO-data=8, RW-data=10 */
     9935            off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1, X86DESCATTR_P | X86DESCATTR_DT | X86_SEL_TYPE_CODE);
     9936            /* sub reg1, X86_SEL_TYPE_WRITE ; EO-code=-2, ER-code=0, RO-data=6, RW-data=8 */
     9937            off = iemNativeEmitSubGpr32ImmEx(pCodeBuf, off, pTlbState->idxReg1, X86_SEL_TYPE_WRITE /* ER-code */);
     9938            /* cmp reg1, X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE */
     9939            AssertCompile(X86_SEL_TYPE_CODE == 8);
     9940            off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, X86_SEL_TYPE_CODE);
     9941            /* ja  tlbmiss */
     9942            off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
     9943        }
     9944
     9945        /*
     9946         * Check the limit.  If this is a write access, we know that it's a
     9947         * data segment and includes the expand_down bit.  For read-only accesses
     9948         * we need to check that code/data=0 and expanddown=1 before continuing.
     9949         */
     9950        if (fAccess & IEM_ACCESS_TYPE_WRITE)
     9951        {
     9952            /* test segattrs, X86_SEL_TYPE_DOWN */
     9953            AssertCompile(X86_SEL_TYPE_DOWN < 128);
     9954            off = iemNativeEmitTestAnyBitsInGpr8Ex(pCodeBuf, off, pTlbState->idxRegSegAttrib, X86_SEL_TYPE_DOWN);
     9955            /* jnz  check_expand_down */
     9956            off = iemNativeEmitJccToFixedEx(pCodeBuf, off, offCheckExpandDown, kIemNativeInstrCond_ne);
     9957        }
     9958        else
     9959        {
     9960            /* mov reg1, segattrs */
     9961            off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxRegSegAttrib);
     9962            /* and reg1, code | down */
     9963            off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1, X86_SEL_TYPE_CODE | X86_SEL_TYPE_DOWN);
     9964            /* cmp reg1, down */
     9965            off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, X86_SEL_TYPE_DOWN);
     9966            /* je check_expand_down */
     9967            off = iemNativeEmitJccToFixedEx(pCodeBuf, off, offCheckExpandDown, kIemNativeInstrCond_e);
     9968        }
     9969
     9970        /* expand_up:
     9971           cmp  seglim, regptr/reg2/imm */
     9972        if (pTlbState->idxRegPtr != UINT8_MAX)
     9973            off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit, cbMem > 1 ? pTlbState->idxReg2 : pTlbState->idxRegPtr);
     9974        else
     9975            off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxRegSegLimit, (uint32_t)pTlbState->uAbsPtr);
     9976        /* jbe  tlbmiss */
     9977        off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_be);
     9978
     9979        /* limitdone: */
     9980        iemNativeFixupFixedJump(pReNative, offFixupLimitDone, off);
     9981    }
     9982
     9983    /* 1b. Add the segment base. We use idxRegMemResult for the ptr register if this step is
     9984           required or if the address is a constant (simplicity). */
     9985    uint8_t const idxRegFlatPtr = iSegReg != UINT8_MAX || pTlbState->idxRegPtr == UINT8_MAX
     9986                                ? idxRegMemResult : pTlbState->idxRegPtr;
     9987    if (iSegReg != UINT8_MAX)
     9988    {
     9989        /** @todo this can be done using LEA as well. */
     9990        if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
     9991        {
     9992            Assert(iSegReg >= X86_SREG_FS);
     9993            /* mov regflat, regptr/imm */
     9994            if (pTlbState->idxRegPtr != UINT8_MAX)
     9995                off = iemNativeEmitLoadGprFromGprEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegPtr);
     9996            else
     9997                off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->uAbsPtr);
     9998            /* add regflat, seg.base */
     9999            off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase);
     10000        }
     10001        else
     10002        {
     10003            /* mov regflat, regptr/imm */
     10004            if (pTlbState->idxRegPtr != UINT8_MAX)
     10005                off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegPtr);
     10006            else
     10007                off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->uAbsPtr);
     10008            /* add regflat, seg.base */
     10009            off = iemNativeEmitAddTwoGprs32Ex(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase);
     10010        }
     10011    }
     10012    else if (pTlbState->idxRegPtr == UINT8_MAX)
     10013        off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->uAbsPtr);
     10014
     10015    /*
     10016     * 2. Check that the address doesn't cross a page boundrary and doesn't have alignment issues.
     10017     *
     10018     * 2a. Alignment check using fAlignMask.
     10019     */
     10020    if (fAlignMask)
     10021    {
     10022        Assert(RT_IS_POWER_OF_TWO(fAlignMask + 1));
     10023        Assert(fAlignMask < 128);
     10024        /* test regflat, fAlignMask */
     10025        off = iemNativeEmitTestAnyBitsInGpr8Ex(pCodeBuf, off, idxRegFlatPtr, fAlignMask);
     10026        /* jnz tlbmiss */
     10027        off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
     10028    }
     10029
     10030    /*
     10031     * 2b. Check that it's not crossing page a boundrary. This is implicit in
     10032     *     the previous test if the alignment is same or larger than the type.
     10033     */
     10034    if (cbMem > fAlignMask + 1)
     10035    {
     10036        /* mov reg1, 0xfff */
     10037        off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, pTlbState->idxReg1, GUEST_PAGE_OFFSET_MASK);
     10038        /* and reg1, regflat */
     10039        off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, pTlbState->idxReg1, idxRegFlatPtr);
     10040        /* neg reg1 */
     10041        off = iemNativeEmitNegGpr32Ex(pCodeBuf, off, pTlbState->idxReg1);
     10042        /* add reg1, 0x1000 */
     10043        off = iemNativeEmitAddGpr32ImmEx(pCodeBuf, off, pTlbState->idxReg1, GUEST_PAGE_SIZE);
     10044        /* cmp reg1, cbMem */
     10045        off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, GUEST_PAGE_SIZE);
     10046        /* ja  tlbmiss */
     10047        off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
     10048    }
     10049
     10050    /*
     10051     * 3. TLB lookup.
     10052     *
     10053     * 3a. Calculate the TLB tag value (IEMTLB_CALC_TAG).
     10054     *     In 64-bit mode we will also check for non-canonical addresses here.
     10055     */
     10056    if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
     10057    {
     10058        /* mov reg1, regflat */
     10059        off = iemNativeEmitLoadGprFromGprEx(pCodeBuf, off, pTlbState->idxReg1, idxRegFlatPtr);
     10060        /* rol reg1, 16 */
     10061        off = iemNativeEmitRotateGprLeftEx(pCodeBuf, off, pTlbState->idxReg1, 16);
     10062        /** @todo Would 'movsx reg2, word reg1' and working on reg2 in dwords be faster? */
     10063        /* inc word reg1 */
     10064        pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
     10065        if (pTlbState->idxReg1 >= 8)
     10066            pCodeBuf[off++] = X86_OP_REX_B;
     10067        pCodeBuf[off++] = 0xff;
     10068        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, pTlbState->idxReg1 & 7);
     10069        /* cmp word reg1, 1 */
     10070        pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
     10071        if (pTlbState->idxReg1 >= 8)
     10072            pCodeBuf[off++] = X86_OP_REX_B;
     10073        pCodeBuf[off++] = 0x83;
     10074        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, pTlbState->idxReg1 & 7);
     10075        pCodeBuf[off++] = 1;
     10076        /* ja  tlbmiss */
     10077        off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
     10078        /* shr reg1, 16 + GUEST_PAGE_SHIFT */
     10079        off = iemNativeEmitShiftGprRightEx(pCodeBuf, off, pTlbState->idxReg1, 16 + GUEST_PAGE_SHIFT);
     10080    }
     10081    else
     10082    {
     10083        /* mov reg1, regflat */
     10084        off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, pTlbState->idxReg1, idxRegFlatPtr);
     10085        /* shr reg1, GUEST_PAGE_SHIFT */
     10086        off = iemNativeEmitShiftGpr32RightEx(pCodeBuf, off, pTlbState->idxReg1, GUEST_PAGE_SHIFT);
     10087    }
     10088    /* or  reg1, [qword pVCpu->iem.s.DataTlb.uTlbRevision] */
     10089    pCodeBuf[off++] = pTlbState->idxReg1 < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R;
     10090    pCodeBuf[off++] = 0x0b; /* OR r64,r/m64 */
     10091    off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, pTlbState->idxReg1, RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbRevision));
     10092
     10093    /*
     10094     * 3b. Calc pTlbe.
     10095     */
     10096    /* movzx reg2, byte reg1 */
     10097    off = iemNativeEmitLoadGprFromGpr8Ex(pCodeBuf, off, pTlbState->idxReg2, pTlbState->idxReg1);
     10098    /* shl   reg2, 5 ; reg2 *= sizeof(IEMTLBENTRY) */
     10099    AssertCompileSize(IEMTLBENTRY, 32);
     10100    off = iemNativeEmitShiftGprLeftEx(pCodeBuf, off, pTlbState->idxReg2, 5);
     10101    /* lea   reg2, [pVCpu->iem.s.DataTlb.aEntries + reg2] */
     10102    AssertCompile(IEMNATIVE_REG_FIXED_PVMCPU < 8);
     10103    pCodeBuf[off++] = pTlbState->idxReg2 < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_X | X86_OP_REX_R;
     10104    pCodeBuf[off++] = 0x8d;
     10105    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, pTlbState->idxReg2 & 7, 4 /*SIB*/);
     10106    pCodeBuf[off++] = X86_SIB_MAKE(IEMNATIVE_REG_FIXED_PVMCPU & 7, pTlbState->idxReg2 & 7, 0);
     10107    pCodeBuf[off++] = RT_BYTE1(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
     10108    pCodeBuf[off++] = RT_BYTE2(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
     10109    pCodeBuf[off++] = RT_BYTE3(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
     10110    pCodeBuf[off++] = RT_BYTE4(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
     10111
     10112    /*
     10113     * 3c. Compare the TLBE.uTag with the one from 2a (reg1).
     10114     */
     10115    /* cmp reg1, [reg2] */
     10116    pCodeBuf[off++] = X86_OP_REX_W | (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_R) | (pTlbState->idxReg2 < 8 ? 0 : X86_OP_REX_B);
     10117    pCodeBuf[off++] = 0x3b;
     10118    off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, uTag));
     10119    /* jne tlbmiss */
     10120    off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
     10121
     10122    /*
     10123     * 4. Check TLB page table level access flags and physical page revision #.
     10124     */
     10125    /* mov reg1, mask */
     10126    AssertCompile(IEMTLBE_F_PT_NO_USER == 4);
     10127    uint64_t const fNoUser = (((pReNative->fExec >> IEM_F_X86_CPL_SHIFT) & IEM_F_X86_CPL_SMASK) + 1) & IEMTLBE_F_PT_NO_USER;
     10128    off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, pTlbState->idxReg1,
     10129                                      IEMTLBE_F_PHYS_REV       | IEMTLBE_F_NO_MAPPINGR3
     10130                                    | IEMTLBE_F_PG_UNASSIGNED  | IEMTLBE_F_PG_NO_READ
     10131                                    | IEMTLBE_F_PT_NO_ACCESSED | fNoUser);
     10132    /* and reg1, [reg2->fFlagsAndPhysRev] */
     10133    pCodeBuf[off++] = X86_OP_REX_W | (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_R) | (pTlbState->idxReg2 < 8 ? 0 : X86_OP_REX_B);
     10134    pCodeBuf[off++] = 0x23;
     10135    off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, fFlagsAndPhysRev));
     10136
     10137    /* cmp reg1, [pVCpu->iem.s.DataTlb.uTlbPhysRev] */
     10138    pCodeBuf[off++] = X86_OP_REX_W | (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_R);
     10139    pCodeBuf[off++] = 0x3b;
     10140    off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1, IEMNATIVE_REG_FIXED_PVMCPU,
     10141                                    RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbPhysRev));
     10142    /* jne tlbmiss */
     10143    off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
     10144
     10145    /*
     10146     * 5. Check that pbMappingR3 isn't NULL (paranoia) and calculate the
     10147     *    resulting pointer.
     10148     */
     10149    /* mov  reg1, [reg2->pbMappingR3] */
     10150    pCodeBuf[off++] = X86_OP_REX_W | (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_R) | (pTlbState->idxReg2 < 8 ? 0 : X86_OP_REX_B);
     10151    pCodeBuf[off++] = 0x8b;
     10152    off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, pbMappingR3));
     10153
     10154    /** @todo eliminate the need for this test? */
     10155    /* test reg1, reg1 */
     10156    pCodeBuf[off++] = X86_OP_REX_W | (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_R | X86_OP_REX_B);
     10157    pCodeBuf[off++] = 0x85;
     10158    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, pTlbState->idxReg1 & 7, pTlbState->idxReg1 & 7);
     10159
     10160    /* jz   tlbmiss */
     10161    off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_e);
     10162
     10163    if (idxRegFlatPtr == idxRegMemResult) /* See step 1b. */
     10164    {
     10165        /* and result, 0xfff */
     10166        off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
     10167    }
     10168    else
     10169    {
     10170        Assert(idxRegFlatPtr == pTlbState->idxRegPtr);
     10171        /* mov result, 0xfff */
     10172        off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
     10173        /* and result, regflat */
     10174        off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, idxRegMemResult, idxRegFlatPtr);
     10175    }
     10176    /* add result, reg1 */
     10177    off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, idxRegMemResult, pTlbState->idxReg1);
     10178
     10179    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     10180
     10181    return off;
     10182}
     10183#endif
    973810184
    973910185
    … …  
    1089911345       Set the the type to stack here so we don't need to do it twice below. */
    1090011346    iemNativeVarSetKindToStack(pReNative, idxVarUnmapInfo);
     11347    uint8_t const idxRegUnmapInfo   = iemNativeVarRegisterAcquire(pReNative, idxVarUnmapInfo, &off);
     11348    /** @todo use a tmp register from TlbState, since they'll be free after tlb
     11349     *        lookup is done. */
    1090111350
    1090211351    /*
    … …  
    1090511354     */
    1090611355    uint16_t const uTlbSeqNo        = pReNative->uTlbSeqNo++;
    10907     uint32_t const idxLabelTlbMiss  = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, UINT32_MAX, uTlbSeqNo);
    10908     uint32_t const idxLabelTlbDone  = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
    1090911356    uint8_t  const idxRegMemResult  = !(pReNative->Core.bmHstRegs & RT_BIT_32(IEMNATIVE_CALL_RET_GREG))
    1091011357                                    ? iemNativeVarRegisterSetAndAcquire(pReNative, idxVarMem, IEMNATIVE_CALL_RET_GREG, &off)
    1091111358                                    : iemNativeVarRegisterAcquire(pReNative, idxVarMem, &off);
    1091211359
    10913     /*
    10914      * First we try to go via the TLB.
    10915      */
    10916 #if defined(RT_ARCH_AMD64) && 1
    10917     /* Immediate addresses are a pain if they wrap around the 32-bit space, so
    10918        always fall back for those.  In 64-bit mode this is less like. */
    10919     if (   pReNative->Core.aVars[idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate
    10920         || (uint64_t)((uint64_t)UINT32_MAX - pReNative->Core.aVars[idxVarGCPtrMem].u.uValue) <= (uint32_t)UINT32_MAX)
    10921     {
    10922         uint8_t const   idxRegPtr       = pReNative->Core.aVars[idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate
    10923                                         ? iemNativeVarRegisterAcquire(pReNative, idxVarGCPtrMem, &off,
    10924                                                                       true /*fInitialized*/, IEMNATIVE_CALL_ARG2_GREG)
    10925                                         : UINT8_MAX;
    10926         uint64_t const  uAbsPtr         = pReNative->Core.aVars[idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate
    10927                                         ? UINT32_MAX
    10928                                         : pReNative->Core.aVars[idxVarGCPtrMem].u.uValue;
    10929         uint8_t const   idxRegSegBase   = iSegReg == UINT8_MAX ? UINT8_MAX
    10930                                         : iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(iSegReg));
    10931         uint8_t const   idxRegSegLimit  = iSegReg == UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT
    10932                                         ? UINT8_MAX
    10933                                         : iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(iSegReg));
    10934         uint8_t const   idxRegSegAttrib = iSegReg == UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT
    10935                                         ? UINT8_MAX
    10936                                         : iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_ATTRIB(iSegReg));
    10937         uint8_t const   idxReg1         = iemNativeRegAllocTmp(pReNative, &off);
    10938         uint8_t const   idxReg2         = iemNativeRegAllocTmp(pReNative, &off);
    10939         uint8_t * const pbCodeBuf       = iemNativeInstrBufEnsure(pReNative, off, 256);
    10940 
    10941         /*
    10942          * 1. Segmentation.
    10943          *
    10944          * 1a. Check segment limit and attributes if non-flat 32-bit code.  This is complicated.
    10945          */
    10946         if (iSegReg != UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT)
    10947         {
    10948             /* If we're accessing more than one byte, put the last address we'll be
    10949                accessing in idxReg2 (64-bit). */
    10950             if (cbMem > 1 && idxRegPtr != UINT8_MAX)
    10951             {
    10952 # if 1
    10953                 Assert(cbMem - 1 <= 127);
    10954                 /* mov reg2, regptr */
    10955                 off = iemNativeEmitLoadGprFromGpr32Ex(pbCodeBuf, off, idxReg2, idxRegPtr);
    10956                 /* add reg2, cbMem-1 */
    10957                 off = iemNativeEmitAddGpr32Imm8Ex(pbCodeBuf, off, idxReg2, cbMem - 1);
    10958 # else
    10959                 /* mov reg2, cbMem-1 */
    10960                 off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxReg2, cbMem - 1);
    10961                 /* add reg2, regptr */
    10962                 off = iemNativeEmitAddTwoGprsEx(pbCodeBuf, off, idxReg2, idxRegPtr);
    10963 # endif
    10964             }
    10965 
    10966             /* Check that we've got a segment loaded and that it allows the access.
    10967                For write access this means a writable data segment.
    10968                For read-only accesses this means a readable code segment or any data segment. */
    10969             if (fAccess & IEM_ACCESS_TYPE_WRITE)
    10970             {
    10971                 uint32_t const fMustBe1 = X86DESCATTR_P        | X86DESCATTR_DT    | X86_SEL_TYPE_WRITE;
    10972                 uint32_t const fMustBe0 = X86DESCATTR_UNUSABLE | X86_SEL_TYPE_CODE;
    10973                 /* mov reg1, must1|must0 */
    10974                 off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxReg1, fMustBe1 | fMustBe0);
    10975                 /* and reg1, segattrs */
    10976                 off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxReg1, idxRegSegAttrib);
    10977                 /* cmp reg1, must1 */
    10978                 off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, fMustBe1);
    10979                 /* jne tlbmiss */
    10980                 off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
    10981             }
    10982             else
    10983             {
    10984                 /*  U  | !P |!DT |!CD | RW |
    10985                     16 |  8 |  4 |  3 |  1 |
    10986                   -------------------------------
    10987                     0  |  0 |  0 |  0 |  0 | execute-only code segment. - must be excluded
    10988                     0  |  0 |  0 |  0 |  1 | execute-read code segment.
    10989                     0  |  0 |  0 |  1 |  0 | read-only data segment.
    10990                     0  |  0 |  0 |  1 |  1 | read-write data segment.   - last valid combination
    10991                 */
    10992                 /* mov reg1, relevant attributes  */
    10993                 off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxReg1,
    10994                                                     X86DESCATTR_UNUSABLE | X86DESCATTR_P | X86DESCATTR_DT
    10995                                                   | X86_SEL_TYPE_CODE    | X86_SEL_TYPE_WRITE);
    10996                 /* and reg1, segattrs */
    10997                 off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxReg1, idxRegSegAttrib);
    10998                 /* xor reg1, X86DESCATTR_P | X86DESCATTR_DT | X86_SEL_TYPE_CODE ; place C=1 RW=0 at the bottom & limit the range.
    10999                                                 ; EO-code=0,  ER-code=2, RO-data=8, RW-data=10 */
    11000                 off = iemNativeEmitXorGpr32ByImmEx(pbCodeBuf, off, idxReg1, X86DESCATTR_P | X86DESCATTR_DT | X86_SEL_TYPE_CODE);
    11001                 /* sub reg1, X86_SEL_TYPE_WRITE ; EO-code=-2, ER-code=0, RO-data=6, RW-data=8 */
    11002                 off = iemNativeEmitSubGpr32ImmEx(pbCodeBuf, off, idxReg1, X86_SEL_TYPE_WRITE /* ER-code */);
    11003                 /* cmp reg1, X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE */
    11004                 AssertCompile(X86_SEL_TYPE_CODE == 8);
    11005                 off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, X86_SEL_TYPE_CODE);
    11006                 /* ja  tlbmiss */
    11007                 off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
    11008             }
    11009 
    11010             /*
    11011              * Check the limit.  If this is a write access, we know that it's a
    11012              * data segment and includes the expand_down bit.  For read-only accesses
    11013              * we need to check that code/data=0 and expanddown=1 before continuing.
    11014              */
    11015             uint32_t offFixupCheckExpandDown;
    11016             if (fAccess & IEM_ACCESS_TYPE_WRITE)
    11017             {
    11018                 /* test segattrs, X86_SEL_TYPE_DOWN */
    11019                 AssertCompile(X86_SEL_TYPE_DOWN < 128);
    11020                 off = iemNativeEmitTestAnyBitsInGpr8Ex(pbCodeBuf, off, idxRegSegAttrib, X86_SEL_TYPE_DOWN);
    11021                 /* jnz  check_expand_down */
    11022                 offFixupCheckExpandDown = off;
    11023                 off = iemNativeEmitJccToFixedEx(pbCodeBuf, off, off /*ASSUMES rel8 suffices*/, kIemNativeInstrCond_ne);
    11024             }
    11025             else
    11026             {
    11027                 /* mov reg1, segattrs */
    11028                 off = iemNativeEmitLoadGprFromGpr32Ex(pbCodeBuf, off, idxReg1, idxRegSegAttrib);
    11029                 /* and reg1, code | down */
    11030                 off = iemNativeEmitAndGpr32ByImmEx(pbCodeBuf, off, idxReg1, X86_SEL_TYPE_CODE | X86_SEL_TYPE_DOWN);
    11031                 /* cmp reg1, down */
    11032                 off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, X86_SEL_TYPE_DOWN);
    11033                 /* je check_expand_down */
    11034                 offFixupCheckExpandDown = off;
    11035                 off = iemNativeEmitJccToFixedEx(pbCodeBuf, off, off /*ASSUMES rel8 suffices*/, kIemNativeInstrCond_e);
    11036             }
    11037 
    11038             /* expand_up:
    11039                cmp  seglim, regptr/reg2/imm */
    11040             if (idxRegPtr != UINT8_MAX)
    11041                 off = iemNativeEmitCmpGprWithGprEx(pbCodeBuf, off, idxRegSegLimit, cbMem > 1 ? idxReg2 : idxRegPtr);
    11042             else
    11043                 off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxRegSegLimit, (uint32_t)uAbsPtr);
    11044             /* jbe  tlbmiss */
    11045             off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_be);
    11046             /* jmp  limitdone */
    11047             uint32_t const offFixupLimitDone = off;
    11048             off = iemNativeEmitJmpToFixedEx(pbCodeBuf, off, off /*ASSUMES rel8 suffices*/);
    11049 
    11050             /* check_expand_down: ; complicted! */
    11051             iemNativeFixupFixedJump(pReNative, offFixupCheckExpandDown, off);
    11052     pbCodeBuf[off++] = 0xcc;
    11053             /* cmp  seglim, regptr */
    11054             if (idxRegPtr != UINT8_MAX)
    11055                 off = iemNativeEmitCmpGprWithGprEx(pbCodeBuf, off, idxRegSegLimit, idxRegPtr);
    11056             else
    11057                 off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxRegSegLimit, (uint32_t)uAbsPtr);
    11058             /* ja  tlbmiss */
    11059             off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
    11060             /* mov  reg1, X86DESCATTR_D (0x4000) */
    11061             off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, X86DESCATTR_D);
    11062             /* and  reg1, segattr */
    11063             off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxReg1, idxRegSegAttrib);
    11064             /* xor  reg1, X86DESCATTR_D */
    11065             off = iemNativeEmitXorGpr32ByImmEx(pbCodeBuf, off, idxReg1, X86DESCATTR_D);
    11066             /* shl  reg1, 2 (16 - 14) */
    11067             AssertCompile((X86DESCATTR_D << 2) == UINT32_C(0x10000));
    11068             off = iemNativeEmitShiftGpr32LeftEx(pbCodeBuf, off, idxReg1, 2);
    11069             /* dec  reg1 (=> 0xffff if D=0; 0xffffffff if D=1) */
    11070             off = iemNativeEmitSubGpr32ImmEx(pbCodeBuf, off, idxReg1, 1);
    11071             /* cmp  reg1, reg2 (64-bit) / imm (32-bit) */
    11072             if (idxRegPtr != UINT8_MAX)
    11073                 off = iemNativeEmitCmpGprWithGprEx(pbCodeBuf, off, idxReg1, idxReg2);
    11074             else
    11075                 off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, (uint32_t)(uAbsPtr + cbMem - 1));
    11076             /* jbe  tlbmiss */
    11077             off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_be);
    11078 
    11079             /* limitdone: */
    11080             iemNativeFixupFixedJump(pReNative, offFixupLimitDone, off);
    11081         }
    11082 
    11083         /* 1b. Add the segment base. We use idxRegMemResult for the ptr register if this step is
    11084                required or if the address is a constant (simplicity). */
    11085         uint8_t const idxRegFlatPtr = iSegReg != UINT8_MAX || idxRegPtr == UINT8_MAX ? idxRegMemResult : idxRegPtr;
    11086         if (iSegReg != UINT8_MAX)
    11087         {
    11088             /** @todo this can be done using LEA as well. */
    11089             if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
    11090             {
    11091                 Assert(iSegReg >= X86_SREG_FS);
    11092                 /* mov regflat, regptr/imm */
    11093                 if (idxRegPtr != UINT8_MAX)
    11094                     off = iemNativeEmitLoadGprFromGprEx(pbCodeBuf, off, idxRegFlatPtr, idxRegPtr);
    11095                 else
    11096                     off = iemNativeEmitLoadGprImmEx(pbCodeBuf, off, idxRegFlatPtr, uAbsPtr);
    11097                 /* add regflat, seg.base */
    11098                 off = iemNativeEmitAddTwoGprsEx(pbCodeBuf, off, idxRegFlatPtr, idxRegSegBase);
    11099             }
    11100             else
    11101             {
    11102                 /* mov regflat, regptr/imm */
    11103                 if (idxRegPtr != UINT8_MAX)
    11104                     off = iemNativeEmitLoadGprFromGpr32Ex(pbCodeBuf, off, idxRegFlatPtr, idxRegPtr);
    11105                 else
    11106                     off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxRegFlatPtr, uAbsPtr);
    11107                 /* add regflat, seg.base */
    11108                 off = iemNativeEmitAddTwoGprs32Ex(pbCodeBuf, off, idxRegFlatPtr, idxRegSegBase);
    11109             }
    11110         }
    11111         else if (idxRegPtr == UINT8_MAX)
    11112             off = iemNativeEmitLoadGprImmEx(pbCodeBuf, off, idxRegFlatPtr, uAbsPtr);
    11113 
    11114         /*
    11115          * 2. Check that the address doesn't cross a page boundrary and doesn't have alignment issues.
    11116          *
    11117          * 2a. Alignment check using fAlignMask.
    11118          */
    11119         if (fAlignMask)
    11120         {
    11121             Assert(RT_IS_POWER_OF_TWO(fAlignMask + 1));
    11122             Assert(fAlignMask < 128);
    11123             /* test regflat, fAlignMask */
    11124             off = iemNativeEmitTestAnyBitsInGpr8Ex(pbCodeBuf, off, idxRegFlatPtr, fAlignMask);
    11125             /* jnz tlbmiss */
    11126             off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
    11127         }
    11128 
    11129         /*
    11130          * 2b. Check that it's not crossing page a boundrary. This is implicit in
    11131          *     the previous test if the alignment is same or larger than the type.
    11132          */
    11133         if (cbMem > fAlignMask + 1)
    11134         {
    11135             /* mov reg1, 0xfff */
    11136             off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxReg1, GUEST_PAGE_OFFSET_MASK);
    11137             /* and reg1, regflat */
    11138             off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxReg1, idxRegFlatPtr);
    11139             /* neg reg1 */
    11140             off = iemNativeEmitNegGpr32Ex(pbCodeBuf, off, idxReg1);
    11141             /* add reg1, 0x1000 */
    11142             off = iemNativeEmitAddGpr32ImmEx(pbCodeBuf, off, idxReg1, GUEST_PAGE_SIZE);
    11143             /* cmp reg1, cbMem */
    11144             off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, GUEST_PAGE_SIZE);
    11145             /* ja  tlbmiss */
    11146             off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
    11147         }
    11148 
    11149         /*
    11150          * 3. TLB lookup.
    11151          *
    11152          * 3a. Calculate the TLB tag value (IEMTLB_CALC_TAG).
    11153          *     In 64-bit mode we will also check for non-canonical addresses here.
    11154          */
    11155         if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
    11156         {
    11157             /* mov reg1, regflat */
    11158             off = iemNativeEmitLoadGprFromGprEx(pbCodeBuf, off, idxReg1, idxRegFlatPtr);
    11159             /* rol reg1, 16 */
    11160             off = iemNativeEmitRotateGprLeftEx(pbCodeBuf, off, idxReg1, 16);
    11161             /** @todo Would 'movsx reg2, word reg1' and working on reg2 in dwords be faster? */
    11162             /* inc word reg1 */
    11163             pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
    11164             if (idxReg1 >= 8)
    11165                 pbCodeBuf[off++] = X86_OP_REX_B;
    11166             pbCodeBuf[off++] = 0xff;
    11167             pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, idxReg1 & 7);
    11168             /* cmp word reg1, 1 */
    11169             pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
    11170             if (idxReg1 >= 8)
    11171                 pbCodeBuf[off++] = X86_OP_REX_B;
    11172             pbCodeBuf[off++] = 0x83;
    11173             pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, idxReg1 & 7);
    11174             pbCodeBuf[off++] = 1;
    11175             /* ja  tlbmiss */
    11176             off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
    11177             /* shr reg1, 16 + GUEST_PAGE_SHIFT */
    11178             off = iemNativeEmitShiftGprRightEx(pbCodeBuf, off, idxReg1, 16 + GUEST_PAGE_SHIFT);
    11179         }
    11180         else
    11181         {
    11182             /* mov reg1, regflat */
    11183             off = iemNativeEmitLoadGprFromGpr32Ex(pbCodeBuf, off, idxReg1, idxRegFlatPtr);
    11184             /* shr reg1, GUEST_PAGE_SHIFT */
    11185             off = iemNativeEmitShiftGpr32RightEx(pbCodeBuf, off, idxReg1, GUEST_PAGE_SHIFT);
    11186         }
    11187         /* or  reg1, [qword pVCpu->iem.s.DataTlb.uTlbRevision] */
    11188         pbCodeBuf[off++] = idxReg1 < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R;
    11189         pbCodeBuf[off++] = 0x0b; /* OR r64,r/m64 */
    11190         off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxReg1, RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbRevision));
    11191 
    11192         /*
    11193          * 3b. Calc pTlbe.
    11194          */
    11195         /* movzx reg2, byte reg1 */
    11196         off = iemNativeEmitLoadGprFromGpr8Ex(pbCodeBuf, off, idxReg2, idxReg1);
    11197         /* shl   reg2, 5 ; reg2 *= sizeof(IEMTLBENTRY) */
    11198         AssertCompileSize(IEMTLBENTRY, 32);
    11199         off = iemNativeEmitShiftGprLeftEx(pbCodeBuf, off, idxReg2, 5);
    11200         /* lea   reg2, [pVCpu->iem.s.DataTlb.aEntries + reg2] */
    11201         AssertCompile(IEMNATIVE_REG_FIXED_PVMCPU < 8);
    11202         pbCodeBuf[off++] = idxReg2 < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_X | X86_OP_REX_R;
    11203         pbCodeBuf[off++] = 0x8d;
    11204         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, idxReg2 & 7, 4 /*SIB*/);
    11205         pbCodeBuf[off++] = X86_SIB_MAKE(IEMNATIVE_REG_FIXED_PVMCPU & 7, idxReg2 & 7, 0);
    11206         pbCodeBuf[off++] = RT_BYTE1(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
    11207         pbCodeBuf[off++] = RT_BYTE2(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
    11208         pbCodeBuf[off++] = RT_BYTE3(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
    11209         pbCodeBuf[off++] = RT_BYTE4(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
    11210 
    11211         /*
    11212          * 3c. Compare the TLBE.uTag with the one from 2a (reg1).
    11213          */
    11214         /* cmp reg1, [reg2] */
    11215         pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R) | (idxReg2 < 8 ? 0 : X86_OP_REX_B);
    11216         pbCodeBuf[off++] = 0x3b;
    11217         off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, idxReg1, idxReg2, RT_UOFFSETOF(IEMTLBENTRY, uTag));
    11218         /* jne tlbmiss */
    11219         off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
    11220 
    11221         /*
    11222          * 4. Check TLB page table level access flags and physical page revision #.
    11223          */
    11224         /* mov reg1, mask */
    11225         AssertCompile(IEMTLBE_F_PT_NO_USER == 4);
    11226         uint64_t const fNoUser = (((pReNative->fExec >> IEM_F_X86_CPL_SHIFT) & IEM_F_X86_CPL_SMASK) + 1) & IEMTLBE_F_PT_NO_USER;
    11227         off = iemNativeEmitLoadGprImmEx(pbCodeBuf, off, idxReg1,
    11228                                           IEMTLBE_F_PHYS_REV       | IEMTLBE_F_NO_MAPPINGR3
    11229                                         | IEMTLBE_F_PG_UNASSIGNED  | IEMTLBE_F_PG_NO_READ
    11230                                         | IEMTLBE_F_PT_NO_ACCESSED | fNoUser);
    11231         /* and reg1, [reg2->fFlagsAndPhysRev] */
    11232         pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R) | (idxReg2 < 8 ? 0 : X86_OP_REX_B);
    11233         pbCodeBuf[off++] = 0x23;
    11234         off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, idxReg1, idxReg2, RT_UOFFSETOF(IEMTLBENTRY, fFlagsAndPhysRev));
    11235 
    11236         /* cmp reg1, [pVCpu->iem.s.DataTlb.uTlbPhysRev] */
    11237         pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R);
    11238         pbCodeBuf[off++] = 0x3b;
    11239         off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, idxReg1, IEMNATIVE_REG_FIXED_PVMCPU,
    11240                                         RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbPhysRev));
    11241         /* jne tlbmiss */
    11242         off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
    11243 
    11244         /*
    11245          * 5. Check that pbMappingR3 isn't NULL (paranoia) and calculate the
    11246          *    resulting pointer.
    11247          */
    11248         /* mov  reg1, [reg2->pbMappingR3] */
    11249         pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R) | (idxReg2 < 8 ? 0 : X86_OP_REX_B);
    11250         pbCodeBuf[off++] = 0x8b;
    11251         off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, idxReg1, idxReg2, RT_UOFFSETOF(IEMTLBENTRY, pbMappingR3));
    11252 
    11253         /** @todo eliminate the need for this test? */
    11254         /* test reg1, reg1 */
    11255         pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R | X86_OP_REX_B);
    11256         pbCodeBuf[off++] = 0x85;
    11257         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxReg1 & 7, idxReg1 & 7);
    11258 
    11259         /* jz   tlbmiss */
    11260         off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbDone, kIemNativeInstrCond_e);
    11261 
    11262         if (idxRegFlatPtr == idxRegMemResult) /* See step 1b. */
    11263         {
    11264             /* and result, 0xfff */
    11265             off = iemNativeEmitAndGpr32ByImmEx(pbCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
    11266         }
    11267         else
    11268         {
    11269             Assert(idxRegFlatPtr == idxRegPtr);
    11270             /* mov result, 0xfff */
    11271             off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
    11272             /* and result, regflat */
    11273             off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxRegMemResult, idxRegFlatPtr);
    11274         }
    11275         /* add result, reg1 */
    11276         off = iemNativeEmitAddTwoGprsEx(pbCodeBuf, off, idxRegMemResult, idxReg1);
    11277 
    11278         if (idxRegPtr != UINT8_MAX)
    11279             iemNativeVarRegisterRelease(pReNative, idxVarGCPtrMem);
    11280         if (idxRegSegBase != UINT8_MAX)
    11281             iemNativeRegFreeTmp(pReNative, idxRegSegBase);
    11282         if (idxRegSegLimit != UINT8_MAX)
    11283         {
    11284             iemNativeRegFreeTmp(pReNative, idxRegSegLimit);
    11285             iemNativeRegFreeTmp(pReNative, idxRegSegAttrib);
    11286         }
    11287         else
    11288             Assert(idxRegSegAttrib == UINT8_MAX);
    11289         iemNativeRegFreeTmp(pReNative, idxReg2);
    11290         iemNativeRegFreeTmp(pReNative, idxReg1);
    11291         IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    11292     }
    11293     /* Problematic absolute address, jmp to tlbmiss.  Caller will be generate
    11294        some unused tail code, but whatever. */
    11295     else
    11296         off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbMiss);
    11297 
    11298     /*
    11299      * Tail code, specific to the MC when the above is moved into a separate function.
    11300      */
    11301     uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 32);
    11302 
    11303     /* [idxVarUnmapInfo] = 0 - allocate register for it. There must be free ones now, so no spilling required. */
    11304     uint32_t const offSaved = off;
    11305     uint8_t const idxRegUnmapInfo = iemNativeVarRegisterAcquire(pReNative, idxVarUnmapInfo, &off);
    11306     AssertStmt(off == offSaved, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_9));
    11307     off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxRegUnmapInfo, 0);
    11308 
    11309     /* jmp tlbdone */
    11310     off = iemNativeEmitJmpToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbDone);
    11311 
    11312     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    11313 #else
    11314     /** @todo arm64 TLB code   */
    11315     RT_NOREF(fAccess, fAlignMask, cbMem);
    11316 #endif
    11317 
    11318     /*
     11360    IEMNATIVEEMITTLBSTATE const TlbState(pReNative, &off, idxVarGCPtrMem, iSegReg, cbMem);
     11361
     11362    uint32_t const idxLabelTlbLookup = !TlbState.fSkip
     11363                                     ? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
     11364                                     : UINT32_MAX;
     11365//off=iemNativeEmitBrk(pReNative, off, 0);
     11366    /*
     11367     * Jump to the TLB lookup code.
     11368     */
     11369    if (!TlbState.fSkip)
     11370        off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
     11371
     11372    /*
     11373     * tlbmiss:
     11374     *
    1131911375     * Call helper to do the fetching.
    1132011376     * We flush all guest register shadow copies here.
    1132111377     */
    11322     iemNativeLabelDefine(pReNative, idxLabelTlbMiss, off);
     11378    uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
    1132311379
    1132411380#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
    … …  
    1135811414    if (idxRegMemResult != IEMNATIVE_CALL_RET_GREG)
    1135911415        off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegMemResult, IEMNATIVE_CALL_RET_GREG);
    11360     iemNativeVarRegisterRelease(pReNative, idxVarMem);
    11361 
    11362 #if defined(RT_ARCH_AMD64)
     11416
    1136311417    Assert(pReNative->Core.aVars[idxVarUnmapInfo].idxReg == idxRegUnmapInfo);
    1136411418    off = iemNativeEmitLoadGprByBpU8(pReNative, off, idxRegUnmapInfo, offBpDispVarUnmapInfo);
     11419
     11420#ifdef IEMNATIVE_WITH_TLB_LOOKUP
     11421    if (!TlbState.fSkip)
     11422    {
     11423        /* end of tlbsmiss - Jump to the done label. */
     11424        uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
     11425        off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
     11426
     11427        /*
     11428         * tlblookup:
     11429         */
     11430        off = iemNativeEmitTlbLookup(pReNative, off, &TlbState, iSegReg, cbMem, fAlignMask, fAccess,
     11431                                     idxLabelTlbLookup, idxLabelTlbMiss, idxRegMemResult);
     11432        TlbState.freeRegsAndReleaseVars(pReNative, idxVarGCPtrMem);
     11433
     11434        /*
     11435         * Lookup tail code, specific to the MC when the above is moved into a separate function.
     11436         */
     11437        /* [idxVarUnmapInfo] = 0 - allocate register for it. There must be free ones now, so no spilling required. */
     11438        off = iemNativeEmitLoadGprImm32(pReNative, off, idxRegUnmapInfo, 0);
     11439
     11440        /*
     11441         * tlbdone:
     11442         */
     11443        iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
     11444    }
     11445#else
     11446    RT_NOREF(fAccess, fAlignMask);
     11447#endif
     11448
    1136511449    iemNativeVarRegisterRelease(pReNative, idxVarUnmapInfo);
    11366 #endif
    11367 
    11368     iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
     11450    iemNativeVarRegisterRelease(pReNative, idxVarMem);
    1136911451
    1137011452    return off;
    … …  
    1187311955                                case kIemNativeLabelType_CheckIrq:
    1187411956                                    pszName = "CheckIrq_CheckVM";
     11957                                    fNumbered = true;
     11958                                    break;
     11959                                case kIemNativeLabelType_TlbLookup:
     11960                                    pszName = "TlbLookup";
    1187511961                                    fNumbered = true;
    1187611962                                    break;

  • trunk/src/VBox/VMM/include/IEMN8veRecompiler.h

    r102717 r102724  
    319319    kIemNativeLabelType_Endif,
    320320    kIemNativeLabelType_CheckIrq,
     321    kIemNativeLabelType_TlbLookup,
    321322    kIemNativeLabelType_TlbMiss,
    322323    kIemNativeLabelType_TlbDone,

  • trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h

    r102720 r102724  
    8282 * Emit a breakpoint instruction.
    8383 */
     84DECL_FORCE_INLINE(uint32_t) iemNativeEmitBrkEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t uInfo)
     85{
     86#ifdef RT_ARCH_AMD64
     87    pCodeBuf[off++] = 0xcc;
     88    RT_NOREF(uInfo);   /** @todo use multibyte nop for info? */
     89
     90#elif defined(RT_ARCH_ARM64)
     91    pCodeBuf[off++] = Armv8A64MkInstrBrk(uInfo & UINT32_C(0xffff));
     92
     93#else
     94# error "error"
     95#endif
     96    return off;
     97}
     98
     99
     100/**
     101 * Emit a breakpoint instruction.
     102 */
    84103DECL_INLINE_THROW(uint32_t) iemNativeEmitBrk(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
    85104{
    86105#ifdef RT_ARCH_AMD64
    87     uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    88     pbCodeBuf[off++] = 0xcc;
    89     RT_NOREF(uInfo);
    90 
    91 #elif defined(RT_ARCH_ARM64)
    92     uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    93     pu32CodeBuf[off++] = Armv8A64MkInstrBrk(uInfo & UINT32_C(0xffff));
    94 
     106    off = iemNativeEmitBrkEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, uInfo);
     107#elif defined(RT_ARCH_ARM64)
     108    off = iemNativeEmitBrkEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, uInfo);
    95109#else
    96110# error "error"
    … …  
    33213335    Assert(idxLabel < pReNative->cLabels);
    33223336
    3323 #ifdef RT_ARCH_AMD64
    3324     /* jcc rel32 */
    3325     pCodeBuf[off++] = 0x0f;
    3326     pCodeBuf[off++] = (uint8_t)enmCond | 0x80;
    3327     iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
    3328     pCodeBuf[off++] = 0x00;
    3329     pCodeBuf[off++] = 0x00;
    3330     pCodeBuf[off++] = 0x00;
    3331     pCodeBuf[off++] = 0x00;
    3332 
    3333 #elif defined(RT_ARCH_ARM64)
    3334     iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
    3335     pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
     3337    uint32_t const offLabel = pReNative->paLabels[idxLabel].off;
     3338#ifdef RT_ARCH_AMD64
     3339    if (offLabel >= off)
     3340    {
     3341        /* jcc rel32 */
     3342        pCodeBuf[off++] = 0x0f;
     3343        pCodeBuf[off++] = (uint8_t)enmCond | 0x80;
     3344        iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
     3345        pCodeBuf[off++] = 0x00;
     3346        pCodeBuf[off++] = 0x00;
     3347        pCodeBuf[off++] = 0x00;
     3348        pCodeBuf[off++] = 0x00;
     3349    }
     3350    else
     3351    {
     3352        int32_t offDisp = offLabel - (off + 2);
     3353        if ((int8_t)offDisp == offDisp)
     3354        {
     3355            /* jcc rel8 */
     3356            pCodeBuf[off++] = (uint8_t)enmCond | 0x70;
     3357            pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
     3358        }
     3359        else
     3360        {
     3361            /* jcc rel32 */
     3362            offDisp -= 4;
     3363            pCodeBuf[off++] = 0x0f;
     3364            pCodeBuf[off++] = (uint8_t)enmCond | 0x80;
     3365            pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
     3366            pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
     3367            pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
     3368            pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
     3369        }
     3370    }
     3371
     3372#elif defined(RT_ARCH_ARM64)
     3373    if (offLabel >= off)
     3374    {
     3375        iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
     3376        pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
     3377    }
     3378    else
     3379    {
     3380        Assert(offLabel - off <= -0x3ffff);
     3381        pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offLabel - off);
     3382    }
    33363383
    33373384#else
Note: See TracChangeset for help on using the changeset viewer.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette