VirtualBox

Changeset 102735 in vbox for trunk/src/VBox/VMM


Ignore:
Timestamp:
Dec 30, 2023 11:59:54 PM (13 months ago)
Author:
vboxsync
Message:

VMM/IEM: Porting the TLB lookup code to ARM. Compiles, but needs debugging. bugref:10371

Location:
trunk/src/VBox/VMM
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

    r102734 r102735  
    99689968*********************************************************************************************************************************/
    99699969
    9970 #if defined(RT_ARCH_AMD64) && 1
     9970#if (defined(RT_ARCH_AMD64) && 1) || (defined(RT_ARCH_ARM64) && 0)
    99719971# define IEMNATIVE_WITH_TLB_LOOKUP
    99729972#endif
    … …  
    99799979typedef struct IEMNATIVEEMITTLBSTATE
    99809980{
    9981     uint64_t const  uAbsPtr;
    99829981    bool const      fSkip;
     9982    uint8_t const   idxRegPtrHlp;   /**< We don't support immediate variables with register assignment, so this a tmp reg alloc. */
    99839983    uint8_t const   idxRegPtr;
    99849984    uint8_t const   idxRegSegBase;
    … …  
    99879987    uint8_t const   idxReg1;
    99889988    uint8_t const   idxReg2;
     9989#if defined(RT_ARCH_ARM64)
     9990    uint8_t const   idxReg3;
     9991#endif
     9992    uint64_t const  uAbsPtr;
    99899993
    99909994    IEMNATIVEEMITTLBSTATE(PIEMRECOMPILERSTATE a_pReNative, uint32_t *a_poff, uint8_t a_idxVarGCPtrMem,
    99919995                          uint8_t a_iSegReg, uint8_t a_cbMem)
    9992         :         uAbsPtr(  a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate
    9993                           ? UINT64_MAX
    9994                           : a_pReNative->Core.aVars[a_idxVarGCPtrMem].u.uValue)
    99959996#ifdef IEMNATIVE_WITH_TLB_LOOKUP
    99969997        /* 32-bit and 64-bit wraparound will require special handling, so skip these for absolute addresses. */
    9997         ,           fSkip(   a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind == kIemNativeVarKind_Immediate
     9998        :           fSkip(   a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind == kIemNativeVarKind_Immediate
    99989999                          &&   (  (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT
    999910000                                ? (uint64_t)(UINT32_MAX - a_cbMem)
    … …  
    1000110002                             < a_pReNative->Core.aVars[a_idxVarGCPtrMem].u.uValue)
    1000210003#else
    10003         ,           fSkip(true)
     10004        :           fSkip(true)
     10005#endif
     10006#if defined(RT_ARCH_AMD64) /* got good immediate encoding, otherwise we just load the address in a reg immediately. */
     10007        ,    idxRegPtrHlp(UINT8_MAX)
     10008#else
     10009        ,    idxRegPtrHlp(   a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate
     10010                          || fSkip
     10011                          ? UINT8_MAX
     10012                          : iemNativeRegAllocTmpImm(a_pReNative, a_poff, a_pReNative->Core.aVars[a_idxVarGCPtrMem].u.uValue) )
    1000410013#endif
    1000510014        ,       idxRegPtr(a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate
    1000610015                          ? iemNativeVarRegisterAcquire(a_pReNative, a_idxVarGCPtrMem, a_poff,
    1000710016                                                        true /*fInitialized*/, IEMNATIVE_CALL_ARG2_GREG)
    10008                           : UINT8_MAX)
     10017                          : idxRegPtrHlp)
    1000910018        ,   idxRegSegBase(a_iSegReg == UINT8_MAX || fSkip
    1001010019                          ? UINT8_MAX
    … …  
    1001810027        ,         idxReg1(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
    1001910028        ,         idxReg2(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
     10029#if defined(RT_ARCH_ARM64)
     10030        ,         idxReg3(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
     10031#endif
     10032        ,         uAbsPtr(  a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate || fSkip
     10033                          ? UINT64_MAX
     10034                          : a_pReNative->Core.aVars[a_idxVarGCPtrMem].u.uValue)
    1002010035
    1002110036    {
    … …  
    1002610041    {
    1002710042        if (idxRegPtr != UINT8_MAX)
    10028             iemNativeVarRegisterRelease(a_pReNative, idxVarGCPtrMem);
     10043        {
     10044            if (idxRegPtrHlp == UINT8_MAX)
     10045                iemNativeVarRegisterRelease(a_pReNative, idxVarGCPtrMem);
     10046            else
     10047            {
     10048                Assert(idxRegPtrHlp == idxRegPtr);
     10049                iemNativeRegFreeTmpImm(a_pReNative, idxRegPtrHlp);
     10050            }
     10051        }
     10052        else
     10053            Assert(idxRegPtrHlp == UINT8_MAX);
    1002910054        if (idxRegSegBase != UINT8_MAX)
    1003010055            iemNativeRegFreeTmp(a_pReNative, idxRegSegBase);
    … …  
    1003610061        else
    1003710062            Assert(idxRegSegAttrib == UINT8_MAX);
     10063#if defined(RT_ARCH_ARM64)
     10064        iemNativeRegFreeTmp(a_pReNative, idxReg3);
     10065#endif
    1003810066        iemNativeRegFreeTmp(a_pReNative, idxReg2);
    1003910067        iemNativeRegFreeTmp(a_pReNative, idxReg1);
     10068
    1004010069    }
    1004110070
    … …  
    1004310072    {
    1004410073        if (!fSkip)
    10045             return RT_BIT_32(idxReg1) | RT_BIT_32(idxReg2);
     10074            return RT_BIT_32(idxReg1)
     10075                 | RT_BIT_32(idxReg2)
     10076#if defined(RT_ARCH_ARM64)
     10077                 | RT_BIT_32(idxReg3)
     10078#endif
     10079                 ;
    1004610080        return 0;
    1004710081    }
    … …  
    1009110125        /* ja  tlbmiss */
    1009210126        off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
    10093         /* mov  reg1, X86DESCATTR_D (0x4000) */
    10094         off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, X86DESCATTR_D);
    10095         /* and  reg1, segattr */
    10096         off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxRegSegAttrib);
     10127        /* reg1 = segattr & X86DESCATTR_D (0x4000) */
     10128        off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxRegSegAttrib, X86DESCATTR_D);
    1009710129        /* xor  reg1, X86DESCATTR_D */
    1009810130        off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1, X86DESCATTR_D);
    … …  
    1011810150     */
    1011910151    iemNativeLabelDefine(pReNative, idxLabelTlbLookup, off);
     10152# if defined(RT_ARCH_ARM64)
     10153    off = iemNativeEmitBrkEx(pCodeBuf, off, 0); /** @todo debug on arm */
     10154# endif
    1012010155
    1012110156    /*
    … …  
    1013010165        if (cbMem > 1 && pTlbState->idxRegPtr != UINT8_MAX)
    1013110166        {
    10132 # if 1
    10133             Assert(cbMem - 1 <= 127);
    10134             /* mov reg2, regptr */
    10135             off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, pTlbState->idxReg2, pTlbState->idxRegPtr);
    10136             /* add reg2, cbMem-1 */
    10137             off = iemNativeEmitAddGpr32Imm8Ex(pCodeBuf, off, pTlbState->idxReg2, cbMem - 1);
    10138 # else
    10139             /* mov reg2, cbMem-1 */
    10140             off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, pTlbState->idxReg2, cbMem - 1);
    10141             /* add reg2, regptr */
    10142             off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, pTlbState->idxReg2, pTlbState->idxRegPtr);
    10143 # endif
     10167            /* reg2 = regptr + cbMem - 1 ; 64-bit result so we can fend of wraparounds/overflows. */
     10168            off = iemNativeEmitGprEqGprPlusImmEx(pCodeBuf, off, pTlbState->idxReg2,/*=*/ pTlbState->idxRegPtr,/*+*/ cbMem - 1);
    1014410169        }
    1014510170
    … …  
    1015110176            uint32_t const fMustBe1 = X86DESCATTR_P        | X86DESCATTR_DT    | X86_SEL_TYPE_WRITE;
    1015210177            uint32_t const fMustBe0 = X86DESCATTR_UNUSABLE | X86_SEL_TYPE_CODE;
    10153             /* mov reg1, must1|must0 */
    10154             off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, pTlbState->idxReg1, fMustBe1 | fMustBe0);
    10155             /* and reg1, segattrs */
    10156             off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxRegSegAttrib);
     10178            /* reg1 = segattrs & (must1|must0) */
     10179            off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1,
     10180                                                  pTlbState->idxRegSegAttrib, fMustBe1 | fMustBe0);
    1015710181            /* cmp reg1, must1 */
     10182            AssertCompile(fMustBe1 <= UINT16_MAX);
    1015810183            off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, fMustBe1);
    1015910184            /* jne tlbmiss */
    … …  
    1017010195                0  |  0 |  0 |  1 |  1 | read-write data segment.   - last valid combination
    1017110196            */
    10172             /* mov reg1, relevant attributes  */
    10173             off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, pTlbState->idxReg1,
    10174                                                 X86DESCATTR_UNUSABLE | X86DESCATTR_P | X86DESCATTR_DT
    10175                                               | X86_SEL_TYPE_CODE    | X86_SEL_TYPE_WRITE);
    10176             /* and reg1, segattrs */
    10177             off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxRegSegAttrib);
     10197            /* reg1 = segattrs & (relevant attributes) */
     10198            off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxRegSegAttrib,
     10199                                                    X86DESCATTR_UNUSABLE | X86DESCATTR_P | X86DESCATTR_DT
     10200                                                  | X86_SEL_TYPE_CODE    | X86_SEL_TYPE_WRITE);
    1017810201            /* xor reg1, X86DESCATTR_P | X86DESCATTR_DT | X86_SEL_TYPE_CODE ; place C=1 RW=0 at the bottom & limit the range.
    1017910202                                            ; EO-code=0,  ER-code=2, RO-data=8, RW-data=10 */
    10180             off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1, X86DESCATTR_P | X86DESCATTR_DT | X86_SEL_TYPE_CODE);
     10203#ifdef RT_ARCH_ARM64
     10204            off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1, X86DESCATTR_DT | X86_SEL_TYPE_CODE);
     10205            off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1, X86DESCATTR_P);
     10206#else
     10207            off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1,
     10208                                               X86DESCATTR_P | X86DESCATTR_DT | X86_SEL_TYPE_CODE);
     10209#endif
    1018110210            /* sub reg1, X86_SEL_TYPE_WRITE ; EO-code=-2, ER-code=0, RO-data=6, RW-data=8 */
    1018210211            off = iemNativeEmitSubGpr32ImmEx(pCodeBuf, off, pTlbState->idxReg1, X86_SEL_TYPE_WRITE /* ER-code */);
    … …  
    1020310232        else
    1020410233        {
    10205             /* mov reg1, segattrs */
    10206             off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxRegSegAttrib);
    10207             /* and reg1, code | down */
    10208             off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1, X86_SEL_TYPE_CODE | X86_SEL_TYPE_DOWN);
     10234            /* reg1 = segattr & (code | down) */
     10235            off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1,
     10236                                                  pTlbState->idxRegSegAttrib, X86_SEL_TYPE_CODE | X86_SEL_TYPE_DOWN);
    1020910237            /* cmp reg1, down */
    1021010238            off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, X86_SEL_TYPE_DOWN);
    … …  
    1023210260    if (iSegReg != UINT8_MAX)
    1023310261    {
    10234         /** @todo this can be done using LEA as well. */
     10262        /* regflat = segbase + regptr/imm */
    1023510263        if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
    1023610264        {
    1023710265            Assert(iSegReg >= X86_SREG_FS);
    10238             /* mov regflat, regptr/imm */
    1023910266            if (pTlbState->idxRegPtr != UINT8_MAX)
    10240                 off = iemNativeEmitLoadGprFromGprEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegPtr);
     10267                off = iemNativeEmitGprEqGprPlusGprEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase, pTlbState->idxRegPtr);
    1024110268            else
    10242                 off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->uAbsPtr);
    10243             /* add regflat, seg.base */
    10244             off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase);
     10269                off = iemNativeEmitGprEqGprPlusImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase, pTlbState->uAbsPtr);
    1024510270        }
     10271        else if (pTlbState->idxRegPtr != UINT8_MAX)
     10272            off = iemNativeEmitGpr32EqGprPlusGprEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase, pTlbState->idxRegPtr);
    1024610273        else
    10247         {
    10248             /* mov regflat, regptr/imm */
    10249             if (pTlbState->idxRegPtr != UINT8_MAX)
    10250                 off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegPtr);
    10251             else
    10252                 off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->uAbsPtr);
    10253             /* add regflat, seg.base */
    10254             off = iemNativeEmitAddTwoGprs32Ex(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase);
    10255         }
     10274            off = iemNativeEmitGpr32EqGprPlusImmEx(pCodeBuf, off, idxRegFlatPtr,
     10275                                                   pTlbState->idxRegSegBase, (uint32_t)pTlbState->uAbsPtr);
    1025610276    }
    1025710277    else if (pTlbState->idxRegPtr == UINT8_MAX)
    … …  
    1027910299    if (cbMem > fAlignMask + 1)
    1028010300    {
    10281         /* mov reg1, 0xfff */
    10282         off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, pTlbState->idxReg1, GUEST_PAGE_OFFSET_MASK);
    10283         /* and reg1, regflat */
    10284         off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, pTlbState->idxReg1, idxRegFlatPtr);
    10285         /* neg reg1 */
    10286         off = iemNativeEmitNegGpr32Ex(pCodeBuf, off, pTlbState->idxReg1);
    10287         /* add reg1, 0x1000 */
    10288         off = iemNativeEmitAddGpr32ImmEx(pCodeBuf, off, pTlbState->idxReg1, GUEST_PAGE_SIZE);
    10289         /* cmp reg1, cbMem */
     10301        /* reg1 = regflat & 0xfff */
     10302        off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1,/*=*/ idxRegFlatPtr,/*&*/ GUEST_PAGE_OFFSET_MASK);
     10303        /* cmp reg1, GUEST_PAGE_SIZE - cbMem */
    1029010304        off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, GUEST_PAGE_SIZE);
    1029110305        /* ja  tlbmiss */
    … …  
    1030110315    if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
    1030210316    {
     10317# if defined(RT_ARCH_AMD64)
    1030310318        /* mov reg1, regflat */
    1030410319        off = iemNativeEmitLoadGprFromGprEx(pCodeBuf, off, pTlbState->idxReg1, idxRegFlatPtr);
    … …  
    1032310338        /* shr reg1, 16 + GUEST_PAGE_SHIFT */
    1032410339        off = iemNativeEmitShiftGprRightEx(pCodeBuf, off, pTlbState->idxReg1, 16 + GUEST_PAGE_SHIFT);
     10340
     10341# elif defined(RT_ARCH_ARM64)
     10342        /* lsr  reg1, regflat, #48 */
     10343        pCodeBuf[off++] = Armv8A64MkInstrLslImm(pTlbState->idxReg1, idxRegFlatPtr, 4);
     10344        /* add  reg1, reg1, #1 */
     10345        pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(pTlbState->idxReg1, pTlbState->idxReg1, 1, false /*f64Bit*/);
     10346        /* tst  reg1, #0xfffe */
     10347        Assert(Armv8A64ConvertImmRImmS2Mask32(14, 31) == 0xfffe);
     10348        pCodeBuf[off++] = Armv8A64MkInstrTstImm(pTlbState->idxReg1, 14, 31,  false /*f64Bit*/);
     10349        /* b.nq tlbmiss */
     10350        off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
     10351
     10352        /* ubfx reg1, regflat, #12, #36 */
     10353        pCodeBuf[off++] = Armv8A64MkInstrUbfx(pTlbState->idxReg1, idxRegFlatPtr, GUEST_PAGE_SHIFT, 48 - GUEST_PAGE_SHIFT);
     10354# else
     10355#  error "Port me"
     10356# endif
    1032510357    }
    1032610358    else
    1032710359    {
    10328         /* mov reg1, regflat */
    10329         off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, pTlbState->idxReg1, idxRegFlatPtr);
    10330         /* shr reg1, GUEST_PAGE_SHIFT */
    10331         off = iemNativeEmitShiftGpr32RightEx(pCodeBuf, off, pTlbState->idxReg1, GUEST_PAGE_SHIFT);
     10360        /* reg1 = (uint32_t)(regflat >> 12) */
     10361        off = iemNativeEmitGpr32EqGprShiftRightImmEx(pCodeBuf, off, pTlbState->idxReg1, idxRegFlatPtr, GUEST_PAGE_SHIFT);
    1033210362    }
    1033310363    /* or  reg1, [qword pVCpu->iem.s.DataTlb.uTlbRevision] */
     10364# if defined(RT_ARCH_AMD64)
    1033410365    pCodeBuf[off++] = pTlbState->idxReg1 < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R;
    1033510366    pCodeBuf[off++] = 0x0b; /* OR r64,r/m64 */
    1033610367    off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, pTlbState->idxReg1, RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbRevision));
     10368# else
     10369    off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg3, RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbRevision));
     10370    off = iemNativeEmitOrGprByGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg3);
     10371# endif
    1033710372
    1033810373    /*
    1033910374     * 3b. Calc pTlbe.
    1034010375     */
     10376# if defined(RT_ARCH_AMD64)
    1034110377    /* movzx reg2, byte reg1 */
    1034210378    off = iemNativeEmitLoadGprFromGpr8Ex(pCodeBuf, off, pTlbState->idxReg2, pTlbState->idxReg1);
    … …  
    1035510391    pCodeBuf[off++] = RT_BYTE4(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
    1035610392
     10393# elif defined(RT_ARCH_ARM64)
     10394    /* reg2 = (reg1 & 0xff) << 5 */
     10395    pCodeBuf[off++] = Armv8A64MkInstrUbfiz(pTlbState->idxReg2, pTlbState->idxReg1, 5, 8);
     10396    /* reg2 += offsetof(VMCPUCC, iem.s.DataTlb.aEntries) */
     10397    off = iemNativeEmitAddGprImmEx(pCodeBuf, off, pTlbState->idxReg2, RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.aEntries),
     10398                                   pTlbState->idxReg3 /*iGprTmp*/);
     10399
     10400# else
     10401#  error "Port me"
     10402# endif
     10403
    1035710404    /*
    1035810405     * 3c. Compare the TLBE.uTag with the one from 2a (reg1).
    1035910406     */
     10407# if defined(RT_ARCH_AMD64)
    1036010408    /* cmp reg1, [reg2] */
    1036110409    pCodeBuf[off++] = X86_OP_REX_W | (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_R) | (pTlbState->idxReg2 < 8 ? 0 : X86_OP_REX_B);
    1036210410    pCodeBuf[off++] = 0x3b;
    1036310411    off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, uTag));
     10412# elif defined(RT_ARCH_ARM64)
     10413    pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, pTlbState->idxReg3,
     10414                                               pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, uTag));
     10415    off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg3);
     10416# else
     10417#  error "Port me"
     10418# endif
    1036410419    /* jne tlbmiss */
    1036510420    off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
    … …  
    1037510430                                    | IEMTLBE_F_PG_UNASSIGNED  | IEMTLBE_F_PG_NO_READ
    1037610431                                    | IEMTLBE_F_PT_NO_ACCESSED | fNoUser);
     10432# if defined(RT_ARCH_AMD64)
    1037710433    /* and reg1, [reg2->fFlagsAndPhysRev] */
    1037810434    pCodeBuf[off++] = X86_OP_REX_W | (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_R) | (pTlbState->idxReg2 < 8 ? 0 : X86_OP_REX_B);
    1037910435    pCodeBuf[off++] = 0x23;
    10380     off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, fFlagsAndPhysRev));
     10436    off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1,
     10437                                    pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, fFlagsAndPhysRev));
    1038110438
    1038210439    /* cmp reg1, [pVCpu->iem.s.DataTlb.uTlbPhysRev] */
    … …  
    1038510442    off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1, IEMNATIVE_REG_FIXED_PVMCPU,
    1038610443                                    RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbPhysRev));
     10444# elif defined(RT_ARCH_ARM64)
     10445    pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, pTlbState->idxReg3,
     10446                                               pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, fFlagsAndPhysRev));
     10447    pCodeBuf[off++] = Armv8A64MkInstrAnd(pTlbState->idxReg1, pTlbState->idxReg1, pTlbState->idxReg3);
     10448    off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg3,  RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbPhysRev));
     10449    off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg3);
     10450# else
     10451#  error "Port me"
     10452# endif
    1038710453    /* jne tlbmiss */
    1038810454    off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
    … …  
    1039310459     */
    1039410460    /* mov  reg1, [reg2->pbMappingR3] */
    10395     pCodeBuf[off++] = X86_OP_REX_W | (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_R) | (pTlbState->idxReg2 < 8 ? 0 : X86_OP_REX_B);
    10396     pCodeBuf[off++] = 0x8b;
    10397     off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, pbMappingR3));
    10398 
     10461    off = iemNativeEmitLoadGprByGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2,
     10462                                      RT_UOFFSETOF(IEMTLBENTRY, pbMappingR3));
     10463    /* if (!reg1) jmp tlbmiss */
    1039910464    /** @todo eliminate the need for this test? */
    10400     /* test reg1, reg1 */
    10401     pCodeBuf[off++] = X86_OP_REX_W | (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_R | X86_OP_REX_B);
    10402     pCodeBuf[off++] = 0x85;
    10403     pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, pTlbState->idxReg1 & 7, pTlbState->idxReg1 & 7);
    10404 
    10405     /* jz   tlbmiss */
    10406     off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_e);
     10465    off = iemNativeEmitTestIfGprIsZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, pTlbState->idxReg1,
     10466                                                      true /*f64Bit*/, idxLabelTlbMiss);
    1040710467
    1040810468    if (idxRegFlatPtr == idxRegMemResult) /* See step 1b. */
    … …  
    1041410474    {
    1041510475        Assert(idxRegFlatPtr == pTlbState->idxRegPtr);
    10416         /* mov result, 0xfff */
    10417         off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
    10418         /* and result, regflat */
    10419         off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, idxRegMemResult, idxRegFlatPtr);
     10476        /* result = regflat & 0xfff */
     10477        off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, idxRegMemResult, idxRegFlatPtr, GUEST_PAGE_OFFSET_MASK);
    1042010478    }
    1042110479    /* add result, reg1 */
    … …  
    1042610484    return off;
    1042710485}
    10428 #endif
     10486#endif /* IEMNATIVE_WITH_TLB_LOOKUP */
    1042910487
    1043010488

  • trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h

    r102734 r102735  
    496496    return off;
    497497}
    498 #elif defined(RT_ARCH_ARM64)
    499 /**
    500  * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
     498
     499#elif defined(RT_ARCH_ARM64)
     500
     501/**
     502 * Common bit of iemNativeEmitLoadGprFromVCpuU64Ex and friends.
     503 *
     504 * @note Loads can use @a iGprReg for large offsets, stores requires a temporary
     505 *       registers (@a iGprTmp).
     506 * @note DON'T try this with prefetch.
    501507 */
    502508DECL_FORCE_INLINE_THROW(uint32_t)
    503 iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
    504                            uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
     509iemNativeEmitGprByVCpuLdStEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu,
     510                             ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData, uint8_t iGprTmp = UINT8_MAX)
    505511{
    506512    /*
    … …  
    510516     */
    511517    if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
     518        /* Use the unsigned variant of ldr Wt, [<Xn|SP>, #off]. */
     519        pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
     520    else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
     521        pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PCPUMCTX,
     522                                                   (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
     523    else if (!ARMV8A64INSTRLDSTTYPE_IS_STORE(enmOperation) || iGprTmp != UINT8_MAX)
     524    {
     525        /* The offset is too large, so we must load it into a register and use
     526           ldr Wt, [<Xn|SP>, (<Wm>|<Xm>)]. */
     527        /** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
     528        if (iGprTmp == UINT8_MAX)
     529            iGprTmp = iGprReg;
     530        off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, offVCpu);
     531        pCodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, iGprTmp);
     532    }
     533    else
     534# ifdef IEM_WITH_THROW_CATCH
     535        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     536# else
     537        AssertReleaseFailedStmt(off = UINT32_MAX);
     538# endif
     539
     540    return off;
     541}
     542
     543/**
     544 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
     545 */
     546DECL_FORCE_INLINE_THROW(uint32_t)
     547iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
     548                           uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
     549{
     550    /*
     551     * There are a couple of ldr variants that takes an immediate offset, so
     552     * try use those if we can, otherwise we have to use the temporary register
     553     * help with the addressing.
     554     */
     555    if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
    512556    {
    513557        /* Use the unsigned variant of ldr Wt, [<Xn|SP>, #off]. */
    … …  
    527571        /** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
    528572        off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
    529 
    530573        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    531574        pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU,
    … …  
    535578    return off;
    536579}
    537 #endif
     580
     581#endif /* RT_ARCH_ARM64 */
    538582
    539583
    … …  
    541585 * Emits a 64-bit GPR load of a VCpu value.
    542586 */
     587DECL_FORCE_INLINE_THROW(uint32_t)
     588iemNativeEmitLoadGprFromVCpuU64Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
     589{
     590#ifdef RT_ARCH_AMD64
     591    /* mov reg64, mem64 */
     592    if (iGpr < 8)
     593        pCodeBuf[off++] = X86_OP_REX_W;
     594    else
     595        pCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
     596    pCodeBuf[off++] = 0x8b;
     597    off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off,iGpr, offVCpu);
     598
     599#elif defined(RT_ARCH_ARM64)
     600    off = iemNativeEmitGprByVCpuLdStEx(pCodeBuf, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
     601
     602#else
     603# error "port me"
     604#endif
     605    return off;
     606}
     607
     608
     609/**
     610 * Emits a 64-bit GPR load of a VCpu value.
     611 */
    543612DECL_INLINE_THROW(uint32_t)
    544613iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
    545614{
    546615#ifdef RT_ARCH_AMD64
    547     /* mov reg64, mem64 */
    548     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
    549     if (iGpr < 8)
    550         pbCodeBuf[off++] = X86_OP_REX_W;
    551     else
    552         pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
    553     pbCodeBuf[off++] = 0x8b;
    554     off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off,iGpr, offVCpu);
     616    off = iemNativeEmitLoadGprFromVCpuU64Ex(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGpr, offVCpu);
    555617    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    556618
    … …  
    16481710}
    16491711
    1650 
    16511712#if defined(RT_ARCH_ARM64)
     1713
     1714/**
     1715 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
     1716 *
     1717 * @note Odd and large @a offDisp values requires a temporary, unless it's a
     1718 *       load and @a iGprReg differs from @a iGprBase.  Will assert / throw if
     1719 *       caller does not heed this.
     1720 *
     1721 * @note DON'T try this with prefetch.
     1722 */
     1723DECL_FORCE_INLINE_THROW(uint32_t)
     1724iemNativeEmitGprByGprLdStEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp,
     1725                            ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData, uint8_t iGprTmp = UINT8_MAX)
     1726{
     1727    if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
     1728    {
     1729        /* Use the unsigned variant of ldr Wt, [<Xn|SP>, #off]. */
     1730        pCodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
     1731    }
     1732    else if (   (   !ARMV8A64INSTRLDSTTYPE_IS_STORE(enmOperation)
     1733                 && iGprReg != iGprBase)
     1734             || iGprTmp != UINT8_MAX)
     1735    {
     1736        /* The offset is too large, so we must load it into a register and use
     1737           ldr Wt, [<Xn|SP>, (<Wm>|<Xm>)]. */
     1738        /** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
     1739        if (iGprTmp == UINT8_MAX)
     1740            iGprTmp = iGprReg;
     1741        off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, (int64_t)offDisp);
     1742        pCodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, iGprTmp);
     1743    }
     1744    else
     1745# ifdef IEM_WITH_THROW_CATCH
     1746        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     1747# else
     1748        AssertReleaseFailedStmt(off = UINT32_MAX);
     1749# endif
     1750    return off;
     1751}
     1752
    16521753/**
    16531754 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
    … …  
    16731774           ldr Wt, [<Xn|SP>, (<Wm>|<Xm>)]. */
    16741775        /** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
    1675         uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)offDisp);
     1776        uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (int64_t)offDisp);
    16761777
    16771778        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    … …  
    16831784    return off;
    16841785}
    1685 #endif
    1686 
     1786
     1787#endif /* RT_ARCH_ARM64 */
    16871788
    16881789/**
    16891790 * Emits a 64-bit GPR load via a GPR base address with a displacement.
     1791 *
     1792 * @note ARM64: Misaligned @a offDisp values and values not in the
     1793 *       -0x7ff8...0x7ff8 range will require a temporary register (@a iGprTmp) if
     1794 *       @a iGprReg and @a iGprBase are the same. Will assert / throw if caller
     1795 *       does not heed this.
     1796 */
     1797DECL_FORCE_INLINE_THROW(uint32_t)
     1798iemNativeEmitLoadGprByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprBase,
     1799                            int32_t offDisp, uint8_t iGprTmp = UINT8_MAX)
     1800{
     1801#ifdef RT_ARCH_AMD64
     1802    /* mov reg64, mem64 */
     1803    pCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprBase < 8 ? 0 : X86_OP_REX_B);
     1804    pCodeBuf[off++] = 0x8b;
     1805    off = iemNativeEmitGprByGprDisp(pCodeBuf, off, iGprDst, iGprBase, offDisp);
     1806    RT_NOREF(iGprTmp);
     1807
     1808#elif defined(RT_ARCH_ARM64)
     1809    off = iemNativeEmitGprByGprLdStEx(pCodeBuf, off, iGprDst, iGprBase, offDisp,
     1810                                      kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t), iGprTmp);
     1811
     1812#else
     1813# error "port me"
     1814#endif
     1815    return off;
     1816}
     1817
     1818
     1819/**
     1820 * Emits a 64-bit GPR load via a GPR base address with a displacement.
    16901821 */
    16911822DECL_INLINE_THROW(uint32_t)
    … …  
    16931824{
    16941825#ifdef RT_ARCH_AMD64
    1695     /* mov reg64, mem64 */
    1696     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
    1697     pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprBase < 8 ? 0 : X86_OP_REX_B);
    1698     pbCodeBuf[off++] = 0x8b;
    1699     off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
     1826    off = iemNativeEmitLoadGprByGprEx(iemNativeInstrBufEnsure(pReNative, off, 8), off, iGprDst, iGprBase, offDisp);
    17001827    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    17011828
    … …  
    18451972
    18461973
    1847 #ifdef RT_ARCH_AMD64
    18481974/**
    18491975 * Emits a 32-bit GPR subtract with a signed immediate subtrahend.
    18501976 *
    18511977 * This will optimize using DEC/INC/whatever, so try avoid flag dependencies.
    1852  */
    1853 DECL_FORCE_INLINE(uint32_t)
    1854 iemNativeEmitSubGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
    1855 {
     1978 *
     1979 * @note ARM64: Larger constants will require a temporary register.  Failing to
     1980 *       specify one when needed will trigger fatal assertion / throw.
     1981 */
     1982DECL_FORCE_INLINE_THROW(uint32_t)
     1983iemNativeEmitSubGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend,
     1984                           uint8_t iGprTmp = UINT8_MAX)
     1985{
     1986#ifdef RT_ARCH_AMD64
    18561987    if (iGprDst >= 8)
    18571988        pCodeBuf[off++] = X86_OP_REX_B;
    … …  
    18852016        pCodeBuf[off++] = RT_BYTE4(iSubtrahend);
    18862017    }
    1887     return off;
    1888 }
    1889 #endif
     2018    RT_NOREF(iGprTmp);
     2019
     2020#elif defined(RT_ARCH_ARM64)
     2021    uint32_t uAbsSubtrahend = RT_ABS(iSubtrahend);
     2022    if (uAbsSubtrahend < 4096)
     2023    {
     2024        if (iSubtrahend >= 0)
     2025            pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend, false /*f64Bit*/);
     2026        else
     2027            pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend, false /*f64Bit*/);
     2028    }
     2029    else if (uAbsSubtrahend <= 0xfff000 && !(uAbsSubtrahend & 0xfff))
     2030    {
     2031        if (iSubtrahend >= 0)
     2032            pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
     2033                                                       false /*f64Bit*/, false /*fSetFlags*/, true /*fShift*/);
     2034        else
     2035            pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsSubtrahend >> 12,
     2036                                                       false /*f64Bit*/, false /*fSetFlags*/, true /*fShift*/);
     2037    }
     2038    else if (iGprTmp != UINT8_MAX)
     2039    {
     2040        off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprTmp, (uint32_t)iSubtrahend);
     2041        pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprTmp);
     2042    }
     2043    else
     2044# ifdef IEM_WITH_THROW_CATCH
     2045        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     2046# else
     2047        AssertReleaseFailedStmt(off = UINT32_MAX);
     2048# endif
     2049
     2050#else
     2051# error "Port me"
     2052#endif
     2053    return off;
     2054}
    18902055
    18912056
    … …  
    19812146 */
    19822147DECL_INLINE_THROW(uint32_t)
    1983 iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
     2148iemNativeEmitAddGprImm8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int8_t iImm8)
    19842149{
    19852150#if defined(RT_ARCH_AMD64)
    19862151    /* add or inc */
    1987     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
    1988     pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
     2152    pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
    19892153    if (iImm8 != 1)
    19902154    {
    1991         pbCodeBuf[off++] = 0x83;
    1992         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
    1993         pbCodeBuf[off++] = (uint8_t)iImm8;
    1994     }
    1995     else
    1996     {
    1997         pbCodeBuf[off++] = 0xff;
    1998         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
    1999     }
    2000 
    2001 #elif defined(RT_ARCH_ARM64)
    2002     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
     2155        pCodeBuf[off++] = 0x83;
     2156        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
     2157        pCodeBuf[off++] = (uint8_t)iImm8;
     2158    }
     2159    else
     2160    {
     2161        pCodeBuf[off++] = 0xff;
     2162        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
     2163    }
     2164
     2165#elif defined(RT_ARCH_ARM64)
    20032166    if (iImm8 >= 0)
    2004         pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint8_t)iImm8);
    2005     else
    2006         pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint8_t)-iImm8);
    2007 
     2167        pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, (uint8_t)iImm8);
     2168    else
     2169        pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, (uint8_t)-iImm8);
     2170
     2171#else
     2172# error "Port me"
     2173#endif
     2174    return off;
     2175}
     2176
     2177
     2178/**
     2179 * Emits a 64-bit GPR additions with a 8-bit signed immediate.
     2180 */
     2181DECL_INLINE_THROW(uint32_t)
     2182iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
     2183{
     2184#if defined(RT_ARCH_AMD64)
     2185    off = iemNativeEmitAddGprImm8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iImm8);
     2186#elif defined(RT_ARCH_ARM64)
     2187    off = iemNativeEmitAddGprImm8Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iImm8);
    20082188#else
    20092189# error "Port me"
    … …  
    20652245#endif
    20662246    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     2247    return off;
     2248}
     2249
     2250
     2251/**
     2252 * Emits a 64-bit GPR additions with a 64-bit signed addend.
     2253 *
     2254 * @note Will assert / throw if @a iGprTmp is not specified when needed.
     2255 */
     2256DECL_FORCE_INLINE_THROW(uint32_t)
     2257iemNativeEmitAddGprImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int64_t iAddend, uint8_t iGprTmp = UINT8_MAX)
     2258{
     2259#if defined(RT_ARCH_AMD64)
     2260    if ((int8_t)iAddend == iAddend)
     2261        return iemNativeEmitAddGprImm8Ex(pCodeBuf, off, iGprDst, (int8_t)iAddend);
     2262
     2263    if ((int32_t)iAddend == iAddend)
     2264    {
     2265        /* add grp, imm32 */
     2266        pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
     2267        pCodeBuf[off++] = 0x81;
     2268        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
     2269        pCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
     2270        pCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
     2271        pCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
     2272        pCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
     2273    }
     2274    else if (iGprTmp != UINT8_MAX)
     2275    {
     2276        off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, iAddend);
     2277
     2278        /* add dst, tmpreg  */
     2279        pCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R)
     2280                        | (iGprTmp < 8 ? 0 : X86_OP_REX_B);
     2281        pCodeBuf[off++] = 0x03;
     2282        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprTmp & 7);
     2283    }
     2284    else
     2285# ifdef IEM_WITH_THROW_CATCH
     2286        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     2287# else
     2288        AssertReleaseFailedStmt(off = UINT32_MAX);
     2289# endif
     2290
     2291#elif defined(RT_ARCH_ARM64)
     2292    uint64_t const uAbsAddend = (uint64_t)RT_ABS(iAddend);
     2293    if (uAbsAddend < 4096)
     2294    {
     2295        if (iAddend >= 0)
     2296            pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend);
     2297        else
     2298            pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend);
     2299    }
     2300    else if (uAbsAddend <= 0xfff000 && !(uAbsAddend & 0xfff))
     2301    {
     2302        if (iAddend >= 0)
     2303            pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend >> 12,
     2304                                                       true /*f64Bit*/, true /*fShift12*/);
     2305        else
     2306            pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, (uint32_t)uAbsAddend >> 12,
     2307                                                       true /*f64Bit*/, true /*fShift12*/);
     2308    }
     2309    else if (iGprTmp != UINT8_MAX)
     2310    {
     2311        off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprTmp, iAddend);
     2312        pCodeBuf[off++] = Armv8A64MkInstrAddReg(iGprDst, iGprDst, iGprTmp);
     2313    }
     2314    else
     2315# ifdef IEM_WITH_THROW_CATCH
     2316        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     2317# else
     2318        AssertReleaseFailedStmt(off = UINT32_MAX);
     2319# endif
     2320
     2321#else
     2322# error "Port me"
     2323#endif
    20672324    return off;
    20682325}
    … …  
    22322489
    22332490
     2491/**
     2492 * Adds two 64-bit GPRs together, storing the result in a third register.
     2493 */
     2494DECL_FORCE_INLINE(uint32_t)
     2495iemNativeEmitGprEqGprPlusGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend1, uint8_t iGprAddend2)
     2496{
     2497#ifdef RT_ARCH_AMD64
     2498    if (iGprDst != iGprAddend1 && iGprDst != iGprAddend2)
     2499    {
     2500        /** @todo consider LEA */
     2501        off = iemNativeEmitLoadGprFromGprEx(pCodeBuf, off, iGprDst, iGprAddend1);
     2502        off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend2);
     2503    }
     2504    else
     2505        off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprDst == iGprAddend1 ? iGprAddend1 : iGprAddend2);
     2506
     2507#elif defined(RT_ARCH_ARM64)
     2508    pCodeBuf[off++] = Armv8A64MkInstrAddReg(iGprDst, iGprAddend1, iGprAddend2);
     2509
     2510#else
     2511# error "Port me!"
     2512#endif
     2513    return off;
     2514}
     2515
     2516
     2517
     2518/**
     2519 * Adds two 32-bit GPRs together, storing the result in a third register.
     2520 * @note Bits 32 thru 63 in @a iGprDst will be zero after the operation.
     2521 */
     2522DECL_FORCE_INLINE(uint32_t)
     2523iemNativeEmitGpr32EqGprPlusGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend1, uint8_t iGprAddend2)
     2524{
     2525#ifdef RT_ARCH_AMD64
     2526    if (iGprDst != iGprAddend1 && iGprDst != iGprAddend2)
     2527    {
     2528        /** @todo consider LEA */
     2529        off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, iGprDst, iGprAddend1);
     2530        off = iemNativeEmitAddTwoGprs32Ex(pCodeBuf, off, iGprDst, iGprAddend2);
     2531    }
     2532    else
     2533        off = iemNativeEmitAddTwoGprs32Ex(pCodeBuf, off, iGprDst, iGprDst == iGprAddend1 ? iGprAddend1 : iGprAddend2);
     2534
     2535#elif defined(RT_ARCH_ARM64)
     2536    pCodeBuf[off++] = Armv8A64MkInstrAddReg(iGprDst, iGprAddend1, iGprAddend2, false /*f64Bit*/);
     2537
     2538#else
     2539# error "Port me!"
     2540#endif
     2541    return off;
     2542}
     2543
     2544
     2545/**
     2546 * Adds a 64-bit GPR and a 64-bit unsigned constant, storing the result in a
     2547 * third register.
     2548 *
     2549 * @note The ARM64 version does not work for non-trivial constants if the
     2550 *       two registers are the same.  Will assert / throw exception.
     2551 */
     2552DECL_FORCE_INLINE_THROW(uint32_t)
     2553iemNativeEmitGprEqGprPlusImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend, int32_t iImmAddend)
     2554{
     2555#ifdef RT_ARCH_AMD64
     2556    /** @todo consider LEA */
     2557    if ((int8_t)iImmAddend == iImmAddend)
     2558    {
     2559        /* mov dst, gpradd */
     2560        off = iemNativeEmitLoadGprFromGprEx(pCodeBuf, off, iGprDst, iGprAddend);
     2561        /* add dst, immadd */
     2562        off = iemNativeEmitAddGprImm8Ex(pCodeBuf, off, iGprDst, (int8_t)iImmAddend);
     2563    }
     2564    else
     2565    {
     2566        /* mov dst, immadd */
     2567        off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprDst, iImmAddend);
     2568        /* add dst, gpradd */
     2569        off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend);
     2570    }
     2571
     2572#elif defined(RT_ARCH_ARM64)
     2573    uint32_t const uAbsImmAddend = RT_ABS(iImmAddend);
     2574    if (uAbsImmAddend < 4096)
     2575    {
     2576        if (iImmAddend >= 0)
     2577            pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprAddend, uAbsImmAddend);
     2578        else
     2579            pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprAddend, uAbsImmAddend);
     2580    }
     2581    else if (uAbsImmAddend <= 0xfff000 && !(uAbsImmAddend & 0xfff))
     2582    {
     2583        if (iImmAddend >= 0)
     2584            pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, true /*f64Bit*/, true /*fShift12*/);
     2585        else
     2586            pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, true /*f64Bit*/, true /*fShift12*/);
     2587    }
     2588    else if (iGprDst != iGprAddend)
     2589    {
     2590        off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, iGprDst, (uint32_t)iImmAddend);
     2591        off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend);
     2592    }
     2593    else
     2594# ifdef IEM_WITH_THROW_CATCH
     2595        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     2596# else
     2597        AssertReleaseFailedStmt(off = UINT32_MAX);
     2598# endif
     2599
     2600#else
     2601# error "Port me!"
     2602#endif
     2603    return off;
     2604}
     2605
     2606
     2607/**
     2608 * Adds a 32-bit GPR and a 32-bit unsigned constant, storing the result in a
     2609 * third register.
     2610 *
     2611 * @note Bits 32 thru 63 in @a iGprDst will be zero after the operation.
     2612 *
     2613 * @note The ARM64 version does not work for non-trivial constants if the
     2614 *       two registers are the same.  Will assert / throw exception.
     2615 */
     2616DECL_FORCE_INLINE_THROW(uint32_t)
     2617iemNativeEmitGpr32EqGprPlusImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend, int32_t iImmAddend)
     2618{
     2619#ifdef RT_ARCH_AMD64
     2620    /** @todo consider LEA */
     2621    if ((int8_t)iImmAddend == iImmAddend)
     2622    {
     2623        /* mov dst, gpradd */
     2624        off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, iGprDst, iGprAddend);
     2625        /* add dst, immadd */
     2626        off = iemNativeEmitAddGpr32Imm8Ex(pCodeBuf, off, iGprDst, (int8_t)iImmAddend);
     2627    }
     2628    else
     2629    {
     2630        /* mov dst, immadd */
     2631        off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, iImmAddend);
     2632        /* add dst, gpradd */
     2633        off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, iGprDst, iGprAddend);
     2634    }
     2635
     2636#elif defined(RT_ARCH_ARM64)
     2637    uint32_t const uAbsImmAddend = RT_ABS(iImmAddend);
     2638    if (uAbsImmAddend < 4096)
     2639    {
     2640        if (iImmAddend >= 0)
     2641            pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprAddend, uAbsImmAddend, false /*f64Bit*/);
     2642        else
     2643            pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprAddend, uAbsImmAddend, false /*f64Bit*/);
     2644    }
     2645    else if (uAbsImmAddend <= 0xfff000 && !(uAbsImmAddend & 0xfff))
     2646    {
     2647        if (iImmAddend >= 0)
     2648            pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, false /*f64Bit*/, true /*fShift12*/);
     2649        else
     2650            pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(iGprDst, iGprDst, uAbsImmAddend >> 12, false /*f64Bit*/, true /*fShift12*/);
     2651    }
     2652    else if (iGprDst != iGprAddend)
     2653    {
     2654        off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, (uint32_t)iImmAddend);
     2655        off = iemNativeEmitAddTwoGprs32Ex(pCodeBuf, off, iGprDst, iGprAddend);
     2656    }
     2657    else
     2658# ifdef IEM_WITH_THROW_CATCH
     2659        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     2660# else
     2661        AssertReleaseFailedStmt(off = UINT32_MAX);
     2662# endif
     2663
     2664#else
     2665# error "Port me!"
     2666#endif
     2667    return off;
     2668}
     2669
     2670
    22342671/*********************************************************************************************************************************
    22352672*   Unary Operations                                                                                                             *
    … …  
    25582995/**
    25592996 * Emits code for AND'ing an 32-bit GPRs with a constant.
     2997 *
    25602998 * @note Bits 32 thru 63 in the destination will be zero after the operation.
    25612999 */
    … …  
    25913029    return off;
    25923030}
     3031
     3032
     3033/**
     3034 * Emits code for AND'ing an 32-bit GPRs with a constant.
     3035 *
     3036 * @note For ARM64 any complicated immediates w/o a AND/ANDS compatible
     3037 *       encoding will assert / throw exception if @a iGprDst and @a iGprSrc are
     3038 *       the same.
     3039 *
     3040 * @note Bits 32 thru 63 in the destination will be zero after the operation.
     3041 */
     3042DECL_FORCE_INLINE_THROW(uint32_t)
     3043iemNativeEmitGpr32EqGprAndImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, uint32_t uImm,
     3044                                bool fSetFlags = false)
     3045{
     3046#if defined(RT_ARCH_AMD64)
     3047    off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, uImm);
     3048    off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, iGprDst, iGprSrc);
     3049    RT_NOREF(fSetFlags);
     3050
     3051#elif defined(RT_ARCH_ARM64)
     3052    uint32_t uImmR     = 0;
     3053    uint32_t uImmNandS = 0;
     3054    if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
     3055    {
     3056        if (!fSetFlags)
     3057            pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /*f64Bit*/);
     3058        else
     3059            pCodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /*f64Bit*/);
     3060    }
     3061    else if (iGprDst != iGprSrc)
     3062    {
     3063        off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, iGprDst, uImm);
     3064        off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, iGprDst, iGprSrc, fSetFlags);
     3065    }
     3066    else
     3067# ifdef IEM_WITH_THROW_CATCH
     3068        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     3069# else
     3070        AssertReleaseFailedStmt(off = UINT32_MAX);
     3071# endif
     3072
     3073#else
     3074# error "Port me"
     3075#endif
     3076    return off;
     3077}
     3078
     3079
     3080/**
     3081 * Emits code for OR'ing two 64-bit GPRs.
     3082 */
     3083DECL_INLINE_THROW(uint32_t)
     3084iemNativeEmitOrGprByGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
     3085{
     3086#if defined(RT_ARCH_AMD64)
     3087    /* or Gv, Ev */
     3088    pCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
     3089    pCodeBuf[off++] = 0x0b;
     3090    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
     3091
     3092#elif defined(RT_ARCH_ARM64)
     3093    pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, iGprDst, iGprSrc);
     3094
     3095#else
     3096# error "Port me"
     3097#endif
     3098    return off;
     3099}
     3100
    25933101
    25943102
    … …  
    29013409#endif
    29023410    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     3411    return off;
     3412}
     3413
     3414
     3415/**
     3416 * Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
     3417 * right and assigning it to a different GPR.
     3418 */
     3419DECL_INLINE_THROW(uint32_t)
     3420iemNativeEmitGpr32EqGprShiftRightImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, uint8_t cShift)
     3421{
     3422    Assert(cShift > 0); Assert(cShift < 32);
     3423#if defined(RT_ARCH_AMD64)
     3424    off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, iGprDst, iGprSrc);
     3425    off = iemNativeEmitShiftGpr32RightEx(pCodeBuf, off, iGprDst, cShift);
     3426
     3427#elif defined(RT_ARCH_ARM64)
     3428    pCodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprSrc, cShift, false /*64Bit*/);
     3429
     3430#else
     3431# error "Port me"
     3432#endif
    29033433    return off;
    29043434}
    … …  
    33213851#elif defined(RT_ARCH_ARM64)
    33223852typedef ARMV8INSTRCOND  IEMNATIVEINSTRCOND;
     3853# define kIemNativeInstrCond_o      todo_conditional_codes
     3854# define kIemNativeInstrCond_no     todo_conditional_codes
     3855# define kIemNativeInstrCond_c      todo_conditional_codes
     3856# define kIemNativeInstrCond_nc     todo_conditional_codes
     3857# define kIemNativeInstrCond_e      kArmv8InstrCond_Eq
     3858# define kIemNativeInstrCond_ne     kArmv8InstrCond_Ne
     3859# define kIemNativeInstrCond_be     kArmv8InstrCond_Ls
     3860# define kIemNativeInstrCond_nbe    kArmv8InstrCond_Hi
     3861# define kIemNativeInstrCond_s      todo_conditional_codes
     3862# define kIemNativeInstrCond_ns     todo_conditional_codes
     3863# define kIemNativeInstrCond_p      todo_conditional_codes
     3864# define kIemNativeInstrCond_np     todo_conditional_codes
     3865# define kIemNativeInstrCond_l      kArmv8InstrCond_Lt
     3866# define kIemNativeInstrCond_nl     kArmv8InstrCond_Ge
     3867# define kIemNativeInstrCond_le     kArmv8InstrCond_Le
     3868# define kIemNativeInstrCond_nle    kArmv8InstrCond_Gt
    33233869#else
    33243870# error "Port me!"
    … …  
    40834629
    40844630/**
     4631 * Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
     4632 *
     4633 * The operand size is given by @a f64Bit.
     4634 */
     4635DECL_FORCE_INLINE_THROW(uint32_t)
     4636iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
     4637                                                     uint8_t iGprSrc, bool f64Bit, bool fJmpIfNotZero, uint32_t idxLabel)
     4638{
     4639    Assert(idxLabel < pReNative->cLabels);
     4640
     4641#ifdef RT_ARCH_AMD64
     4642    /* test reg32,reg32  / test reg64,reg64 */
     4643    if (f64Bit)
     4644        pCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_R | X86_OP_REX_B);
     4645    else if (iGprSrc >= 8)
     4646        pCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
     4647    pCodeBuf[off++] = 0x85;
     4648    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
     4649
     4650    /* jnz idxLabel  */
     4651    off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabel,
     4652                                    fJmpIfNotZero ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);
     4653
     4654#elif defined(RT_ARCH_ARM64)
     4655    if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
     4656        pCodeBuf[off++] = Armv8A64MkInstrCbzCbnz(fJmpIfNotZero, (int32_t)(pReNative->paLabels[idxLabel].off - off),
     4657                                                 iGprSrc, f64Bit);
     4658    else
     4659    {
     4660        iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
     4661        pCodeBuf[off++] = Armv8A64MkInstrCbzCbnz(fJmpIfNotZero, 0, iGprSrc, f64Bit);
     4662    }
     4663
     4664#else
     4665# error "Port me!"
     4666#endif
     4667    return off;
     4668}
     4669
     4670
     4671/**
     4672 * Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
     4673 *
     4674 * The operand size is given by @a f64Bit.
     4675 */
     4676DECL_FORCE_INLINE_THROW(uint32_t)
     4677iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,
     4678                                                   bool f64Bit, bool fJmpIfNotZero, uint32_t idxLabel)
     4679{
     4680#ifdef RT_ARCH_AMD64
     4681    off = iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 3 + 6),
     4682                                                               off, iGprSrc, f64Bit, fJmpIfNotZero, idxLabel);
     4683#elif defined(RT_ARCH_ARM64)
     4684    off = iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 1),
     4685                                                               off, iGprSrc, f64Bit, fJmpIfNotZero, idxLabel);
     4686#else
     4687# error "Port me!"
     4688#endif
     4689    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     4690    return off;
     4691}
     4692
     4693
     4694/**
    40854695 * Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
    40864696 *
    40874697 * The operand size is given by @a f64Bit.
    40884698 */
     4699DECL_FORCE_INLINE_THROW(uint32_t)
     4700iemNativeEmitTestIfGprIsZeroAndJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
     4701                                            uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
     4702{
     4703    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
     4704                                                                f64Bit, false /*fJmpIfNotZero*/, idxLabel);
     4705}
     4706
     4707
     4708/**
     4709 * Emits code that jumps to @a idxLabel if @a iGprSrc is zero.
     4710 *
     4711 * The operand size is given by @a f64Bit.
     4712 */
    40894713DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    40904714                                                                      uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
    40914715{
    4092     Assert(idxLabel < pReNative->cLabels);
    4093 
    4094 #ifdef RT_ARCH_AMD64
    4095     /* test reg32,reg32  / test reg64,reg64 */
    4096     uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
    4097     if (f64Bit)
    4098         pbCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_R | X86_OP_REX_B);
    4099     else if (iGprSrc >= 8)
    4100         pbCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
    4101     pbCodeBuf[off++] = 0x85;
    4102     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
    4103     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    4104 
    4105     /* jz idxLabel  */
    4106     off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);
    4107 
    4108 #elif defined(RT_ARCH_ARM64)
    4109     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    4110     iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
    4111     pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /*fJmpIfNotZero*/, 0, iGprSrc, f64Bit);
    4112     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    4113 
    4114 #else
    4115 # error "Port me!"
    4116 #endif
    4117     return off;
     4716    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, false /*fJmpIfNotZero*/, idxLabel);
    41184717}
    41194718
    … …  
    41384737 * The operand size is given by @a f64Bit.
    41394738 */
     4739DECL_FORCE_INLINE_THROW(uint32_t)
     4740iemNativeEmitTestIfGprIsNotZeroAndJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
     4741                                               uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
     4742{
     4743    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
     4744                                                                f64Bit, true /*fJmpIfNotZero*/, idxLabel);
     4745}
     4746
     4747
     4748/**
     4749 * Emits code that jumps to @a idxLabel if @a iGprSrc is not zero.
     4750 *
     4751 * The operand size is given by @a f64Bit.
     4752 */
    41404753DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    41414754                                                                         uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)
    41424755{
    4143     Assert(idxLabel < pReNative->cLabels);
    4144 
    4145 #ifdef RT_ARCH_AMD64
    4146     /* test reg32,reg32  / test reg64,reg64 */
    4147     uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
    4148     if (f64Bit)
    4149         pbCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_R | X86_OP_REX_B);
    4150     else if (iGprSrc >= 8)
    4151         pbCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
    4152     pbCodeBuf[off++] = 0x85;
    4153     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
    4154     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    4155 
    4156     /* jnz idxLabel  */
    4157     off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);
    4158 
    4159 #elif defined(RT_ARCH_ARM64)
    4160     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    4161     iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
    4162     pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(true /*fJmpIfNotZero*/, 0, iGprSrc, f64Bit);
    4163     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    4164 
    4165 #else
    4166 # error "Port me!"
    4167 #endif
    4168     return off;
     4756    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, true /*fJmpIfNotZero*/, idxLabel);
    41694757}
    41704758
Note: See TracChangeset for help on using the changeset viewer.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette