VirtualBox

Changeset 102717 in vbox


Ignore:
Timestamp:
Dec 27, 2023 7:45:49 PM (11 months ago)
Author:
vboxsync
Message:

VBox/VMM: Outlined native TLB lookup code for IEM_MC_MEM_MAP_XXXX on x86 hosts. Untested+disabled. bugref:10371

Location:
trunk
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • trunk/include/iprt/armv8.h

    r102688 r102717  
    31323132
    31333133/**
     3134 * A64: Encodes an EXTR instruction with an immediate.
     3135 *
     3136 * @returns The encoded instruction.
     3137 * @param   iRegResult  The register to store the result in. ZR is valid.
     3138 * @param   iRegLow     The register holding the least significant bits in the
     3139 *                      extraction. ZR is valid.
     3140 * @param   iRegHigh    The register holding the most significant bits in the
     3141 *                      extraction. ZR is valid.
     3142 * @param   uLsb        The bit number of the least significant bit, or where in
     3143 *                      @a iRegLow to start the
     3144 *                      extraction.
     3145 * @param   f64Bit      true for 64-bit GRPs (default), false for 32-bit GPRs.
     3146 */
     3147DECL_FORCE_INLINE(uint32_t) Armv8A64MkInstrExtrImm(uint32_t iRegResult, uint32_t iRegLow, uint32_t iRegHigh, uint32_t uLsb,
     3148                                                   bool f64Bit = true)
     3149{
     3150    Assert(uLsb < (uint32_t)(f64Bit ? 64 : 32)); Assert(iRegHigh < 32); Assert(iRegLow < 32); Assert(iRegResult < 32);
     3151    return ((uint32_t)f64Bit       << 31)
     3152         | UINT32_C(0x13800000)
     3153         | ((uint32_t)f64Bit       << 22) /*N*/
     3154         | (iRegHigh               << 16)
     3155         | (uLsb                   << 10)
     3156         | (iRegLow                <<  5)
     3157         | iRegResult;
     3158}
     3159
     3160
     3161/** A64: Rotates the value of a register (alias for EXTR). */
     3162DECL_FORCE_INLINE(uint32_t) Armv8A64MkInstrRorImm(uint32_t iRegResult, uint32_t iRegSrc, uint32_t cShift, bool f64Bit = true)
     3163{
     3164    return Armv8A64MkInstrExtrImm(iRegResult, iRegSrc, iRegSrc, cShift, f64Bit);
     3165}
     3166
     3167
     3168/**
    31343169 * A64: Encodes either add, adds, sub or subs with unsigned 12-bit immediate.
    31353170 *

  • trunk/include/iprt/x86.h

    r102646 r102717  
    39943994
    39953995/** @name X86DESCATTR masks
     3996 * Fields X86DESCGENERIC::u4Type thru X86DESCGENERIC::u1Granularity (or
     3997 * bits[55:40] if you like).  The X86DESCATTR_UNUSABLE bit is an Intel addition.
    39963998 * @{ */
    39973999#define X86DESCATTR_TYPE            UINT32_C(0x0000000f)
    3998 #define X86DESCATTR_DT              UINT32_C(0x00000010)
     4000#define X86DESCATTR_DT              UINT32_C(0x00000010)    /**< Descriptor type: 0=system, 1=code/data */
    39994001#define X86DESCATTR_DPL             UINT32_C(0x00000060)
    4000 #define X86DESCATTR_DPL_SHIFT       5 /**< Shift count for the DPL value. */
     4002#define X86DESCATTR_DPL_SHIFT       5                       /**< Shift count for the DPL bitfield. */
    40014003#define X86DESCATTR_P               UINT32_C(0x00000080)
    40024004#define X86DESCATTR_LIMIT_HIGH      UINT32_C(0x00000f00)

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

    r102701 r102717  
    30573057    /* [kIemNativeGstReg_SegLimitFirst + 4] = */        { CPUMCTX_OFF_AND_SIZE(aSRegs[4].u32Limit), "fs_limit", },
    30583058    /* [kIemNativeGstReg_SegLimitFirst + 5] = */        { CPUMCTX_OFF_AND_SIZE(aSRegs[5].u32Limit), "gs_limit", },
     3059    /* [kIemNativeGstReg_SegAttribFirst + 0] = */       { CPUMCTX_OFF_AND_SIZE(aSRegs[0].Attr.u),   "es_attrib", },
     3060    /* [kIemNativeGstReg_SegAttribFirst + 1] = */       { CPUMCTX_OFF_AND_SIZE(aSRegs[1].Attr.u),   "cs_attrib", },
     3061    /* [kIemNativeGstReg_SegAttribFirst + 2] = */       { CPUMCTX_OFF_AND_SIZE(aSRegs[2].Attr.u),   "ss_attrib", },
     3062    /* [kIemNativeGstReg_SegAttribFirst + 3] = */       { CPUMCTX_OFF_AND_SIZE(aSRegs[3].Attr.u),   "ds_attrib", },
     3063    /* [kIemNativeGstReg_SegAttribFirst + 4] = */       { CPUMCTX_OFF_AND_SIZE(aSRegs[4].Attr.u),   "fs_attrib", },
     3064    /* [kIemNativeGstReg_SegAttribFirst + 5] = */       { CPUMCTX_OFF_AND_SIZE(aSRegs[5].Attr.u),   "gs_attrib", },
    30593065#undef CPUMCTX_OFF_AND_SIZE
    30603066};
    … …  
    36403646DECL_HIDDEN_THROW(uint8_t)
    36413647iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, IEMNATIVEGSTREG enmGstReg,
    3642                                 IEMNATIVEGSTREGUSE enmIntendedUse, bool fNoVolatileRegs /*=false*/)
     3648                                IEMNATIVEGSTREGUSE enmIntendedUse /*= kIemNativeGstRegUse_ReadOnly*/,
     3649                                bool fNoVolatileRegs /*= false*/)
    36433650{
    36443651    Assert(enmGstReg < kIemNativeGstReg_End && g_aGstShadowInfo[enmGstReg].cb != 0);
    … …  
    67496756        Log11(("iemNativeVarRegisterAcquire: idxVar=%u idxReg=%u (matching arg %u)\n", idxVar, idxReg, uArgNo));
    67506757    }
    6751     else if (   idxRegPref < RT_ELEMENTS(pReNative->Core.aHstRegs)
     6758    else if (   idxRegPref >= RT_ELEMENTS(pReNative->Core.aHstRegs)
    67526759             || (pReNative->Core.bmHstRegs & RT_BIT_32(idxRegPref)))
    67536760    {
    … …  
    1090110908     * First we try to go via the TLB.
    1090210909     */
    10903 //pReNative->pInstrBuf[off++] = 0xcc;
    10904     /** @todo later. */
     10910#if defined(RT_ARCH_AMD64) && 0 /* untested code sketch */
     10911    uint8_t const   idxRegPtr       = iemNativeVarRegisterAcquire(pReNative, idxVarGCPtrMem, &off,
     10912                                                                  true /*fInitialized*/, IEMNATIVE_CALL_ARG2_GREG);
     10913    uint8_t const   idxRegSegBase   = iSegReg == UINT8_MAX ? UINT8_MAX
     10914                                    : iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(iSegReg));
     10915    uint8_t const   idxRegSegLimit  = iSegReg == UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT
     10916                                    ? UINT8_MAX
     10917                                    : iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(iSegReg));
     10918    uint8_t const   idxRegSegAttrib = iSegReg == UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT
     10919                                    ? UINT8_MAX
     10920                                    : iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_ATTRIB(iSegReg));
     10921    uint8_t const   idxReg1         = iemNativeRegAllocTmp(pReNative, &off);
     10922    uint8_t const   idxReg2         = iemNativeRegAllocTmp(pReNative, &off);
     10923    uint8_t * const pbCodeBuf       = iemNativeInstrBufEnsure(pReNative, off, 256);
     10924    pbCodeBuf[off++] = 0xcc;
     10925
     10926    /*
     10927     * 1. Segmentation.
     10928     *
     10929     * 1a. Check segment limit and attributes if non-flat 32-bit code.  This is complicated.
     10930     */
     10931    if (iSegReg != UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT)
     10932    {
     10933        /* If we're accessing more than one byte, put the last address we'll be
     10934           accessing in idxReg2 (64-bit). */
     10935        if (cbMem > 1)
     10936        {
     10937            /* mov reg2, cbMem-1 */
     10938            off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxReg2, cbMem - 1);
     10939            /* add reg2, regptr */
     10940            off = iemNativeEmitAddTwoGprsEx(pbCodeBuf, off, idxReg2, idxRegPtr);
     10941        }
     10942
     10943        /* Check that we've got a segment loaded and that it allows the access.
     10944           For write access this means a writable data segment.
     10945           For read-only accesses this means a readable code segment or any data segment. */
     10946        if (fAccess & IEM_ACCESS_TYPE_WRITE)
     10947        {
     10948            uint32_t const fMustBe1 = X86DESCATTR_P        | X86DESCATTR_DT    | X86_SEL_TYPE_WRITE;
     10949            uint32_t const fMustBe0 = X86DESCATTR_UNUSABLE | X86_SEL_TYPE_CODE;
     10950            /* mov reg1, must1|must0 */
     10951            off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxReg1, fMustBe1 | fMustBe0);
     10952            /* and reg1, segattrs */
     10953            off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxReg1, idxRegSegAttrib);
     10954            /* cmp reg1, must1 */
     10955            off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, fMustBe1);
     10956            /* jne tlbmiss */
     10957            off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
     10958        }
     10959        else
     10960        {
     10961            /*  U  | !P |!DT |!CD | RW |
     10962                16 |  8 |  4 |  3 |  1 |
     10963              -------------------------------
     10964                0  |  0 |  0 |  0 |  0 | execute-only code segment. - must be excluded
     10965                0  |  0 |  0 |  0 |  1 | execute-read code segment.
     10966                0  |  0 |  0 |  1 |  0 | read-only data segment.
     10967                0  |  0 |  0 |  1 |  1 | read-write data segment.   - last valid combination
     10968            */
     10969            /* mov reg1, relevant attributes  */
     10970            off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1,
     10971                                                   X86DESCATTR_UNUSABLE | X86DESCATTR_P | X86DESCATTR_DT
     10972                                                 | X86_SEL_TYPE_CODE    | X86_SEL_TYPE_WRITE);
     10973            /* and reg1, segattrs */
     10974            off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxReg1, idxRegSegAttrib);
     10975            /* xor reg1, X86DESCATTR_P | X86DESCATTR_DT | X86_SEL_TYPE_CODE ; place C=1 RW=0 at the bottom & limit the range. */
     10976            off = iemNativeEmitXorGpr32ByImmEx(pbCodeBuf, off, idxReg1, X86DESCATTR_P | X86DESCATTR_DT | X86_SEL_TYPE_CODE);
     10977            /* sub reg1, X86_SEL_TYPE_WRITE ; ER-code=0, EO-code=0xffffffff, R0-data=7, RW-data=9 */
     10978            off = iemNativeEmitSubGpr32ImmEx(pbCodeBuf, off, idxReg1, X86_SEL_TYPE_WRITE);
     10979            /* cmp reg1, X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE */
     10980            AssertCompile((X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE) - 1 == 9);
     10981            off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, (X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE) - 1);
     10982            /* ja  tlbmiss */
     10983            off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
     10984        }
     10985
     10986        /*
     10987         * Check the limit.  If this is a write access, we know that it's a
     10988         * data segment and includes the expand_down bit.  For read-only accesses
     10989         * we need to check that code/data=0 and expanddown=1 before continuing.
     10990         */
     10991        uint32_t offFixupCheckExpandDown;
     10992        if (fAccess & IEM_ACCESS_TYPE_WRITE)
     10993        {
     10994            /* test segattrs, X86_SEL_TYPE_DOWN */
     10995            AssertCompile(X86_SEL_TYPE_DOWN < 128);
     10996            off = iemNativeEmitTestAnyBitsInGpr8Ex(pbCodeBuf, off, idxRegSegAttrib, X86_SEL_TYPE_DOWN);
     10997            /* jnz  check_expand_down */
     10998            offFixupCheckExpandDown = off;
     10999            off = iemNativeEmitJccToFixedEx(pbCodeBuf, off, off /*ASSUMES rel8 suffices*/, kIemNativeInstrCond_ne);
     11000        }
     11001        else
     11002        {
     11003            /* mov reg1, segattrs */
     11004            off = iemNativeEmitLoadGprFromGpr32Ex(pbCodeBuf, off, idxReg1, idxRegSegAttrib);
     11005            /* and reg1, code | down */
     11006            off = iemNativeEmitAndGpr32ByImmEx(pbCodeBuf, off, idxReg1, X86_SEL_TYPE_CODE | X86_SEL_TYPE_DOWN);
     11007            /* cmp reg1, down */
     11008            off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, X86_SEL_TYPE_DOWN);
     11009            /* je check_expand_down */
     11010            offFixupCheckExpandDown = off;
     11011            off = iemNativeEmitJccToFixedEx(pbCodeBuf, off, off /*ASSUMES rel8 suffices*/, kIemNativeInstrCond_e);
     11012        }
     11013
     11014        /* expand_up:
     11015           cmp  regptr/reg2, seglim */
     11016        off = iemNativeEmitCmpGprWithGprEx(pbCodeBuf, off, cbMem > 1 ? idxReg2 : idxRegPtr, idxRegSegLimit);
     11017        /* ja   tlbmiss */
     11018        off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
     11019        /* jmp  limitdone */
     11020        uint32_t const offFixupLimitDone = off;
     11021        off = iemNativeEmitJmpToFixedEx(pbCodeBuf, off, off /*ASSUMES rel8 suffices*/);
     11022
     11023        /* check_expand_down: ; complicted! */
     11024        iemNativeFixupFixedJump(pReNative, offFixupCheckExpandDown, off);
     11025        /* cmp  regptr, seglim */
     11026        off = iemNativeEmitCmpGprWithGprEx(pbCodeBuf, off, idxRegPtr, idxRegSegLimit);
     11027        /* jbe  tlbmiss */
     11028        off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_be);
     11029        /* mov  reg1, X86DESCATTR_D (0x4000) */
     11030        off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, X86DESCATTR_D);
     11031        /* and  reg1, segattr */
     11032        off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxReg1, idxRegSegAttrib);
     11033        /* xor  reg1, X86DESCATTR_D */
     11034        off = iemNativeEmitXorGpr32ByImmEx(pbCodeBuf, off, idxReg1, X86DESCATTR_D);
     11035        /* shl  reg1, 2 (16 - 14) */
     11036        AssertCompile((X86DESCATTR_D << 2) == UINT32_C(0x10000));
     11037        off = iemNativeEmitShiftGpr32LeftEx(pbCodeBuf, off, idxReg1, 2);
     11038        /* dec  reg1 (=> 0xffff if D=0; 0xffffffff if D=1) */
     11039        off = iemNativeEmitSubGpr32ImmEx(pbCodeBuf, off, idxReg1, 1);
     11040        /* cmp  reg2, reg1 (64-bit) */
     11041        off = iemNativeEmitCmpGpr32WithGprEx(pbCodeBuf, off, idxReg2, idxReg1);
     11042        /* ja   tlbmiss */
     11043        off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
     11044
     11045        /* limitdone: */
     11046        iemNativeFixupFixedJump(pReNative, offFixupLimitDone, off);
     11047    }
     11048
     11049    /* 1b. Add the segment base. We use idxRegMemResult for the ptr register if this step is required. */
     11050    uint8_t const idxRegFlatPtr = iSegReg != UINT8_MAX ? idxRegMemResult : idxRegPtr;
     11051    if (iSegReg != UINT8_MAX)
     11052    {
     11053        if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
     11054        {
     11055            Assert(iSegReg >= X86_SREG_FS);
     11056            /* mov regflat, regptr */
     11057            off = iemNativeEmitLoadGprFromGprEx(pbCodeBuf, off, idxRegFlatPtr, idxRegPtr);
     11058            /* add regflat, seg.base */
     11059            off = iemNativeEmitAddTwoGprsEx(pbCodeBuf, off, idxRegFlatPtr, idxRegSegBase);
     11060        }
     11061        else
     11062        {
     11063            /* mov regflat, regptr */
     11064            off = iemNativeEmitLoadGprFromGpr32Ex(pbCodeBuf, off, idxRegFlatPtr, idxRegPtr);
     11065            /* add regflat, seg.base */
     11066            off = iemNativeEmitAddTwoGprs32Ex(pbCodeBuf, off, idxRegFlatPtr, idxRegSegBase);
     11067        }
     11068    }
     11069
     11070    /*
     11071     * 2. Check that the address doesn't cross a page boundrary and doesn't have alignment issues.
     11072     *
     11073     * 2a. Alignment check using fAlignMask.
     11074     */
     11075    Assert(RT_IS_POWER_OF_TWO(fAlignMask + 1));
     11076    Assert(fAlignMask < 128);
     11077    /* test regflat, fAlignMask */
     11078    off = iemNativeEmitTestAnyBitsInGpr8Ex(pbCodeBuf, off, idxRegFlatPtr, fAlignMask);
     11079    /* jnz tlbmiss */
     11080    off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
     11081
     11082    /*
     11083     * 2b. Check that it's not crossing page a boundrary. This is implicit in
     11084     *     the previous test if the alignment is same or larger than the type.
     11085     */
     11086    if (cbMem > fAlignMask + 1)
     11087    {
     11088        /* mov reg1, 0xfff */
     11089        off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxReg1, GUEST_PAGE_OFFSET_MASK);
     11090        /* and reg1, regflat */
     11091        off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxReg1, idxRegFlatPtr);
     11092        /* neg reg1 */
     11093        off = iemNativeEmitNegGpr32Ex(pbCodeBuf, off, idxReg1);
     11094        /* add reg1, 0x1000 */
     11095        off = iemNativeEmitAddGpr32ImmEx(pbCodeBuf, off, idxReg1, GUEST_PAGE_SIZE);
     11096        /* cmp reg1, cbMem */
     11097        off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, GUEST_PAGE_SIZE);
     11098        /* ja  tlbmiss */
     11099        off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
     11100    }
     11101
     11102    /*
     11103     * 3. TLB lookup.
     11104     *
     11105     * 3a. Calculate the TLB tag value (IEMTLB_CALC_TAG).
     11106     *     In 64-bit mode we will also check for non-canonical addresses here.
     11107     */
     11108    if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
     11109    {
     11110        /* mov reg1, regflat */
     11111        off = iemNativeEmitLoadGprFromGprEx(pbCodeBuf, off, idxReg1, idxRegFlatPtr);
     11112        /* rol reg1, 16 */
     11113        off = iemNativeEmitRotateGprLeftEx(pbCodeBuf, off, idxReg1, 16);
     11114        /** @todo Would 'movsx reg2, word reg1' and working on reg2 in dwords be faster? */
     11115        /* inc word reg1 */
     11116        pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
     11117        if (idxReg1 >= 8)
     11118            pbCodeBuf[off++] = X86_OP_REX_B;
     11119        pbCodeBuf[off++] = 0xff;
     11120        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, idxReg1 & 7);
     11121        /* cmp word reg1, 1 */
     11122        pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
     11123        if (idxReg1 >= 8)
     11124            pbCodeBuf[off++] = X86_OP_REX_B;
     11125        pbCodeBuf[off++] = 0x83;
     11126        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, idxReg1 & 7);
     11127        pbCodeBuf[off++] = 1;
     11128        /* ja  tlbmiss */
     11129        off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
     11130        /* shr reg1, 16 + GUEST_PAGE_SHIFT */
     11131        off = iemNativeEmitShiftGprRightEx(pbCodeBuf, off, idxReg1, 16 + GUEST_PAGE_SHIFT);
     11132    }
     11133    else
     11134    {
     11135        /* mov reg1, regflat */
     11136        off = iemNativeEmitLoadGprFromGpr32Ex(pbCodeBuf, off, idxReg1, idxRegFlatPtr);
     11137        /* shr reg1, GUEST_PAGE_SHIFT */
     11138        off = iemNativeEmitShiftGpr32RightEx(pbCodeBuf, off, idxReg1, GUEST_PAGE_SHIFT);
     11139    }
     11140    /* or  reg1, [qword pVCpu->iem.s.DataTlb.uTlbRevision] */
     11141    pbCodeBuf[off++] = idxReg1 < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R;
     11142    pbCodeBuf[off++] = 0x0b; /* OR r64,r/m64 */
     11143    off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxReg1, RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbRevision));
     11144
     11145    /*
     11146     * 3b. Calc pTlbe.
     11147     */
     11148    /* movzx reg2, byte reg1 */
     11149    off = iemNativeEmitLoadGprFromGpr8Ex(pbCodeBuf, off, idxReg2, idxReg1);
     11150    /* shl   reg2, 5 ; reg2 *= sizeof(IEMTLBENTRY) */
     11151    AssertCompileSize(IEMTLBENTRY, 32);
     11152    off = iemNativeEmitShiftGprLeftEx(pbCodeBuf, off, idxReg2, 5);
     11153    /* lea   reg2, [pVCpu->iem.s.DataTlb.aEntries + reg2] */
     11154    AssertCompile(IEMNATIVE_REG_FIXED_PVMCPU < 8);
     11155    pbCodeBuf[off++] = idxReg2 < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_X | X86_OP_REX_R;
     11156    pbCodeBuf[off++] = 0x8d;
     11157    pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, idxReg2 & 7, 4 /*SIB*/);
     11158    pbCodeBuf[off++] = X86_SIB_MAKE(IEMNATIVE_REG_FIXED_PVMCPU & 7, idxReg2 & 7, 0);
     11159    pbCodeBuf[off++] = RT_BYTE1(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
     11160    pbCodeBuf[off++] = RT_BYTE2(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
     11161    pbCodeBuf[off++] = RT_BYTE3(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
     11162    pbCodeBuf[off++] = RT_BYTE4(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
     11163
     11164    /*
     11165     * 3c. Compare the TLBE.uTag with the one from 2a (reg1).
     11166     */
     11167    /* cmp reg1, [reg2] */
     11168    pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R) | (idxReg2 < 8 ? 0 : X86_OP_REX_B);
     11169    pbCodeBuf[off++] = 0x3b;
     11170    off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, idxReg1, idxReg2, RT_UOFFSETOF(IEMTLBENTRY, uTag));
     11171    /* jne tlbmiss */
     11172    off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
     11173
     11174    /*
     11175     * 4. Check TLB page table level access flags and physical page revision #.
     11176     */
     11177    /* mov reg1, mask */
     11178    AssertCompile(IEMTLBE_F_PT_NO_USER == 4);
     11179    uint64_t const fNoUser = (((pReNative->fExec >> IEM_F_X86_CPL_SHIFT) & IEM_F_X86_CPL_SMASK) + 1) & IEMTLBE_F_PT_NO_USER;
     11180    off = iemNativeEmitLoadGprImmEx(pbCodeBuf, off, idxReg1,
     11181                                      IEMTLBE_F_PHYS_REV       | IEMTLBE_F_NO_MAPPINGR3
     11182                                    | IEMTLBE_F_PG_UNASSIGNED  | IEMTLBE_F_PG_NO_READ
     11183                                    | IEMTLBE_F_PT_NO_ACCESSED | fNoUser);
     11184    /* and reg1, [reg2->fFlagsAndPhysRev] */
     11185    pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R) | (idxReg2 < 8 ? 0 : X86_OP_REX_B);
     11186    pbCodeBuf[off++] = 0x23;
     11187    off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, idxReg1, idxReg2, RT_UOFFSETOF(IEMTLBENTRY, fFlagsAndPhysRev));
     11188
     11189    /* cmp reg1, [pVCpu->iem.s.DataTlb.uTlbPhysRev] */
     11190    pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R);
     11191    pbCodeBuf[off++] = 0x3b;
     11192    off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, idxReg1, IEMNATIVE_REG_FIXED_PVMCPU,
     11193                                    RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbPhysRev));
     11194    /* jne tlbmiss */
     11195    off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
     11196
     11197    /*
     11198     * 5. Check that pbMappingR3 isn't NULL (paranoia) and calculate the
     11199     *    resulting pointer.
     11200     */
     11201    /* mov  reg1, [reg2->pbMappingR3] */
     11202    pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R) | (idxReg2 < 8 ? 0 : X86_OP_REX_B);
     11203    pbCodeBuf[off++] = 0x8b;
     11204    off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, idxRegMemResult, idxReg2, RT_UOFFSETOF(IEMTLBENTRY, pbMappingR3));
     11205
     11206    /** @todo eliminate the need for this test? */
     11207    /* test reg1, reg1 */
     11208    pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R | X86_OP_REX_B);
     11209    pbCodeBuf[off++] = 0x85;
     11210    pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxReg1 & 7, idxReg1 & 7);
     11211
     11212    /* jz   tlbmiss */
     11213    off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbDone, kIemNativeInstrCond_e);
     11214
     11215    if (idxRegFlatPtr == idxRegMemResult) /* See step 1b. */
     11216    {
     11217        /* and result, 0xfff */
     11218        off = iemNativeEmitAndGpr32ByImmEx(pbCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
     11219    }
     11220    else
     11221    {
     11222        Assert(idxRegFlatPtr == idxRegPtr);
     11223        /* mov result, 0xfff */
     11224        off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
     11225        /* and result, regflat */
     11226        off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxRegMemResult, idxRegFlatPtr);
     11227    }
     11228    /* add result, reg1 */
     11229    off = iemNativeEmitAddTwoGprsEx(pbCodeBuf, off, idxRegMemResult, idxReg1);
     11230
     11231    /* jmp tlbdone */
     11232    off = iemNativeEmitJmpToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbDone);
     11233
     11234    iemNativeVarRegisterRelease(pReNative, idxRegPtr);
     11235    iemNativeRegFree(pReNative, idxRegSegBase);
     11236    iemNativeRegFree(pReNative, idxRegSegLimit);
     11237    iemNativeRegFree(pReNative, idxRegSegAttrib);
     11238    iemNativeRegFree(pReNative, idxReg2);
     11239    iemNativeRegFree(pReNative, idxReg1);
     11240
     11241#else
     11242    /** @todo arm64 TLB code   */
    1090511243    RT_NOREF(fAccess, fAlignMask, cbMem);
     11244#endif
    1090611245
    1090711246    /*

  • trunk/src/VBox/VMM/include/IEMN8veRecompiler.h

    r102699 r102717  
    388388    kIemNativeGstReg_SegLimitFirst,
    389389    kIemNativeGstReg_SegLimitLast  = kIemNativeGstReg_SegLimitFirst + 5,
     390    kIemNativeGstReg_SegAttribFirst,
     391    kIemNativeGstReg_SegAttribLast = kIemNativeGstReg_SegAttribFirst + 5,
    390392    kIemNativeGstReg_End
    391393} IEMNATIVEGSTREG;
    … …  
    393395/** @name Helpers for converting register numbers to IEMNATIVEGSTREG values.
    394396 * @{  */
    395 #define IEMNATIVEGSTREG_GPR(a_iGpr)             ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst      + (a_iGpr)    ))
    396 #define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg)      ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst   + (a_iSegReg) ))
    397 #define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg)     ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst  + (a_iSegReg) ))
    398 #define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg)    ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst + (a_iSegReg) ))
     397#define IEMNATIVEGSTREG_GPR(a_iGpr)             ((IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst       + (a_iGpr)    ))
     398#define IEMNATIVEGSTREG_SEG_SEL(a_iSegReg)      ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegSelFirst    + (a_iSegReg) ))
     399#define IEMNATIVEGSTREG_SEG_BASE(a_iSegReg)     ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegBaseFirst   + (a_iSegReg) ))
     400#define IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg)    ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegLimitFirst  + (a_iSegReg) ))
     401#define IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg)   ((IEMNATIVEGSTREG)(kIemNativeGstReg_SegAttribFirst + (a_iSegReg) ))
    399402/** @} */
    400403
    … …  
    814817                                                    bool fPreferVolatile = true);
    815818DECL_HIDDEN_THROW(uint8_t)  iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
    816                                                             IEMNATIVEGSTREG enmGstReg, IEMNATIVEGSTREGUSE enmIntendedUse,
     819                                                            IEMNATIVEGSTREG enmGstReg,
     820                                                            IEMNATIVEGSTREGUSE enmIntendedUse = kIemNativeGstRegUse_ReadOnly,
    817821                                                            bool fNoVoltileRegs = false);
    818822DECL_HIDDEN_THROW(uint8_t)  iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,

  • trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h

    r102700 r102717  
    793793 * Emits a gprdst = gprsrc load.
    794794 */
     795DECL_FORCE_INLINE(uint32_t)
     796iemNativeEmitLoadGprFromGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
     797{
     798#ifdef RT_ARCH_AMD64
     799    /* mov gprdst, gprsrc */
     800    if ((iGprDst | iGprSrc) >= 8)
     801        pCodeBuf[off++] = iGprDst < 8  ? X86_OP_REX_W | X86_OP_REX_B
     802                        : iGprSrc >= 8 ? X86_OP_REX_W | X86_OP_REX_R | X86_OP_REX_B
     803                        :                X86_OP_REX_W | X86_OP_REX_R;
     804    else
     805        pCodeBuf[off++] = X86_OP_REX_W;
     806    pCodeBuf[off++] = 0x8b;
     807    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
     808
     809#elif defined(RT_ARCH_ARM64)
     810    /* mov dst, src;   alias for: orr dst, xzr, src */
     811    p32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_XZR, iGprSrc);
     812
     813#else
     814# error "port me"
     815#endif
     816    return off;
     817}
     818
     819
     820/**
     821 * Emits a gprdst = gprsrc load.
     822 */
    795823DECL_INLINE_THROW(uint32_t)
    796824iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
    797825{
    798826#ifdef RT_ARCH_AMD64
    799     /* mov gprdst, gprsrc */
    800     uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
    801     if ((iGprDst | iGprSrc) >= 8)
    802         pbCodeBuf[off++] = iGprDst < 8  ? X86_OP_REX_W | X86_OP_REX_B
    803                          : iGprSrc >= 8 ? X86_OP_REX_W | X86_OP_REX_R | X86_OP_REX_B
    804                          :                X86_OP_REX_W | X86_OP_REX_R;
    805     else
    806         pbCodeBuf[off++] = X86_OP_REX_W;
    807     pbCodeBuf[off++] = 0x8b;
    808     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
    809 
    810 #elif defined(RT_ARCH_ARM64)
    811     /* mov dst, src;   alias for: orr dst, xzr, src */
    812     uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    813     pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_XZR, iGprSrc);
    814 
     827    off = iemNativeEmitLoadGprFromGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc);
     828#elif defined(RT_ARCH_ARM64)
     829    off = iemNativeEmitLoadGprFromGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
    815830#else
    816831# error "port me"
    … …  
    825840 * @note Bits 63 thru 32 are cleared.
    826841 */
     842DECL_FORCE_INLINE(uint32_t)
     843iemNativeEmitLoadGprFromGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
     844{
     845#ifdef RT_ARCH_AMD64
     846    /* mov gprdst, gprsrc */
     847    if ((iGprDst | iGprSrc) >= 8)
     848        pCodeBuf[off++] = iGprDst < 8  ? X86_OP_REX_B
     849                        : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B
     850                        :                X86_OP_REX_R;
     851    pCodeBuf[off++] = 0x8b;
     852    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
     853
     854#elif defined(RT_ARCH_ARM64)
     855    /* mov dst32, src32;   alias for: orr dst32, wzr, src32 */
     856    pCodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_WZR, iGprSrc, false /*f64bit*/);
     857
     858#else
     859# error "port me"
     860#endif
     861    return off;
     862}
     863
     864
     865/**
     866 * Emits a gprdst = gprsrc[31:0] load.
     867 * @note Bits 63 thru 32 are cleared.
     868 */
    827869DECL_INLINE_THROW(uint32_t)
    828870iemNativeEmitLoadGprFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
    829871{
    830872#ifdef RT_ARCH_AMD64
    831     /* mov gprdst, gprsrc */
    832     uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
    833     if ((iGprDst | iGprSrc) >= 8)
    834         pbCodeBuf[off++] = iGprDst < 8  ? X86_OP_REX_B
    835                          : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B
    836                          :                X86_OP_REX_R;
    837     pbCodeBuf[off++] = 0x8b;
    838     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
    839 
    840 #elif defined(RT_ARCH_ARM64)
    841     /* mov dst32, src32;   alias for: orr dst32, wzr, src32 */
    842     uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    843     pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_WZR, iGprSrc, false /*f64bit*/);
    844 
     873    off = iemNativeEmitLoadGprFromGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc);
     874#elif defined(RT_ARCH_ARM64)
     875    off = iemNativeEmitLoadGprFromGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
    845876#else
    846877# error "port me"
    … …  
    892923 * @note Bits 63 thru 8 are cleared.
    893924 */
     925DECL_FORCE_INLINE(uint32_t)
     926iemNativeEmitLoadGprFromGpr8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
     927{
     928#ifdef RT_ARCH_AMD64
     929    /* movzx Gv,Eb */
     930    if (iGprDst >= 8 || iGprSrc >= 8)
     931        pCodeBuf[off++] = iGprDst < 8  ? X86_OP_REX_B
     932                        : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B
     933                        :                X86_OP_REX_R;
     934    else if (iGprSrc >= 4)
     935        pCodeBuf[off++] = X86_OP_REX;
     936    pCodeBuf[off++] = 0x0f;
     937    pCodeBuf[off++] = 0xb6;
     938    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
     939
     940#elif defined(RT_ARCH_ARM64)
     941    /* and gprdst, gprsrc, #0xff */
     942    Assert(Armv8A64ConvertImmRImmS2Mask32(0x07, 0) == UINT8_MAX);
     943    pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x07, 0, false /*f64Bit*/);
     944
     945#else
     946# error "port me"
     947#endif
     948    return off;
     949}
     950
     951
     952/**
     953 * Emits a gprdst = gprsrc[7:0] load.
     954 * @note Bits 63 thru 8 are cleared.
     955 */
    894956DECL_INLINE_THROW(uint32_t)
    895957iemNativeEmitLoadGprFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
    896958{
    897959#ifdef RT_ARCH_AMD64
    898     /* movzx Gv,Eb */
    899     uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
    900     if (iGprDst >= 8 || iGprSrc >= 8)
    901         pbCodeBuf[off++] = iGprDst < 8  ? X86_OP_REX_B
    902                          : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B
    903                          :                X86_OP_REX_R;
    904     else if (iGprSrc >= 4)
    905         pbCodeBuf[off++] = X86_OP_REX;
    906     pbCodeBuf[off++] = 0x0f;
    907     pbCodeBuf[off++] = 0xb6;
    908     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
    909 
    910 #elif defined(RT_ARCH_ARM64)
    911     /* and gprdst, gprsrc, #0xff */
    912     uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    913 # if 1
    914     Assert(Armv8A64ConvertImmRImmS2Mask32(0x07, 0) == UINT8_MAX);
    915     pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x07, 0, false /*f64Bit*/);
    916 # else
    917     Assert(Armv8A64ConvertImmRImmS2Mask64(0x47, 0) == UINT8_MAX);
    918     pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x47, 0);
    919 # endif
    920 
     960    off = iemNativeEmitLoadGprFromGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iGprSrc);
     961#elif defined(RT_ARCH_ARM64)
     962    off = iemNativeEmitLoadGprFromGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc);
    921963#else
    922964# error "port me"
    … …  
    16961738
    16971739
     1740/**
     1741 * Emits subtracting a 32-bit GPR from another, storing the result in the first.
     1742 * @note The AMD64 version sets flags.
     1743 */
     1744DECL_FORCE_INLINE(uint32_t)
     1745iemNativeEmitSubTwoGprs32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
     1746{
     1747#if defined(RT_ARCH_AMD64)
     1748    /* sub Gv,Ev */
     1749    if (iGprDst >= 8 || iGprSubtrahend >= 8)
     1750        pCodeBuf[off++] = (iGprDst        < 8 ? 0 : X86_OP_REX_R)
     1751                        | (iGprSubtrahend < 8 ? 0 : X86_OP_REX_B);
     1752    pCodeBuf[off++] = 0x2b;
     1753    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSubtrahend & 7);
     1754
     1755#elif defined(RT_ARCH_ARM64)
     1756    pCodeBuf[off++] = Armv8A64MkInstrSubReg(iGprDst, iGprDst, iGprSubtrahend, false /*f64Bit*/);
     1757
     1758#else
     1759# error "Port me"
     1760#endif
     1761    return off;
     1762}
     1763
     1764
     1765/**
     1766 * Emits subtracting a 32-bit GPR from another, storing the result in the first.
     1767 * @note The AMD64 version sets flags.
     1768 */
     1769DECL_INLINE_THROW(uint32_t)
     1770iemNativeEmitSubTwoGprs32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSubtrahend)
     1771{
     1772#if defined(RT_ARCH_AMD64)
     1773    off = iemNativeEmitSubTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSubtrahend);
     1774#elif defined(RT_ARCH_ARM64)
     1775    off = iemNativeEmitSubTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSubtrahend);
     1776#else
     1777# error "Port me"
     1778#endif
     1779    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     1780    return off;
     1781}
     1782
     1783
    16981784#ifdef RT_ARCH_AMD64
    16991785/**
    … …  
    17301816
    17311817
     1818#ifdef RT_ARCH_AMD64
     1819/**
     1820 * Emits a 32-bit GPR subtract with a signed immediate subtrahend.
     1821 *
     1822 * This will optimize using DEC/INC/whatever, so try avoid flag dependencies.
     1823 */
     1824DECL_FORCE_INLINE(uint32_t)
     1825iemNativeEmitSubGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
     1826{
     1827    if (iGprDst >= 8)
     1828        pCodeBuf[off++] = X86_OP_REX_B;
     1829    if (iSubtrahend == 1)
     1830    {
     1831        /* dec r/m32 */
     1832        pCodeBuf[off++] = 0xff;
     1833        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 1, iGprDst & 7);
     1834    }
     1835    else if (iSubtrahend == -1)
     1836    {
     1837        /* inc r/m32 */
     1838        pCodeBuf[off++] = 0xff;
     1839        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
     1840    }
     1841    else if (iSubtrahend < 128 && iSubtrahend >= -128)
     1842    {
     1843        /* sub r/m32, imm8 */
     1844        pCodeBuf[off++] = 0x83;
     1845        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
     1846        pCodeBuf[off++] = (uint8_t)iSubtrahend;
     1847    }
     1848    else
     1849    {
     1850        /* sub r/m32, imm32 */
     1851        pCodeBuf[off++] = 0x81;
     1852        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
     1853        pCodeBuf[off++] = RT_BYTE1(iSubtrahend);
     1854        pCodeBuf[off++] = RT_BYTE2(iSubtrahend);
     1855        pCodeBuf[off++] = RT_BYTE3(iSubtrahend);
     1856        pCodeBuf[off++] = RT_BYTE4(iSubtrahend);
     1857    }
     1858    return off;
     1859}
     1860#endif
     1861
     1862
    17321863/**
    17331864 * Emits adding a 64-bit GPR to another, storing the result in the first.
    17341865 * @note The AMD64 version sets flags.
    17351866 */
    1736 DECL_INLINE_THROW(uint32_t)
    1737 iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
     1867DECL_FORCE_INLINE(uint32_t)
     1868iemNativeEmitAddTwoGprsEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
    17381869{
    17391870#if defined(RT_ARCH_AMD64)
    17401871    /* add Gv,Ev */
    1741     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
    1742     pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R)
    1743                      | (iGprAddend < 8 ? 0 : X86_OP_REX_B);
    1744     pbCodeBuf[off++] = 0x03;
    1745     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
    1746 
    1747 #elif defined(RT_ARCH_ARM64)
    1748     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    1749     pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iGprAddend);
    1750 
     1872    pCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R)
     1873                    | (iGprAddend < 8 ? 0 : X86_OP_REX_B);
     1874    pCodeBuf[off++] = 0x03;
     1875    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
     1876
     1877#elif defined(RT_ARCH_ARM64)
     1878    pCodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iGprAddend);
     1879
     1880#else
     1881# error "Port me"
     1882#endif
     1883    return off;
     1884}
     1885
     1886
     1887/**
     1888 * Emits adding a 64-bit GPR to another, storing the result in the first.
     1889 * @note The AMD64 version sets flags.
     1890 */
     1891DECL_INLINE_THROW(uint32_t)
     1892iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
     1893{
     1894#if defined(RT_ARCH_AMD64)
     1895    off = iemNativeEmitAddTwoGprsEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprAddend);
     1896#elif defined(RT_ARCH_ARM64)
     1897    off = iemNativeEmitAddTwoGprsEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprAddend);
     1898#else
     1899# error "Port me"
     1900#endif
     1901    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     1902    return off;
     1903}
     1904
     1905
     1906/**
     1907 * Emits adding a 64-bit GPR to another, storing the result in the first.
     1908 * @note The AMD64 version sets flags.
     1909 */
     1910DECL_FORCE_INLINE(uint32_t)
     1911iemNativeEmitAddTwoGprs32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
     1912{
     1913#if defined(RT_ARCH_AMD64)
     1914    /* add Gv,Ev */
     1915    if (iGprDst >= 8 || iGprAddend >= 8)
     1916        pCodeBuf[off++] = (iGprDst    >= 8 ? X86_OP_REX_R : 0)
     1917                        | (iGprAddend >= 8 ? X86_OP_REX_B : 0);
     1918    pCodeBuf[off++] = 0x03;
     1919    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);
     1920
     1921#elif defined(RT_ARCH_ARM64)
     1922    pCodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iGprAddend, false /*f64Bit*/);
     1923
     1924#else
     1925# error "Port me"
     1926#endif
     1927    return off;
     1928}
     1929
     1930
     1931/**
     1932 * Emits adding a 64-bit GPR to another, storing the result in the first.
     1933 * @note The AMD64 version sets flags.
     1934 */
     1935DECL_INLINE_THROW(uint32_t)
     1936iemNativeEmitAddTwoGprs32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)
     1937{
     1938#if defined(RT_ARCH_AMD64)
     1939    off = iemNativeEmitAddTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprAddend);
     1940#elif defined(RT_ARCH_ARM64)
     1941    off = iemNativeEmitAddTwoGprs32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprAddend);
    17511942#else
    17521943# error "Port me"
    … …  
    17981989 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
    17991990 */
    1800 DECL_INLINE_THROW(uint32_t)
    1801 iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
     1991DECL_FORCE_INLINE(uint32_t)
     1992iemNativeEmitAddGpr32Imm8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int8_t iImm8)
    18021993{
    18031994#if defined(RT_ARCH_AMD64)
    18041995    /* add or inc */
    1805     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
    18061996    if (iGprDst >= 8)
    1807         pbCodeBuf[off++] = X86_OP_REX_B;
     1997        pCodeBuf[off++] = X86_OP_REX_B;
    18081998    if (iImm8 != 1)
    18091999    {
    1810         pbCodeBuf[off++] = 0x83;
    1811         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
    1812         pbCodeBuf[off++] = (uint8_t)iImm8;
    1813     }
    1814     else
    1815     {
    1816         pbCodeBuf[off++] = 0xff;
    1817         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
    1818     }
    1819 
    1820 #elif defined(RT_ARCH_ARM64)
    1821     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
     2000        pCodeBuf[off++] = 0x83;
     2001        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
     2002        pCodeBuf[off++] = (uint8_t)iImm8;
     2003    }
     2004    else
     2005    {
     2006        pCodeBuf[off++] = 0xff;
     2007        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
     2008    }
     2009
     2010#elif defined(RT_ARCH_ARM64)
    18222011    if (iImm8 >= 0)
    1823         pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint8_t)iImm8, false /*f64Bit*/);
    1824     else
    1825         pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint8_t)-iImm8, false /*f64Bit*/);
    1826 
     2012        pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint8_t)iImm8, false /*f64Bit*/);
     2013    else
     2014        pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint8_t)-iImm8, false /*f64Bit*/);
     2015
     2016#else
     2017# error "Port me"
     2018#endif
     2019    return off;
     2020}
     2021
     2022
     2023/**
     2024 * Emits a 32-bit GPR additions with a 8-bit signed immediate.
     2025 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
     2026 */
     2027DECL_INLINE_THROW(uint32_t)
     2028iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
     2029{
     2030#if defined(RT_ARCH_AMD64)
     2031    off = iemNativeEmitAddGpr32Imm8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, iImm8);
     2032#elif defined(RT_ARCH_ARM64)
     2033    off = iemNativeEmitAddGpr32Imm8Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iImm8);
    18272034#else
    18282035# error "Port me"
    … …  
    19022109 * Emits a 32-bit GPR additions with a 32-bit signed immediate.
    19032110 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
    1904  */
    1905 DECL_INLINE_THROW(uint32_t)
    1906 iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
     2111 * @note For ARM64 the iAddend value must be in the range 0x000..0xfff,
     2112 *       or that range shifted 12 bits to the left (e.g. 0x1000..0xfff000 with
     2113 *       the lower 12 bits always zero).  The negative ranges are also allowed,
     2114 *       making it behave like a subtraction.  If the constant does not conform,
     2115 *       bad stuff will happen.
     2116 */
     2117DECL_FORCE_INLINE_THROW(uint32_t)
     2118iemNativeEmitAddGpr32ImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, int32_t iAddend)
    19072119{
    19082120#if defined(RT_ARCH_AMD64)
    19092121    if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)
    1910         return iemNativeEmitAddGpr32Imm8(pReNative, off, iGprDst, (int8_t)iAddend);
     2122        return iemNativeEmitAddGpr32Imm8Ex(pCodeBuf, off, iGprDst, (int8_t)iAddend);
    19112123
    19122124    /* add grp, imm32 */
    1913     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
    19142125    if (iGprDst >= 8)
    1915         pbCodeBuf[off++] = X86_OP_REX_B;
    1916     pbCodeBuf[off++] = 0x81;
    1917     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
    1918     pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
    1919     pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
    1920     pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
    1921     pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
     2126        pCodeBuf[off++] = X86_OP_REX_B;
     2127    pCodeBuf[off++] = 0x81;
     2128    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
     2129    pCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);
     2130    pCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);
     2131    pCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);
     2132    pCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);
     2133
     2134#elif defined(RT_ARCH_ARM64)
     2135    uint32_t const uAbsAddend = (uint32_t)RT_ABS(iAddened);
     2136    if (uAbsAddend <= 0xfff)
     2137    {
     2138        if (iAddend >= 0)
     2139            pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, uAbsAddend, false /*f64Bit*/);
     2140        else
     2141            pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/,  iGprDst, iGprDst, uAbsAddend, false /*f64Bit*/);
     2142    }
     2143    else if (uAbsAddend <= 0xfff000 && !(uAbsAddend & 0xfff))
     2144    {
     2145        if (iAddend >= 0)
     2146            pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, uAbsAddend >> 12,
     2147                                                          false /*f64Bit*/, true /*fShift12*/);
     2148        else
     2149            pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/,  iGprDst, iGprDst, uAbsAddend >> 12,
     2150                                                          false /*f64Bit*/, true /*fShift12*/);
     2151    }
     2152    else
     2153# ifdef IEM_WITH_THROW_CATCH
     2154        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     2155# else
     2156        AssertReleaseFailedStmt(off = UINT32_MAX);
     2157# endif
     2158
     2159#else
     2160# error "Port me"
     2161#endif
     2162    return off;
     2163}
     2164
     2165
     2166/**
     2167 * Emits a 32-bit GPR additions with a 32-bit signed immediate.
     2168 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
     2169 */
     2170DECL_INLINE_THROW(uint32_t)
     2171iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)
     2172{
     2173#if defined(RT_ARCH_AMD64)
     2174    off = iemNativeEmitAddGpr32ImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprDst, iAddend);
    19222175
    19232176#elif defined(RT_ARCH_ARM64)
    … …  
    19552208
    19562209/**
    1957  * Emits code for clearing bits 16 thru 63 in the GPR.
    1958  */
    1959 DECL_INLINE_THROW(uint32_t)
    1960 iemNativeEmitNegGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
     2210 * Emits code for two complement negation of a 64-bit GPR.
     2211 */
     2212DECL_FORCE_INLINE_THROW(uint32_t)
     2213iemNativeEmitNegGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst)
    19612214{
    19622215#if defined(RT_ARCH_AMD64)
    19632216    /* neg Ev */
    1964     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
    1965     pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
    1966     pbCodeBuf[off++] = 0xf7;
    1967     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 3, iGprDst & 7);
     2217    pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
     2218    pCodeBuf[off++] = 0xf7;
     2219    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 3, iGprDst & 7);
    19682220
    19692221#elif defined(RT_ARCH_ARM64)
    19702222    /* sub dst, xzr, dst */
    1971     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    1972     pu32CodeBuf[off++] = Armv8A64MkInstrNeg(iGprDst);
    1973 
     2223    pCodeBuf[off++] = Armv8A64MkInstrNeg(iGprDst);
     2224
     2225#else
     2226# error "Port me"
     2227#endif
     2228    return off;
     2229}
     2230
     2231
     2232/**
     2233 * Emits code for two complement negation of a 64-bit GPR.
     2234 */
     2235DECL_INLINE_THROW(uint32_t)
     2236iemNativeEmitNegGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
     2237{
     2238#if defined(RT_ARCH_AMD64)
     2239    off = iemNativeEmitNegGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst);
     2240#elif defined(RT_ARCH_ARM64)
     2241    off = iemNativeEmitNegGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst);
     2242#else
     2243# error "Port me"
     2244#endif
     2245    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     2246    return off;
     2247}
     2248
     2249
     2250/**
     2251 * Emits code for two complement negation of a 32-bit GPR.
     2252 * @note bit 32 thru 63 are set to zero.
     2253 */
     2254DECL_FORCE_INLINE_THROW(uint32_t)
     2255iemNativeEmitNegGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst)
     2256{
     2257#if defined(RT_ARCH_AMD64)
     2258    /* neg Ev */
     2259    if (iGprDst >= 8)
     2260        pCodeBuf[off++] = X86_OP_REX_B;
     2261    pCodeBuf[off++] = 0xf7;
     2262    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 3, iGprDst & 7);
     2263
     2264#elif defined(RT_ARCH_ARM64)
     2265    /* sub dst, xzr, dst */
     2266    pCodeBuf[off++] = Armv8A64MkInstrNeg(iGprDst, false /*f64Bit*/);
     2267
     2268#else
     2269# error "Port me"
     2270#endif
     2271    return off;
     2272}
     2273
     2274
     2275/**
     2276 * Emits code for two complement negation of a 32-bit GPR.
     2277 * @note bit 32 thru 63 are set to zero.
     2278 */
     2279DECL_INLINE_THROW(uint32_t)
     2280iemNativeEmitNegGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)
     2281{
     2282#if defined(RT_ARCH_AMD64)
     2283    off = iemNativeEmitNegGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst);
     2284#elif defined(RT_ARCH_ARM64)
     2285    off = iemNativeEmitNegGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst);
    19742286#else
    19752287# error "Port me"
    … …  
    20512363 * Emits code for AND'ing two 32-bit GPRs.
    20522364 */
    2053 DECL_INLINE_THROW(uint32_t)
    2054 iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
     2365DECL_FORCE_INLINE(uint32_t)
     2366iemNativeEmitAndGpr32ByGpr32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
    20552367{
    20562368#if defined(RT_ARCH_AMD64)
    20572369    /* and Gv, Ev */
    2058     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
    20592370    if (iGprDst >= 8 || iGprSrc >= 8)
    2060         pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
    2061     pbCodeBuf[off++] = 0x23;
    2062     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
     2371        pCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);
     2372    pCodeBuf[off++] = 0x23;
     2373    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
    20632374    RT_NOREF(fSetFlags);
    20642375
    20652376#elif defined(RT_ARCH_ARM64)
    2066     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    20672377    if (!fSetFlags)
    2068         pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /*f64Bit*/);
    2069     else
    2070         pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc, false /*f64Bit*/);
    2071 
     2378        pCodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /*f64Bit*/);
     2379    else
     2380        pCodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc, false /*f64Bit*/);
     2381
     2382#else
     2383# error "Port me"
     2384#endif
     2385    return off;
     2386}
     2387
     2388
     2389/**
     2390 * Emits code for AND'ing two 32-bit GPRs.
     2391 */
     2392DECL_INLINE_THROW(uint32_t)
     2393iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)
     2394{
     2395#if defined(RT_ARCH_AMD64)
     2396    off = iemNativeEmitAndGpr32ByGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprDst, iGprSrc, fSetFlags);
     2397#elif defined(RT_ARCH_ARM64)
     2398    off = iemNativeEmitAndGpr32ByGpr32Ex(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, iGprSrc, fSetFlags);
    20722399#else
    20732400# error "Port me"
    … …  
    21472474/**
    21482475 * Emits code for AND'ing an 32-bit GPRs with a constant.
    2149  */
    2150 DECL_INLINE_THROW(uint32_t)
    2151 iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
     2476 * @note Bits 32 thru 63 in the destination will be zero after the operation.
     2477 * @note For ARM64 this only supports @a uImm values that can be expressed using
     2478 *       the two 6-bit immediates of the AND/ANDS instructions.  The caller must
     2479 *       make sure this is possible!
     2480 */
     2481DECL_FORCE_INLINE_THROW(uint32_t)
     2482iemNativeEmitAndGpr32ByImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
    21522483{
    21532484#if defined(RT_ARCH_AMD64)
    21542485    /* and Ev, imm */
    2155     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
    21562486    if (iGprDst >= 8)
    2157         pbCodeBuf[off++] = X86_OP_REX_B;
     2487        pCodeBuf[off++] = X86_OP_REX_B;
    21582488    if ((int32_t)uImm == (int8_t)uImm)
    21592489    {
    2160         pbCodeBuf[off++] = 0x83;
    2161         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
    2162         pbCodeBuf[off++] = (uint8_t)uImm;
    2163     }
    2164     else
    2165     {
    2166         pbCodeBuf[off++] = 0x81;
    2167         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
    2168         pbCodeBuf[off++] = RT_BYTE1(uImm);
    2169         pbCodeBuf[off++] = RT_BYTE2(uImm);
    2170         pbCodeBuf[off++] = RT_BYTE3(uImm);
    2171         pbCodeBuf[off++] = RT_BYTE4(uImm);
     2490        pCodeBuf[off++] = 0x83;
     2491        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
     2492        pCodeBuf[off++] = (uint8_t)uImm;
     2493    }
     2494    else
     2495    {
     2496        pCodeBuf[off++] = 0x81;
     2497        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
     2498        pCodeBuf[off++] = RT_BYTE1(uImm);
     2499        pCodeBuf[off++] = RT_BYTE2(uImm);
     2500        pCodeBuf[off++] = RT_BYTE3(uImm);
     2501        pCodeBuf[off++] = RT_BYTE4(uImm);
    21722502    }
    21732503    RT_NOREF(fSetFlags);
     2504
     2505#elif defined(RT_ARCH_ARM64)
     2506    uint32_t uImmR     = 0;
     2507    uint32_t uImmNandS = 0;
     2508    if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
     2509    {
     2510        if (!fSetFlags)
     2511            pCodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /*f64Bit*/);
     2512        else
     2513            pCodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /*f64Bit*/);
     2514    }
     2515    else
     2516# ifdef IEM_WITH_THROW_CATCH
     2517        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     2518# else
     2519        AssertReleaseFailedStmt(off = UINT32_MAX);
     2520# endif
     2521
     2522#else
     2523# error "Port me"
     2524#endif
     2525    return off;
     2526}
     2527
     2528
     2529/**
     2530 * Emits code for AND'ing an 32-bit GPRs with a constant.
     2531 * @note Bits 32 thru 63 in the destination will be zero after the operation.
     2532 */
     2533DECL_INLINE_THROW(uint32_t)
     2534iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)
     2535{
     2536#if defined(RT_ARCH_AMD64)
     2537    off = iemNativeEmitAndGpr32ByImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprDst, uImm, fSetFlags);
    21742538
    21752539#elif defined(RT_ARCH_ARM64)
    … …  
    22512615
    22522616
     2617/**
     2618 * Emits code for XOR'ing an 32-bit GPRs with a constant.
     2619 * @note Bits 32 thru 63 in the destination will be zero after the operation.
     2620 * @note For ARM64 this only supports @a uImm values that can be expressed using
     2621 *       the two 6-bit immediates of the EOR instructions.  The caller must make
     2622 *       sure this is possible!
     2623 */
     2624DECL_FORCE_INLINE_THROW(uint32_t)
     2625iemNativeEmitXorGpr32ByImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint32_t uImm)
     2626{
     2627#if defined(RT_ARCH_AMD64)
     2628    /* and Ev, imm */
     2629    if (iGprDst >= 8)
     2630        pCodeBuf[off++] = X86_OP_REX_B;
     2631    if ((int32_t)uImm == (int8_t)uImm)
     2632    {
     2633        pCodeBuf[off++] = 0x83;
     2634        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 6, iGprDst & 7);
     2635        pCodeBuf[off++] = (uint8_t)uImm;
     2636    }
     2637    else
     2638    {
     2639        pCodeBuf[off++] = 0x81;
     2640        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 6, iGprDst & 7);
     2641        pCodeBuf[off++] = RT_BYTE1(uImm);
     2642        pCodeBuf[off++] = RT_BYTE2(uImm);
     2643        pCodeBuf[off++] = RT_BYTE3(uImm);
     2644        pCodeBuf[off++] = RT_BYTE4(uImm);
     2645    }
     2646
     2647#elif defined(RT_ARCH_ARM64)
     2648    uint32_t uImmR     = 0;
     2649    uint32_t uImmNandS = 0;
     2650    if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
     2651        pCodeBuf[off++] = Armv8A64MkInstrEorImm(iGprDst, iGprDst, uImmNandS, uImmR, false /*f64Bit*/);
     2652    else
     2653# ifdef IEM_WITH_THROW_CATCH
     2654        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     2655# else
     2656        AssertReleaseFailedStmt(off = UINT32_MAX);
     2657# endif
     2658
     2659#else
     2660# error "Port me"
     2661#endif
     2662    return off;
     2663}
     2664
     2665
    22532666/*********************************************************************************************************************************
    22542667*   Shifting                                                                                                                     *
    … …  
    22582671 * Emits code for shifting a GPR a fixed number of bits to the left.
    22592672 */
    2260 DECL_INLINE_THROW(uint32_t)
    2261 iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
     2673DECL_FORCE_INLINE(uint32_t)
     2674iemNativeEmitShiftGprLeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
    22622675{
    22632676    Assert(cShift > 0 && cShift < 64);
    … …  
    22652678#if defined(RT_ARCH_AMD64)
    22662679    /* shl dst, cShift */
    2267     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
    2268     pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
     2680    pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
    22692681    if (cShift != 1)
    22702682    {
    2271         pbCodeBuf[off++] = 0xc1;
    2272         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
    2273         pbCodeBuf[off++] = cShift;
    2274     }
    2275     else
    2276     {
    2277         pbCodeBuf[off++] = 0xd1;
    2278         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
    2279     }
    2280 
    2281 #elif defined(RT_ARCH_ARM64)
    2282     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    2283     pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);
     2683        pCodeBuf[off++] = 0xc1;
     2684        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
     2685        pCodeBuf[off++] = cShift;
     2686    }
     2687    else
     2688    {
     2689        pCodeBuf[off++] = 0xd1;
     2690        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
     2691    }
     2692
     2693#elif defined(RT_ARCH_ARM64)
     2694    pCodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);
    22842695
    22852696#else
    22862697# error "Port me"
    22872698#endif
     2699    return off;
     2700}
     2701
     2702
     2703/**
     2704 * Emits code for shifting a GPR a fixed number of bits to the left.
     2705 */
     2706DECL_INLINE_THROW(uint32_t)
     2707iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
     2708{
     2709#if defined(RT_ARCH_AMD64)
     2710    off = iemNativeEmitShiftGprLeftEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
     2711#elif defined(RT_ARCH_ARM64)
     2712    off = iemNativeEmitShiftGprLeftEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
     2713#else
     2714# error "Port me"
     2715#endif
    22882716    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    22892717    return off;
    … …  
    22942722 * Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
    22952723 */
    2296 DECL_INLINE_THROW(uint32_t)
    2297 iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
     2724DECL_FORCE_INLINE(uint32_t)
     2725iemNativeEmitShiftGpr32LeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
    22982726{
    22992727    Assert(cShift > 0 && cShift < 32);
    … …  
    23012729#if defined(RT_ARCH_AMD64)
    23022730    /* shl dst, cShift */
    2303     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
    23042731    if (iGprDst >= 8)
    2305         pbCodeBuf[off++] = X86_OP_REX_B;
     2732        pCodeBuf[off++] = X86_OP_REX_B;
    23062733    if (cShift != 1)
    23072734    {
    2308         pbCodeBuf[off++] = 0xc1;
    2309         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
    2310         pbCodeBuf[off++] = cShift;
    2311     }
    2312     else
    2313     {
    2314         pbCodeBuf[off++] = 0xd1;
    2315         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
    2316     }
    2317 
    2318 #elif defined(RT_ARCH_ARM64)
    2319     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    2320     pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /*64Bit*/);
     2735        pCodeBuf[off++] = 0xc1;
     2736        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
     2737        pCodeBuf[off++] = cShift;
     2738    }
     2739    else
     2740    {
     2741        pCodeBuf[off++] = 0xd1;
     2742        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
     2743    }
     2744
     2745#elif defined(RT_ARCH_ARM64)
     2746    pCodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /*64Bit*/);
    23212747
    23222748#else
    23232749# error "Port me"
    23242750#endif
     2751    return off;
     2752}
     2753
     2754
     2755/**
     2756 * Emits code for shifting a 32-bit GPR a fixed number of bits to the left.
     2757 */
     2758DECL_INLINE_THROW(uint32_t)
     2759iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
     2760{
     2761#if defined(RT_ARCH_AMD64)
     2762    off = iemNativeEmitShiftGpr32LeftEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
     2763#elif defined(RT_ARCH_ARM64)
     2764    off = iemNativeEmitShiftGpr32LeftEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
     2765#else
     2766# error "Port me"
     2767#endif
    23252768    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    23262769    return off;
    … …  
    23312774 * Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
    23322775 */
    2333 DECL_INLINE_THROW(uint32_t)
    2334 iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
     2776DECL_FORCE_INLINE(uint32_t)
     2777iemNativeEmitShiftGprRightEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
    23352778{
    23362779    Assert(cShift > 0 && cShift < 64);
    … …  
    23382781#if defined(RT_ARCH_AMD64)
    23392782    /* shr dst, cShift */
    2340     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
    2341     pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
     2783    pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
    23422784    if (cShift != 1)
    23432785    {
    2344         pbCodeBuf[off++] = 0xc1;
    2345         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
    2346         pbCodeBuf[off++] = cShift;
    2347     }
    2348     else
    2349     {
    2350         pbCodeBuf[off++] = 0xd1;
    2351         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
    2352     }
    2353 
    2354 #elif defined(RT_ARCH_ARM64)
    2355     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    2356     pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
    2357 
     2786        pCodeBuf[off++] = 0xc1;
     2787        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
     2788        pCodeBuf[off++] = cShift;
     2789    }
     2790    else
     2791    {
     2792        pCodeBuf[off++] = 0xd1;
     2793        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
     2794    }
     2795
     2796#elif defined(RT_ARCH_ARM64)
     2797    pCodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);
     2798
     2799#else
     2800# error "Port me"
     2801#endif
     2802    return off;
     2803}
     2804
     2805
     2806/**
     2807 * Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.
     2808 */
     2809DECL_INLINE_THROW(uint32_t)
     2810iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
     2811{
     2812#if defined(RT_ARCH_AMD64)
     2813    off = iemNativeEmitShiftGprRightEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
     2814#elif defined(RT_ARCH_ARM64)
     2815    off = iemNativeEmitShiftGprRightEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
    23582816#else
    23592817# error "Port me"
    … …  
    23682826 * right.
    23692827 */
    2370 DECL_INLINE_THROW(uint32_t)
    2371 iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
     2828DECL_FORCE_INLINE(uint32_t)
     2829iemNativeEmitShiftGpr32RightEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
    23722830{
    23732831    Assert(cShift > 0 && cShift < 32);
    … …  
    23752833#if defined(RT_ARCH_AMD64)
    23762834    /* shr dst, cShift */
    2377     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
    23782835    if (iGprDst >= 8)
    2379         pbCodeBuf[off++] = X86_OP_REX_B;
     2836        pCodeBuf[off++] = X86_OP_REX_B;
    23802837    if (cShift != 1)
    23812838    {
    2382         pbCodeBuf[off++] = 0xc1;
    2383         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
    2384         pbCodeBuf[off++] = cShift;
    2385     }
    2386     else
    2387     {
    2388         pbCodeBuf[off++] = 0xd1;
    2389         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
    2390     }
    2391 
    2392 #elif defined(RT_ARCH_ARM64)
    2393     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    2394     pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /*64Bit*/);
     2839        pCodeBuf[off++] = 0xc1;
     2840        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
     2841        pCodeBuf[off++] = cShift;
     2842    }
     2843    else
     2844    {
     2845        pCodeBuf[off++] = 0xd1;
     2846        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
     2847    }
     2848
     2849#elif defined(RT_ARCH_ARM64)
     2850    pCodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /*64Bit*/);
    23952851
    23962852#else
    23972853# error "Port me"
    23982854#endif
    2399     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     2855    return off;
     2856}
     2857
     2858
     2859/**
     2860 * Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the
     2861 * right.
     2862 */
     2863DECL_INLINE_THROW(uint32_t)
     2864iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)
     2865{
     2866#if defined(RT_ARCH_AMD64)
     2867    off = iemNativeEmitShiftGpr32RightEx(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprDst, cShift);
     2868#elif defined(RT_ARCH_ARM64)
     2869    off = iemNativeEmitShiftGpr32RightEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprDst, cShift);
     2870#else
     2871# error "Port me"
     2872#endif
     2873    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     2874    return off;
     2875}
     2876
     2877
     2878/**
     2879 * Emits code for rotating a GPR a fixed number of bits to the left.
     2880 */
     2881DECL_FORCE_INLINE(uint32_t)
     2882iemNativeEmitRotateGprLeftEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprDst, uint8_t cShift)
     2883{
     2884    Assert(cShift > 0 && cShift < 64);
     2885
     2886#if defined(RT_ARCH_AMD64)
     2887    /* rol dst, cShift */
     2888    pCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;
     2889    if (cShift != 1)
     2890    {
     2891        pCodeBuf[off++] = 0xc1;
     2892        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
     2893        pCodeBuf[off++] = cShift;
     2894    }
     2895    else
     2896    {
     2897        pCodeBuf[off++] = 0xd1;
     2898        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
     2899    }
     2900
     2901#elif defined(RT_ARCH_ARM64)
     2902    pCodeBuf[off++] = Armv8A64MkInstrRorImm(iGprDst, iGprDst, cShift);
     2903
     2904#else
     2905# error "Port me"
     2906#endif
    24002907    return off;
    24012908}
    … …  
    24292936 * with conditional instruction.
    24302937 */
     2938DECL_FORCE_INLINE(uint32_t)
     2939iemNativeEmitCmpGprWithGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
     2940{
     2941#ifdef RT_ARCH_AMD64
     2942    /* cmp Gv, Ev */
     2943    pCodeBuf[off++] = X86_OP_REX_W | (iGprLeft >= 8 ? X86_OP_REX_R : 0) | (iGprRight >= 8 ? X86_OP_REX_B : 0);
     2944    pCodeBuf[off++] = 0x3b;
     2945    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
     2946
     2947#elif defined(RT_ARCH_ARM64)
     2948    pCodeBuf[off++] = Armv8A64MkInstrCmpReg(iGprLeft, iGprRight);
     2949
     2950#else
     2951# error "Port me!"
     2952#endif
     2953    return off;
     2954}
     2955
     2956
     2957/**
     2958 * Emits a compare of two 64-bit GPRs, settings status flags/whatever for use
     2959 * with conditional instruction.
     2960 */
    24312961DECL_INLINE_THROW(uint32_t)
    24322962iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
    24332963{
    24342964#ifdef RT_ARCH_AMD64
    2435     /* cmp Gv, Ev */
    2436     uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
    2437     pbCodeBuf[off++] = X86_OP_REX_W | (iGprLeft >= 8 ? X86_OP_REX_R : 0) | (iGprRight >= 8 ? X86_OP_REX_B : 0);
    2438     pbCodeBuf[off++] = 0x3b;
    2439     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
    2440     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    2441 
    2442 #elif defined(RT_ARCH_ARM64)
    2443     off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /*f64Bit*/);
    2444 
     2965    off = iemNativeEmitCmpGprWithGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprLeft, iGprRight);
     2966#elif defined(RT_ARCH_ARM64)
     2967    off = iemNativeEmitCmpGprWithGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprLeft, iGprRight);
    24452968#else
    24462969# error "Port me!"
    24472970#endif
     2971    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    24482972    return off;
    24492973}
    … …  
    24542978 * with conditional instruction.
    24552979 */
     2980DECL_FORCE_INLINE(uint32_t)
     2981iemNativeEmitCmpGpr32WithGprEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
     2982{
     2983#ifdef RT_ARCH_AMD64
     2984    /* cmp Gv, Ev */
     2985    if (iGprLeft >= 8 || iGprRight >= 8)
     2986        pCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) | (iGprRight >= 8 ? X86_OP_REX_B : 0);
     2987    pCodeBuf[off++] = 0x3b;
     2988    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
     2989
     2990#elif defined(RT_ARCH_ARM64)
     2991    pCodeBuf[off++] = Armv8A64MkInstrCmpReg(iGprLeft, iGprRight, false /*f64Bit*/);
     2992
     2993#else
     2994# error "Port me!"
     2995#endif
     2996    return off;
     2997}
     2998
     2999
     3000/**
     3001 * Emits a compare of two 32-bit GPRs, settings status flags/whatever for use
     3002 * with conditional instruction.
     3003 */
    24563004DECL_INLINE_THROW(uint32_t)
    24573005iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
    24583006{
    24593007#ifdef RT_ARCH_AMD64
    2460     /* cmp Gv, Ev */
    2461     uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
    2462     if (iGprLeft >= 8 || iGprRight >= 8)
    2463         pbCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) | (iGprRight >= 8 ? X86_OP_REX_B : 0);
    2464     pbCodeBuf[off++] = 0x3b;
    2465     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);
    2466     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    2467 
    2468 #elif defined(RT_ARCH_ARM64)
    2469     off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /*f64Bit*/);
    2470 
     3008    off = iemNativeEmitCmpGpr32WithGprEx(iemNativeInstrBufEnsure(pReNative, off, 3), off, iGprLeft, iGprRight);
     3009#elif defined(RT_ARCH_ARM64)
     3010    off = iemNativeEmitCmpGpr32WithGprEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, iGprLeft, iGprRight);
    24713011#else
    24723012# error "Port me!"
    24733013#endif
     3014    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    24743015    return off;
    24753016}
    … …  
    25483089 * Emits a compare of a 32-bit GPR with a constant value, settings status
    25493090 * flags/whatever for use with conditional instruction.
     3091 *
     3092 * @note On ARM64 the @a uImm value must be in the range 0x000..0xfff or that
     3093 *       shifted 12 bits to the left (e.g. 0x1000..0xfff0000 with the lower 12
     3094 *       bits all zero).  Will release assert or throw exception if the caller
     3095 *       violates this restriction.
     3096 */
     3097DECL_FORCE_INLINE_THROW(uint32_t)
     3098iemNativeEmitCmpGpr32WithImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft, uint32_t uImm)
     3099{
     3100#ifdef RT_ARCH_AMD64
     3101    if (iGprLeft >= 8)
     3102        pCodeBuf[off++] = X86_OP_REX_B;
     3103    if (uImm <= UINT32_C(0xff))
     3104    {
     3105        /* cmp Ev, Ib */
     3106        pCodeBuf[off++] = 0x83;
     3107        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
     3108        pCodeBuf[off++] = (uint8_t)uImm;
     3109    }
     3110    else
     3111    {
     3112        /* cmp Ev, imm */
     3113        pCodeBuf[off++] = 0x81;
     3114        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
     3115        pCodeBuf[off++] = RT_BYTE1(uImm);
     3116        pCodeBuf[off++] = RT_BYTE2(uImm);
     3117        pCodeBuf[off++] = RT_BYTE3(uImm);
     3118        pCodeBuf[off++] = RT_BYTE4(uImm);
     3119    }
     3120
     3121#elif defined(RT_ARCH_ARM64)
     3122    /** @todo guess there are clevere things we can do here...   */
     3123    if (uImm < _4K)
     3124        pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
     3125                                                      false /*64Bit*/, true /*fSetFlags*/);
     3126    else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)
     3127        pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
     3128                                                      false /*64Bit*/, true /*fSetFlags*/, true /*fShift12*/);
     3129    else
     3130# ifdef IEM_WITH_THROW_CATCH
     3131        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     3132# else
     3133        AssertReleaseFailedStmt(off = UINT32_MAX);
     3134# endif
     3135
     3136#else
     3137# error "Port me!"
     3138#endif
     3139    return off;
     3140}
     3141
     3142
     3143/**
     3144 * Emits a compare of a 32-bit GPR with a constant value, settings status
     3145 * flags/whatever for use with conditional instruction.
    25503146 */
    25513147DECL_INLINE_THROW(uint32_t)
    … …  
    25533149{
    25543150#ifdef RT_ARCH_AMD64
    2555     uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
    2556     if (iGprLeft >= 8)
    2557         pbCodeBuf[off++] = X86_OP_REX_B;
    2558     if (uImm <= UINT32_C(0xff))
    2559     {
    2560         /* cmp Ev, Ib */
    2561         pbCodeBuf[off++] = 0x83;
    2562         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
    2563         pbCodeBuf[off++] = (uint8_t)uImm;
    2564     }
    2565     else
    2566     {
    2567         /* cmp Ev, imm */
    2568         pbCodeBuf[off++] = 0x81;
    2569         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
    2570         IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    2571         pbCodeBuf[off++] = RT_BYTE1(uImm);
    2572         pbCodeBuf[off++] = RT_BYTE2(uImm);
    2573         pbCodeBuf[off++] = RT_BYTE3(uImm);
    2574         pbCodeBuf[off++] = RT_BYTE4(uImm);
    2575     }
     3151    off = iemNativeEmitCmpGpr32WithImmEx(iemNativeInstrBufEnsure(pReNative, off, 7), off, iGprLeft, uImm);
    25763152
    25773153#elif defined(RT_ARCH_ARM64)
    … …  
    26143190 * Emits a JMP rel32 / B imm19 to the given label.
    26153191 */
    2616 DECL_INLINE_THROW(uint32_t)
    2617 iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
     3192DECL_FORCE_INLINE_THROW(uint32_t)
     3193iemNativeEmitJmpToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t idxLabel)
    26183194{
    26193195    Assert(idxLabel < pReNative->cLabels);
    26203196
    26213197#ifdef RT_ARCH_AMD64
    2622     uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
    26233198    if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
    26243199    {
    … …  
    26263201        if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)
    26273202        {
    2628             pbCodeBuf[off++] = 0xeb;                /* jmp rel8 */
    2629             pbCodeBuf[off++] = (uint8_t)offRel;
     3203            pCodeBuf[off++] = 0xeb;                 /* jmp rel8 */
     3204            pCodeBuf[off++] = (uint8_t)offRel;
    26303205        }
    26313206        else
    26323207        {
    26333208            offRel -= 3;
    2634             pbCodeBuf[off++] = 0xe9;                /* jmp rel32 */
    2635             pbCodeBuf[off++] = RT_BYTE1(offRel);
    2636             pbCodeBuf[off++] = RT_BYTE2(offRel);
    2637             pbCodeBuf[off++] = RT_BYTE3(offRel);
    2638             pbCodeBuf[off++] = RT_BYTE4(offRel);
     3209            pCodeBuf[off++] = 0xe9;                 /* jmp rel32 */
     3210            pCodeBuf[off++] = RT_BYTE1(offRel);
     3211            pCodeBuf[off++] = RT_BYTE2(offRel);
     3212            pCodeBuf[off++] = RT_BYTE3(offRel);
     3213            pCodeBuf[off++] = RT_BYTE4(offRel);
    26393214        }
    26403215    }
    26413216    else
    26423217    {
    2643         pbCodeBuf[off++] = 0xe9;                    /* jmp rel32 */
     3218        pCodeBuf[off++] = 0xe9;                     /* jmp rel32 */
    26443219        iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
    2645         pbCodeBuf[off++] = 0xfe;
    2646         pbCodeBuf[off++] = 0xff;
    2647         pbCodeBuf[off++] = 0xff;
    2648         pbCodeBuf[off++] = 0xff;
    2649     }
    2650     pbCodeBuf[off++] = 0xcc;                        /* int3 poison */
    2651 
    2652 #elif defined(RT_ARCH_ARM64)
    2653     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
     3220        pCodeBuf[off++] = 0xfe;
     3221        pCodeBuf[off++] = 0xff;
     3222        pCodeBuf[off++] = 0xff;
     3223        pCodeBuf[off++] = 0xff;
     3224    }
     3225    pCodeBuf[off++] = 0xcc;                         /* int3 poison */
     3226
     3227#elif defined(RT_ARCH_ARM64)
    26543228    if (pReNative->paLabels[idxLabel].off != UINT32_MAX)
    2655         pu32CodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxLabel].off - off);
     3229        pCodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxLabel].off - off);
    26563230    else
    26573231    {
    26583232        iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm26At0);
    2659         pu32CodeBuf[off++] = Armv8A64MkInstrB(-1);
    2660     }
    2661 
     3233        pCodeBuf[off++] = Armv8A64MkInstrB(-1);
     3234    }
     3235
     3236#else
     3237# error "Port me!"
     3238#endif
     3239    return off;
     3240}
     3241
     3242
     3243/**
     3244 * Emits a JMP rel32 / B imm19 to the given label.
     3245 */
     3246DECL_INLINE_THROW(uint32_t)
     3247iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)
     3248{
     3249#ifdef RT_ARCH_AMD64
     3250    off = iemNativeEmitJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 6), off, idxLabel);
     3251#elif defined(RT_ARCH_ARM64)
     3252    off = iemNativeEmitJmpToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 1), off, idxLabel);
    26623253#else
    26633254# error "Port me!"
    … …  
    27093300 * Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
    27103301 */
     3302DECL_FORCE_INLINE_THROW(uint32_t)
     3303iemNativeEmitJccToLabelEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
     3304                          uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
     3305{
     3306    Assert(idxLabel < pReNative->cLabels);
     3307
     3308#ifdef RT_ARCH_AMD64
     3309    /* jcc rel32 */
     3310    pCodeBuf[off++] = 0x0f;
     3311    pCodeBuf[off++] = (uint8_t)enmCond | 0x80;
     3312    iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
     3313    pCodeBuf[off++] = 0x00;
     3314    pCodeBuf[off++] = 0x00;
     3315    pCodeBuf[off++] = 0x00;
     3316    pCodeBuf[off++] = 0x00;
     3317
     3318#elif defined(RT_ARCH_ARM64)
     3319    iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
     3320    pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
     3321
     3322#else
     3323# error "Port me!"
     3324#endif
     3325    return off;
     3326}
     3327
     3328
     3329/**
     3330 * Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
     3331 */
    27113332DECL_INLINE_THROW(uint32_t)
    27123333iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)
    27133334{
    2714     Assert(idxLabel < pReNative->cLabels);
    2715 
    2716 #ifdef RT_ARCH_AMD64
    2717     /* jcc rel32 */
    2718     uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
    2719     pbCodeBuf[off++] = 0x0f;
    2720     pbCodeBuf[off++] = (uint8_t)enmCond | 0x80;
    2721     iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);
    2722     pbCodeBuf[off++] = 0x00;
    2723     pbCodeBuf[off++] = 0x00;
    2724     pbCodeBuf[off++] = 0x00;
    2725     pbCodeBuf[off++] = 0x00;
    2726 
    2727 #elif defined(RT_ARCH_ARM64)
    2728     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    2729     iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);
    2730     pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);
    2731 
     3335#ifdef RT_ARCH_AMD64
     3336    off = iemNativeEmitJccToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 6), off, idxLabel, enmCond);
     3337#elif defined(RT_ARCH_ARM64)
     3338    off = iemNativeEmitJccToLabelEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 1), off, idxLabel, enmCond);
    27323339#else
    27333340# error "Port me!"
    … …  
    29133520 *       bytes long.
    29143521 */
     3522DECL_FORCE_INLINE(uint32_t)
     3523iemNativeEmitJccToFixedEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t offTarget, IEMNATIVEINSTRCOND enmCond)
     3524{
     3525#ifdef RT_ARCH_AMD64
     3526    /* jcc rel8 / rel32 */
     3527    int32_t offDisp = (int32_t)(offTarget - (off + 2));
     3528    if (offDisp < 128 && offDisp >= -128)
     3529    {
     3530        pCodeBuf[off++] = (uint8_t)enmCond | 0x70;
     3531        pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
     3532    }
     3533    else
     3534    {
     3535        offDisp -= 4;
     3536        pCodeBuf[off++] = 0x0f;
     3537        pCodeBuf[off++] = (uint8_t)enmCond | 0x80;
     3538        pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
     3539        pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
     3540        pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
     3541        pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
     3542    }
     3543
     3544#elif defined(RT_ARCH_ARM64)
     3545    pCodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, (int32_t)(offTarget - off));
     3546
     3547#else
     3548# error "Port me!"
     3549#endif
     3550    return off;
     3551}
     3552
     3553
     3554/**
     3555 * Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.
     3556 *
     3557 * @note The @a offTarget is the absolute jump target (unit is IEMNATIVEINSTR).
     3558 *
     3559 *       Only use hardcoded jumps forward when emitting for exactly one
     3560 *       platform, otherwise apply iemNativeFixupFixedJump() to ensure hitting
     3561 *       the right target address on all platforms!
     3562 *
     3563 *       Please also note that on x86 it is necessary pass off + 256 or higher
     3564 *       for @a offTarget one believe the intervening code is more than 127
     3565 *       bytes long.
     3566 */
    29153567DECL_INLINE_THROW(uint32_t)
    29163568iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget, IEMNATIVEINSTRCOND enmCond)
    29173569{
    29183570#ifdef RT_ARCH_AMD64
    2919     /* jcc rel8 / rel32 */
    2920     uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
    2921     int32_t         offDisp   = (int32_t)(offTarget - (off + 2));
    2922     if (offDisp < 128 && offDisp >= -128)
    2923     {
    2924         pbCodeBuf[off++] = (uint8_t)enmCond | 0x70;
    2925         pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
    2926     }
    2927     else
    2928     {
    2929         offDisp -= 4;
    2930         pbCodeBuf[off++] = 0x0f;
    2931         pbCodeBuf[off++] = (uint8_t)enmCond | 0x80;
    2932         pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
    2933         pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
    2934         pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
    2935         pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
    2936     }
    2937 
    2938 #elif defined(RT_ARCH_ARM64)
    2939     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    2940     pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, (int32_t)(offTarget - off));
    2941 
     3571    off = iemNativeEmitJccToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 6), off, offTarget, enmCond);
     3572#elif defined(RT_ARCH_ARM64)
     3573    off = iemNativeEmitJccToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, offTarget, enmCond);
    29423574#else
    29433575# error "Port me!"
    … …  
    30213653 * See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
    30223654 */
     3655DECL_FORCE_INLINE(uint32_t) iemNativeEmitJmpToFixedEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint32_t offTarget)
     3656{
     3657#ifdef RT_ARCH_AMD64
     3658    /* jmp rel8 or rel32 */
     3659    int32_t offDisp = offTarget - (off + 2);
     3660    if (offDisp < 128 && offDisp >= -128)
     3661    {
     3662        pCodeBuf[off++] = 0xeb;
     3663        pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
     3664    }
     3665    else
     3666    {
     3667        offDisp -= 3;
     3668        pCodeBuf[off++] = 0xe9;
     3669        pCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
     3670        pCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
     3671        pCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
     3672        pCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
     3673    }
     3674
     3675#elif defined(RT_ARCH_ARM64)
     3676    pCodeBuf[off++] = Armv8A64MkInstrB((int32_t)(offTarget - off));
     3677
     3678#else
     3679# error "Port me!"
     3680#endif
     3681    return off;
     3682}
     3683
     3684
     3685/**
     3686 * Emits a JMP rel32/rel8 / B imm26 with a fixed displacement.
     3687 *
     3688 * See notes on @a offTarget in the iemNativeEmitJccToFixed() documentation.
     3689 */
    30233690DECL_INLINE_THROW(uint32_t) iemNativeEmitJmpToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t offTarget)
    30243691{
    30253692#ifdef RT_ARCH_AMD64
    3026     /* jmp rel8 or rel32 */
    3027     uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);
    3028     int32_t         offDisp   = offTarget - (off + 2);
    3029     if (offDisp < 128 && offDisp >= -128)
    3030     {
    3031         pbCodeBuf[off++] = 0xeb;
    3032         pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
    3033     }
    3034     else
    3035     {
    3036         offDisp -= 3;
    3037         pbCodeBuf[off++] = 0xe9;
    3038         pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
    3039         pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
    3040         pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
    3041         pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
    3042     }
    3043 
    3044 #elif defined(RT_ARCH_ARM64)
    3045     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
    3046     pu32CodeBuf[off++] = Armv8A64MkInstrB((int32_t)(offTarget - off));
    3047 
     3693    off = iemNativeEmitJmpToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 5), off, offTarget);
     3694#elif defined(RT_ARCH_ARM64)
     3695    off = iemNativeEmitJmpToFixedEx(iemNativeInstrBufEnsure(pReNative, off, 1), off, offTarget);
    30483696#else
    30493697# error "Port me!"
    … …  
    32623910 * Emits a test for any of the bits from @a fBits in the lower 8 bits of
    32633911 * @a iGprSrc, setting CPU flags accordingly.
     3912 *
     3913 * @note For ARM64 this only supports @a fBits values that can be expressed
     3914 *       using the two 6-bit immediates of the ANDS instruction.  The caller
     3915 *       must make sure this is possible!
     3916 */
     3917DECL_FORCE_INLINE_THROW(uint32_t)
     3918iemNativeEmitTestAnyBitsInGpr8Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprSrc, uint8_t fBits)
     3919{
     3920    Assert(fBits != 0);
     3921
     3922    /* test Eb, imm8 */
     3923#ifdef RT_ARCH_AMD64
     3924    if (iGprSrc >= 4)
     3925        pCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
     3926    pCodeBuf[off++] = 0xf6;
     3927    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
     3928    pCodeBuf[off++] = fBits;
     3929
     3930#elif defined(RT_ARCH_ARM64)
     3931    /* ands xzr, src, [tmp|#imm] */
     3932    uint32_t uImmR     = 0;
     3933    uint32_t uImmNandS = 0;
     3934    if (Armv8A64ConvertMask32ToImmRImmS(fBits, &uImmNandS, &uImmR))
     3935        pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprSrc, uImmNandS, uImmR, false /*f64Bit*/);
     3936    else
     3937# ifdef IEM_WITH_THROW_CATCH
     3938        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
     3939# else
     3940        AssertReleaseFailedStmt(off = UINT32_MAX);
     3941# endif
     3942
     3943#else
     3944# error "Port me!"
     3945#endif
     3946    return off;
     3947}
     3948
     3949
     3950/**
     3951 * Emits a test for any of the bits from @a fBits in the lower 8 bits of
     3952 * @a iGprSrc, setting CPU flags accordingly.
    32643953 */
    32653954DECL_INLINE_THROW(uint32_t)
    … …  
    32693958
    32703959#ifdef RT_ARCH_AMD64
    3271     /* test Eb, imm8 */
    3272     uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
    3273     if (iGprSrc >= 4)
    3274         pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
    3275     pbCodeBuf[off++] = 0xf6;
    3276     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
    3277     pbCodeBuf[off++] = fBits;
    3278 
    3279 #elif defined(RT_ARCH_ARM64)
    3280 
     3960    off = iemNativeEmitTestAnyBitsInGpr8Ex(iemNativeInstrBufEnsure(pReNative, off, 4), off, iGprSrc, fBits);
     3961
     3962#elif defined(RT_ARCH_ARM64)
    32813963    /* ands xzr, src, [tmp|#imm] */
    32823964    uint32_t uImmR     = 0;
Note: See TracChangeset for help on using the changeset viewer.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette