VirtualBox

Changeset 102720 in vbox for trunk/src/VBox/VMM


Ignore:
Timestamp:
Dec 28, 2023 12:28:00 AM (13 months ago)
Author:
vboxsync
Message:

VMM/IEM: Enabled the memmap TLB code on x86. bugref:10371

Location:
trunk/src/VBox/VMM
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

    r102717 r102720  
    67966796    if (idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS)
    67976797    {
     6798        Assert(fInitialized);
    67986799        int32_t const offDispBp = iemNativeStackCalcBpDisp(idxStackSlot);
    67996800        switch (pReNative->Core.aVars[idxVar].cbVar)
    … …  
    1089410895    off = iemNativeRegMoveAndFreeAndFlushAtCall(pReNative, off, 0 /* vacate all non-volatile regs */);
    1089510896
     10897    /* The bUnmapInfo variable will get a register in the tlb-hit code path,
     10898       while the tlb-miss codepath will temporarily put it on the stack.
     10899       Set the the type to stack here so we don't need to do it twice below. */
     10900    iemNativeVarSetKindToStack(pReNative, idxVarUnmapInfo);
     10901
    1089610902    /*
    1089710903     * Define labels and allocate the result register (trying for the return
    … …  
    1090810914     * First we try to go via the TLB.
    1090910915     */
    10910 #if defined(RT_ARCH_AMD64) && 0 /* untested code sketch */
    10911     uint8_t const   idxRegPtr       = iemNativeVarRegisterAcquire(pReNative, idxVarGCPtrMem, &off,
    10912                                                                   true /*fInitialized*/, IEMNATIVE_CALL_ARG2_GREG);
    10913     uint8_t const   idxRegSegBase   = iSegReg == UINT8_MAX ? UINT8_MAX
    10914                                     : iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(iSegReg));
    10915     uint8_t const   idxRegSegLimit  = iSegReg == UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT
    10916                                     ? UINT8_MAX
    10917                                     : iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(iSegReg));
    10918     uint8_t const   idxRegSegAttrib = iSegReg == UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT
    10919                                     ? UINT8_MAX
    10920                                     : iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_ATTRIB(iSegReg));
    10921     uint8_t const   idxReg1         = iemNativeRegAllocTmp(pReNative, &off);
    10922     uint8_t const   idxReg2         = iemNativeRegAllocTmp(pReNative, &off);
    10923     uint8_t * const pbCodeBuf       = iemNativeInstrBufEnsure(pReNative, off, 256);
     10916#if defined(RT_ARCH_AMD64) && 1
     10917    /* Immediate addresses are a pain if they wrap around the 32-bit space, so
     10918       always fall back for those.  In 64-bit mode this is less like. */
     10919    if (   pReNative->Core.aVars[idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate
     10920        || (uint64_t)((uint64_t)UINT32_MAX - pReNative->Core.aVars[idxVarGCPtrMem].u.uValue) <= (uint32_t)UINT32_MAX)
     10921    {
     10922        uint8_t const   idxRegPtr       = pReNative->Core.aVars[idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate
     10923                                        ? iemNativeVarRegisterAcquire(pReNative, idxVarGCPtrMem, &off,
     10924                                                                      true /*fInitialized*/, IEMNATIVE_CALL_ARG2_GREG)
     10925                                        : UINT8_MAX;
     10926        uint64_t const  uAbsPtr         = pReNative->Core.aVars[idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate
     10927                                        ? UINT32_MAX
     10928                                        : pReNative->Core.aVars[idxVarGCPtrMem].u.uValue;
     10929        uint8_t const   idxRegSegBase   = iSegReg == UINT8_MAX ? UINT8_MAX
     10930                                        : iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(iSegReg));
     10931        uint8_t const   idxRegSegLimit  = iSegReg == UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT
     10932                                        ? UINT8_MAX
     10933                                        : iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(iSegReg));
     10934        uint8_t const   idxRegSegAttrib = iSegReg == UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT
     10935                                        ? UINT8_MAX
     10936                                        : iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_ATTRIB(iSegReg));
     10937        uint8_t const   idxReg1         = iemNativeRegAllocTmp(pReNative, &off);
     10938        uint8_t const   idxReg2         = iemNativeRegAllocTmp(pReNative, &off);
     10939        uint8_t * const pbCodeBuf       = iemNativeInstrBufEnsure(pReNative, off, 256);
     10940
     10941        /*
     10942         * 1. Segmentation.
     10943         *
     10944         * 1a. Check segment limit and attributes if non-flat 32-bit code.  This is complicated.
     10945         */
     10946        if (iSegReg != UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT)
     10947        {
     10948            /* If we're accessing more than one byte, put the last address we'll be
     10949               accessing in idxReg2 (64-bit). */
     10950            if (cbMem > 1 && idxRegPtr != UINT8_MAX)
     10951            {
     10952# if 1
     10953                Assert(cbMem - 1 <= 127);
     10954                /* mov reg2, regptr */
     10955                off = iemNativeEmitLoadGprFromGpr32Ex(pbCodeBuf, off, idxReg2, idxRegPtr);
     10956                /* add reg2, cbMem-1 */
     10957                off = iemNativeEmitAddGpr32Imm8Ex(pbCodeBuf, off, idxReg2, cbMem - 1);
     10958# else
     10959                /* mov reg2, cbMem-1 */
     10960                off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxReg2, cbMem - 1);
     10961                /* add reg2, regptr */
     10962                off = iemNativeEmitAddTwoGprsEx(pbCodeBuf, off, idxReg2, idxRegPtr);
     10963# endif
     10964            }
     10965
     10966            /* Check that we've got a segment loaded and that it allows the access.
     10967               For write access this means a writable data segment.
     10968               For read-only accesses this means a readable code segment or any data segment. */
     10969            if (fAccess & IEM_ACCESS_TYPE_WRITE)
     10970            {
     10971                uint32_t const fMustBe1 = X86DESCATTR_P        | X86DESCATTR_DT    | X86_SEL_TYPE_WRITE;
     10972                uint32_t const fMustBe0 = X86DESCATTR_UNUSABLE | X86_SEL_TYPE_CODE;
     10973                /* mov reg1, must1|must0 */
     10974                off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxReg1, fMustBe1 | fMustBe0);
     10975                /* and reg1, segattrs */
     10976                off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxReg1, idxRegSegAttrib);
     10977                /* cmp reg1, must1 */
     10978                off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, fMustBe1);
     10979                /* jne tlbmiss */
     10980                off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
     10981            }
     10982            else
     10983            {
     10984                /*  U  | !P |!DT |!CD | RW |
     10985                    16 |  8 |  4 |  3 |  1 |
     10986                  -------------------------------
     10987                    0  |  0 |  0 |  0 |  0 | execute-only code segment. - must be excluded
     10988                    0  |  0 |  0 |  0 |  1 | execute-read code segment.
     10989                    0  |  0 |  0 |  1 |  0 | read-only data segment.
     10990                    0  |  0 |  0 |  1 |  1 | read-write data segment.   - last valid combination
     10991                */
     10992                /* mov reg1, relevant attributes  */
     10993                off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxReg1,
     10994                                                    X86DESCATTR_UNUSABLE | X86DESCATTR_P | X86DESCATTR_DT
     10995                                                  | X86_SEL_TYPE_CODE    | X86_SEL_TYPE_WRITE);
     10996                /* and reg1, segattrs */
     10997                off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxReg1, idxRegSegAttrib);
     10998                /* xor reg1, X86DESCATTR_P | X86DESCATTR_DT | X86_SEL_TYPE_CODE ; place C=1 RW=0 at the bottom & limit the range.
     10999                                                ; EO-code=0,  ER-code=2, RO-data=8, RW-data=10 */
     11000                off = iemNativeEmitXorGpr32ByImmEx(pbCodeBuf, off, idxReg1, X86DESCATTR_P | X86DESCATTR_DT | X86_SEL_TYPE_CODE);
     11001                /* sub reg1, X86_SEL_TYPE_WRITE ; EO-code=-2, ER-code=0, RO-data=6, RW-data=8 */
     11002                off = iemNativeEmitSubGpr32ImmEx(pbCodeBuf, off, idxReg1, X86_SEL_TYPE_WRITE /* ER-code */);
     11003                /* cmp reg1, X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE */
     11004                AssertCompile(X86_SEL_TYPE_CODE == 8);
     11005                off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, X86_SEL_TYPE_CODE);
     11006                /* ja  tlbmiss */
     11007                off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
     11008            }
     11009
     11010            /*
     11011             * Check the limit.  If this is a write access, we know that it's a
     11012             * data segment and includes the expand_down bit.  For read-only accesses
     11013             * we need to check that code/data=0 and expanddown=1 before continuing.
     11014             */
     11015            uint32_t offFixupCheckExpandDown;
     11016            if (fAccess & IEM_ACCESS_TYPE_WRITE)
     11017            {
     11018                /* test segattrs, X86_SEL_TYPE_DOWN */
     11019                AssertCompile(X86_SEL_TYPE_DOWN < 128);
     11020                off = iemNativeEmitTestAnyBitsInGpr8Ex(pbCodeBuf, off, idxRegSegAttrib, X86_SEL_TYPE_DOWN);
     11021                /* jnz  check_expand_down */
     11022                offFixupCheckExpandDown = off;
     11023                off = iemNativeEmitJccToFixedEx(pbCodeBuf, off, off /*ASSUMES rel8 suffices*/, kIemNativeInstrCond_ne);
     11024            }
     11025            else
     11026            {
     11027                /* mov reg1, segattrs */
     11028                off = iemNativeEmitLoadGprFromGpr32Ex(pbCodeBuf, off, idxReg1, idxRegSegAttrib);
     11029                /* and reg1, code | down */
     11030                off = iemNativeEmitAndGpr32ByImmEx(pbCodeBuf, off, idxReg1, X86_SEL_TYPE_CODE | X86_SEL_TYPE_DOWN);
     11031                /* cmp reg1, down */
     11032                off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, X86_SEL_TYPE_DOWN);
     11033                /* je check_expand_down */
     11034                offFixupCheckExpandDown = off;
     11035                off = iemNativeEmitJccToFixedEx(pbCodeBuf, off, off /*ASSUMES rel8 suffices*/, kIemNativeInstrCond_e);
     11036            }
     11037
     11038            /* expand_up:
     11039               cmp  seglim, regptr/reg2/imm */
     11040            if (idxRegPtr != UINT8_MAX)
     11041                off = iemNativeEmitCmpGprWithGprEx(pbCodeBuf, off, idxRegSegLimit, cbMem > 1 ? idxReg2 : idxRegPtr);
     11042            else
     11043                off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxRegSegLimit, (uint32_t)uAbsPtr);
     11044            /* jbe  tlbmiss */
     11045            off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_be);
     11046            /* jmp  limitdone */
     11047            uint32_t const offFixupLimitDone = off;
     11048            off = iemNativeEmitJmpToFixedEx(pbCodeBuf, off, off /*ASSUMES rel8 suffices*/);
     11049
     11050            /* check_expand_down: ; complicted! */
     11051            iemNativeFixupFixedJump(pReNative, offFixupCheckExpandDown, off);
    1092411052    pbCodeBuf[off++] = 0xcc;
    10925 
    10926     /*
    10927      * 1. Segmentation.
    10928      *
    10929      * 1a. Check segment limit and attributes if non-flat 32-bit code.  This is complicated.
    10930      */
    10931     if (iSegReg != UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT)
    10932     {
    10933         /* If we're accessing more than one byte, put the last address we'll be
    10934            accessing in idxReg2 (64-bit). */
    10935         if (cbMem > 1)
     11053            /* cmp  seglim, regptr */
     11054            if (idxRegPtr != UINT8_MAX)
     11055                off = iemNativeEmitCmpGprWithGprEx(pbCodeBuf, off, idxRegSegLimit, idxRegPtr);
     11056            else
     11057                off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxRegSegLimit, (uint32_t)uAbsPtr);
     11058            /* ja  tlbmiss */
     11059            off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
     11060            /* mov  reg1, X86DESCATTR_D (0x4000) */
     11061            off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, X86DESCATTR_D);
     11062            /* and  reg1, segattr */
     11063            off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxReg1, idxRegSegAttrib);
     11064            /* xor  reg1, X86DESCATTR_D */
     11065            off = iemNativeEmitXorGpr32ByImmEx(pbCodeBuf, off, idxReg1, X86DESCATTR_D);
     11066            /* shl  reg1, 2 (16 - 14) */
     11067            AssertCompile((X86DESCATTR_D << 2) == UINT32_C(0x10000));
     11068            off = iemNativeEmitShiftGpr32LeftEx(pbCodeBuf, off, idxReg1, 2);
     11069            /* dec  reg1 (=> 0xffff if D=0; 0xffffffff if D=1) */
     11070            off = iemNativeEmitSubGpr32ImmEx(pbCodeBuf, off, idxReg1, 1);
     11071            /* cmp  reg1, reg2 (64-bit) / imm (32-bit) */
     11072            if (idxRegPtr != UINT8_MAX)
     11073                off = iemNativeEmitCmpGprWithGprEx(pbCodeBuf, off, idxReg1, idxReg2);
     11074            else
     11075                off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, (uint32_t)(uAbsPtr + cbMem - 1));
     11076            /* jbe  tlbmiss */
     11077            off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_be);
     11078
     11079            /* limitdone: */
     11080            iemNativeFixupFixedJump(pReNative, offFixupLimitDone, off);
     11081        }
     11082
     11083        /* 1b. Add the segment base. We use idxRegMemResult for the ptr register if this step is
     11084               required or if the address is a constant (simplicity). */
     11085        uint8_t const idxRegFlatPtr = iSegReg != UINT8_MAX || idxRegPtr == UINT8_MAX ? idxRegMemResult : idxRegPtr;
     11086        if (iSegReg != UINT8_MAX)
    1093611087        {
    10937             /* mov reg2, cbMem-1 */
    10938             off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxReg2, cbMem - 1);
    10939             /* add reg2, regptr */
    10940             off = iemNativeEmitAddTwoGprsEx(pbCodeBuf, off, idxReg2, idxRegPtr);
     11088            /** @todo this can be done using LEA as well. */
     11089            if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
     11090            {
     11091                Assert(iSegReg >= X86_SREG_FS);
     11092                /* mov regflat, regptr/imm */
     11093                if (idxRegPtr != UINT8_MAX)
     11094                    off = iemNativeEmitLoadGprFromGprEx(pbCodeBuf, off, idxRegFlatPtr, idxRegPtr);
     11095                else
     11096                    off = iemNativeEmitLoadGprImmEx(pbCodeBuf, off, idxRegFlatPtr, uAbsPtr);
     11097                /* add regflat, seg.base */
     11098                off = iemNativeEmitAddTwoGprsEx(pbCodeBuf, off, idxRegFlatPtr, idxRegSegBase);
     11099            }
     11100            else
     11101            {
     11102                /* mov regflat, regptr/imm */
     11103                if (idxRegPtr != UINT8_MAX)
     11104                    off = iemNativeEmitLoadGprFromGpr32Ex(pbCodeBuf, off, idxRegFlatPtr, idxRegPtr);
     11105                else
     11106                    off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxRegFlatPtr, uAbsPtr);
     11107                /* add regflat, seg.base */
     11108                off = iemNativeEmitAddTwoGprs32Ex(pbCodeBuf, off, idxRegFlatPtr, idxRegSegBase);
     11109            }
    1094111110        }
    10942 
    10943         /* Check that we've got a segment loaded and that it allows the access.
    10944            For write access this means a writable data segment.
    10945            For read-only accesses this means a readable code segment or any data segment. */
    10946         if (fAccess & IEM_ACCESS_TYPE_WRITE)
     11111        else if (idxRegPtr == UINT8_MAX)
     11112            off = iemNativeEmitLoadGprImmEx(pbCodeBuf, off, idxRegFlatPtr, uAbsPtr);
     11113
     11114        /*
     11115         * 2. Check that the address doesn't cross a page boundrary and doesn't have alignment issues.
     11116         *
     11117         * 2a. Alignment check using fAlignMask.
     11118         */
     11119        if (fAlignMask)
    1094711120        {
    10948             uint32_t const fMustBe1 = X86DESCATTR_P        | X86DESCATTR_DT    | X86_SEL_TYPE_WRITE;
    10949             uint32_t const fMustBe0 = X86DESCATTR_UNUSABLE | X86_SEL_TYPE_CODE;
    10950             /* mov reg1, must1|must0 */
    10951             off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxReg1, fMustBe1 | fMustBe0);
    10952             /* and reg1, segattrs */
    10953             off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxReg1, idxRegSegAttrib);
    10954             /* cmp reg1, must1 */
    10955             off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, fMustBe1);
    10956             /* jne tlbmiss */
     11121            Assert(RT_IS_POWER_OF_TWO(fAlignMask + 1));
     11122            Assert(fAlignMask < 128);
     11123            /* test regflat, fAlignMask */
     11124            off = iemNativeEmitTestAnyBitsInGpr8Ex(pbCodeBuf, off, idxRegFlatPtr, fAlignMask);
     11125            /* jnz tlbmiss */
    1095711126            off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
     11127        }
     11128
     11129        /*
     11130         * 2b. Check that it's not crossing page a boundrary. This is implicit in
     11131         *     the previous test if the alignment is same or larger than the type.
     11132         */
     11133        if (cbMem > fAlignMask + 1)
     11134        {
     11135            /* mov reg1, 0xfff */
     11136            off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxReg1, GUEST_PAGE_OFFSET_MASK);
     11137            /* and reg1, regflat */
     11138            off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxReg1, idxRegFlatPtr);
     11139            /* neg reg1 */
     11140            off = iemNativeEmitNegGpr32Ex(pbCodeBuf, off, idxReg1);
     11141            /* add reg1, 0x1000 */
     11142            off = iemNativeEmitAddGpr32ImmEx(pbCodeBuf, off, idxReg1, GUEST_PAGE_SIZE);
     11143            /* cmp reg1, cbMem */
     11144            off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, GUEST_PAGE_SIZE);
     11145            /* ja  tlbmiss */
     11146            off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
     11147        }
     11148
     11149        /*
     11150         * 3. TLB lookup.
     11151         *
     11152         * 3a. Calculate the TLB tag value (IEMTLB_CALC_TAG).
     11153         *     In 64-bit mode we will also check for non-canonical addresses here.
     11154         */
     11155        if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
     11156        {
     11157            /* mov reg1, regflat */
     11158            off = iemNativeEmitLoadGprFromGprEx(pbCodeBuf, off, idxReg1, idxRegFlatPtr);
     11159            /* rol reg1, 16 */
     11160            off = iemNativeEmitRotateGprLeftEx(pbCodeBuf, off, idxReg1, 16);
     11161            /** @todo Would 'movsx reg2, word reg1' and working on reg2 in dwords be faster? */
     11162            /* inc word reg1 */
     11163            pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
     11164            if (idxReg1 >= 8)
     11165                pbCodeBuf[off++] = X86_OP_REX_B;
     11166            pbCodeBuf[off++] = 0xff;
     11167            pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, idxReg1 & 7);
     11168            /* cmp word reg1, 1 */
     11169            pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
     11170            if (idxReg1 >= 8)
     11171                pbCodeBuf[off++] = X86_OP_REX_B;
     11172            pbCodeBuf[off++] = 0x83;
     11173            pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, idxReg1 & 7);
     11174            pbCodeBuf[off++] = 1;
     11175            /* ja  tlbmiss */
     11176            off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
     11177            /* shr reg1, 16 + GUEST_PAGE_SHIFT */
     11178            off = iemNativeEmitShiftGprRightEx(pbCodeBuf, off, idxReg1, 16 + GUEST_PAGE_SHIFT);
    1095811179        }
    1095911180        else
    1096011181        {
    10961             /*  U  | !P |!DT |!CD | RW |
    10962                 16 |  8 |  4 |  3 |  1 |
    10963               -------------------------------
    10964                 0  |  0 |  0 |  0 |  0 | execute-only code segment. - must be excluded
    10965                 0  |  0 |  0 |  0 |  1 | execute-read code segment.
    10966                 0  |  0 |  0 |  1 |  0 | read-only data segment.
    10967                 0  |  0 |  0 |  1 |  1 | read-write data segment.   - last valid combination
    10968             */
    10969             /* mov reg1, relevant attributes  */
    10970             off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1,
    10971                                                    X86DESCATTR_UNUSABLE | X86DESCATTR_P | X86DESCATTR_DT
    10972                                                  | X86_SEL_TYPE_CODE    | X86_SEL_TYPE_WRITE);
    10973             /* and reg1, segattrs */
    10974             off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxReg1, idxRegSegAttrib);
    10975             /* xor reg1, X86DESCATTR_P | X86DESCATTR_DT | X86_SEL_TYPE_CODE ; place C=1 RW=0 at the bottom & limit the range. */
    10976             off = iemNativeEmitXorGpr32ByImmEx(pbCodeBuf, off, idxReg1, X86DESCATTR_P | X86DESCATTR_DT | X86_SEL_TYPE_CODE);
    10977             /* sub reg1, X86_SEL_TYPE_WRITE ; ER-code=0, EO-code=0xffffffff, R0-data=7, RW-data=9 */
    10978             off = iemNativeEmitSubGpr32ImmEx(pbCodeBuf, off, idxReg1, X86_SEL_TYPE_WRITE);
    10979             /* cmp reg1, X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE */
    10980             AssertCompile((X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE) - 1 == 9);
    10981             off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, (X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE) - 1);
    10982             /* ja  tlbmiss */
    10983             off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
     11182            /* mov reg1, regflat */
     11183            off = iemNativeEmitLoadGprFromGpr32Ex(pbCodeBuf, off, idxReg1, idxRegFlatPtr);
     11184            /* shr reg1, GUEST_PAGE_SHIFT */
     11185            off = iemNativeEmitShiftGpr32RightEx(pbCodeBuf, off, idxReg1, GUEST_PAGE_SHIFT);
    1098411186        }
     11187        /* or  reg1, [qword pVCpu->iem.s.DataTlb.uTlbRevision] */
     11188        pbCodeBuf[off++] = idxReg1 < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R;
     11189        pbCodeBuf[off++] = 0x0b; /* OR r64,r/m64 */
     11190        off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxReg1, RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbRevision));
    1098511191
    1098611192        /*
    10987          * Check the limit.  If this is a write access, we know that it's a
    10988          * data segment and includes the expand_down bit.  For read-only accesses
    10989          * we need to check that code/data=0 and expanddown=1 before continuing.
     11193         * 3b. Calc pTlbe.
    1099011194         */
    10991         uint32_t offFixupCheckExpandDown;
    10992         if (fAccess & IEM_ACCESS_TYPE_WRITE)
     11195        /* movzx reg2, byte reg1 */
     11196        off = iemNativeEmitLoadGprFromGpr8Ex(pbCodeBuf, off, idxReg2, idxReg1);
     11197        /* shl   reg2, 5 ; reg2 *= sizeof(IEMTLBENTRY) */
     11198        AssertCompileSize(IEMTLBENTRY, 32);
     11199        off = iemNativeEmitShiftGprLeftEx(pbCodeBuf, off, idxReg2, 5);
     11200        /* lea   reg2, [pVCpu->iem.s.DataTlb.aEntries + reg2] */
     11201        AssertCompile(IEMNATIVE_REG_FIXED_PVMCPU < 8);
     11202        pbCodeBuf[off++] = idxReg2 < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_X | X86_OP_REX_R;
     11203        pbCodeBuf[off++] = 0x8d;
     11204        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, idxReg2 & 7, 4 /*SIB*/);
     11205        pbCodeBuf[off++] = X86_SIB_MAKE(IEMNATIVE_REG_FIXED_PVMCPU & 7, idxReg2 & 7, 0);
     11206        pbCodeBuf[off++] = RT_BYTE1(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
     11207        pbCodeBuf[off++] = RT_BYTE2(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
     11208        pbCodeBuf[off++] = RT_BYTE3(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
     11209        pbCodeBuf[off++] = RT_BYTE4(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
     11210
     11211        /*
     11212         * 3c. Compare the TLBE.uTag with the one from 2a (reg1).
     11213         */
     11214        /* cmp reg1, [reg2] */
     11215        pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R) | (idxReg2 < 8 ? 0 : X86_OP_REX_B);
     11216        pbCodeBuf[off++] = 0x3b;
     11217        off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, idxReg1, idxReg2, RT_UOFFSETOF(IEMTLBENTRY, uTag));
     11218        /* jne tlbmiss */
     11219        off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
     11220
     11221        /*
     11222         * 4. Check TLB page table level access flags and physical page revision #.
     11223         */
     11224        /* mov reg1, mask */
     11225        AssertCompile(IEMTLBE_F_PT_NO_USER == 4);
     11226        uint64_t const fNoUser = (((pReNative->fExec >> IEM_F_X86_CPL_SHIFT) & IEM_F_X86_CPL_SMASK) + 1) & IEMTLBE_F_PT_NO_USER;
     11227        off = iemNativeEmitLoadGprImmEx(pbCodeBuf, off, idxReg1,
     11228                                          IEMTLBE_F_PHYS_REV       | IEMTLBE_F_NO_MAPPINGR3
     11229                                        | IEMTLBE_F_PG_UNASSIGNED  | IEMTLBE_F_PG_NO_READ
     11230                                        | IEMTLBE_F_PT_NO_ACCESSED | fNoUser);
     11231        /* and reg1, [reg2->fFlagsAndPhysRev] */
     11232        pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R) | (idxReg2 < 8 ? 0 : X86_OP_REX_B);
     11233        pbCodeBuf[off++] = 0x23;
     11234        off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, idxReg1, idxReg2, RT_UOFFSETOF(IEMTLBENTRY, fFlagsAndPhysRev));
     11235
     11236        /* cmp reg1, [pVCpu->iem.s.DataTlb.uTlbPhysRev] */
     11237        pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R);
     11238        pbCodeBuf[off++] = 0x3b;
     11239        off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, idxReg1, IEMNATIVE_REG_FIXED_PVMCPU,
     11240                                        RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbPhysRev));
     11241        /* jne tlbmiss */
     11242        off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
     11243
     11244        /*
     11245         * 5. Check that pbMappingR3 isn't NULL (paranoia) and calculate the
     11246         *    resulting pointer.
     11247         */
     11248        /* mov  reg1, [reg2->pbMappingR3] */
     11249        pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R) | (idxReg2 < 8 ? 0 : X86_OP_REX_B);
     11250        pbCodeBuf[off++] = 0x8b;
     11251        off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, idxReg1, idxReg2, RT_UOFFSETOF(IEMTLBENTRY, pbMappingR3));
     11252
     11253        /** @todo eliminate the need for this test? */
     11254        /* test reg1, reg1 */
     11255        pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R | X86_OP_REX_B);
     11256        pbCodeBuf[off++] = 0x85;
     11257        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxReg1 & 7, idxReg1 & 7);
     11258
     11259        /* jz   tlbmiss */
     11260        off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbDone, kIemNativeInstrCond_e);
     11261
     11262        if (idxRegFlatPtr == idxRegMemResult) /* See step 1b. */
    1099311263        {
    10994             /* test segattrs, X86_SEL_TYPE_DOWN */
    10995             AssertCompile(X86_SEL_TYPE_DOWN < 128);
    10996             off = iemNativeEmitTestAnyBitsInGpr8Ex(pbCodeBuf, off, idxRegSegAttrib, X86_SEL_TYPE_DOWN);
    10997             /* jnz  check_expand_down */
    10998             offFixupCheckExpandDown = off;
    10999             off = iemNativeEmitJccToFixedEx(pbCodeBuf, off, off /*ASSUMES rel8 suffices*/, kIemNativeInstrCond_ne);
     11264            /* and result, 0xfff */
     11265            off = iemNativeEmitAndGpr32ByImmEx(pbCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
    1100011266        }
    1100111267        else
    1100211268        {
    11003             /* mov reg1, segattrs */
    11004             off = iemNativeEmitLoadGprFromGpr32Ex(pbCodeBuf, off, idxReg1, idxRegSegAttrib);
    11005             /* and reg1, code | down */
    11006             off = iemNativeEmitAndGpr32ByImmEx(pbCodeBuf, off, idxReg1, X86_SEL_TYPE_CODE | X86_SEL_TYPE_DOWN);
    11007             /* cmp reg1, down */
    11008             off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, X86_SEL_TYPE_DOWN);
    11009             /* je check_expand_down */
    11010             offFixupCheckExpandDown = off;
    11011             off = iemNativeEmitJccToFixedEx(pbCodeBuf, off, off /*ASSUMES rel8 suffices*/, kIemNativeInstrCond_e);
     11269            Assert(idxRegFlatPtr == idxRegPtr);
     11270            /* mov result, 0xfff */
     11271            off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
     11272            /* and result, regflat */
     11273            off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxRegMemResult, idxRegFlatPtr);
    1101211274        }
    11013 
    11014         /* expand_up:
    11015            cmp  regptr/reg2, seglim */
    11016         off = iemNativeEmitCmpGprWithGprEx(pbCodeBuf, off, cbMem > 1 ? idxReg2 : idxRegPtr, idxRegSegLimit);
    11017         /* ja   tlbmiss */
    11018         off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
    11019         /* jmp  limitdone */
    11020         uint32_t const offFixupLimitDone = off;
    11021         off = iemNativeEmitJmpToFixedEx(pbCodeBuf, off, off /*ASSUMES rel8 suffices*/);
    11022 
    11023         /* check_expand_down: ; complicted! */
    11024         iemNativeFixupFixedJump(pReNative, offFixupCheckExpandDown, off);
    11025         /* cmp  regptr, seglim */
    11026         off = iemNativeEmitCmpGprWithGprEx(pbCodeBuf, off, idxRegPtr, idxRegSegLimit);
    11027         /* jbe  tlbmiss */
    11028         off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_be);
    11029         /* mov  reg1, X86DESCATTR_D (0x4000) */
    11030         off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, X86DESCATTR_D);
    11031         /* and  reg1, segattr */
    11032         off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxReg1, idxRegSegAttrib);
    11033         /* xor  reg1, X86DESCATTR_D */
    11034         off = iemNativeEmitXorGpr32ByImmEx(pbCodeBuf, off, idxReg1, X86DESCATTR_D);
    11035         /* shl  reg1, 2 (16 - 14) */
    11036         AssertCompile((X86DESCATTR_D << 2) == UINT32_C(0x10000));
    11037         off = iemNativeEmitShiftGpr32LeftEx(pbCodeBuf, off, idxReg1, 2);
    11038         /* dec  reg1 (=> 0xffff if D=0; 0xffffffff if D=1) */
    11039         off = iemNativeEmitSubGpr32ImmEx(pbCodeBuf, off, idxReg1, 1);
    11040         /* cmp  reg2, reg1 (64-bit) */
    11041         off = iemNativeEmitCmpGpr32WithGprEx(pbCodeBuf, off, idxReg2, idxReg1);
    11042         /* ja   tlbmiss */
    11043         off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
    11044 
    11045         /* limitdone: */
    11046         iemNativeFixupFixedJump(pReNative, offFixupLimitDone, off);
    11047     }
    11048 
    11049     /* 1b. Add the segment base. We use idxRegMemResult for the ptr register if this step is required. */
    11050     uint8_t const idxRegFlatPtr = iSegReg != UINT8_MAX ? idxRegMemResult : idxRegPtr;
    11051     if (iSegReg != UINT8_MAX)
    11052     {
    11053         if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
     11275        /* add result, reg1 */
     11276        off = iemNativeEmitAddTwoGprsEx(pbCodeBuf, off, idxRegMemResult, idxReg1);
     11277
     11278        if (idxRegPtr != UINT8_MAX)
     11279            iemNativeVarRegisterRelease(pReNative, idxVarGCPtrMem);
     11280        if (idxRegSegBase != UINT8_MAX)
     11281            iemNativeRegFreeTmp(pReNative, idxRegSegBase);
     11282        if (idxRegSegLimit != UINT8_MAX)
    1105411283        {
    11055             Assert(iSegReg >= X86_SREG_FS);
    11056             /* mov regflat, regptr */
    11057             off = iemNativeEmitLoadGprFromGprEx(pbCodeBuf, off, idxRegFlatPtr, idxRegPtr);
    11058             /* add regflat, seg.base */
    11059             off = iemNativeEmitAddTwoGprsEx(pbCodeBuf, off, idxRegFlatPtr, idxRegSegBase);
     11284            iemNativeRegFreeTmp(pReNative, idxRegSegLimit);
     11285            iemNativeRegFreeTmp(pReNative, idxRegSegAttrib);
    1106011286        }
    1106111287        else
    11062         {
    11063             /* mov regflat, regptr */
    11064             off = iemNativeEmitLoadGprFromGpr32Ex(pbCodeBuf, off, idxRegFlatPtr, idxRegPtr);
    11065             /* add regflat, seg.base */
    11066             off = iemNativeEmitAddTwoGprs32Ex(pbCodeBuf, off, idxRegFlatPtr, idxRegSegBase);
    11067         }
    11068     }
    11069 
    11070     /*
    11071      * 2. Check that the address doesn't cross a page boundrary and doesn't have alignment issues.
    11072      *
    11073      * 2a. Alignment check using fAlignMask.
    11074      */
    11075     Assert(RT_IS_POWER_OF_TWO(fAlignMask + 1));
    11076     Assert(fAlignMask < 128);
    11077     /* test regflat, fAlignMask */
    11078     off = iemNativeEmitTestAnyBitsInGpr8Ex(pbCodeBuf, off, idxRegFlatPtr, fAlignMask);
    11079     /* jnz tlbmiss */
    11080     off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
    11081 
    11082     /*
    11083      * 2b. Check that it's not crossing page a boundrary. This is implicit in
    11084      *     the previous test if the alignment is same or larger than the type.
    11085      */
    11086     if (cbMem > fAlignMask + 1)
    11087     {
    11088         /* mov reg1, 0xfff */
    11089         off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxReg1, GUEST_PAGE_OFFSET_MASK);
    11090         /* and reg1, regflat */
    11091         off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxReg1, idxRegFlatPtr);
    11092         /* neg reg1 */
    11093         off = iemNativeEmitNegGpr32Ex(pbCodeBuf, off, idxReg1);
    11094         /* add reg1, 0x1000 */
    11095         off = iemNativeEmitAddGpr32ImmEx(pbCodeBuf, off, idxReg1, GUEST_PAGE_SIZE);
    11096         /* cmp reg1, cbMem */
    11097         off = iemNativeEmitCmpGpr32WithImmEx(pbCodeBuf, off, idxReg1, GUEST_PAGE_SIZE);
    11098         /* ja  tlbmiss */
    11099         off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
    11100     }
    11101 
    11102     /*
    11103      * 3. TLB lookup.
    11104      *
    11105      * 3a. Calculate the TLB tag value (IEMTLB_CALC_TAG).
    11106      *     In 64-bit mode we will also check for non-canonical addresses here.
    11107      */
    11108     if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
    11109     {
    11110         /* mov reg1, regflat */
    11111         off = iemNativeEmitLoadGprFromGprEx(pbCodeBuf, off, idxReg1, idxRegFlatPtr);
    11112         /* rol reg1, 16 */
    11113         off = iemNativeEmitRotateGprLeftEx(pbCodeBuf, off, idxReg1, 16);
    11114         /** @todo Would 'movsx reg2, word reg1' and working on reg2 in dwords be faster? */
    11115         /* inc word reg1 */
    11116         pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
    11117         if (idxReg1 >= 8)
    11118             pbCodeBuf[off++] = X86_OP_REX_B;
    11119         pbCodeBuf[off++] = 0xff;
    11120         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, idxReg1 & 7);
    11121         /* cmp word reg1, 1 */
    11122         pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
    11123         if (idxReg1 >= 8)
    11124             pbCodeBuf[off++] = X86_OP_REX_B;
    11125         pbCodeBuf[off++] = 0x83;
    11126         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, idxReg1 & 7);
    11127         pbCodeBuf[off++] = 1;
    11128         /* ja  tlbmiss */
    11129         off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
    11130         /* shr reg1, 16 + GUEST_PAGE_SHIFT */
    11131         off = iemNativeEmitShiftGprRightEx(pbCodeBuf, off, idxReg1, 16 + GUEST_PAGE_SHIFT);
    11132     }
     11288            Assert(idxRegSegAttrib == UINT8_MAX);
     11289        iemNativeRegFreeTmp(pReNative, idxReg2);
     11290        iemNativeRegFreeTmp(pReNative, idxReg1);
     11291        IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     11292    }
     11293    /* Problematic absolute address, jmp to tlbmiss.  Caller will be generate
     11294       some unused tail code, but whatever. */
    1113311295    else
    11134     {
    11135         /* mov reg1, regflat */
    11136         off = iemNativeEmitLoadGprFromGpr32Ex(pbCodeBuf, off, idxReg1, idxRegFlatPtr);
    11137         /* shr reg1, GUEST_PAGE_SHIFT */
    11138         off = iemNativeEmitShiftGpr32RightEx(pbCodeBuf, off, idxReg1, GUEST_PAGE_SHIFT);
    11139     }
    11140     /* or  reg1, [qword pVCpu->iem.s.DataTlb.uTlbRevision] */
    11141     pbCodeBuf[off++] = idxReg1 < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R;
    11142     pbCodeBuf[off++] = 0x0b; /* OR r64,r/m64 */
    11143     off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxReg1, RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbRevision));
    11144 
    11145     /*
    11146      * 3b. Calc pTlbe.
    11147      */
    11148     /* movzx reg2, byte reg1 */
    11149     off = iemNativeEmitLoadGprFromGpr8Ex(pbCodeBuf, off, idxReg2, idxReg1);
    11150     /* shl   reg2, 5 ; reg2 *= sizeof(IEMTLBENTRY) */
    11151     AssertCompileSize(IEMTLBENTRY, 32);
    11152     off = iemNativeEmitShiftGprLeftEx(pbCodeBuf, off, idxReg2, 5);
    11153     /* lea   reg2, [pVCpu->iem.s.DataTlb.aEntries + reg2] */
    11154     AssertCompile(IEMNATIVE_REG_FIXED_PVMCPU < 8);
    11155     pbCodeBuf[off++] = idxReg2 < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_X | X86_OP_REX_R;
    11156     pbCodeBuf[off++] = 0x8d;
    11157     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, idxReg2 & 7, 4 /*SIB*/);
    11158     pbCodeBuf[off++] = X86_SIB_MAKE(IEMNATIVE_REG_FIXED_PVMCPU & 7, idxReg2 & 7, 0);
    11159     pbCodeBuf[off++] = RT_BYTE1(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
    11160     pbCodeBuf[off++] = RT_BYTE2(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
    11161     pbCodeBuf[off++] = RT_BYTE3(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
    11162     pbCodeBuf[off++] = RT_BYTE4(RT_UOFFSETOF(VMCPUCC,  iem.s.DataTlb.aEntries));
    11163 
    11164     /*
    11165      * 3c. Compare the TLBE.uTag with the one from 2a (reg1).
    11166      */
    11167     /* cmp reg1, [reg2] */
    11168     pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R) | (idxReg2 < 8 ? 0 : X86_OP_REX_B);
    11169     pbCodeBuf[off++] = 0x3b;
    11170     off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, idxReg1, idxReg2, RT_UOFFSETOF(IEMTLBENTRY, uTag));
    11171     /* jne tlbmiss */
    11172     off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
    11173 
    11174     /*
    11175      * 4. Check TLB page table level access flags and physical page revision #.
    11176      */
    11177     /* mov reg1, mask */
    11178     AssertCompile(IEMTLBE_F_PT_NO_USER == 4);
    11179     uint64_t const fNoUser = (((pReNative->fExec >> IEM_F_X86_CPL_SHIFT) & IEM_F_X86_CPL_SMASK) + 1) & IEMTLBE_F_PT_NO_USER;
    11180     off = iemNativeEmitLoadGprImmEx(pbCodeBuf, off, idxReg1,
    11181                                       IEMTLBE_F_PHYS_REV       | IEMTLBE_F_NO_MAPPINGR3
    11182                                     | IEMTLBE_F_PG_UNASSIGNED  | IEMTLBE_F_PG_NO_READ
    11183                                     | IEMTLBE_F_PT_NO_ACCESSED | fNoUser);
    11184     /* and reg1, [reg2->fFlagsAndPhysRev] */
    11185     pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R) | (idxReg2 < 8 ? 0 : X86_OP_REX_B);
    11186     pbCodeBuf[off++] = 0x23;
    11187     off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, idxReg1, idxReg2, RT_UOFFSETOF(IEMTLBENTRY, fFlagsAndPhysRev));
    11188 
    11189     /* cmp reg1, [pVCpu->iem.s.DataTlb.uTlbPhysRev] */
    11190     pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R);
    11191     pbCodeBuf[off++] = 0x3b;
    11192     off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, idxReg1, IEMNATIVE_REG_FIXED_PVMCPU,
    11193                                     RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb.uTlbPhysRev));
    11194     /* jne tlbmiss */
    11195     off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
    11196 
    11197     /*
    11198      * 5. Check that pbMappingR3 isn't NULL (paranoia) and calculate the
    11199      *    resulting pointer.
    11200      */
    11201     /* mov  reg1, [reg2->pbMappingR3] */
    11202     pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R) | (idxReg2 < 8 ? 0 : X86_OP_REX_B);
    11203     pbCodeBuf[off++] = 0x8b;
    11204     off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, idxRegMemResult, idxReg2, RT_UOFFSETOF(IEMTLBENTRY, pbMappingR3));
    11205 
    11206     /** @todo eliminate the need for this test? */
    11207     /* test reg1, reg1 */
    11208     pbCodeBuf[off++] = X86_OP_REX_W | (idxReg1 < 8 ? 0 : X86_OP_REX_R | X86_OP_REX_B);
    11209     pbCodeBuf[off++] = 0x85;
    11210     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxReg1 & 7, idxReg1 & 7);
    11211 
    11212     /* jz   tlbmiss */
    11213     off = iemNativeEmitJccToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbDone, kIemNativeInstrCond_e);
    11214 
    11215     if (idxRegFlatPtr == idxRegMemResult) /* See step 1b. */
    11216     {
    11217         /* and result, 0xfff */
    11218         off = iemNativeEmitAndGpr32ByImmEx(pbCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
    11219     }
    11220     else
    11221     {
    11222         Assert(idxRegFlatPtr == idxRegPtr);
    11223         /* mov result, 0xfff */
    11224         off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
    11225         /* and result, regflat */
    11226         off = iemNativeEmitAndGpr32ByGpr32Ex(pbCodeBuf, off, idxRegMemResult, idxRegFlatPtr);
    11227     }
    11228     /* add result, reg1 */
    11229     off = iemNativeEmitAddTwoGprsEx(pbCodeBuf, off, idxRegMemResult, idxReg1);
     11296        off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbMiss);
     11297
     11298    /*
     11299     * Tail code, specific to the MC when the above is moved into a separate function.
     11300     */
     11301    uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 32);
     11302
     11303    /* [idxVarUnmapInfo] = 0 - allocate register for it. There must be free ones now, so no spilling required. */
     11304    uint32_t const offSaved = off;
     11305    uint8_t const idxRegUnmapInfo = iemNativeVarRegisterAcquire(pReNative, idxVarUnmapInfo, &off);
     11306    AssertStmt(off == offSaved, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_9));
     11307    off = iemNativeEmitLoadGpr32ImmEx(pbCodeBuf, off, idxRegUnmapInfo, 0);
    1123011308
    1123111309    /* jmp tlbdone */
    1123211310    off = iemNativeEmitJmpToLabelEx(pReNative, pbCodeBuf, off, idxLabelTlbDone);
    1123311311
    11234     iemNativeVarRegisterRelease(pReNative, idxRegPtr);
    11235     iemNativeRegFree(pReNative, idxRegSegBase);
    11236     iemNativeRegFree(pReNative, idxRegSegLimit);
    11237     iemNativeRegFree(pReNative, idxRegSegAttrib);
    11238     iemNativeRegFree(pReNative, idxReg2);
    11239     iemNativeRegFree(pReNative, idxReg1);
    11240 
     11312    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    1124111313#else
    1124211314    /** @todo arm64 TLB code   */
    … …  
    1126611338    off = iemNativeEmitLoadArgGregFromImmOrStackVar(pReNative, off, IEMNATIVE_CALL_ARG2_GREG, idxVarGCPtrMem);
    1126711339
    11268     /* IEMNATIVE_CALL_ARG1_GREG = &idxVarUnmapInfo */
    11269     iemNativeVarSetKindToStack(pReNative, idxVarUnmapInfo);
     11340    /* IEMNATIVE_CALL_ARG1_GREG = &idxVarUnmapInfo; stackslot address, load any register with result after the call. */
     11341#if 0
    1127011342    off = iemNativeEmitLoadArgGregWithVarAddr(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, idxVarUnmapInfo, true /*fFlushShadows*/);
     11343#else
     11344    int32_t const offBpDispVarUnmapInfo = iemNativeStackCalcBpDisp(iemNativeVarGetStackSlot(pReNative, idxVarUnmapInfo));
     11345    off = iemNativeEmitLeaGprByBp(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, offBpDispVarUnmapInfo);
     11346#endif
    1127111347
    1127211348    /* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
    … …  
    1127711353
    1127811354    /*
    11279      * Put the result in the right register .
     11355     * Put the output in the right registers.
    1128011356     */
    1128111357    Assert(idxRegMemResult == pReNative->Core.aVars[idxVarMem].idxReg);
    … …  
    1128311359        off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegMemResult, IEMNATIVE_CALL_RET_GREG);
    1128411360    iemNativeVarRegisterRelease(pReNative, idxVarMem);
     11361
     11362#if defined(RT_ARCH_AMD64)
     11363    Assert(pReNative->Core.aVars[idxVarUnmapInfo].idxReg == idxRegUnmapInfo);
     11364    off = iemNativeEmitLoadGprByBpU8(pReNative, off, idxRegUnmapInfo, offBpDispVarUnmapInfo);
     11365    iemNativeVarRegisterRelease(pReNative, idxVarUnmapInfo);
     11366#endif
    1128511367
    1128611368    iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);

  • trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h

    r102718 r102720  
    174174 *      - ARM64: 4 instruction words (16 bytes).
    175175 */
    176 DECLINLINE(uint32_t) iemNativeEmitLoadGprImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint64_t uImm64)
     176DECL_FORCE_INLINE(uint32_t)
     177iemNativeEmitLoadGprImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint64_t uImm64)
    177178{
    178179#ifdef RT_ARCH_AMD64
    … …  
    191192            pCodeBuf[off++] = X86_OP_REX_B;
    192193        pCodeBuf[off++] = 0xb8 + (iGpr & 7);
     194        pCodeBuf[off++] = RT_BYTE1(uImm64);
     195        pCodeBuf[off++] = RT_BYTE2(uImm64);
     196        pCodeBuf[off++] = RT_BYTE3(uImm64);
     197        pCodeBuf[off++] = RT_BYTE4(uImm64);
     198    }
     199    else if (uImm64 == (uint64_t)(int32_t)uImm64)
     200    {
     201        /* mov gpr, sx(imm32) */
     202        if (iGpr < 8)
     203            pCodeBuf[off++] = X86_OP_REX_W;
     204        else
     205            pCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_B;
     206        pCodeBuf[off++] = 0xc7;
     207        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGpr & 7);
    193208        pCodeBuf[off++] = RT_BYTE1(uImm64);
    194209        pCodeBuf[off++] = RT_BYTE2(uImm64);
    … …  
    31013116    if (iGprLeft >= 8)
    31023117        pCodeBuf[off++] = X86_OP_REX_B;
    3103     if (uImm <= UINT32_C(0xff))
     3118    if (uImm <= UINT32_C(0x7f))
    31043119    {
    31053120        /* cmp Ev, Ib */
Note: See TracChangeset for help on using the changeset viewer.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette