VirtualBox

Changeset 102768 in vbox for trunk/src/VBox/VMM/VMMAll


Ignore:
Timestamp:
Jan 4, 2024 10:39:04 PM (15 months ago)
Author:
vboxsync
svn:sync-xref-src-repo-rev:
160949
Message:

VMM/IEM: Emit native TLB lookup in IEM_MC_*PUSH*. bugreg:10371

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

    r102767 r102768  
    104104
    105105/*
     106 * TLB Lookup config.
     107 */
     108#if (defined(RT_ARCH_AMD64) && 1) || (defined(RT_ARCH_ARM64) && 1)
     109# define IEMNATIVE_WITH_TLB_LOOKUP
     110#endif
     111#ifdef IEMNATIVE_WITH_TLB_LOOKUP
     112# define IEMNATIVE_WITH_TLB_LOOKUP_PUSH
     113#endif
     114
     115
     116/*
    106117 * Narrow down configs here to avoid wasting time on unused configs here.
    107118 * Note! Same checks in IEMAllThrdRecompiler.cpp.
    … …  
    141152DECL_FORCE_INLINE(void) iemNativeRegClearGstRegShadowingOne(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg,
    142153                                                            IEMNATIVEGSTREG enmGstReg, uint32_t off);
     154DECL_INLINE_THROW(void) iemNativeVarRegisterRelease(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
    143155
    144156
    … …  
    17421754 * Used by TB code to store unsigned 8-bit data w/ segmentation.
    17431755 */
     1756IEMNATIVE_WITH_TLB_LOOKUP_PUSH
    17441757IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemStoreDataU8,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint8_t iSegReg, uint8_t u8Value))
    17451758{
    … …  
    17811794IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpStackStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value))
    17821795{
    1783 #if 0
     1796#ifdef IEMNATIVE_WITH_TLB_LOOKUP_PUSH
    17841797    iemMemStoreStackU16SafeJmp(pVCpu, GCPtrMem, u16Value);
    17851798#else
    … …  
    17941807IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpStackStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value))
    17951808{
    1796 #if 0
     1809#ifdef IEMNATIVE_WITH_TLB_LOOKUP_PUSH
    17971810    iemMemStoreStackU32SafeJmp(pVCpu, GCPtrMem, u32Value);
    17981811#else
    … …  
    18091822IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpStackStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value))
    18101823{
    1811 #if 0
     1824#ifdef IEMNATIVE_WITH_TLB_LOOKUP_PUSH
    18121825    iemMemStoreStackU32SRegSafeJmp(pVCpu, GCPtrMem, u32Value);
    18131826#else
    … …  
    18221835IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpStackStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value))
    18231836{
    1824 #if 0
     1837#ifdef IEMNATIVE_WITH_TLB_LOOKUP_PUSH
    18251838    iemMemStoreStackU64SafeJmp(pVCpu, GCPtrMem, u64Value);
    18261839#else
    … …  
    20082021IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpStackFlatStoreU16,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint16_t u16Value))
    20092022{
    2010 #if 0
     2023#ifdef IEMNATIVE_WITH_TLB_LOOKUP_PUSH
    20112024    iemMemStoreStackU16SafeJmp(pVCpu, GCPtrMem, u16Value);
    20122025#else
    … …  
    20212034IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpStackFlatStoreU32,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value))
    20222035{
    2023 #if 0
     2036#ifdef IEMNATIVE_WITH_TLB_LOOKUP_PUSH
    20242037    iemMemStoreStackU32SafeJmp(pVCpu, GCPtrMem, u32Value);
    20252038#else
    … …  
    20362049IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpStackFlatStoreU32SReg,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t u32Value))
    20372050{
    2038 #if 0
     2051#ifdef IEMNATIVE_WITH_TLB_LOOKUP_PUSH
    20392052    iemMemStoreStackU32SRegSafeJmp(pVCpu, GCPtrMem, u32Value);
    20402053#else
    … …  
    20492062IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpStackFlatStoreU64,(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint64_t u64Value))
    20502063{
    2051 #if 0
     2064#ifdef IEMNATIVE_WITH_TLB_LOOKUP_PUSH
    20522065    iemMemStoreStackU64SafeJmp(pVCpu, GCPtrMem, u64Value);
    20532066#else
    … …  
    58695882    off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxPcReg, RT_UOFFSETOF(VMCPU, cpum.GstCtx.rip));
    58705883
     5884    iemNativeVarRegisterRelease(pReNative, idxVarPc);
    58715885    /** @todo implictly free the variable? */
    58725886
    … …  
    73217335    Assert(   pReNative->Core.aVars[idxVar].enmKind >= kIemNativeVarKind_Invalid  /* Including invalid as we may have unused */
    73227336           && pReNative->Core.aVars[idxVar].enmKind <  kIemNativeVarKind_End);    /* variables in conditional branches. */
     7337    Assert(!pReNative->Core.aVars[idxVar].fRegAcquired);
    73237338
    73247339    /* Free the host register first if any assigned. */
    … …  
    1004810063*********************************************************************************************************************************/
    1004910064
    10050 #if (defined(RT_ARCH_AMD64) && 1) || (defined(RT_ARCH_ARM64) && 1)
    10051 # define IEMNATIVE_WITH_TLB_LOOKUP
    10052 #endif
    10053 
    10054 
    1005510065/**
    1005610066 * This must be instantiate *before* branching off to the lookup code,
    … …  
    1009210102                          : iemNativeRegAllocTmpImm(a_pReNative, a_poff, a_pReNative->Core.aVars[a_idxVarGCPtrMem].u.uValue) )
    1009310103#endif
    10094         ,       idxRegPtr(a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate
     10104        ,       idxRegPtr(a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate && !fSkip
    1009510105                          ? iemNativeVarRegisterAcquire(a_pReNative, a_idxVarGCPtrMem, a_poff,
    1009610106                                                        true /*fInitialized*/, IEMNATIVE_CALL_ARG2_GREG)
    … …  
    1011510125
    1011610126    {
     10127        RT_NOREF(a_cbMem, a_offDisp);
     10128    }
     10129
     10130    /* Alternative constructor for PUSH and POP where we don't have a GCPtrMem
     10131       variable, only a register derived from the guest RSP. */
     10132    IEMNATIVEEMITTLBSTATE(PIEMRECOMPILERSTATE a_pReNative, uint8_t a_idxRegPtr, uint32_t *a_poff,
     10133                          uint8_t a_iSegReg, uint8_t a_cbMem)
     10134#ifdef IEMNATIVE_WITH_TLB_LOOKUP
     10135        :           fSkip(false)
     10136#else
     10137        :           fSkip(true)
     10138#endif
     10139        ,    idxRegPtrHlp(UINT8_MAX)
     10140        ,       idxRegPtr(a_idxRegPtr)
     10141        ,   idxRegSegBase(a_iSegReg == UINT8_MAX || fSkip
     10142                          ? UINT8_MAX
     10143                          : iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_BASE(a_iSegReg)))
     10144        ,  idxRegSegLimit((a_iSegReg == UINT8_MAX || (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT) || fSkip
     10145                          ? UINT8_MAX
     10146                          : iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg)))
     10147        , idxRegSegAttrib((a_iSegReg == UINT8_MAX || (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT) || fSkip
     10148                          ? UINT8_MAX
     10149                          : iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg)))
     10150        ,         idxReg1(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
     10151        ,         idxReg2(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
     10152#if defined(RT_ARCH_ARM64)
     10153        ,         idxReg3(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
     10154#endif
     10155        ,         uAbsPtr(UINT64_MAX)
     10156
     10157    {
    1011710158        RT_NOREF_PV(a_cbMem);
    1011810159    }
    1011910160
    10120     void freeRegsAndReleaseVars(PIEMRECOMPILERSTATE a_pReNative, uint8_t idxVarGCPtrMem) const
     10161    void freeRegsAndReleaseVars(PIEMRECOMPILERSTATE a_pReNative, uint8_t idxVarGCPtrMem = UINT8_MAX) const
    1012110162    {
    1012210163        if (idxRegPtr != UINT8_MAX)
    1012310164        {
    1012410165            if (idxRegPtrHlp == UINT8_MAX)
    10125                 iemNativeVarRegisterRelease(a_pReNative, idxVarGCPtrMem);
     10166            {
     10167                if (idxVarGCPtrMem != UINT8_MAX)
     10168                    iemNativeVarRegisterRelease(a_pReNative, idxVarGCPtrMem);
     10169            }
    1012610170            else
    1012710171            {
    … …  
    1096911013    if (!TlbState.fSkip)
    1097011014    {
    10971         /* end of tlbsmiss - Jump to the done label. */
     11015        /* end of TlbMiss - Jump to the done label. */
    1097211016        uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
    1097311017        off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
    … …  
    1110811152    }
    1110911153#else
    11110     RT_NOREF(fAccess, fAlignMask, idxLabelTlbMiss);
     11154    RT_NOREF(fAlignMask, idxLabelTlbMiss);
    1111111155#endif
    1111211156
    … …  
    1153511579    uint8_t const cbMem       = RT_BYTE1(cBitsVarAndFlat) / 8;
    1153611580    uint8_t const cBitsFlat   = RT_BYTE2(cBitsVarAndFlat);      RT_NOREF(cBitsFlat);
    11537     bool const    fSeg        = RT_BYTE3(cBitsVarAndFlat) != 0; RT_NOREF(fSeg);
     11581    bool const    fIsSegReg   = RT_BYTE3(cBitsVarAndFlat) != 0; RT_NOREF(fIsSegReg);
    1153811582    uint8_t const idxRegRsp   = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(X86_GREG_xSP),
    1153911583                                                                kIemNativeGstRegUse_ForUpdate, true /*fNoVolatileRegs*/);
    … …  
    1158211626     * we're skipping lookup).
    1158311627     */
    11584     //IEMNATIVEEMITTLBSTATE const TlbState(pReNative, &off, idxVarGCPtrMem, iSegReg, cbMem);
     11628    uint8_t const  iSegReg           = cBitsFlat != 0 ? UINT8_MAX : X86_SREG_SS;
     11629    IEMNATIVEEMITTLBSTATE const TlbState(pReNative, idxRegEffSp, &off, iSegReg, cbMem);
    1158511630    uint16_t const uTlbSeqNo         = pReNative->uTlbSeqNo++;
    1158611631    uint32_t const idxLabelTlbMiss   = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, UINT32_MAX, uTlbSeqNo);
    11587     uint32_t const idxLabelTlbDone   = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
    11588     uint32_t const idxLabelTlbLookup = !true//!TlbState.fSkip
     11632    uint32_t const idxLabelTlbLookup = !TlbState.fSkip
    1158911633                                     ? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
    1159011634                                     : UINT32_MAX;
    11591 
    11592     if (!true)//TlbState.fSkip)
     11635    uint8_t const  idxRegValue       =    !TlbState.fSkip
     11636                                       && pReNative->Core.aVars[idxVarValue].enmKind != kIemNativeVarKind_Immediate
     11637                                     ? iemNativeVarRegisterAcquire(pReNative, idxVarValue, &off, true /*fInitialized*/,
     11638                                                                   IEMNATIVE_CALL_ARG2_GREG /*idxRegPref*/)
     11639                                     : UINT8_MAX;
     11640    uint8_t const  idxRegMemResult   = !TlbState.fSkip ? iemNativeRegAllocTmp(pReNative, &off) : UINT8_MAX;
     11641
     11642
     11643    if (!TlbState.fSkip)
    1159311644        off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
    1159411645    else
    … …  
    1162811679
    1162911680    /* Save variables in volatile registers. */
    11630     uint32_t const fHstRegsNotToSave = 0/*TlbState.getRegsNotToSave()*/
     11681    uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave()
     11682                                     | (idxRegMemResult < RT_ELEMENTS(pReNative->Core.aHstRegs) ? RT_BIT_32(idxRegMemResult) : 0)
    1163111683                                     | (idxRegEffSp != idxRegRsp ? RT_BIT_32(idxRegEffSp) : 0)
    11632                                      | (  pReNative->Core.aVars[idxVarValue].idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs)
    11633                                         ? RT_BIT_32(pReNative->Core.aVars[idxVarValue].idxReg) : 0);
     11684                                     | (idxRegValue < RT_ELEMENTS(pReNative->Core.aHstRegs) ? RT_BIT_32(idxRegValue) : 0);
    1163411685    off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
    1163511686
    11636     if (   pReNative->Core.aVars[idxVarValue].idxReg == IEMNATIVE_CALL_ARG1_GREG
     11687    if (   idxRegValue == IEMNATIVE_CALL_ARG1_GREG
    1163711688        && idxRegEffSp == IEMNATIVE_CALL_ARG2_GREG)
    1163811689    {
    … …  
    1167011721    /* Restore variables and guest shadow registers to volatile registers. */
    1167111722    off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
    11672     off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off, 0/*TlbState.getActiveRegsWithShadows()*/);
    11673 
    11674     /* The value variable is implictly flushed. */
    11675     iemNativeVarFreeLocal(pReNative, idxVarValue);
    11676 
    11677     /*
    11678      * TlbDone:
    11679      *
    11680      * Commit the new RSP value.
    11681      */
    11682     iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
     11723    off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off, TlbState.getActiveRegsWithShadows());
     11724
     11725#ifdef IEMNATIVE_WITH_TLB_LOOKUP
     11726    if (!TlbState.fSkip)
     11727    {
     11728        /* end of TlbMiss - Jump to the done label. */
     11729        uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
     11730        off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
     11731
     11732        /*
     11733         * TlbLookup:
     11734         */
     11735        off = iemNativeEmitTlbLookup(pReNative, off, &TlbState, iSegReg, cbMem, cbMem - 1, IEM_ACCESS_TYPE_WRITE,
     11736                                     idxLabelTlbLookup, idxLabelTlbMiss, idxRegMemResult);
     11737
     11738        /*
     11739         * Emit code to do the actual storing / fetching.
     11740         */
     11741        PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
     11742        if (idxRegValue != UINT8_MAX)
     11743        {
     11744            switch (cbMem)
     11745            {
     11746                case 2:
     11747                    off = iemNativeEmitStoreGpr16ByGprEx(pCodeBuf, off, idxRegValue, idxRegMemResult);
     11748                    break;
     11749                case 4:
     11750                    if (!fIsSegReg)
     11751                        off = iemNativeEmitStoreGpr32ByGprEx(pCodeBuf, off, idxRegValue, idxRegMemResult);
     11752                    else
     11753                        off = iemNativeEmitStoreGpr16ByGprEx(pCodeBuf, off, idxRegValue, idxRegMemResult);
     11754                    break;
     11755                case 8:
     11756                    off = iemNativeEmitStoreGpr64ByGprEx(pCodeBuf, off, idxRegValue, idxRegMemResult);
     11757                    break;
     11758                default:
     11759                    AssertFailed();
     11760            }
     11761        }
     11762        else
     11763        {
     11764            switch (cbMem)
     11765            {
     11766                case 2:
     11767                    off = iemNativeEmitStoreImm16ByGprEx(pCodeBuf, off,
     11768                                                         (uint16_t)pReNative->Core.aVars[idxVarValue].u.uValue,
     11769                                                         idxRegMemResult, TlbState.idxReg1);
     11770                    break;
     11771                case 4:
     11772                    Assert(!fIsSegReg);
     11773                    off = iemNativeEmitStoreImm32ByGprEx(pCodeBuf, off,
     11774                                                         (uint32_t)pReNative->Core.aVars[idxVarValue].u.uValue,
     11775                                                         idxRegMemResult, TlbState.idxReg1);
     11776                    break;
     11777                case 8:
     11778                    off = iemNativeEmitStoreImm64ByGprEx(pCodeBuf, off, pReNative->Core.aVars[idxVarValue].u.uValue,
     11779                                                         idxRegMemResult, TlbState.idxReg1);
     11780                    break;
     11781                default:
     11782                    AssertFailed();
     11783            }
     11784        }
     11785
     11786        iemNativeRegFreeTmp(pReNative, idxRegMemResult);
     11787        TlbState.freeRegsAndReleaseVars(pReNative);
     11788
     11789        /*
     11790         * TlbDone:
     11791         *
     11792         * Commit the new RSP value.
     11793         */
     11794        iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
     11795    }
     11796#endif /* IEMNATIVE_WITH_TLB_LOOKUP */
    1168311797
    1168411798    off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxRegRsp, RT_UOFFSETOF_DYN(VMCPU, cpum.GstCtx.rsp));
    … …  
    1168611800    if (idxRegEffSp != idxRegRsp)
    1168711801        iemNativeRegFreeTmp(pReNative, idxRegEffSp);
     11802
     11803    /* The value variable is implictly flushed. */
     11804    if (idxRegValue != UINT8_MAX)
     11805        iemNativeVarRegisterRelease(pReNative, idxVarValue);
     11806    iemNativeVarFreeLocal(pReNative, idxVarValue);
    1168811807
    1168911808    return off;
Note: See TracChangeset for help on using the changeset viewer.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette