VirtualBox

Changeset 106453 in vbox for trunk/src/VBox/VMM/VMMAll


Ignore:
Timestamp:
Oct 17, 2024 1:54:35 PM (7 months ago)
Author:
vboxsync
svn:sync-xref-src-repo-rev:
165272
Message:

VMM/IEM: Eliminated the IEMNATIVE_WITH_SIMD_REG_ALLOCATOR define. Fixed bug in iemNativeEmitMemFetchStoreDataCommon where a SIMD register was masked in calls to iemNativeVarSaveVolatileRegsPreHlpCall and friends. Fixed theoretical loop-forever bugs in iemNativeSimdRegAllocFindFree & iemNativeRegAllocFindFree. bugref:10720

Location:
trunk/src/VBox/VMM/VMMAll
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompFuncs.h

    r106445 r106453  
    8080#endif
    8181
    82 #if defined(IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS) && !defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR)
    83 # error "IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS requires IEMNATIVE_WITH_SIMD_REG_ALLOCATOR"
    84 #endif
    85 
    8682
    8783/*********************************************************************************************************************************
    … …  
    190186 * RIP updates, since these are the most common ones.
    191187 */
    192 DECL_INLINE_THROW(uint32_t)
    193 iemNativeRegFlushPendingSpecificWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEGSTREGREF enmClass, uint8_t idxReg)
     188template<IEMNATIVEGSTREGREF a_enmClass>
     189DECL_INLINE_THROW(uint32_t)
     190iemNativeRegFlushPendingSpecificWrite(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxReg)
    194191{
    195192#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
    … …  
    198195
    199196#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
    200 #if 0 /** @todo r=aeichner EFLAGS writeback delay. */
    201     if (   enmClass == kIemNativeGstRegRef_EFlags
    202         && pReNative->Core.bmGstRegShadowDirty & RT_BIT_64(kIemNativeGstReg_EFlags))
    203         off = iemNativeRegFlushPendingWrite(pReNative, off, kIemNativeGstReg_EFlags);
    204 #else
     197# if 0 /** @todo r=aeichner EFLAGS writeback delay. */
     198    if RT_CONSTEXPR_IF(a_enmClass == kIemNativeGstRegRef_EFlags)
     199        if (pReNative->Core.bmGstRegShadowDirty & RT_BIT_64(kIemNativeGstReg_EFlags))
     200            off = iemNativeRegFlushPendingWrite(pReNative, off, kIemNativeGstReg_EFlags);
     201# else
    205202    Assert(!(pReNative->Core.bmGstRegShadowDirty & RT_BIT_64(kIemNativeGstReg_EFlags)));
    206 #endif
    207 
    208     if (   enmClass == kIemNativeGstRegRef_Gpr
    209         && pReNative->Core.bmGstRegShadowDirty & RT_BIT_64(idxReg))
    210         off = iemNativeRegFlushPendingWrite(pReNative, off, IEMNATIVEGSTREG_GPR(idxReg));
    211 #endif
    212 
    213 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    214     if (   enmClass == kIemNativeGstRegRef_XReg
    215         && pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(idxReg))
    216     {
    217         off = iemNativeSimdRegFlushPendingWrite(pReNative, off, IEMNATIVEGSTSIMDREG_SIMD(idxReg));
    218         /* Flush the shadows as the register needs to be reloaded (there is no guarantee right now, that the referenced register doesn't change). */
    219         uint8_t const idxHstSimdReg = pReNative->Core.aidxGstSimdRegShadows[idxReg];
    220 
    221         iemNativeSimdRegClearGstSimdRegShadowing(pReNative, idxHstSimdReg, off);
    222         iemNativeSimdRegFlushGuestShadows(pReNative, RT_BIT_64(IEMNATIVEGSTSIMDREG_SIMD(idxReg)));
    223     }
    224 #endif
    225     RT_NOREF(pReNative, enmClass, idxReg);
     203# endif
     204
     205    if RT_CONSTEXPR_IF(a_enmClass == kIemNativeGstRegRef_Gpr)
     206        if (pReNative->Core.bmGstRegShadowDirty & RT_BIT_64(idxReg))
     207            off = iemNativeRegFlushPendingWrite(pReNative, off, IEMNATIVEGSTREG_GPR(idxReg));
     208#endif
     209
     210    if RT_CONSTEXPR_IF(a_enmClass == kIemNativeGstRegRef_XReg)
     211        if (pReNative->Core.bmGstSimdRegShadows & RT_BIT_64(idxReg))
     212        {
     213            off = iemNativeSimdRegFlushPendingWrite(pReNative, off, IEMNATIVEGSTSIMDREG_SIMD(idxReg));
     214            /* Flush the shadows as the register needs to be reloaded (there is no
     215               guarantee right now, that the referenced register doesn't change). */
     216            uint8_t const idxHstSimdReg = pReNative->Core.aidxGstSimdRegShadows[idxReg];
     217
     218            iemNativeSimdRegClearGstSimdRegShadowing(pReNative, idxHstSimdReg, off);
     219            iemNativeSimdRegFlushGuestShadows(pReNative, RT_BIT_64(IEMNATIVEGSTSIMDREG_SIMD(idxReg)));
     220        }
     221
    226222    return off;
    227223}
    … …  
    25552551iemNativeEmitMaybeRaiseDeviceNotAvailable(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr)
    25562552{
    2557 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    25582553    STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativeMaybeDeviceNotAvailXcptCheckPotential);
    25592554
    25602555    if (!(pReNative->fSimdRaiseXcptChecksEmitted & IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE))
    25612556    {
    2562 #endif
    25632557        /*
    25642558         * Make sure we don't have any outstanding guest register writes as we may
    … …  
    25892583        iemNativeRegFreeTmp(pReNative, idxCr0Reg);
    25902584
    2591 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    25922585        pReNative->fSimdRaiseXcptChecksEmitted |= IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE;
    25932586    }
    25942587    else
    25952588        STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativeMaybeDeviceNotAvailXcptCheckOmitted);
    2596 #endif
    25972589
    25982590    return off;
    … …  
    26142606iemNativeEmitMaybeRaiseWaitDeviceNotAvailable(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr)
    26152607{
    2616 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    26172608    STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativeMaybeWaitDeviceNotAvailXcptCheckPotential);
    26182609
    26192610    if (!(pReNative->fSimdRaiseXcptChecksEmitted & IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_WAIT_DEVICE_NOT_AVAILABLE))
    26202611    {
    2621 #endif
    26222612        /*
    26232613         * Make sure we don't have any outstanding guest register writes as we may
    … …  
    26482638        iemNativeRegFreeTmp(pReNative, idxCr0Reg);
    26492639
    2650 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    26512640        pReNative->fSimdRaiseXcptChecksEmitted |= IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_WAIT_DEVICE_NOT_AVAILABLE;
    26522641    }
    26532642    else
    26542643        STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativeMaybeWaitDeviceNotAvailXcptCheckOmitted);
    2655 #endif
    26562644
    26572645    return off;
    … …  
    27182706iemNativeEmitMaybeRaiseSseRelatedXcpt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr)
    27192707{
    2720 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    27212708    STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativeMaybeSseXcptCheckPotential);
    27222709
    27232710    if (!(pReNative->fSimdRaiseXcptChecksEmitted & IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE))
    27242711    {
    2725 #endif
    27262712        /*
    27272713         * Make sure we don't have any outstanding guest register writes as we may
    … …  
    27902776        iemNativeRegFreeTmp(pReNative, idxCr4Reg);
    27912777
    2792 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    27932778        pReNative->fSimdRaiseXcptChecksEmitted |= IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE;
    27942779    }
    27952780    else
    27962781        STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativeMaybeSseXcptCheckOmitted);
    2797 #endif
    27982782
    27992783    return off;
    … …  
    28152799iemNativeEmitMaybeRaiseAvxRelatedXcpt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr)
    28162800{
    2817 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    28182801    STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativeMaybeAvxXcptCheckPotential);
    28192802
    28202803    if (!(pReNative->fSimdRaiseXcptChecksEmitted & IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX))
    28212804    {
    2822 #endif
    28232805        /*
    28242806         * Make sure we don't have any outstanding guest register writes as we may
    … …  
    28982880        iemNativeRegFreeTmp(pReNative, idxCr4Reg);
    28992881        iemNativeRegFreeTmp(pReNative, idxXcr0Reg);
    2900 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
     2882
    29012883        pReNative->fSimdRaiseXcptChecksEmitted |= IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX;
    29022884    }
    29032885    else
    29042886        STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativeMaybeAvxXcptCheckOmitted);
    2905 #endif
    29062887
    29072888    return off;
    … …  
    41224103    IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(pReNative, off, X86_EFL_STATUS_BITS);
    41234104
    4124 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    41254105    /* Clear the appropriate IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_XXX flags
    41264106       when a calls clobber any of the relevant control registers. */
    4127 # if 1
     4107#if 1
    41284108    if (!(fGstShwFlush & (RT_BIT_64(kIemNativeGstReg_Cr0) | RT_BIT_64(kIemNativeGstReg_Cr4) | RT_BIT_64(kIemNativeGstReg_Xcr0))))
    41294109    {
    … …  
    41424122                                                    | IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE);
    41434123
    4144 # else
     4124#else
    41454125    if (pfnCImpl == (uintptr_t)iemCImpl_xsetbv) /* Modifies xcr0 which only the AVX check uses. */
    41464126        pReNative->fSimdRaiseXcptChecksEmitted &= ~IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_AVX;
    … …  
    41544134                                                    | IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_SSE
    41554135                                                    | IEMNATIVE_SIMD_RAISE_XCPT_CHECKS_EMITTED_MAYBE_DEVICE_NOT_AVAILABLE);
    4156 # endif
    4157 
    4158 # ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
     4136#endif
     4137
     4138#ifdef IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS
    41594139    /* Mark the host floating point control register as not synced if MXCSR is modified. */
    41604140    if (fGstShwFlush & RT_BIT_64(kIemNativeGstReg_MxCsr))
    41614141        pReNative->fSimdRaiseXcptChecksEmitted &= ~IEMNATIVE_SIMD_HOST_FP_CTRL_REG_SYNCED;
    4162 # endif
    41634142#endif
    41644143
    … …  
    47344713
    47354714
    4736 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    47374715#define IEM_MC_FETCH_GREG_PAIR_U64(a_u128Dst, a_iGRegLo, a_iGRegHi) \
    47384716    off = iemNativeEmitFetchGregPairU64(pReNative, off, a_u128Dst, a_iGRegLo, a_iGRegHi)
    … …  
    47614739    return off;
    47624740}
    4763 #endif
    47644741
    47654742
    … …  
    51975174
    51985175
    5199 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    52005176#define IEM_MC_STORE_GREG_PAIR_U64(a_iGRegLo, a_iGRegHi, a_u128Value) \
    52015177    off = iemNativeEmitStoreGregPairU64(pReNative, off, a_iGRegLo, a_iGRegHi, a_u128Value)
    … …  
    52245200    return off;
    52255201}
    5226 #endif
    52275202
    52285203
    … …  
    61856160
    61866161    /* If we've delayed writing back the register value, flush it now. */
    6187     off = iemNativeRegFlushPendingSpecificWrite(pReNative, off, kIemNativeGstRegRef_Gpr, iGRegEx & 15);
     6162    off = iemNativeRegFlushPendingSpecificWrite<kIemNativeGstRegRef_Gpr>(pReNative, off, iGRegEx & 15);
    61886163
    61896164    /* If it's not a const reference we need to flush the shadow copy of the register now. */
    … …  
    62336208
    62346209    /* If we've delayed writing back the register value, flush it now. */
    6235     off = iemNativeRegFlushPendingSpecificWrite(pReNative, off, kIemNativeGstRegRef_Gpr, iGReg);
     6210    off = iemNativeRegFlushPendingSpecificWrite<kIemNativeGstRegRef_Gpr>(pReNative, off, iGReg);
    62366211
    62376212    /* If it's not a const reference we need to flush the shadow copy of the register now. */
    … …  
    62816256
    62826257    /* If we've delayed writing back the register value, flush it now. */
    6283     off = iemNativeRegFlushPendingSpecificWrite(pReNative, off, kIemNativeGstRegRef_EFlags, 0);
     6258    off = iemNativeRegFlushPendingSpecificWrite<kIemNativeGstRegRef_EFlags>(pReNative, off, 0);
    62846259
    62856260    /* If there is a shadow copy of guest EFLAGS, flush it now. */
    … …  
    63086283    off = iemNativeEmitRefXregXxx(pReNative, off, a_pXmmDst, a_iXReg, true /*fConst*/)
    63096284
    6310 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    63116285/* Just being paranoid here. */
    6312 # ifndef _MSC_VER /* MSC can't compile this, doesn't like [0]. Added reduced version afterwards. */
     6286#ifndef _MSC_VER /* MSC can't compile this, doesn't like [0]. Added reduced version afterwards. */
    63136287AssertCompile2MemberOffsets(CPUMCTX, XState.x87.aXMM[0], XState.x87.aXMM[0].au64[0]);
    63146288AssertCompile2MemberOffsets(CPUMCTX, XState.x87.aXMM[0], XState.x87.aXMM[0].au32[0]);
    63156289AssertCompile2MemberOffsets(CPUMCTX, XState.x87.aXMM[0], XState.x87.aXMM[0].ar64[0]);
    63166290AssertCompile2MemberOffsets(CPUMCTX, XState.x87.aXMM[0], XState.x87.aXMM[0].ar32[0]);
    6317 # endif
     6291#endif
    63186292AssertCompileMemberOffset(X86XMMREG, au64, 0);
    63196293AssertCompileMemberOffset(X86XMMREG, au32, 0);
    … …  
    63216295AssertCompileMemberOffset(X86XMMREG, ar32, 0);
    63226296
    6323 # define IEM_MC_REF_XREG_U32_CONST(a_pu32Dst, a_iXReg) \
     6297#define IEM_MC_REF_XREG_U32_CONST(a_pu32Dst, a_iXReg) \
    63246298    off = iemNativeEmitRefXregXxx(pReNative, off, a_pu32Dst, a_iXReg, true /*fConst*/)
    6325 # define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
     6299#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
    63266300    off = iemNativeEmitRefXregXxx(pReNative, off, a_pu64Dst, a_iXReg, true /*fConst*/)
    6327 # define IEM_MC_REF_XREG_R32_CONST(a_pr32Dst, a_iXReg) \
     6301#define IEM_MC_REF_XREG_R32_CONST(a_pr32Dst, a_iXReg) \
    63286302    off = iemNativeEmitRefXregXxx(pReNative, off, a_pr32Dst, a_iXReg, true /*fConst*/)
    6329 # define IEM_MC_REF_XREG_R64_CONST(a_pr64Dst, a_iXReg) \
     6303#define IEM_MC_REF_XREG_R64_CONST(a_pr64Dst, a_iXReg) \
    63306304    off = iemNativeEmitRefXregXxx(pReNative, off, a_pr64Dst, a_iXReg, true /*fConst*/)
    6331 #endif
    63326305
    63336306/** Handles IEM_MC_REF_XREG_xxx[_CONST]. */
    … …  
    63406313
    63416314    /* If we've delayed writing back the register value, flush it now. */
    6342     off = iemNativeRegFlushPendingSpecificWrite(pReNative, off, kIemNativeGstRegRef_XReg, iXReg);
    6343 
    6344 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
     6315    off = iemNativeRegFlushPendingSpecificWrite<kIemNativeGstRegRef_XReg>(pReNative, off, iXReg);
     6316
    63456317    /* If it's not a const reference we need to flush the shadow copy of the register now. */
    63466318    if (!fConst)
    63476319        iemNativeSimdRegFlushGuestShadows(pReNative, RT_BIT_64(IEMNATIVEGSTSIMDREG_SIMD(iXReg)));
    6348 #else
    6349     RT_NOREF(fConst);
    6350 #endif
    63516320
    63526321    return off;
    … …  
    71787147               IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
    71797148    Assert(!a_fFlat ? iSegReg < 6 : iSegReg == UINT8_MAX);
    7180 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    71817149    AssertCompile(   a_cbMem == 1 || a_cbMem == 2 || a_cbMem == 4 || a_cbMem == 8
    71827150                  || a_cbMem == sizeof(RTUINT128U) || a_cbMem == sizeof(RTUINT256U));
    7183 #else
    7184     AssertCompile(a_cbMem == 1 || a_cbMem == 2 || a_cbMem == 4 || a_cbMem == 8);
    7185 #endif
    71867151    AssertCompile(!(a_fAlignMaskAndCtl & ~(UINT32_C(0xff) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE)));
    71877152    AssertCompile(IEMNATIVE_CALL_ARG_GREG_COUNT >= 4);
    … …  
    72347199                Assert(a_fAlignMaskAndCtl <= 7);
    72357200                break;
    7236 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    72377201            case sizeof(RTUINT128U):
    72387202                Assert(   (   a_enmOp == kIemNativeEmitMemOp_Fetch
    … …  
    72607224                       : a_fAlignMaskAndCtl <= 31);
    72617225                break;
    7262 #endif
    72637226        }
    72647227    }
    … …  
    73087271                Assert(a_fAlignMaskAndCtl <= 7);
    73097272                break;
    7310 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    73117273            case sizeof(RTUINT128U):
    73127274                Assert(   (   a_enmOp == kIemNativeEmitMemOp_Fetch
    … …  
    73347296                       : a_fAlignMaskAndCtl <= 31);
    73357297                break;
    7336 #endif
    73377298        }
    73387299    }
    … …  
    73747335     */
    73757336    uint16_t const uTlbSeqNo         = pReNative->uTlbSeqNo++;
    7376 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    7377     uint8_t        idxRegValueFetch;
    7378     if RT_CONSTEXPR_IF(a_cbMem == sizeof(RTUINT128U) || a_cbMem == sizeof(RTUINT256U))
    7379         idxRegValueFetch = a_enmOp == kIemNativeEmitMemOp_Store ? UINT8_MAX
    7380                          : iemNativeVarSimdRegisterAcquire(pReNative, idxVarValue, &off);
    7381     else
    7382         idxRegValueFetch = a_enmOp == kIemNativeEmitMemOp_Store ? UINT8_MAX
    7383                          : !(pReNative->Core.bmHstRegs & RT_BIT_32(IEMNATIVE_CALL_RET_GREG))
    7384                          ? iemNativeVarRegisterSetAndAcquire(pReNative, idxVarValue, IEMNATIVE_CALL_RET_GREG, &off)
    7385                          : iemNativeVarRegisterAcquire(pReNative, idxVarValue, &off);
    7386 #else
    7387     uint8_t  const idxRegValueFetch  = a_enmOp == kIemNativeEmitMemOp_Store ? UINT8_MAX
     7337    RT_CONSTEXPR
     7338    bool const     fSimdRegValues    = a_cbMem == sizeof(RTUINT128U) || a_cbMem == sizeof(RTUINT256U);
     7339    uint8_t const  idxRegValueFetch  = a_enmOp == kIemNativeEmitMemOp_Store ? UINT8_MAX
     7340                                     : fSimdRegValues
     7341                                     ? iemNativeVarSimdRegisterAcquire(pReNative, idxVarValue, &off)
    73887342                                     : !(pReNative->Core.bmHstRegs & RT_BIT_32(IEMNATIVE_CALL_RET_GREG))
    73897343                                     ? iemNativeVarRegisterSetAndAcquire(pReNative, idxVarValue, IEMNATIVE_CALL_RET_GREG, &off)
    73907344                                     : iemNativeVarRegisterAcquire(pReNative, idxVarValue, &off);
    7391 #endif
    73927345    IEMNATIVEEMITTLBSTATE const TlbState(pReNative, &off, idxVarGCPtrMem, iSegReg, a_fFlat, a_cbMem, offDisp);
    7393 
    7394 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    7395     uint8_t  idxRegValueStore = UINT8_MAX;
    7396 
    7397     if RT_CONSTEXPR_IF(a_cbMem == sizeof(RTUINT128U) || a_cbMem == sizeof(RTUINT256U))
    7398         idxRegValueStore =    a_enmOp == kIemNativeEmitMemOp_Store
    7399                            && !TlbState.fSkip
    7400                            && pVarValue->enmKind != kIemNativeVarKind_Immediate
    7401                          ? iemNativeVarSimdRegisterAcquire(pReNative, idxVarValue, &off, true /*fInitialized*/)
    7402                          : UINT8_MAX;
    7403     else
    7404         idxRegValueStore  =    a_enmOp == kIemNativeEmitMemOp_Store
    7405                             && !TlbState.fSkip
    7406                             && pVarValue->enmKind != kIemNativeVarKind_Immediate
    7407                           ? iemNativeVarRegisterAcquireInited(pReNative, idxVarValue, &off)
    7408                           : UINT8_MAX;
    7409 
    7410 #else
    7411     uint8_t  const idxRegValueStore  =    a_enmOp == kIemNativeEmitMemOp_Store
    7412                                        && !TlbState.fSkip
    7413                                        && pVarValue->enmKind != kIemNativeVarKind_Immediate
    7414                                      ? iemNativeVarRegisterAcquireInited(pReNative, idxVarValue, &off)
    7415                                      : UINT8_MAX;
    7416 #endif
     7346    uint8_t const  idxRegValueStore =     a_enmOp != kIemNativeEmitMemOp_Store
     7347                                       || TlbState.fSkip
     7348                                       || pVarValue->enmKind == kIemNativeVarKind_Immediate
     7349                                     ? UINT8_MAX
     7350                                     : fSimdRegValues
     7351                                     ? iemNativeVarSimdRegisterAcquire(pReNative, idxVarValue, &off, true /*fInitialized*/)
     7352                                     : iemNativeVarRegisterAcquireInited(pReNative, idxVarValue, &off);
    74177353    uint32_t const idxRegMemResult   = !TlbState.fSkip ? iemNativeRegAllocTmp(pReNative, &off) : UINT8_MAX;
    74187354    uint32_t const idxLabelTlbLookup = !TlbState.fSkip
    … …  
    74687404#ifndef IEMNATIVE_WITH_FREE_AND_FLUSH_VOLATILE_REGS_AT_TLB_LOOKUP
    74697405    /* Save variables in volatile registers. */
    7470     uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave()
    7471                                      | (idxRegMemResult  != UINT8_MAX ? RT_BIT_32(idxRegMemResult)  : 0)
    7472                                      | (idxRegValueFetch != UINT8_MAX ? RT_BIT_32(idxRegValueFetch) : 0);
    7473     off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
     7406    uint32_t const fHstGprsNotToSave = TlbState.getRegsNotToSave()
     7407                                     | (idxRegMemResult  < 32                    ? RT_BIT_32(idxRegMemResult)        : 0)
     7408#ifdef _MSC_VER /* Workaround for stupid compiler (2019). */
     7409                                     | (idxRegValueFetch < 32 && !fSimdRegValues ? RT_BIT_32(idxRegValueFetch & 0x1f) : 0);
     7410#else
     7411                                     | (idxRegValueFetch < 32 && !fSimdRegValues ? RT_BIT_32(idxRegValueFetch)        : 0);
     7412#endif
     7413    off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstGprsNotToSave);
    74747414#endif
    74757415
    74767416    /* IEMNATIVE_CALL_ARG2/3_GREG = uValue (idxVarValue) - if store */
    74777417    uint32_t fVolGregMask = IEMNATIVE_CALL_VOLATILE_GREG_MASK;
    7478 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    7479     if RT_CONSTEXPR_IF(a_cbMem == sizeof(RTUINT128U) || a_cbMem == sizeof(RTUINT256U))
     7418    if RT_CONSTEXPR_IF(fSimdRegValues)
    74807419    {
    74817420        /*
    … …  
    74947433        fVolGregMask &= ~RT_BIT_32(idxRegArgValue);
    74957434    }
    7496     else
    7497 #endif
    7498     if RT_CONSTEXPR_IF(a_enmOp == kIemNativeEmitMemOp_Store)
     7435    else if RT_CONSTEXPR_IF(a_enmOp == kIemNativeEmitMemOp_Store)
    74997436    {
    75007437        uint8_t const idxRegArgValue = a_fFlat ? IEMNATIVE_CALL_ARG2_GREG : IEMNATIVE_CALL_ARG3_GREG;
    … …  
    75257462#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
    75267463    /* Do delayed EFLAGS calculations. */
    7527     if RT_CONSTEXPR_IF(a_enmOp == kIemNativeEmitMemOp_Store || a_cbMem == sizeof(RTUINT128U) || a_cbMem == sizeof(RTUINT256U))
     7464    if RT_CONSTEXPR_IF(a_enmOp == kIemNativeEmitMemOp_Store || fSimdRegValues)
    75287465    {
    75297466        if RT_CONSTEXPR_IF(a_fFlat)
    75307467            off = iemNativeDoPostponedEFlagsAtTlbMiss<  RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG)
    75317468                                                      | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG)>(pReNative, off, &TlbState,
    7532                                                                                              fHstRegsNotToSave);
     7469                                                                                             fHstGprsNotToSave);
    75337470        else
    75347471            off = iemNativeDoPostponedEFlagsAtTlbMiss<  RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG)
    75357472                                                      | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG)
    75367473                                                      | RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG)>(pReNative, off, &TlbState,
    7537                                                                                              fHstRegsNotToSave);
     7474                                                                                             fHstGprsNotToSave);
    75387475    }
    75397476    else if RT_CONSTEXPR_IF(a_fFlat)
    75407477        off = iemNativeDoPostponedEFlagsAtTlbMiss<  RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG)>(pReNative, off, &TlbState,
    7541                                                                                          fHstRegsNotToSave);
     7478                                                                                         fHstGprsNotToSave);
    75427479    else
    75437480        off = iemNativeDoPostponedEFlagsAtTlbMiss<  RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG)
    75447481                                                  | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG)>(pReNative, off, &TlbState,
    7545                                                                                          fHstRegsNotToSave);
     7482                                                                                         fHstGprsNotToSave);
    75467483#endif
    75477484
    … …  
    75577494    if RT_CONSTEXPR_IF(a_enmOp != kIemNativeEmitMemOp_Store)
    75587495    {
    7559 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    7560         if RT_CONSTEXPR_IF(a_cbMem == sizeof(RTUINT128U) || a_cbMem == sizeof(RTUINT256U))
     7496        if RT_CONSTEXPR_IF(fSimdRegValues)
    75617497        {
    75627498            Assert(a_enmOp == kIemNativeEmitMemOp_Fetch);
    … …  
    75667502        }
    75677503        else
    7568 #endif
    75697504        {
    75707505            Assert(idxRegValueFetch == pVarValue->idxReg);
    … …  
    75767511#ifndef IEMNATIVE_WITH_FREE_AND_FLUSH_VOLATILE_REGS_AT_TLB_LOOKUP
    75777512    /* Restore variables and guest shadow registers to volatile registers. */
    7578     off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
     7513    off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstGprsNotToSave);
    75797514    off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off, TlbState.getActiveRegsWithShadows());
    75807515#endif
    … …  
    76447579                            off = iemNativeEmitStoreGpr64ByGprEx(pCodeBuf, off, idxRegValueStore, idxRegMemResult);
    76457580                            break;
    7646 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    76477581                        case sizeof(RTUINT128U):
    76487582                            off = iemNativeEmitStoreVecRegByGprU128Ex(pCodeBuf, off, idxRegValueStore, idxRegMemResult);
    … …  
    76517585                            off = iemNativeEmitStoreVecRegByGprU256Ex(pCodeBuf, off, idxRegValueStore, idxRegMemResult);
    76527586                            break;
    7653 #endif
    76547587                        default:
    76557588                            AssertFailed();
    … …  
    77007633                        off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
    77017634                        break;
    7702 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    77037635                    case sizeof(RTUINT128U):
    77047636                        /*
    … …  
    77157647                        off = iemNativeEmitLoadVecRegByGprU256Ex(pCodeBuf, off, idxRegValueFetch, idxRegMemResult);
    77167648                        break;
    7717 #endif
    77187649                    default:
    77197650                        AssertFailed();
    … …  
    80007931            pReNative, off, a_r64Dst, UINT8_MAX, a_GCPtrMem, (uintptr_t)iemNativeHlpMemFlatFetchDataU64, pCallEntry->idxInstr)
    80017932
    8002 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
     7933
    80037934/* 128-bit segmented: */
    80047935#define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
    … …  
    80928023            pReNative, off, a_uYmmDst, UINT8_MAX, a_GCPtrMem, (uintptr_t)iemNativeHlpMemFlatFetchDataU256NoAc, pCallEntry->idxInstr)
    80938024
    8094 #endif
    80958025
    80968026
    … …  
    81878117
    81888118
    8189 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    8190 # define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
     8119#define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
    81918120    off = iemNativeEmitMemFetchStoreDataCommon<sizeof(RTUINT128U), \
    81928121                                               (sizeof(RTUINT128U) - 1U) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE, \
    … …  
    81948123            pReNative, off, a_u128Value, a_iSeg, a_GCPtrMem, (uintptr_t)iemNativeHlpMemStoreDataU128AlignedSse, pCallEntry->idxInstr)
    81958124
    8196 # define IEM_MC_STORE_MEM_U128_NO_AC(a_iSeg, a_GCPtrMem, a_u128Value) \
     8125#define IEM_MC_STORE_MEM_U128_NO_AC(a_iSeg, a_GCPtrMem, a_u128Value) \
    81978126    off = iemNativeEmitMemFetchStoreDataCommon<sizeof(RTUINT128U), sizeof(RTUINT128U) - 1, kIemNativeEmitMemOp_Store>(\
    81988127            pReNative, off, a_u128Value, a_iSeg, a_GCPtrMem, (uintptr_t)iemNativeHlpMemStoreDataU128NoAc, pCallEntry->idxInstr)
    81998128
    8200 # define IEM_MC_STORE_MEM_U256_NO_AC(a_iSeg, a_GCPtrMem, a_u256Value) \
     8129#define IEM_MC_STORE_MEM_U256_NO_AC(a_iSeg, a_GCPtrMem, a_u256Value) \
    82018130    off = iemNativeEmitMemFetchStoreDataCommon<sizeof(RTUINT256U), sizeof(RTUINT256U) - 1, kIemNativeEmitMemOp_Store>(\
    82028131            pReNative, off, a_u256Value, a_iSeg, a_GCPtrMem, (uintptr_t)iemNativeHlpMemStoreDataU256NoAc, pCallEntry->idxInstr)
    82038132
    8204 # define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
     8133#define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
    82058134    off = iemNativeEmitMemFetchStoreDataCommon<sizeof(RTUINT256U), \
    82068135                                               (sizeof(RTUINT256U) - 1U) | IEM_MEMMAP_F_ALIGN_GP, \
    … …  
    82098138
    82108139
    8211 # define IEM_MC_STORE_MEM_FLAT_U128_ALIGN_SSE(a_GCPtrMem, a_u128Value) \
     8140#define IEM_MC_STORE_MEM_FLAT_U128_ALIGN_SSE(a_GCPtrMem, a_u128Value) \
    82128141    off = iemNativeEmitMemFetchStoreDataCommon<sizeof(RTUINT128U), \
    82138142                                               (sizeof(RTUINT128U) - 1U) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE, \
    … …  
    82168145            pCallEntry->idxInstr)
    82178146
    8218 # define IEM_MC_STORE_MEM_FLAT_U128_NO_AC(a_GCPtrMem, a_u128Value) \
     8147#define IEM_MC_STORE_MEM_FLAT_U128_NO_AC(a_GCPtrMem, a_u128Value) \
    82198148    off = iemNativeEmitMemFetchStoreDataCommon<sizeof(RTUINT128U), sizeof(RTUINT128U) - 1, kIemNativeEmitMemOp_Store, true>(\
    82208149            pReNative, off, a_u128Value, UINT8_MAX, a_GCPtrMem, (uintptr_t)iemNativeHlpMemFlatStoreDataU128NoAc, pCallEntry->idxInstr)
    82218150
    8222 # define IEM_MC_STORE_MEM_FLAT_U256_NO_AC(a_GCPtrMem, a_u256Value) \
     8151#define IEM_MC_STORE_MEM_FLAT_U256_NO_AC(a_GCPtrMem, a_u256Value) \
    82238152    off = iemNativeEmitMemFetchStoreDataCommon<sizeof(RTUINT256U), sizeof(RTUINT256U) - 1, kIemNativeEmitMemOp_Store, true>(\
    82248153            pReNative, off, a_u256Value, UINT8_MAX, a_GCPtrMem, (uintptr_t)iemNativeHlpMemFlatStoreDataU256NoAc, pCallEntry->idxInstr)
    82258154
    8226 # define IEM_MC_STORE_MEM_FLAT_U256_ALIGN_AVX(a_GCPtrMem, a_u256Value) \
     8155#define IEM_MC_STORE_MEM_FLAT_U256_ALIGN_AVX(a_GCPtrMem, a_u256Value) \
    82278156    off = iemNativeEmitMemFetchStoreDataCommon<sizeof(RTUINT256U), \
    82288157                                               (sizeof(RTUINT256U) - 1U) | IEM_MEMMAP_F_ALIGN_GP, kIemNativeEmitMemOp_Store, \
    82298158                                               true>(\
    82308159            pReNative, off, a_u256Value, UINT8_MAX, a_GCPtrMem, (uintptr_t)iemNativeHlpMemFlatStoreDataU256AlignedAvx, pCallEntry->idxInstr)
    8231 #endif
    82328160
    82338161
    … …  
    98089736
    98099737
    9810 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    9811 
    9812 
    98139738/*********************************************************************************************************************************
    98149739*   Emitters for SSE/AVX specific operations.                                                                                    *
    … …  
    1110211027
    1110311028
    11104 #endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
    11105 
    1110611029
    1110711030/*********************************************************************************************************************************

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

    r106408 r106453  
    711711
    712712
    713 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    714713/**
    715714 * Used by TB code to load 128-bit data w/ segmentation.
    … …  
    775774#endif
    776775}
    777 #endif
    778776
    779777
    … …  
    830828
    831829
    832 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    833830/**
    834831 * Used by TB code to store unsigned 128-bit data w/ segmentation.
    … …  
    881878#endif
    882879}
    883 #endif
    884 
    885880
    886881
    … …  
    11271122
    11281123
    1129 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    11301124/**
    11311125 * Used by TB code to load unsigned 128-bit data w/ flat address.
    … …  
    11911185#endif
    11921186}
    1193 #endif
    11941187
    11951188
    … …  
    12461239
    12471240
    1248 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    12491241/**
    12501242 * Used by TB code to store unsigned 128-bit data w/ flat address.
    … …  
    12971289#endif
    12981290}
    1299 #endif
    1300 
    13011291
    13021292
    … …  
    20872077# endif
    20882078#endif
    2089 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    20902079    pReNative->fSimdRaiseXcptChecksEmitted = 0;
    2091 #endif
    20922080    pReNative->Core.bmHstRegs              = IEMNATIVE_REG_FIXED_MASK
    20932081#if IEMNATIVE_HST_GREG_COUNT < 32
    … …  
    21752163#endif
    21762164
    2177 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    21782165    pReNative->Core.bmHstSimdRegs          = IEMNATIVE_SIMD_REG_FIXED_MASK
    2179 # if IEMNATIVE_HST_SIMD_REG_COUNT < 32
     2166#if IEMNATIVE_HST_SIMD_REG_COUNT < 32
    21802167                                           | ~(RT_BIT(IEMNATIVE_HST_SIMD_REG_COUNT) - 1U)
    2181 # endif
     2168#endif
    21822169                                           ;
    21832170    pReNative->Core.bmHstSimdRegsWithGstShadow   = 0;
    … …  
    22042191#ifdef IEMNATIVE_SIMD_REG_FIXED_TMP0
    22052192    pReNative->Core.aHstSimdRegs[IEMNATIVE_SIMD_REG_FIXED_TMP0].enmWhat = kIemNativeWhat_FixedTmp;
    2206 #endif
    2207 
    22082193#endif
    22092194
    … …  
    26882673    pEntry->GuestRegShadowing.idxHstReg     = idxHstReg;
    26892674    pEntry->GuestRegShadowing.idxHstRegPrev = idxHstRegPrev;
    2690 #ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
     2675# ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
    26912676    Assert(   idxHstReg != UINT8_MAX
    26922677           || !(pReNative->Core.bmGstRegShadowDirty & RT_BIT_64(enmGstReg)));
    2693 #endif
    2694 }
    2695 
    2696 
    2697 # ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
     2678# endif
     2679}
     2680
     2681
    26982682/**
    26992683 * Debug Info: Record info about guest register shadowing.
    … …  
    27102694    pEntry->GuestSimdRegShadowing.idxHstSimdRegPrev = idxHstSimdRegPrev;
    27112695}
    2712 # endif
    27132696
    27142697
    … …  
    27262709# endif
    27272710
    2728 # if defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK) || defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR)
    27292711
    27302712/**
    … …  
    27882770}
    27892771
    2790 # endif /* defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK) || defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR) */
    27912772
    27922773# ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
    … …  
    32843265        {
    32853266            uint32_t const idxVar = ASMBitFirstSetU32(fVars) - 1;
    3286             uint8_t const  idxReg = pReNative->Core.aVars[idxVar].idxReg;
    3287 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    3288             if (pReNative->Core.aVars[idxVar].fSimdReg) /* Need to ignore SIMD variables here or we end up freeing random registers. */
    3289                 continue;
    3290 #endif
    3291 
    3292             if (   idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs)
    3293                 && (RT_BIT_32(idxReg) & fRegMask)
    3294                 && (  iLoop == 0
    3295                     ? pReNative->Core.aVars[idxVar].enmKind != kIemNativeVarKind_Stack
    3296                     : pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack)
    3297                 && !pReNative->Core.aVars[idxVar].fRegAcquired)
     3267            if (!pReNative->Core.aVars[idxVar].fSimdReg) /* (this is the GPR allocator) */
    32983268            {
    3299                 Assert(pReNative->Core.bmHstRegs & RT_BIT_32(idxReg));
    3300                 Assert(   (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxReg].fGstRegShadows)
    3301                        == pReNative->Core.aHstRegs[idxReg].fGstRegShadows);
    3302                 Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
    3303                 Assert(   RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxReg))
    3304                        == RT_BOOL(pReNative->Core.aHstRegs[idxReg].fGstRegShadows));
     3269                uint8_t const  idxReg = pReNative->Core.aVars[idxVar].idxReg;
     3270                if (   idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs)
     3271                    && (RT_BIT_32(idxReg) & fRegMask)
     3272                    && (  iLoop == 0
     3273                        ? pReNative->Core.aVars[idxVar].enmKind != kIemNativeVarKind_Stack
     3274                        : pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack)
     3275                    && !pReNative->Core.aVars[idxVar].fRegAcquired)
     3276                {
     3277                    Assert(pReNative->Core.bmHstRegs & RT_BIT_32(idxReg));
     3278                    Assert(   (pReNative->Core.bmGstRegShadows & pReNative->Core.aHstRegs[idxReg].fGstRegShadows)
     3279                           == pReNative->Core.aHstRegs[idxReg].fGstRegShadows);
     3280                    Assert(pReNative->Core.bmGstRegShadows < RT_BIT_64(kIemNativeGstReg_End));
     3281                    Assert(   RT_BOOL(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxReg))
     3282                           == RT_BOOL(pReNative->Core.aHstRegs[idxReg].fGstRegShadows));
    33053283#ifdef IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK
    3306                 Assert(!(pReNative->Core.aHstRegs[idxReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
    3307 #endif
    3308 
    3309                 if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack)
    3310                 {
    3311                     uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, IEMNATIVE_VAR_IDX_PACK(idxVar));
    3312                     *poff = iemNativeEmitStoreGprByBp(pReNative, *poff, iemNativeStackCalcBpDisp(idxStackSlot), idxReg);
     3284                    Assert(!(pReNative->Core.aHstRegs[idxReg].fGstRegShadows & pReNative->Core.bmGstRegShadowDirty));
     3285#endif
     3286
     3287                    if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack)
     3288                    {
     3289                        uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, IEMNATIVE_VAR_IDX_PACK(idxVar));
     3290                        *poff = iemNativeEmitStoreGprByBp(pReNative, *poff, iemNativeStackCalcBpDisp(idxStackSlot), idxReg);
     3291                    }
     3292
     3293                    pReNative->Core.aVars[idxVar].idxReg    = UINT8_MAX;
     3294                    pReNative->Core.bmHstRegs              &= ~RT_BIT_32(idxReg);
     3295
     3296                    pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxReg);
     3297                    pReNative->Core.bmGstRegShadows        &= ~pReNative->Core.aHstRegs[idxReg].fGstRegShadows;
     3298                    pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
     3299                    return idxReg;
    33133300                }
    3314 
    3315                 pReNative->Core.aVars[idxVar].idxReg    = UINT8_MAX;
    3316                 pReNative->Core.bmHstRegs              &= ~RT_BIT_32(idxReg);
    3317 
    3318                 pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxReg);
    3319                 pReNative->Core.bmGstRegShadows        &= ~pReNative->Core.aHstRegs[idxReg].fGstRegShadows;
    3320                 pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
    3321                 return idxReg;
    33223301            }
    33233302            fVars &= ~RT_BIT_32(idxVar);
    … …  
    33453324    IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
    33463325    Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].idxReg == idxRegOld);
    3347 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    33483326    Assert(!pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
    3349 #endif
    33503327    RT_NOREF(pszCaller);
    33513328
    … …  
    42014178                                   IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_REG_IPE_5));
    42024179                        Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].idxReg == idxReg);
    4203 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    42044180                        Assert(!pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
    4205 #endif
    42064181
    42074182                        if (pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].enmKind != kIemNativeVarKind_Stack)
    … …  
    43254300    IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
    43264301    Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].idxReg == idxHstReg);
    4327 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    43284302    Assert(!pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fSimdReg);
    4329 #endif
    43304303
    43314304    pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].idxReg = UINT8_MAX;
    … …  
    43584331
    43594332
    4360 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    4361 # if defined(LOG_ENABLED) || defined(IEMNATIVE_WITH_TB_DEBUG_INFO)
     4333#if defined(LOG_ENABLED) || defined(IEMNATIVE_WITH_TB_DEBUG_INFO)
    43624334/** Host CPU SIMD register names. */
    43634335DECL_HIDDEN_CONST(const char * const) g_apszIemNativeHstSimdRegNames[] =
    43644336{
    4365 #  ifdef RT_ARCH_AMD64
     4337# ifdef RT_ARCH_AMD64
    43664338    "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15"
    4367 #  elif RT_ARCH_ARM64
     4339# elif RT_ARCH_ARM64
    43684340    "v0",  "v1",  "v2",  "v3",  "v4",  "v5",  "v6",  "v7",  "v8",  "v9",  "v10", "v11", "v12", "v13", "v14", "v15",
    43694341    "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31",
    4370 #  else
    4371 #   error "port me"
    4372 #  endif
     4342# else
     4343#  error "port me"
     4344# endif
    43734345};
    4374 # endif
     4346#endif
    43754347
    43764348
    … …  
    46914663    return off;
    46924664}
    4693 #endif
    46944665
    46954666
    … …  
    47514722                    PIEMNATIVEVAR const pVar   = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)];
    47524723                    Assert(pVar->idxReg == idxReg);
    4753 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    47544724                    Assert(!pVar->fSimdReg);
    4755 #endif
    47564725                    if (!(RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(idxVar)) & fKeepVars))
    47574726                    {
    … …  
    48294798    }
    48304799
    4831 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    4832     /* Now for the SIMD registers, no argument support for now. */
     4800    /*
     4801     * Now for the SIMD registers, no argument support for now.
     4802     */
    48334803    off = iemNativeSimdRegMoveAndFreeAndFlushAtCall(pReNative, off, 0 /*cArgs*/, fKeepVars);
    4834 #endif
    48354804
    48364805    return off;
    … …  
    50315000*   SIMD register allocator (largely code duplication of the GPR allocator for now but might diverge)                            *
    50325001*********************************************************************************************************************************/
    5033 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    50345002
    50355003/**
    … …  
    52955263        {
    52965264            uint32_t const idxVar = ASMBitFirstSetU32(fVars) - 1;
    5297             uint8_t const  idxReg = pReNative->Core.aVars[idxVar].idxReg;
    5298             if (!pReNative->Core.aVars[idxVar].fSimdReg) /* Ignore non SIMD variables here. */
    5299                 continue;
    5300 
    5301             if (   idxReg < RT_ELEMENTS(pReNative->Core.aHstSimdRegs)
    5302                 && (RT_BIT_32(idxReg) & fRegMask)
    5303                 && (  iLoop == 0
    5304                     ? pReNative->Core.aVars[idxVar].enmKind != kIemNativeVarKind_Stack
    5305                     : pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack)
    5306                 && !pReNative->Core.aVars[idxVar].fRegAcquired)
     5265            if (pReNative->Core.aVars[idxVar].fSimdReg) /* (this is the SIMD allocator) */
    53075266            {
    5308                 Assert(pReNative->Core.bmHstSimdRegs & RT_BIT_32(idxReg));
    5309                 Assert(   (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxReg].fGstRegShadows)
    5310                        == pReNative->Core.aHstSimdRegs[idxReg].fGstRegShadows);
    5311                 Assert(pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
    5312                 Assert(   RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxReg))
    5313                        == RT_BOOL(pReNative->Core.aHstSimdRegs[idxReg].fGstRegShadows));
    5314 
    5315                 if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack)
     5267                uint8_t const idxReg = pReNative->Core.aVars[idxVar].idxReg;
     5268                if (   idxReg < RT_ELEMENTS(pReNative->Core.aHstSimdRegs)
     5269                    && (RT_BIT_32(idxReg) & fRegMask)
     5270                    && (  iLoop == 0
     5271                        ? pReNative->Core.aVars[idxVar].enmKind != kIemNativeVarKind_Stack
     5272                        : pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack)
     5273                    && !pReNative->Core.aVars[idxVar].fRegAcquired)
    53165274                {
    5317                     uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, IEMNATIVE_VAR_IDX_PACK(idxVar));
    5318                     *poff = iemNativeEmitStoreGprByBp(pReNative, *poff, iemNativeStackCalcBpDisp(idxStackSlot), idxReg);
     5275                    Assert(pReNative->Core.bmHstSimdRegs & RT_BIT_32(idxReg));
     5276                    Assert(   (pReNative->Core.bmGstSimdRegShadows & pReNative->Core.aHstSimdRegs[idxReg].fGstRegShadows)
     5277                           == pReNative->Core.aHstSimdRegs[idxReg].fGstRegShadows);
     5278                    Assert(pReNative->Core.bmGstSimdRegShadows < RT_BIT_64(kIemNativeGstSimdReg_End));
     5279                    Assert(   RT_BOOL(pReNative->Core.bmHstSimdRegsWithGstShadow & RT_BIT_32(idxReg))
     5280                           == RT_BOOL(pReNative->Core.aHstSimdRegs[idxReg].fGstRegShadows));
     5281
     5282                    if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack)
     5283                    {
     5284                        uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, IEMNATIVE_VAR_IDX_PACK(idxVar));
     5285                        *poff = iemNativeEmitStoreGprByBp(pReNative, *poff, iemNativeStackCalcBpDisp(idxStackSlot), idxReg);
     5286                    }
     5287
     5288                    pReNative->Core.aVars[idxVar].idxReg        = UINT8_MAX;
     5289                    pReNative->Core.bmHstSimdRegs              &= ~RT_BIT_32(idxReg);
     5290
     5291                    pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxReg);
     5292                    pReNative->Core.bmGstSimdRegShadows        &= ~pReNative->Core.aHstRegs[idxReg].fGstRegShadows;
     5293                    pReNative->Core.aHstSimdRegs[idxReg].fGstRegShadows = 0;
     5294                    return idxReg;
    53195295                }
    5320 
    5321                 pReNative->Core.aVars[idxVar].idxReg        = UINT8_MAX;
    5322                 pReNative->Core.bmHstSimdRegs              &= ~RT_BIT_32(idxReg);
    5323 
    5324                 pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxReg);
    5325                 pReNative->Core.bmGstSimdRegShadows        &= ~pReNative->Core.aHstRegs[idxReg].fGstRegShadows;
    5326                 pReNative->Core.aHstSimdRegs[idxReg].fGstRegShadows = 0;
    5327                 return idxReg;
    53285296            }
    53295297            fVars &= ~RT_BIT_32(idxVar);
    … …  
    55465514        || pReNative->Core.aHstSimdRegs[idxHstSimdRegSrc].enmLoaded == kIemNativeGstSimdRegLdStSz_256)
    55475515    {
    5548 # ifdef RT_ARCH_ARM64
     5516#ifdef RT_ARCH_ARM64
    55495517        /* ASSUMES that there are two adjacent 128-bit registers available for the 256-bit value. */
    55505518        Assert(!(idxHstSimdRegDst & 0x1)); Assert(!(idxHstSimdRegSrc & 0x1));
    5551 # endif
     5519#endif
    55525520
    55535521        if (idxHstSimdRegDst != idxHstSimdRegSrc)
    … …  
    57565724            || enmIntendedUse == kIemNativeGstRegUse_ForUpdate)
    57575725        {
    5758 # if defined(IEMNATIVE_WITH_TB_DEBUG_INFO) && defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK)
     5726#if defined(IEMNATIVE_WITH_TB_DEBUG_INFO) && defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK)
    57595727            iemNativeDbgInfoAddNativeOffset(pReNative, *poff);
    57605728            iemNativeDbgInfoAddGuestRegDirty(pReNative, true /*fSimdReg*/, enmGstSimdReg, idxSimdReg);
    5761 # endif
     5729#endif
    57625730
    57635731            if (enmLoadSz == kIemNativeGstSimdRegLdStSz_Low128)
    … …  
    57925760        || enmIntendedUse == kIemNativeGstRegUse_ForUpdate)
    57935761    {
    5794 # if defined(IEMNATIVE_WITH_TB_DEBUG_INFO) && defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK)
     5762#if defined(IEMNATIVE_WITH_TB_DEBUG_INFO) && defined(IEMNATIVE_WITH_DELAYED_REGISTER_WRITEBACK)
    57955763        iemNativeDbgInfoAddNativeOffset(pReNative, *poff);
    57965764        iemNativeDbgInfoAddGuestRegDirty(pReNative, true /*fSimdReg*/, enmGstSimdReg, idxRegNew);
    5797 # endif
     5765#endif
    57985766
    57995767        if (enmLoadSz == kIemNativeGstSimdRegLdStSz_Low128)
    … …  
    58805848    }
    58815849}
    5882 #endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
    58835850
    58845851
    … …  
    59725939#endif
    59735940
    5974 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    5975     off = iemNativeSimdRegFlushDirtyGuest(pReNative, off, ~fGstSimdShwExcept);
    5976 #endif
    5977 
    5978     return off;
     5941    return iemNativeSimdRegFlushDirtyGuest(pReNative, off, ~fGstSimdShwExcept);
    59795942}
    59805943
    … …  
    61756138
    61766139
    6177 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    61786140/**
    61796141 * Loads the guest shadow SIMD register @a enmGstSimdReg into host SIMD reg @a idxHstSimdReg.
    … …  
    62096171    }
    62106172}
    6211 #endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
    62126173
    62136174#ifdef VBOX_STRICT
    … …  
    64036364}
    64046365
    6405 # ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    6406 #  ifdef RT_ARCH_AMD64
     6366# ifdef RT_ARCH_AMD64
    64076367/**
    64086368 * Helper for AMD64 to emit code which checks the low 128-bits of the given SIMD register against the given vCPU offset.
    64096369 */
    6410 DECL_FORCE_INLINE_THROW(uint32_t) iemNativeEmitGuestSimdRegValueCheckVCpuU128(uint8_t * const pbCodeBuf, uint32_t off, uint8_t idxSimdReg, uint32_t offVCpu)
     6370DECL_FORCE_INLINE_THROW(uint32_t)
     6371iemNativeEmitGuestSimdRegValueCheckVCpuU128(uint8_t * const pbCodeBuf, uint32_t off, uint8_t idxSimdReg, uint32_t offVCpu)
    64116372{
    64126373    /* pcmpeqq vectmp0, [gstreg] (ASSUMES SSE4.1) */
    … …  
    64696430    return off;
    64706431}
    6471 #  endif
     6432# endif /* RT_ARCH_AMD64 */
    64726433
    64736434
    … …  
    64946455        return off;
    64956456
    6496 #  ifdef RT_ARCH_AMD64
     6457# ifdef RT_ARCH_AMD64
    64976458    if (enmLoadSz == kIemNativeGstSimdRegLdStSz_Low128 || enmLoadSz == kIemNativeGstSimdRegLdStSz_256)
    64986459    {
    … …  
    65256486                                                          g_aGstSimdShadowInfo[enmGstSimdReg].offYmm);
    65266487    }
    6527 #  elif defined(RT_ARCH_ARM64)
     6488
     6489# elif defined(RT_ARCH_ARM64)
    65286490    /* mov vectmp0, [gstreg] */
    65296491    off = iemNativeEmitLoadSimdRegWithGstShadowSimdReg(pReNative, off, IEMNATIVE_SIMD_REG_FIXED_TMP0, enmGstSimdReg, enmLoadSz);
    … …  
    65616523    }
    65626524
    6563 #  else
    6564 #   error "Port me!"
    6565 #  endif
     6525# else
     6526#  error "Port me!"
     6527# endif
    65666528
    65676529    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    65686530    return off;
    65696531}
    6570 # endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
    65716532
    65726533
    … …  
    65866547    off = iemNativeEmitCmpGpr32WithImm(pReNative, off, idxRegTmp, fExec & IEMTB_F_KEY_MASK);
    65876548
    6588 #ifdef RT_ARCH_AMD64
     6549# ifdef RT_ARCH_AMD64
    65896550    uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
    65906551
    … …  
    71527113    pReNative->Core.aVars[idxVar].fRegAcquired   = false;
    71537114    pReNative->Core.aVars[idxVar].u.uValue       = 0;
    7154 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    71557115    pReNative->Core.aVars[idxVar].fSimdReg       = false;
    7156 #endif
    71577116    return idxVar;
    71587117}
    … …  
    75807539    }
    75817540    iemNativeRegMarkAllocated(pReNative, idxReg, kIemNativeWhat_Var, idxVar);
    7582     pVar->idxReg = idxReg;
    7583 
    7584 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
     7541    pVar->idxReg   = idxReg;
    75857542    pVar->fSimdReg = false;
    7586 #endif
    75877543
    75887544    /*
    … …  
    76677623
    76687624
    7669 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    76707625/**
    76717626 * Makes sure variable @a idxVar has a SIMD register assigned to it and that it stays
    … …  
    76837638 * @param  idxRegPref   Preferred SIMD register number or UINT8_MAX.
    76847639 */
     7640/** @todo Create variants for the last two params like we've done for the
     7641 *        GPR variant? */
    76857642DECL_HIDDEN_THROW(uint8_t) iemNativeVarSimdRegisterAcquire(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff,
    76867643                                                           bool fInitialized /*= false*/, uint8_t idxRegPref /*= UINT8_MAX*/)
    … …  
    76927649    Assert(!pVar->fRegAcquired);
    76937650
     7651/** @todo inline this bit?   */
    76947652    uint8_t idxReg = pVar->idxReg;
    76957653    if (idxReg < RT_ELEMENTS(pReNative->Core.aHstSimdRegs))
    … …  
    77627720    }
    77637721    iemNativeSimdRegMarkAllocated(pReNative, idxReg, kIemNativeWhat_Var, idxVar);
    7764 
     7722    pVar->idxReg   = idxReg;
    77657723    pVar->fSimdReg = true;
    7766     pVar->idxReg = idxReg;
    77677724
    77687725    /*
    … …  
    77897746    return idxReg;
    77907747}
    7791 #endif
    77927748
    77937749
    … …  
    79297885 * @param   pReNative           The recompiler state.
    79307886 * @param   off                 The code buffer position.
    7931  * @param   fHstRegsNotToSave   Set of registers not to save & restore.
     7887 * @param   fHstGprNotToSave    Set of GPRs not to save & restore.
    79327888 */
    79337889DECL_HIDDEN_THROW(uint32_t)
    7934 iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fHstRegsNotToSave)
    7935 {
    7936     uint32_t fHstRegs = pReNative->Core.bmHstRegs & IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK & ~fHstRegsNotToSave;
     7890iemNativeVarSaveVolatileRegsPreHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fHstGprNotToSave)
     7891{
     7892    uint32_t fHstRegs = pReNative->Core.bmHstRegs & IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK & ~fHstGprNotToSave;
    79377893    if (fHstRegs)
    79387894    {
    … …  
    79917947        } while (fHstRegs);
    79927948    }
    7993 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    79947949
    79957950    /*
    … …  
    80558010        } while (fHstRegs);
    80568011    }
    8057 #endif
    80588012    return off;
    80598013}
    … …  
    80668020 * @param   pReNative           The recompiler state.
    80678021 * @param   off                 The code buffer position.
    8068  * @param   fHstRegsNotToSave   Set of registers not to save & restore.
     8022 * @param   fHstGprNotToSave   Set of registers not to save & restore.
    80698023 * @see     iemNativeVarSaveVolatileRegsPreHlpCall(),
    80708024 *          iemNativeRegRestoreGuestShadowsInVolatileRegs()
    80718025 */
    80728026DECL_HIDDEN_THROW(uint32_t)
    8073 iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fHstRegsNotToSave)
    8074 {
    8075     uint32_t fHstRegs = pReNative->Core.bmHstRegs & IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK & ~fHstRegsNotToSave;
     8027iemNativeVarRestoreVolatileRegsPostHlpCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fHstGprNotToSave)
     8028{
     8029    /*
     8030     * GPRs
     8031     */
     8032    uint32_t fHstRegs = pReNative->Core.bmHstRegs & IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK & ~fHstGprNotToSave;
    80768033    if (fHstRegs)
    80778034    {
    … …  
    81288085        } while (fHstRegs);
    81298086    }
    8130 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
     8087
     8088    /*
     8089     * SIMD registers.
     8090     */
    81318091    fHstRegs = pReNative->Core.bmHstSimdRegs & (IEMNATIVE_CALL_VOLATILE_SIMD_REG_MASK & ~IEMNATIVE_SIMD_REG_FIXED_MASK);
    81328092    if (fHstRegs)
    … …  
    81818141        } while (fHstRegs);
    81828142    }
    8183 #endif
    81848143    return off;
    81858144}
    … …  
    82278186    /* Free the host register first if any assigned. */
    82288187    uint8_t const idxHstReg = pReNative->Core.aVars[idxVar].idxReg;
    8229 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    8230     if (   idxHstReg != UINT8_MAX
    8231         && pReNative->Core.aVars[idxVar].fSimdReg)
    8232     {
    8233         Assert(idxHstReg < RT_ELEMENTS(pReNative->Core.aHstSimdRegs));
    8234         Assert(pReNative->Core.aHstSimdRegs[idxHstReg].idxVar == IEMNATIVE_VAR_IDX_PACK(idxVar));
    8235         pReNative->Core.aHstSimdRegs[idxHstReg].idxVar = UINT8_MAX;
    8236         pReNative->Core.bmHstSimdRegs &= ~RT_BIT_32(idxHstReg);
    8237     }
    8238     else
    8239 #endif
    8240     if (idxHstReg < RT_ELEMENTS(pReNative->Core.aHstRegs))
    8241     {
    8242         Assert(pReNative->Core.aHstRegs[idxHstReg].idxVar == IEMNATIVE_VAR_IDX_PACK(idxVar));
    8243         pReNative->Core.aHstRegs[idxHstReg].idxVar = UINT8_MAX;
    8244         pReNative->Core.bmHstRegs &= ~RT_BIT_32(idxHstReg);
     8188    if (idxHstReg != UINT8_MAX)
     8189    {
     8190        if (!pReNative->Core.aVars[idxVar].fSimdReg)
     8191        {
     8192            Assert(idxHstReg < RT_ELEMENTS(pReNative->Core.aHstRegs));
     8193            Assert(pReNative->Core.aHstRegs[idxHstReg].idxVar == IEMNATIVE_VAR_IDX_PACK(idxVar));
     8194            pReNative->Core.aHstRegs[idxHstReg].idxVar = UINT8_MAX;
     8195            pReNative->Core.bmHstRegs &= ~RT_BIT_32(idxHstReg);
     8196        }
     8197        else
     8198        {
     8199            Assert(idxHstReg < RT_ELEMENTS(pReNative->Core.aHstSimdRegs));
     8200            Assert(pReNative->Core.aHstSimdRegs[idxHstReg].idxVar == IEMNATIVE_VAR_IDX_PACK(idxVar));
     8201            pReNative->Core.aHstSimdRegs[idxHstReg].idxVar = UINT8_MAX;
     8202            pReNative->Core.bmHstSimdRegs &= ~RT_BIT_32(idxHstReg);
     8203        }
    82458204    }
    82468205
    … …  
    84468405            {
    84478406                uint8_t const idxRegOld = pReNative->Core.aVars[idxVar].idxReg;
    8448 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    8449                 if (   idxRegOld != UINT8_MAX
    8450                     && pReNative->Core.aVars[idxVar].fSimdReg)
     8407                if (idxRegOld != UINT8_MAX)
    84518408                {
    8452                     Assert(idxRegOld < RT_ELEMENTS(pReNative->Core.aHstSimdRegs));
    8453                     Assert(pReNative->Core.aVars[idxVar].cbVar == sizeof(RTUINT128U) || pReNative->Core.aVars[idxVar].cbVar == sizeof(RTUINT256U));
    8454 
    8455                     uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, IEMNATIVE_VAR_IDX_PACK(idxVar));
    8456                     Log12(("iemNativeEmitCallCommon: spilling idxVar=%d/%#x/idxReg=%d (referred to by %d) onto the stack (slot %#x bp+%d, off=%#x)\n",
    8457                            idxVar, IEMNATIVE_VAR_IDX_PACK(idxVar), idxRegOld, pReNative->Core.aVars[idxVar].idxReferrerVar,
    8458                            idxStackSlot, iemNativeStackCalcBpDisp(idxStackSlot), off));
    8459                     if (pReNative->Core.aVars[idxVar].cbVar == sizeof(RTUINT128U))
    8460                         off = iemNativeEmitStoreVecRegByBpU128(pReNative, off, iemNativeStackCalcBpDisp(idxStackSlot), idxRegOld);
     8409                    if (!pReNative->Core.aVars[idxVar].fSimdReg)
     8410                    {
     8411                        Assert(idxRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
     8412
     8413                        uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, IEMNATIVE_VAR_IDX_PACK(idxVar));
     8414                        Log12(("iemNativeEmitCallCommon: spilling idxVar=%d/%#x/idxReg=%d (referred to by %d) onto the stack (slot %#x bp+%d, off=%#x)\n",
     8415                               idxVar, IEMNATIVE_VAR_IDX_PACK(idxVar), idxRegOld, pReNative->Core.aVars[idxVar].idxReferrerVar,
     8416                               idxStackSlot, iemNativeStackCalcBpDisp(idxStackSlot), off));
     8417                        off = iemNativeEmitStoreGprByBp(pReNative, off, iemNativeStackCalcBpDisp(idxStackSlot), idxRegOld);
     8418
     8419                        pReNative->Core.aVars[idxVar].idxReg    = UINT8_MAX;
     8420                        pReNative->Core.bmHstRegs              &= ~RT_BIT_32(idxRegOld);
     8421                        pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegOld);
     8422                        pReNative->Core.bmGstRegShadows        &= ~pReNative->Core.aHstRegs[idxRegOld].fGstRegShadows;
     8423                        pReNative->Core.aHstRegs[idxRegOld].fGstRegShadows = 0;
     8424                    }
    84618425                    else
    8462                         off = iemNativeEmitStoreVecRegByBpU256(pReNative, off, iemNativeStackCalcBpDisp(idxStackSlot), idxRegOld);
    8463 
    8464                     Assert(!(   (pReNative->Core.bmGstSimdRegShadowDirtyLo128 | pReNative->Core.bmGstSimdRegShadowDirtyHi128)
    8465                               & pReNative->Core.aHstSimdRegs[idxRegOld].fGstRegShadows));
    8466 
    8467                     pReNative->Core.aVars[idxVar].idxReg       = UINT8_MAX;
    8468                     pReNative->Core.bmHstSimdRegs              &= ~RT_BIT_32(idxRegOld);
    8469                     pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxRegOld);
    8470                     pReNative->Core.bmGstSimdRegShadows        &= ~pReNative->Core.aHstSimdRegs[idxRegOld].fGstRegShadows;
    8471                     pReNative->Core.aHstSimdRegs[idxRegOld].fGstRegShadows = 0;
    8472                 }
    8473                 else
    8474 #endif
    8475                 if (idxRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs))
    8476                 {
    8477                     uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, IEMNATIVE_VAR_IDX_PACK(idxVar));
    8478                     Log12(("iemNativeEmitCallCommon: spilling idxVar=%d/%#x/idxReg=%d (referred to by %d) onto the stack (slot %#x bp+%d, off=%#x)\n",
    8479                            idxVar, IEMNATIVE_VAR_IDX_PACK(idxVar), idxRegOld, pReNative->Core.aVars[idxVar].idxReferrerVar,
    8480                            idxStackSlot, iemNativeStackCalcBpDisp(idxStackSlot), off));
    8481                     off = iemNativeEmitStoreGprByBp(pReNative, off, iemNativeStackCalcBpDisp(idxStackSlot), idxRegOld);
    8482 
    8483                     pReNative->Core.aVars[idxVar].idxReg    = UINT8_MAX;
    8484                     pReNative->Core.bmHstRegs              &= ~RT_BIT_32(idxRegOld);
    8485                     pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegOld);
    8486                     pReNative->Core.bmGstRegShadows        &= ~pReNative->Core.aHstRegs[idxRegOld].fGstRegShadows;
    8487                     pReNative->Core.aHstRegs[idxRegOld].fGstRegShadows = 0;
     8426                    {
     8427                        Assert(idxRegOld < RT_ELEMENTS(pReNative->Core.aHstSimdRegs));
     8428                        Assert(   pReNative->Core.aVars[idxVar].cbVar == sizeof(RTUINT128U)
     8429                               || pReNative->Core.aVars[idxVar].cbVar == sizeof(RTUINT256U));
     8430
     8431                        uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, IEMNATIVE_VAR_IDX_PACK(idxVar));
     8432                        Log12(("iemNativeEmitCallCommon: spilling idxVar=%d/%#x/idxReg=%d (referred to by %d) onto the stack (slot %#x bp+%d, off=%#x)\n",
     8433                               idxVar, IEMNATIVE_VAR_IDX_PACK(idxVar), idxRegOld, pReNative->Core.aVars[idxVar].idxReferrerVar,
     8434                               idxStackSlot, iemNativeStackCalcBpDisp(idxStackSlot), off));
     8435                        if (pReNative->Core.aVars[idxVar].cbVar == sizeof(RTUINT128U))
     8436                            off = iemNativeEmitStoreVecRegByBpU128(pReNative, off,
     8437                                                                   iemNativeStackCalcBpDisp(idxStackSlot), idxRegOld);
     8438                        else
     8439                            off = iemNativeEmitStoreVecRegByBpU256(pReNative, off,
     8440                                                                   iemNativeStackCalcBpDisp(idxStackSlot), idxRegOld);
     8441
     8442                        Assert(!(   (pReNative->Core.bmGstSimdRegShadowDirtyLo128 | pReNative->Core.bmGstSimdRegShadowDirtyHi128)
     8443                                  & pReNative->Core.aHstSimdRegs[idxRegOld].fGstRegShadows));
     8444
     8445                        pReNative->Core.aVars[idxVar].idxReg       = UINT8_MAX;
     8446                        pReNative->Core.bmHstSimdRegs              &= ~RT_BIT_32(idxRegOld);
     8447                        pReNative->Core.bmHstSimdRegsWithGstShadow &= ~RT_BIT_32(idxRegOld);
     8448                        pReNative->Core.bmGstSimdRegShadows        &= ~pReNative->Core.aHstSimdRegs[idxRegOld].fGstRegShadows;
     8449                        pReNative->Core.aHstSimdRegs[idxRegOld].fGstRegShadows = 0;
     8450                    }
    84888451                }
    84898452            }
    … …  
    86218584                        int32_t const offBpDispOther = iemNativeStackCalcBpDisp(idxStackSlot);
    86228585                        uint8_t const idxRegOther    = pReNative->Core.aVars[idxOtherVar].idxReg;
    8623 # ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    8624                         bool    const fSimdReg       = pReNative->Core.aVars[idxOtherVar].fSimdReg;
    8625                         uint8_t const cbVar          = pReNative->Core.aVars[idxOtherVar].cbVar;
    8626                         if (   fSimdReg
    8627                             && idxRegOther != UINT8_MAX)
     8586                        if (idxRegOther != UINT8_MAX)
    86288587                        {
    8629                             Assert(idxRegOther < RT_ELEMENTS(pReNative->Core.aHstSimdRegs));
    8630                             if (cbVar == sizeof(RTUINT128U))
    8631                                 off = iemNativeEmitStoreVecRegByBpU128(pReNative, off, offBpDispOther, idxRegOther);
     8588                            if (!pReNative->Core.aVars[idxOtherVar].fSimdReg)
     8589                            {
     8590                                Assert(idxRegOther < RT_ELEMENTS(pReNative->Core.aHstRegs));
     8591                                off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDispOther, idxRegOther);
     8592                                iemNativeRegFreeVar(pReNative, idxRegOther, true); /** @todo const ref? */
     8593                                Assert(pReNative->Core.aVars[idxOtherVar].idxReg == UINT8_MAX);
     8594                            }
    86328595                            else
    8633                                 off = iemNativeEmitStoreVecRegByBpU256(pReNative, off, offBpDispOther, idxRegOther);
    8634                             iemNativeSimdRegFreeVar(pReNative, idxRegOther, true); /** @todo const ref? */
    8635                             Assert(pReNative->Core.aVars[idxOtherVar].idxReg == UINT8_MAX);
    8636                         }
    8637                         else
    8638 # endif
    8639                         if (idxRegOther < RT_ELEMENTS(pReNative->Core.aHstRegs))
    8640                         {
    8641                             off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDispOther, idxRegOther);
    8642                             iemNativeRegFreeVar(pReNative, idxRegOther, true); /** @todo const ref? */
    8643                             Assert(pReNative->Core.aVars[idxOtherVar].idxReg == UINT8_MAX);
     8596                            {
     8597                                Assert(idxRegOther < RT_ELEMENTS(pReNative->Core.aHstSimdRegs));
     8598                                if (pReNative->Core.aVars[idxOtherVar].cbVar == sizeof(RTUINT128U))
     8599                                    off = iemNativeEmitStoreVecRegByBpU128(pReNative, off, offBpDispOther, idxRegOther);
     8600                                else
     8601                                    off = iemNativeEmitStoreVecRegByBpU256(pReNative, off, offBpDispOther, idxRegOther);
     8602                                iemNativeSimdRegFreeVar(pReNative, idxRegOther, true); /** @todo const ref? */
     8603                                Assert(pReNative->Core.aVars[idxOtherVar].idxReg == UINT8_MAX);
     8604                            }
    86448605                        }
    86458606                        Assert(   pReNative->Core.aVars[idxOtherVar].idxStackSlot != UINT8_MAX
    … …  
    87238684                            int32_t const offBpDispOther = iemNativeStackCalcBpDisp(idxStackSlot);
    87248685                            uint8_t const idxRegOther    = pReNative->Core.aVars[idxOtherVar].idxReg;
    8725 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    8726                             bool    const fSimdReg       = pReNative->Core.aVars[idxOtherVar].fSimdReg;
    8727                             uint8_t const cbVar          = pReNative->Core.aVars[idxOtherVar].cbVar;
    8728                             if (   fSimdReg
    8729                                 && idxRegOther != UINT8_MAX)
     8686                            if (idxRegOther != UINT8_MAX)
    87308687                            {
    8731                                 Assert(idxRegOther < RT_ELEMENTS(pReNative->Core.aHstSimdRegs));
    8732                                 if (cbVar == sizeof(RTUINT128U))
    8733                                     off = iemNativeEmitStoreVecRegByBpU128(pReNative, off, offBpDispOther, idxRegOther);
     8688                                if (!pReNative->Core.aVars[idxOtherVar].fSimdReg)
     8689                                {
     8690                                    Assert(idxRegOther < RT_ELEMENTS(pReNative->Core.aHstRegs));
     8691                                    off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDispOther, idxRegOther);
     8692                                    iemNativeRegFreeVar(pReNative, idxRegOther, true); /** @todo const ref? */
     8693                                    Assert(pReNative->Core.aVars[idxOtherVar].idxReg == UINT8_MAX);
     8694                                }
    87348695                                else
    8735                                     off = iemNativeEmitStoreVecRegByBpU256(pReNative, off, offBpDispOther, idxRegOther);
    8736                                 iemNativeSimdRegFreeVar(pReNative, idxRegOther, true); /** @todo const ref? */
    8737                                 Assert(pReNative->Core.aVars[idxOtherVar].idxReg == UINT8_MAX);
    8738                             }
    8739                             else
    8740 #endif
    8741                             if (idxRegOther < RT_ELEMENTS(pReNative->Core.aHstRegs))
    8742                             {
    8743                                 off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDispOther, idxRegOther);
    8744                                 iemNativeRegFreeVar(pReNative, idxRegOther, true); /** @todo const ref? */
    8745                                 Assert(pReNative->Core.aVars[idxOtherVar].idxReg == UINT8_MAX);
     8696                                {
     8697                                    Assert(idxRegOther < RT_ELEMENTS(pReNative->Core.aHstSimdRegs));
     8698                                    if (pReNative->Core.aVars[idxOtherVar].cbVar == sizeof(RTUINT128U))
     8699                                        off = iemNativeEmitStoreVecRegByBpU128(pReNative, off, offBpDispOther, idxRegOther);
     8700                                    else
     8701                                        off = iemNativeEmitStoreVecRegByBpU256(pReNative, off, offBpDispOther, idxRegOther);
     8702                                    iemNativeSimdRegFreeVar(pReNative, idxRegOther, true); /** @todo const ref? */
     8703                                    Assert(pReNative->Core.aVars[idxOtherVar].idxReg == UINT8_MAX);
     8704                                }
    87468705                            }
    87478706                            Assert(   pReNative->Core.aVars[idxOtherVar].idxStackSlot != UINT8_MAX
    … …  
    89858944        ENTRY(cpum.GstCtx.aXcr[0]),
    89868945        ENTRY(cpum.GstCtx.aXcr[1]),
    8987 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    89888946        ENTRY(cpum.GstCtx.XState.x87.MXCSR),
    89898947        ENTRY(cpum.GstCtx.XState.x87.aXMM[0]),
    … …  
    90198977        ENTRY(cpum.GstCtx.XState.u.YmmHi.aYmmHi[14]),
    90208978        ENTRY(cpum.GstCtx.XState.u.YmmHi.aYmmHi[15])
    9021 #endif
    90228979#undef ENTRY
    90238980    };
    … …  
    94949451                        }
    94959452
    9496 # ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    94979453                        case kIemTbDbgEntryType_GuestSimdRegShadowing:
    94989454                        {
    … …  
    95119467                            continue;
    95129468                        }
    9513 # endif
    95149469
    95159470                        case kIemTbDbgEntryType_Label:

  • trunk/src/VBox/VMM/VMMAll/IEMAllThrdRecompiler.cpp

    r106402 r106453  
    114114#endif
    115115
    116 #if defined(IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS) && !defined(IEMNATIVE_WITH_SIMD_REG_ALLOCATOR)
    117 # error "IEMNATIVE_WITH_SIMD_FP_NATIVE_EMITTERS requires IEMNATIVE_WITH_SIMD_REG_ALLOCATOR"
    118 #endif
    119116
    120117

  • trunk/src/VBox/VMM/VMMAll/target-x86/IEMAllN8veEmit-x86.h

    r106407 r106453  
    22332233
    22342234
    2235 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
    22362235/*********************************************************************************************************************************
    22372236*   SIMD emitters.                                                                                                               *
    … …  
    32183217IEMNATIVE_NATIVE_EMIT_FP_3OP_U128(subps, kArmv8VecInstrFpOp_Sub, kArmv8VecInstrFpSz_4x_Single, 0, 0x5c);
    32193218
    3220 #endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
    32213219
    32223220#endif /* !VMM_INCLUDED_SRC_VMMAll_target_x86_IEMAllN8veEmit_x86_h */
Note: See TracChangeset for help on using the changeset viewer.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette