VirtualBox

Changeset 105271 in vbox


Ignore:
Timestamp:
Jul 11, 2024 10:30:56 AM (5 months ago)
Author:
vboxsync
Message:

VMM/IEM: Replaced IEMNATIVEEXITREASON with IEMNATIVELABELTYPE, since it's always been a super set of it. Some source code width adjustments. bugref:10677

Location:
trunk/src/VBox/VMM
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompBltIn.cpp

    r105261 r105271  
    254254                                   ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC), true /*fSetFlags*/);
    255255    /* Return VINF_IEM_REEXEC_BREAK if other FFs are set. */
    256     off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_ReturnBreakFF);
     256    off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_ReturnBreakFF);
    257257
    258258    /* So, it's only interrupt releated FFs and we need to see if IRQs are being
    … …  
    260260    off = iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(pReNative, off, idxEflReg, X86_EFL_IF_BIT, idxLabelVmCheck);
    261261    off = iemNativeEmitTestAnyBitsInGprAndTbExitIfNoneSet(pReNative, off, idxEflReg, CPUMCTX_INHIBIT_SHADOW,
    262                                                           kIemNativeExitReason_ReturnBreakFF);
     262                                                          kIemNativeLabelType_ReturnBreakFF);
    263263
    264264    /* We've got shadow flags set, so we must check that the PC they are valid
    … …  
    268268    off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.uRipInhibitInt));
    269269    off = iemNativeEmitTestIfGprNotEqualGprAndTbExit(pReNative, off, idxTmpReg, idxPcReg,
    270                                                      kIemNativeExitReason_ReturnBreakFF);
     270                                                     kIemNativeLabelType_ReturnBreakFF);
    271271
    272272    /*
    … …  
    278278    off = iemNativeEmitLoadGprByGprU32(pReNative, off, idxTmpReg, idxTmpReg, RT_UOFFSETOF(VMCC, fGlobalForcedActions));
    279279    off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, VM_FF_ALL_MASK, true /*fSetFlags*/);
    280     off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_ReturnBreakFF);
     280    off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_ReturnBreakFF);
    281281
    282282    /** @todo STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatCheckIrqBreaks); */
    … …  
    321321    off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, IEMTB_F_KEY_MASK);
    322322    off = iemNativeEmitTestIfGpr32NotEqualImmAndTbExit(pReNative, off, idxTmpReg, fExpectedExec & IEMTB_F_KEY_MASK,
    323                                                        kIemNativeExitReason_ReturnBreak);
     323                                                       kIemNativeLabelType_ReturnBreak);
    324324    iemNativeRegFreeTmp(pReNative, idxTmpReg);
    325325
    … …  
    450450
    451451    /* 3. Jump if greater. */
    452     off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeExitReason_RaiseGp0);
     452    off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeLabelType_RaiseGp0);
    453453
    454454    iemNativeRegFreeTmp(pReNative, idxRegCsLim);
    … …  
    538538    /* Compare the two and jump out if we're too close to the limit. */
    539539    off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegLeft, idxRegRight);
    540     off = iemNativeEmitJlTbExit(pReNative, off, kIemNativeExitReason_NeedCsLimChecking);
     540    off = iemNativeEmitJlTbExit(pReNative, off, kIemNativeLabelType_NeedCsLimChecking);
    541541
    542542    iemNativeRegFreeTmp(pReNative, idxRegRight);
    … …  
    618618                offConsolidatedJump = off; \
    619619                if (BP_ON_OBSOLETION) pbCodeBuf[off++] = 0xcc; \
    620                 off = iemNativeEmitTbExitEx(pReNative, pbCodeBuf, off, kIemNativeExitReason_ObsoleteTb); \
     620                off = iemNativeEmitTbExitEx(pReNative, pbCodeBuf, off, kIemNativeLabelType_ObsoleteTb); \
    621621            } \
    622622        } while (0)
    … …  
    862862                if (fPendingJmp)
    863863                {
    864                     off = iemNativeEmitJccTbExitEx(pReNative, pu32CodeBuf, off, kIemNativeExitReason_ObsoleteTb,
     864                    off = iemNativeEmitJccTbExitEx(pReNative, pu32CodeBuf, off, kIemNativeLabelType_ObsoleteTb,
    865865                                                   kArmv8InstrCond_Ne);
    866866                    fPendingJmp = false;
    … …  
    899899                                                            ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
    900900
    901                 off = iemNativeEmitJccTbExitEx(pReNative, pu32CodeBuf, off, kIemNativeExitReason_ObsoleteTb,
     901                off = iemNativeEmitJccTbExitEx(pReNative, pu32CodeBuf, off, kIemNativeLabelType_ObsoleteTb,
    902902                                               kArmv8InstrCond_Ne);
    903903
    … …  
    10331033     */
    10341034    if (fPendingJmp)
    1035         off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_ObsoleteTb);
     1035        off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_ObsoleteTb);
    10361036
    10371037    RT_NOREF(pu32CodeBuf, cbLeft, offPage, pbOpcodes, offConsolidatedJump);
    … …  
    11941194    /* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal. */
    11951195    off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
    1196     off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeExitReason_CheckBranchMiss);
     1196    off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeLabelType_CheckBranchMiss);
    11971197
    11981198    /* 4. Add iem.s.GCPhysInstrBuf and compare with GCPhysRangePageWithOffset. */
    … …  
    12261226                                             + offRange;
    12271227    off = iemNativeEmitTestIfGprNotEqualImmAndTbExit(pReNative, off, idxRegTmp, GCPhysRangePageWithOffset,
    1228                                                      kIemNativeExitReason_CheckBranchMiss);
     1228                                                     kIemNativeLabelType_CheckBranchMiss);
    12291229
    12301230    iemNativeRegFreeTmp(pReNative, idxRegTmp);
    … …  
    13361336    RTGCPHYS const GCPhysNewPage = iemTbGetRangePhysPageAddr(pTb, idxRange);
    13371337    off = iemNativeEmitTestIfGprNotEqualImmAndTbExit(pReNative, off, idxRegGCPhys, GCPhysNewPage,
    1338                                                      kIemNativeExitReason_ObsoleteTb);
     1338                                                     kIemNativeLabelType_ObsoleteTb);
    13391339
    13401340    iemNativeRegFreeTmp(pReNative, idxRegGCPhys);
    … …  
    15471547    off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegTmp2, GCPhysRangePageWithOffset);
    15481548    off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegTmp, idxRegTmp2);
    1549     off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_CheckBranchMiss);
     1549    off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_CheckBranchMiss);
    15501550    uint32_t const offFixedJumpToEnd = off;
    15511551    off = iemNativeEmitJmpToFixed(pReNative, off, off + 512 /* force rel32 */);
    … …  
    15601560    /* Check that we haven't been here before. */
    15611561    off = iemNativeEmitTestIfGprIsNotZeroAndTbExit(pReNative, off, idxRegTmp2,  false /*f64Bit*/,
    1562                                                    kIemNativeExitReason_CheckBranchMiss);
     1562                                                   kIemNativeLabelType_CheckBranchMiss);
    15631563
    15641564    /* Jump to the TLB lookup code. */

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompFuncs.h

    r105251 r105271  
    247247    pReNative->fMc    = 0; \
    248248    pReNative->fCImpl = (a_fFlags); \
    249     return iemNativeEmitCImplCall0(pReNative, off, pCallEntry->idxInstr, a_fGstShwFlush, (uintptr_t)a_pfnCImpl, a_cbInstr) /** @todo not used ... */
     249    return iemNativeEmitCImplCall0(pReNative, off, pCallEntry->idxInstr, a_fGstShwFlush, (uintptr_t)a_pfnCImpl, \
     250                                   a_cbInstr) /** @todo not used ... */
    250251
    251252
    … …  
    319320#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
    320321    AssertMsg(   pReNative->idxCurCall == 0
    321               || IEMLIVENESS_STATE_IS_INPUT_EXPECTED(iemNativeLivenessGetStateByGstRegEx(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], IEMLIVENESSBIT_IDX_EFL_OTHER)),
    322               ("Efl_Other - %u\n", iemNativeLivenessGetStateByGstRegEx(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1], IEMLIVENESSBIT_IDX_EFL_OTHER)));
     322              || IEMLIVENESS_STATE_IS_INPUT_EXPECTED(iemNativeLivenessGetStateByGstRegEx(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1],
     323                                                                                         IEMLIVENESSBIT_IDX_EFL_OTHER)),
     324              ("Efl_Other - %u\n", iemNativeLivenessGetStateByGstRegEx(&pReNative->paLivenessEntries[pReNative->idxCurCall - 1],
     325                                                                       IEMLIVENESSBIT_IDX_EFL_OTHER)));
    323326#endif
    324327
    … …  
    334337    off = iemNativeEmitTestAnyBitsInGprAndTbExitIfAnySet(pReNative, off, idxEflReg,
    335338                                                         X86_EFL_TF | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK,
    336                                                          kIemNativeExitReason_ReturnWithFlags);
     339                                                         kIemNativeLabelType_ReturnWithFlags);
    337340    off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxEflReg, ~(uint32_t)(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW));
    338341    off = iemNativeEmitStoreGprToVCpuU32(pReNative, off, idxEflReg, RT_UOFFSETOF(VMCPU, cpum.GstCtx.eflags));
    … …  
    394397                                             RT_UOFFSETOF(VMCPU, iem.s.ppTbLookupEntryR3));
    395398
    396         return iemNativeEmitTbExit(pReNative, off, kIemNativeExitReason_ReturnBreak);
     399        return iemNativeEmitTbExit(pReNative, off, kIemNativeLabelType_ReturnBreak);
    397400
    398401#else
    … …  
    440443
    441444                if (pReNative->idxLastCheckIrqCallNo != UINT32_MAX)
    442                     return iemNativeEmitTbExit(pReNative, off, kIemNativeExitReason_ReturnBreakViaLookup);
    443                 return iemNativeEmitTbExit(pReNative, off, kIemNativeExitReason_ReturnBreakViaLookupWithIrq);
     445                    return iemNativeEmitTbExit(pReNative, off, kIemNativeLabelType_ReturnBreakViaLookup);
     446                return iemNativeEmitTbExit(pReNative, off, kIemNativeLabelType_ReturnBreakViaLookupWithIrq);
    444447            }
    445448        }
    446449        if (pReNative->idxLastCheckIrqCallNo != UINT32_MAX)
    447             return iemNativeEmitTbExit(pReNative, off, kIemNativeExitReason_ReturnBreakViaLookupWithTlb);
    448         return iemNativeEmitTbExit(pReNative, off, kIemNativeExitReason_ReturnBreakViaLookupWithTlbAndIrq);
     450            return iemNativeEmitTbExit(pReNative, off, kIemNativeLabelType_ReturnBreakViaLookupWithTlb);
     451        return iemNativeEmitTbExit(pReNative, off, kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq);
    449452#endif
    450453    }
    … …  
    15941597
    15951598/*********************************************************************************************************************************
    1596 *   Emitters for changing PC/RIP/EIP/IP with a RETN (Iw) instruction (IEM_MC_RETN_AND_FINISH) (requires stack emmiters).    *
     1599*   Emitters for changing PC/RIP/EIP/IP with a RETN (Iw) instruction (IEM_MC_RETN_AND_FINISH) (requires stack emmiters).         *
    15971600*********************************************************************************************************************************/
    15981601
    … …  
    17191722                                        : (uintptr_t)iemNativeHlpStackFetchU16;
    17201723    uint8_t   const idxRegRsp       = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_GPR(X86_GREG_xSP),
    1721                                                                       fFlat ? kIemNativeGstRegUse_ForUpdate : kIemNativeGstRegUse_Calculation,
     1724                                                                      fFlat ? kIemNativeGstRegUse_ForUpdate
     1725                                                                            : kIemNativeGstRegUse_Calculation,
    17221726                                                                      true /*fNoVolatileRegs*/);
    17231727    uint8_t   const idxRegEffSp     = fFlat ? idxRegRsp : iemNativeRegAllocTmp(pReNative, &off);
    17241728    /** @todo can do a better job picking the register here. For cbMem >= 4 this
    17251729     *        will be the resulting register value. */
    1726     uint8_t   const idxRegMemResult = iemNativeRegAllocTmp(pReNative, &off); /* pointer then value; arm64 SP += 2/4 helper too.  */
     1730    uint8_t   const idxRegMemResult = iemNativeRegAllocTmp(pReNative, &off); /* pointer then value; arm64 SP += 2/4 helper too. */
    17271731
    17281732    uint32_t        offFixupJumpToUseOtherBitSp = UINT32_MAX;
    … …  
    19561960
    19571961        /* Allocate a temporary CR0 register. */
    1958         uint8_t const idxCr0Reg       = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Cr0, kIemNativeGstRegUse_ReadOnly);
     1962        uint8_t const idxCr0Reg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Cr0,
     1963                                                                  kIemNativeGstRegUse_ReadOnly);
    19591964
    19601965        /*
    … …  
    19631968         */
    19641969        /* Test and jump. */
    1965         off = iemNativeEmitTestAnyBitsInGprAndTbExitIfAnySet(pReNative, off, idxCr0Reg, X86_CR0_EM | X86_CR0_TS, kIemNativeExitReason_RaiseNm);
     1970        off = iemNativeEmitTestAnyBitsInGprAndTbExitIfAnySet(pReNative, off, idxCr0Reg, X86_CR0_EM | X86_CR0_TS,
     1971                                                             kIemNativeLabelType_RaiseNm);
    19661972
    19671973        /* Free but don't flush the CR0 register. */
    … …  
    20132019
    20142020        /* Allocate a temporary CR0 register. */
    2015         uint8_t const idxCr0Reg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Cr0, kIemNativeGstRegUse_Calculation);
     2021        uint8_t const idxCr0Reg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Cr0,
     2022                                                                  kIemNativeGstRegUse_Calculation);
    20162023
    20172024        /*
    … …  
    20212028        off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxCr0Reg, X86_CR0_MP | X86_CR0_TS);
    20222029        /* Test and jump. */
    2023         off = iemNativeEmitTestIfGpr32EqualsImmAndTbExit(pReNative, off, idxCr0Reg, X86_CR0_MP | X86_CR0_TS, kIemNativeExitReason_RaiseNm);
     2030        off = iemNativeEmitTestIfGpr32EqualsImmAndTbExit(pReNative, off, idxCr0Reg, X86_CR0_MP | X86_CR0_TS,
     2031                                                         kIemNativeLabelType_RaiseNm);
    20242032
    20252033        /* Free the CR0 register. */
    … …  
    20652073
    20662074    /* Allocate a temporary FSW register. */
    2067     uint8_t const idxFpuFswReg    = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_FpuFsw, kIemNativeGstRegUse_ReadOnly);
     2075    uint8_t const idxFpuFswReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_FpuFsw,
     2076                                                                 kIemNativeGstRegUse_ReadOnly);
    20682077
    20692078    /*
    … …  
    20722081     */
    20732082    /* Test and jump. */
    2074     off = iemNativeEmitTestBitInGprAndTbExitIfSet(pReNative, off, idxFpuFswReg, X86_FSW_ES_BIT, kIemNativeExitReason_RaiseMf);
     2083    off = iemNativeEmitTestBitInGprAndTbExitIfSet(pReNative, off, idxFpuFswReg, X86_FSW_ES_BIT, kIemNativeLabelType_RaiseMf);
    20752084
    20762085    /* Free but don't flush the FSW register. */
    … …  
    21362145        off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off,   idxTmpReg, X86_CR0_EM | X86_CR0_TS | X86_CR4_OSFXSR);
    21372146        off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off,   idxTmpReg, X86_CR4_OSFXSR);
    2138         off = iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, kIemNativeExitReason_RaiseSseRelated, kIemNativeInstrCond_ne);
     2147        off = iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, kIemNativeLabelType_RaiseSseRelated, kIemNativeInstrCond_ne);
    21392148
    21402149#elif defined(RT_ARCH_ARM64)
    … …  
    21542163        /* -> idxTmpReg[0]=~OSFXSR; idxTmpReg[2]=EM; idxTmpReg[3]=TS; (the rest is zero) */
    21552164        off = iemNativeEmitTestIfGprIsNotZeroAndTbExitEx(pReNative, pCodeBuf, off, idxTmpReg, false /*f64Bit*/,
    2156                                                          kIemNativeExitReason_RaiseSseRelated);
     2165                                                         kIemNativeLabelType_RaiseSseRelated);
    21572166
    21582167#else
    … …  
    22422251        off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off,                idxTmpReg, ((XSAVE_C_YMM | XSAVE_C_SSE) << 2) | 2);
    22432252        /* -> idxTmpReg[0]=CR0.TS idxTmpReg[1]=~CR4.OSXSAVE; idxTmpReg[2]=0; idxTmpReg[3]=~SSE; idxTmpReg[4]=~YMM; */
    2244         off = iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, kIemNativeExitReason_RaiseAvxRelated, kIemNativeInstrCond_ne);
     2253        off = iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, kIemNativeLabelType_RaiseAvxRelated, kIemNativeInstrCond_ne);
    22452254
    22462255#elif defined(RT_ARCH_ARM64)
    … …  
    22612270        /* -> idxTmpReg[0]=CR0.TS; idxTmpReg[1]=~CR4.OSXSAVE; idxTmpReg[2]=~SSE; idxTmpReg[3]=~YMM; (the rest is zero) */
    22622271        off = iemNativeEmitTestIfGprIsNotZeroAndTbExitEx(pReNative, pCodeBuf, off, idxTmpReg, false /*f64Bit*/,
    2263                                                          kIemNativeExitReason_RaiseAvxRelated);
     2272                                                         kIemNativeLabelType_RaiseAvxRelated);
    22642273
    22652274#else
    … …  
    23022311#endif
    23032312
    2304     uint8_t const idxRegMxCsr = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_MxCsr, kIemNativeGstRegUse_ReadOnly);
     2313    uint8_t const idxRegMxCsr = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_MxCsr,
     2314                                                                kIemNativeGstRegUse_ReadOnly);
    23052315    uint8_t const idxRegTmp   = iemNativeRegAllocTmp(pReNative, &off);
    23062316
    … …  
    23162326    off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, idxRegTmp, idxRegMxCsr);
    23172327    off = iemNativeEmitTestAnyBitsInGprAndTbExitIfAnySet(pReNative, off, idxRegTmp, X86_MXCSR_XCPT_FLAGS,
    2318                                                          kIemNativeExitReason_RaiseSseAvxFpRelated);
     2328                                                         kIemNativeLabelType_RaiseSseAvxFpRelated);
    23192329
    23202330    /* Free but don't flush the MXCSR register. */
    … …  
    23532363
    23542364    /* raise \#DE exception unconditionally. */
    2355     return iemNativeEmitTbExit(pReNative, off, kIemNativeExitReason_RaiseDe);
     2365    return iemNativeEmitTbExit(pReNative, off, kIemNativeLabelType_RaiseDe);
    23562366}
    23572367
    … …  
    23712381 */
    23722382DECL_INLINE_THROW(uint32_t)
    2373 iemNativeEmitRaiseGp0IfEffAddrUnaligned(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr, uint8_t idxVarEffAddr, uint8_t cbAlign)
     2383iemNativeEmitRaiseGp0IfEffAddrUnaligned(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr,
     2384                                        uint8_t idxVarEffAddr, uint8_t cbAlign)
    23742385{
    23752386    IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarEffAddr);
    … …  
    23902401
    23912402    off = iemNativeEmitTestAnyBitsInGprAndTbExitIfAnySet(pReNative, off, idxVarReg, cbAlign - 1,
    2392                                                          kIemNativeExitReason_RaiseGp0);
     2403                                                         kIemNativeLabelType_RaiseGp0);
    23932404
    23942405    iemNativeVarRegisterRelease(pReNative, idxVarEffAddr);
    … …  
    34303441#endif
    34313442    fGstShwFlush = iemNativeCImplFlagsToGuestShadowFlushMask(pReNative->fCImpl, fGstShwFlush | RT_BIT_64(kIemNativeGstReg_Pc));
    3432     if (!(pReNative->fMc & IEM_MC_F_WITHOUT_FLAGS)) /** @todo We don't emit with-flags/without-flags variations for CIMPL calls.  */
     3443    if (!(pReNative->fMc & IEM_MC_F_WITHOUT_FLAGS)) /** @todo We don't emit with-flags/without-flags variations for CIMPL calls. */
    34333444        fGstShwFlush |= RT_BIT_64(kIemNativeGstReg_EFlags);
    34343445    iemNativeRegFlushGuestShadows(pReNative, fGstShwFlush);
    … …  
    52105221
    52115222/** Handles IEM_MC_SET_EFL_BIT/IEM_MC_CLEAR_EFL_BIT/IEM_MC_FLIP_EFL_BIT. */
    5212 DECL_INLINE_THROW(uint32_t) iemNativeEmitModifyEFlagsBit(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflBit, IEMNATIVEMITEFLOP enmOp)
     5223DECL_INLINE_THROW(uint32_t)
     5224iemNativeEmitModifyEFlagsBit(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fEflBit, IEMNATIVEMITEFLOP enmOp)
    52135225{
    52145226    uint8_t const idxEflReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
    … …  
    66796691         */
    66806692        /* Allocate a temporary PC register. */
    6681         uint8_t const idxPcReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc, kIemNativeGstRegUse_ForUpdate);
     6693        uint8_t const idxPcReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
     6694                                                                 kIemNativeGstRegUse_ForUpdate);
    66826695
    66836696        /* Restore the original value. */
    … …  
    80108023            AssertMsg(   pReNative->idxCurCall == 0
    80118024                      || IEMLIVENESS_STATE_IS_CLOBBER_EXPECTED(iemNativeLivenessGetPrevStateByGstReg(pReNative, IEMNATIVEGSTREG_GPR(idxGReg))),
    8012                       ("%s - %u\n", g_aGstShadowInfo[idxGReg].pszName, iemNativeLivenessGetPrevStateByGstReg(pReNative, IEMNATIVEGSTREG_GPR(idxGReg))));
     8025                      ("%s - %u\n", g_aGstShadowInfo[idxGReg].pszName,
     8026                       iemNativeLivenessGetPrevStateByGstReg(pReNative, IEMNATIVEGSTREG_GPR(idxGReg))));
    80138027#endif
    80148028            iemNativeRegClearAndMarkAsGstRegShadow(pReNative, idxRegMemResult,  IEMNATIVEGSTREG_GPR(idxGReg), off);
    … …  
    88668880        /* Allocate destination and source register. */
    88678881        uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXRegDst),
    8868                                                                               kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ForFullWrite);
     8882                                                                              kIemNativeGstSimdRegLdStSz_Low128,
     8883                                                                              kIemNativeGstRegUse_ForFullWrite);
    88698884        uint8_t const idxSimdRegSrc = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXRegSrc),
    8870                                                                               kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ReadOnly);
     8885                                                                              kIemNativeGstSimdRegLdStSz_Low128,
     8886                                                                              kIemNativeGstRegUse_ReadOnly);
    88718887
    88728888        off = iemNativeEmitSimdLoadVecRegFromVecRegU128(pReNative, off, idxSimdRegDst, idxSimdRegSrc);
    … …  
    89208936
    89218937    uint8_t const idxSimdRegSrc = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
    8922                                                                           kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ReadOnly);
     8938                                                                          kIemNativeGstSimdRegLdStSz_Low128,
     8939                                                                          kIemNativeGstRegUse_ReadOnly);
    89238940
    89248941    iemNativeVarSetKindToStack(pReNative, idxDstVar);
    … …  
    89498966
    89508967    uint8_t const idxSimdRegSrc = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
    8951                                                                           kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ReadOnly);
     8968                                                                          kIemNativeGstSimdRegLdStSz_Low128,
     8969                                                                          kIemNativeGstRegUse_ReadOnly);
    89528970
    89538971    iemNativeVarSetKindToStack(pReNative, idxDstVar);
    … …  
    89758993
    89768994    uint8_t const idxSimdRegSrc = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
    8977                                                                           kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ReadOnly);
     8995                                                                          kIemNativeGstSimdRegLdStSz_Low128,
     8996                                                                          kIemNativeGstRegUse_ReadOnly);
    89788997
    89798998    iemNativeVarSetKindToStack(pReNative, idxDstVar);
    … …  
    90019020
    90029021    uint8_t const idxSimdRegSrc = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
    9003                                                                           kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ReadOnly);
     9022                                                                          kIemNativeGstSimdRegLdStSz_Low128,
     9023                                                                          kIemNativeGstRegUse_ReadOnly);
    90049024
    90059025    iemNativeVarSetKindToStack(pReNative, idxDstVar);
    … …  
    90329052
    90339053    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
    9034                                                                           kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ForFullWrite);
     9054                                                                          kIemNativeGstSimdRegLdStSz_Low128,
     9055                                                                          kIemNativeGstRegUse_ForFullWrite);
    90359056    uint8_t const idxVarReg     = iemNativeVarSimdRegisterAcquire(pReNative, idxSrcVar, &off, true /*fInitialized*/);
    90369057
    … …  
    90839104
    90849105    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
    9085                                                                           kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ForUpdate);
     9106                                                                          kIemNativeGstSimdRegLdStSz_Low128,
     9107                                                                          kIemNativeGstRegUse_ForUpdate);
    90869108    uint8_t const idxVarReg     = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off, true /*fInitialized*/);
    90879109
    … …  
    91149136
    91159137    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
    9116                                                                           kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ForUpdate);
     9138                                                                          kIemNativeGstSimdRegLdStSz_Low128,
     9139                                                                          kIemNativeGstRegUse_ForUpdate);
    91179140    uint8_t const idxVarReg     = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off, true /*fInitialized*/);
    91189141
    … …  
    91409163
    91419164    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
    9142                                                                           kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ForUpdate);
     9165                                                                          kIemNativeGstSimdRegLdStSz_Low128,
     9166                                                                          kIemNativeGstRegUse_ForUpdate);
    91439167    uint8_t const idxVarReg     = iemNativeVarRegisterAcquire(pReNative, idxDstVar, &off, true /*fInitialized*/);
    91449168
    … …  
    91679191
    91689192    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
    9169                                                                           kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ForUpdate);
     9193                                                                          kIemNativeGstSimdRegLdStSz_Low128,
     9194                                                                          kIemNativeGstRegUse_ForUpdate);
    91709195    uint8_t const idxVarReg     = iemNativeVarSimdRegisterAcquire(pReNative, idxSrcVar, &off, true /*fInitialized*/);
    91719196
    … …  
    91899214{
    91909215    /*
    9191      * The iYRegSrc == iYRegDst case needs to be treated differently here, because if iYRegDst gets allocated first for the full write
    9192      * it won't load the actual value from CPUMCTX. When allocating iYRegSrc afterwards it will get duplicated from the already
    9193      * allocated host register for iYRegDst containing garbage. This will be catched by the guest register value checking in debug builds.
     9216     * The iYRegSrc == iYRegDst case needs to be treated differently here, because
     9217     * if iYRegDst gets allocated first for the full write  it won't load the
     9218     * actual value from CPUMCTX.  When allocating iYRegSrc afterwards it will get
     9219     * duplicated from the already allocated host register for iYRegDst containing
     9220     * garbage.  This will be catched by the guest register value checking in debug
     9221     * builds.
    91949222     */
    91959223    if (iYRegDst != iYRegSrc)
    … …  
    92149242        /* This effectively only clears the upper 128-bits of the register. */
    92159243        uint8_t const idxSimdReg = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYRegDst),
    9216                                                                            kIemNativeGstSimdRegLdStSz_High128, kIemNativeGstRegUse_ForFullWrite);
     9244                                                                           kIemNativeGstSimdRegLdStSz_High128,
     9245                                                                           kIemNativeGstRegUse_ForFullWrite);
    92179246
    92189247        off = iemNativeEmitSimdZeroVecRegHighU128(pReNative, off, idxSimdReg);
    … …  
    92349263{
    92359264    /*
    9236      * The iYRegSrc == iYRegDst case needs to be treated differently here, because if iYRegDst gets allocated first for the full write
    9237      * it won't load the actual value from CPUMCTX. When allocating iYRegSrc afterwards it will get duplicated from the already
    9238      * allocated host register for iYRegDst containing garbage. This will be catched by the guest register value checking in debug builds.
    9239      * iYRegSrc == iYRegDst would effectively only clear any upper 256-bits for a zmm register we don't support yet, so this is just a nop.
     9265     * The iYRegSrc == iYRegDst case needs to be treated differently here, because
     9266     * if iYRegDst gets allocated first for the full write it won't load the
     9267     * actual value from CPUMCTX.  When allocating iYRegSrc afterwards it will get
     9268     * duplicated from the already allocated host register for iYRegDst containing
     9269     * garbage. This will be catched by the guest register value checking in debug
     9270     * builds. iYRegSrc == iYRegDst would effectively only clear any upper 256-bits
     9271     * for a zmm register we don't support yet, so this is just a nop.
    92409272     */
    92419273    if (iYRegDst != iYRegSrc)
    … …  
    92439275        /* Allocate destination and source register. */
    92449276        uint8_t const idxSimdRegSrc = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYRegSrc),
    9245                                                                               kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ReadOnly);
     9277                                                                              kIemNativeGstSimdRegLdStSz_256,
     9278                                                                              kIemNativeGstRegUse_ReadOnly);
    92469279        uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYRegDst),
    9247                                                                               kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
     9280                                                                              kIemNativeGstSimdRegLdStSz_256,
     9281                                                                              kIemNativeGstRegUse_ForFullWrite);
    92489282
    92499283        off = iemNativeEmitSimdLoadVecRegFromVecRegU256(pReNative, off, idxSimdRegDst, idxSimdRegSrc);
    … …  
    93579391{
    93589392    uint8_t const idxSimdReg = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYReg),
    9359                                                                        kIemNativeGstSimdRegLdStSz_High128, kIemNativeGstRegUse_ForFullWrite);
     9393                                                                       kIemNativeGstSimdRegLdStSz_High128,
     9394                                                                       kIemNativeGstRegUse_ForFullWrite);
    93609395
    93619396    off = iemNativeEmitSimdZeroVecRegHighU128(pReNative, off, idxSimdReg);
    … …  
    94119446
    94129447    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYReg),
    9413                                                                           kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
     9448                                                                          kIemNativeGstSimdRegLdStSz_256,
     9449                                                                          kIemNativeGstRegUse_ForFullWrite);
    94149450
    94159451    uint8_t const idxVarReg = iemNativeVarSimdRegisterAcquire(pReNative, idxSrcVar, &off);
    … …  
    94379473
    94389474    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
    9439                                                                           kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
     9475                                                                          kIemNativeGstSimdRegLdStSz_256,
     9476                                                                          kIemNativeGstRegUse_ForFullWrite);
    94409477
    94419478    uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
    … …  
    94629499
    94639500    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
    9464                                                                           kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
     9501                                                                          kIemNativeGstSimdRegLdStSz_256,
     9502                                                                          kIemNativeGstRegUse_ForFullWrite);
    94659503
    94669504    uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
    … …  
    94889526
    94899527    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
    9490                                                                           kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
     9528                                                                          kIemNativeGstSimdRegLdStSz_256,
     9529                                                                          kIemNativeGstRegUse_ForFullWrite);
    94919530
    94929531    uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
    … …  
    95149553
    95159554    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
    9516                                                                           kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
     9555                                                                          kIemNativeGstSimdRegLdStSz_256,
     9556                                                                          kIemNativeGstRegUse_ForFullWrite);
    95179557
    95189558    uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
    … …  
    95409580
    95419581    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYReg),
    9542                                                                           kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
     9582                                                                          kIemNativeGstSimdRegLdStSz_256,
     9583                                                                          kIemNativeGstRegUse_ForFullWrite);
    95439584
    95449585    uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
    … …  
    95649605
    95659606    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYReg),
    9566                                                                           kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
     9607                                                                          kIemNativeGstSimdRegLdStSz_256,
     9608                                                                          kIemNativeGstRegUse_ForFullWrite);
    95679609
    95689610    uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
    … …  
    95889630
    95899631    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYReg),
    9590                                                                           kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
     9632                                                                          kIemNativeGstSimdRegLdStSz_256,
     9633                                                                          kIemNativeGstRegUse_ForFullWrite);
    95919634
    95929635    uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
    … …  
    96139656
    96149657    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYReg),
    9615                                                                           kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
     9658                                                                          kIemNativeGstSimdRegLdStSz_256,
     9659                                                                          kIemNativeGstRegUse_ForFullWrite);
    96169660
    96179661    uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
    … …  
    96389682
    96399683    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYReg),
    9640                                                                           kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
     9684                                                                          kIemNativeGstSimdRegLdStSz_256,
     9685                                                                          kIemNativeGstRegUse_ForFullWrite);
    96419686
    96429687    uint8_t const idxVarReg = iemNativeVarSimdRegisterAcquire(pReNative, idxSrcVar, &off);
    … …  
    96639708
    96649709    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYReg),
    9665                                                                           kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
     9710                                                                          kIemNativeGstSimdRegLdStSz_256,
     9711                                                                          kIemNativeGstRegUse_ForFullWrite);
    96669712
    96679713    uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
    … …  
    96899735
    96909736    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYReg),
    9691                                                                           kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
     9737                                                                          kIemNativeGstSimdRegLdStSz_256,
     9738                                                                          kIemNativeGstRegUse_ForFullWrite);
    96929739
    96939740    uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
    … …  
    97159762
    97169763    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYRegDst),
    9717                                                                           kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
     9764                                                                          kIemNativeGstSimdRegLdStSz_256,
     9765                                                                          kIemNativeGstRegUse_ForFullWrite);
    97189766    uint8_t const idxSimdRegSrcHx = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYRegSrcHx),
    9719                                                                             kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ReadOnly);
     9767                                                                            kIemNativeGstSimdRegLdStSz_Low128,
     9768                                                                            kIemNativeGstRegUse_ReadOnly);
    97209769    uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
    97219770
    … …  
    97449793
    97459794    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYRegDst),
    9746                                                                           kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
     9795                                                                          kIemNativeGstSimdRegLdStSz_256,
     9796                                                                          kIemNativeGstRegUse_ForFullWrite);
    97479797    uint8_t const idxSimdRegSrcHx = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYRegSrcHx),
    9748                                                                             kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ReadOnly);
     9798                                                                            kIemNativeGstSimdRegLdStSz_Low128,
     9799                                                                            kIemNativeGstRegUse_ReadOnly);
    97499800    uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxSrcVar, &off);
    97509801
    … …  
    97719822{
    97729823    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iXReg),
    9773                                                                           kIemNativeGstSimdRegLdStSz_Low128, kIemNativeGstRegUse_ForUpdate);
     9824                                                                          kIemNativeGstSimdRegLdStSz_Low128,
     9825                                                                          kIemNativeGstRegUse_ForUpdate);
    97749826
    97759827    /** @todo r=aeichner For certain bit combinations we could reduce the number of emitted instructions. */
    … …  
    98049856
    98059857    uint8_t const idxSimdRegSrc = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYRegSrc),
    9806                                                                           kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ReadOnly);
     9858                                                                          kIemNativeGstSimdRegLdStSz_256,
     9859                                                                          kIemNativeGstRegUse_ReadOnly);
    98079860    uint8_t const idxVarReg = iemNativeVarSimdRegisterAcquire(pReNative, idxDstVar, &off);
    98089861
    … …  
    98319884
    98329885    uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(iYRegDst),
    9833                                                                           kIemNativeGstSimdRegLdStSz_256, kIemNativeGstRegUse_ForFullWrite);
     9886                                                                          kIemNativeGstSimdRegLdStSz_256,
     9887                                                                          kIemNativeGstRegUse_ForFullWrite);
    98349888    uint8_t const idxVarRegSrc  = iemNativeVarSimdRegisterAcquire(pReNative, idxSrcVar, &off, true /*fInitalized*/);
    98359889
    … …  
    997410028     * Do all the call setup and cleanup.
    997510029     */
    9976     off = iemNativeEmitCallCommon(pReNative, off, cArgs + IEM_SSE_AIMPL_HIDDEN_ARGS, IEM_SSE_AIMPL_HIDDEN_ARGS, false /*fFlushPendingWrites*/);
     10030    off = iemNativeEmitCallCommon(pReNative, off, cArgs + IEM_SSE_AIMPL_HIDDEN_ARGS, IEM_SSE_AIMPL_HIDDEN_ARGS,
     10031                                  false /*fFlushPendingWrites*/);
    997710032
    997810033    /*
    … …  
    1002010075/** Emits code for IEM_MC_CALL_SSE_AIMPL_3. */
    1002110076DECL_INLINE_THROW(uint32_t)
    10022 iemNativeEmitCallSseAImpl3(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t pfnAImpl, uint8_t idxArg0, uint8_t idxArg1, uint8_t idxArg2)
     10077iemNativeEmitCallSseAImpl3(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t pfnAImpl,
     10078                           uint8_t idxArg0, uint8_t idxArg1, uint8_t idxArg2)
    1002310079{
    1002410080    IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg0, 0 + IEM_SSE_AIMPL_HIDDEN_ARGS);
    … …  
    1005110107/** Emits code for IEM_MC_CALL_AVX_AIMPL_3. */
    1005210108DECL_INLINE_THROW(uint32_t)
    10053 iemNativeEmitCallAvxAImpl3(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t pfnAImpl, uint8_t idxArg0, uint8_t idxArg1, uint8_t idxArg2)
     10109iemNativeEmitCallAvxAImpl3(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t pfnAImpl,
     10110                           uint8_t idxArg0, uint8_t idxArg1, uint8_t idxArg2)
    1005410111{
    1005510112    IEMNATIVE_ASSERT_ARG_VAR_IDX(pReNative, idxArg0, 0 + IEM_AVX_AIMPL_HIDDEN_ARGS);
    … …  
    1005810115    return iemNativeEmitCallSseAvxAImplCommon(pReNative, off, pfnAImpl, 3);
    1005910116}
     10117
     10118
    1006010119#endif /* IEMNATIVE_WITH_SIMD_REG_ALLOCATOR */
    1006110120

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

    r105261 r105271  
    25142514 */
    25152515DECL_HIDDEN_THROW(void)
    2516 iemNativeAddTbExitFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, IEMNATIVEEXITREASON enmExitReason)
    2517 {
     2516iemNativeAddTbExitFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, IEMNATIVELABELTYPE enmExitReason)
     2517{
     2518    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
     2519
    25182520    /*
    25192521     * Make sure we've room.
    … …  
    25402542     */
    25412543    paTbExitFixups[cTbExitFixups].off            = offWhere;
    2542     paTbExitFixups[cTbExitFixups].enmExitReason  = (uint32_t)enmExitReason;
     2544    paTbExitFixups[cTbExitFixups].enmExitReason  = enmExitReason;
    25432545    pReNative->cTbExitFixups = cTbExitFixups + 1;
    25442546}
    … …  
    62366238
    62376239    /* Jump to non-zero status return path. */
    6238     off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_NonZeroRetOrPassUp);
     6240    off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_NonZeroRetOrPassUp);
    62396241
    62406242    /* done. */
    … …  
    62546256
    62556257    off = iemNativeEmitTestIfGprIsNotZeroAndTbExitEx(pReNative, pu32CodeBuf, off, ARMV8_A64_REG_X4, true /*f64Bit*/,
    6256                                                      kIemNativeExitReason_NonZeroRetOrPassUp);
     6258                                                     kIemNativeLabelType_NonZeroRetOrPassUp);
    62576259
    62586260#else
    … …  
    63026304    off = iemNativeEmitAddGpr32Imm(pReNative, off, iTmpReg, (int32_t)0x8000);
    63036305    off = iemNativeEmitShiftGprRight(pReNative, off, iTmpReg, 16);
    6304     off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_RaiseGp0);
     6306    off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_RaiseGp0);
    63056307
    63066308    iemNativeRegFreeTmp(pReNative, iTmpReg);
    … …  
    63216323    off = iemNativeEmitAddTwoGprs(pReNative, off, iTmpReg, idxAddrReg);
    63226324    off = iemNativeEmitCmpArm64(pReNative, off, ARMV8_A64_REG_XZR, iTmpReg, true /*f64Bit*/, 48 /*cShift*/, kArmv8A64InstrShift_Lsr);
    6323     off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_RaiseGp0);
     6325    off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_RaiseGp0);
    63246326
    63256327    iemNativeRegFreeTmp(pReNative, iTmpReg);
    … …  
    63646366
    63656367    off = iemNativeEmitCmpGpr32WithGpr(pReNative, off, idxAddrReg, idxRegCsLim);
    6366     off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeExitReason_RaiseGp0);
     6368    off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeLabelType_RaiseGp0);
    63676369
    63686370    iemNativeRegFreeTmp(pReNative, idxRegCsLim);
    … …  
    96609662#  error "port me"
    96619663# endif
    9662     uint32_t aoffLabels[kIemNativeExitReason_Max] = {0};
     9664    uint32_t aoffLabels[kIemNativeLabelType_LastTbExit + 1] = {0};
    96639665    int      rc  = VINF_SUCCESS;
    96649666    uint32_t off = 0;
    … …  
    96699671         * Emit the epilog code.
    96709672         */
    9671         aoffLabels[kIemNativeExitReason_Return] = off;
     9673        aoffLabels[kIemNativeLabelType_Return] = off;
    96729674        off = iemNativeEmitCoreEpilog(pReNative, off);
    96739675
    … …  
    96779679        static struct
    96789680        {
    9679             IEMNATIVEEXITREASON enmExitReason;
    9680             uint32_t (*pfnEmitCore)(PIEMRECOMPILERSTATE pReNative, uint32_t off);
     9681            IEMNATIVELABELTYPE enmExitReason;
     9682            uint32_t         (*pfnEmitCore)(PIEMRECOMPILERSTATE pReNative, uint32_t off);
    96819683        } const s_aSpecialWithEpilogs[] =
    96829684        {
    9683             { kIemNativeExitReason_NonZeroRetOrPassUp,    iemNativeEmitCoreRcFiddling           },
    9684             { kIemNativeExitReason_ReturnBreak,           iemNativeEmitCoreReturnBreak          },
    9685             { kIemNativeExitReason_ReturnBreakFF,         iemNativeEmitCoreReturnBreakFF        },
    9686             { kIemNativeExitReason_ReturnWithFlags,       iemNativeEmitCoreReturnWithFlags      },
     9685            { kIemNativeLabelType_NonZeroRetOrPassUp,   iemNativeEmitCoreRcFiddling           },
     9686            { kIemNativeLabelType_ReturnBreak,          iemNativeEmitCoreReturnBreak          },
     9687            { kIemNativeLabelType_ReturnBreakFF,        iemNativeEmitCoreReturnBreakFF        },
     9688            { kIemNativeLabelType_ReturnWithFlags,      iemNativeEmitCoreReturnWithFlags      },
    96879689        };
    96889690        for (uint32_t i = 0; i < RT_ELEMENTS(s_aSpecialWithEpilogs); i++)
    … …  
    97009702        static struct
    97019703        {
    9702             IEMNATIVEEXITREASON  enmExitReason;
     9704            IEMNATIVELABELTYPE   enmExitReason;
    97039705            uintptr_t            pfnHelper;
    97049706        } const s_aViaLookup[] =
    97059707        {
    9706             {   kIemNativeExitReason_ReturnBreakViaLookup,
     9708            {   kIemNativeLabelType_ReturnBreakViaLookup,
    97079709                (uintptr_t)iemNativeHlpReturnBreakViaLookup<false /*a_fWithIrqCheck*/>          },
    9708             {   kIemNativeExitReason_ReturnBreakViaLookupWithIrq,
     9710            {   kIemNativeLabelType_ReturnBreakViaLookupWithIrq,
    97099711                (uintptr_t)iemNativeHlpReturnBreakViaLookup<true /*a_fWithIrqCheck*/>           },
    9710             {   kIemNativeExitReason_ReturnBreakViaLookupWithTlb,
     9712            {   kIemNativeLabelType_ReturnBreakViaLookupWithTlb,
    97119713                (uintptr_t)iemNativeHlpReturnBreakViaLookupWithTlb<false /*a_fWithIrqCheck*/>   },
    9712             {   kIemNativeExitReason_ReturnBreakViaLookupWithTlbAndIrq,
     9714            {   kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq,
    97139715                (uintptr_t)iemNativeHlpReturnBreakViaLookupWithTlb<true /*a_fWithIrqCheck*/>    },
    97149716        };
    9715         uint32_t const offReturnBreak = aoffLabels[kIemNativeExitReason_ReturnBreak]; Assert(offReturnBreak != 0);
     9717        uint32_t const offReturnBreak = aoffLabels[kIemNativeLabelType_ReturnBreak]; Assert(offReturnBreak != 0);
    97169718        for (uint32_t i = 0; i < RT_ELEMENTS(s_aViaLookup); i++)
    97179719        {
    … …  
    97299731        static struct
    97309732        {
    9731             IEMNATIVEEXITREASON             enmExitReason;
     9733            IEMNATIVELABELTYPE              enmExitReason;
    97329734            bool                            fWithEpilog;
    97339735            PFNIEMNATIVESIMPLETAILLABELCALL pfnCallback;
    97349736        } const s_aSimpleTailLabels[] =
    97359737        {
    9736             {   kIemNativeExitReason_RaiseDe,               false, iemNativeHlpExecRaiseDe },
    9737             {   kIemNativeExitReason_RaiseUd,               false, iemNativeHlpExecRaiseUd },
    9738             {   kIemNativeExitReason_RaiseSseRelated,       false, iemNativeHlpExecRaiseSseRelated },
    9739             {   kIemNativeExitReason_RaiseAvxRelated,       false, iemNativeHlpExecRaiseAvxRelated },
    9740             {   kIemNativeExitReason_RaiseSseAvxFpRelated,  false, iemNativeHlpExecRaiseSseAvxFpRelated },
    9741             {   kIemNativeExitReason_RaiseNm,               false, iemNativeHlpExecRaiseNm },
    9742             {   kIemNativeExitReason_RaiseGp0,              false, iemNativeHlpExecRaiseGp0 },
    9743             {   kIemNativeExitReason_RaiseMf,               false, iemNativeHlpExecRaiseMf },
    9744             {   kIemNativeExitReason_RaiseXf,               false, iemNativeHlpExecRaiseXf },
    9745             {   kIemNativeExitReason_ObsoleteTb,            true,  iemNativeHlpObsoleteTb },
    9746             {   kIemNativeExitReason_NeedCsLimChecking,     true,  iemNativeHlpNeedCsLimChecking },
    9747             {   kIemNativeExitReason_CheckBranchMiss,       true,  iemNativeHlpCheckBranchMiss },
     9738            {   kIemNativeLabelType_RaiseDe,                false, iemNativeHlpExecRaiseDe },
     9739            {   kIemNativeLabelType_RaiseUd,                false, iemNativeHlpExecRaiseUd },
     9740            {   kIemNativeLabelType_RaiseSseRelated,        false, iemNativeHlpExecRaiseSseRelated },
     9741            {   kIemNativeLabelType_RaiseAvxRelated,        false, iemNativeHlpExecRaiseAvxRelated },
     9742            {   kIemNativeLabelType_RaiseSseAvxFpRelated,   false, iemNativeHlpExecRaiseSseAvxFpRelated },
     9743            {   kIemNativeLabelType_RaiseNm,                false, iemNativeHlpExecRaiseNm },
     9744            {   kIemNativeLabelType_RaiseGp0,               false, iemNativeHlpExecRaiseGp0 },
     9745            {   kIemNativeLabelType_RaiseMf,                false, iemNativeHlpExecRaiseMf },
     9746            {   kIemNativeLabelType_RaiseXf,                false, iemNativeHlpExecRaiseXf },
     9747            {   kIemNativeLabelType_ObsoleteTb,             true,  iemNativeHlpObsoleteTb },
     9748            {   kIemNativeLabelType_NeedCsLimChecking,      true,  iemNativeHlpNeedCsLimChecking },
     9749            {   kIemNativeLabelType_CheckBranchMiss,        true,  iemNativeHlpCheckBranchMiss },
    97489750        };
    97499751        for (uint32_t i = 0; i < RT_ELEMENTS(s_aSimpleTailLabels); i++)
    … …  
    97599761            /* jump back to the return sequence / generate a return sequence. */
    97609762            if (!s_aSimpleTailLabels[i].fWithEpilog)
    9761                 off = iemNativeEmitJmpToFixed(pReNative, off, aoffLabels[kIemNativeExitReason_Return]);
     9763                off = iemNativeEmitJmpToFixed(pReNative, off, aoffLabels[kIemNativeLabelType_Return]);
    97629764            else
    97639765                off = iemNativeEmitCoreEpilog(pReNative, off);
    … …  
    97679769# ifdef VBOX_STRICT
    97689770        /* Make sure we've generate code for all labels. */
    9769         for (uint32_t i = kIemNativeExitReason_Invalid + 1; i < RT_ELEMENTS(aoffLabels); i++)
    9770             Assert(aoffLabels[i] != 0 || i == kIemNativeExitReason_Return);
     9771        for (uint32_t i = kIemNativeLabelType_Invalid + 1; i < RT_ELEMENTS(aoffLabels); i++)
     9772            Assert(aoffLabels[i] != 0 || i == kIemNativeLabelType_Return);
    97719773#endif
    97729774    }
    … …  
    98029804     * Initialize the context.
    98039805     */
    9804     AssertCompile(kIemNativeExitReason_Invalid == 0);
     9806    AssertCompile(kIemNativeLabelType_Invalid == 0);
    98059807    AssertCompile(RT_ELEMENTS(pCtx->apExitLabels) == RT_ELEMENTS(aoffLabels));
    9806     pCtx->apExitLabels[kIemNativeExitReason_Invalid] = 0;
    9807     for (uint32_t i = kIemNativeExitReason_Invalid + 1; i < RT_ELEMENTS(pCtx->apExitLabels); i++)
    9808     {
    9809         Assert(aoffLabels[i] != 0 || i == kIemNativeExitReason_Return);
     9808    pCtx->apExitLabels[kIemNativeLabelType_Invalid] = 0;
     9809    for (uint32_t i = kIemNativeLabelType_Invalid + 1; i < RT_ELEMENTS(pCtx->apExitLabels); i++)
     9810    {
     9811        Assert(aoffLabels[i] != 0 || i == kIemNativeLabelType_Return);
    98109812        pCtx->apExitLabels[i] = &paFinalCommonCodeRx[aoffLabels[i]];
    98119813        Log10(("    apExitLabels[%u]=%p %s\n", i, pCtx->apExitLabels[i], iemNativeGetLabelName((IEMNATIVELABELTYPE)i, true)));
    … …  
    1006810070         */
    1006910071        //off = iemNativeEmitBrk(pReNative, off, 0x1227);
    10070         off = iemNativeEmitTbExit(pReNative, off, kIemNativeExitReason_Return);
     10072        off = iemNativeEmitTbExit(pReNative, off, kIemNativeLabelType_Return);
    1007110073#endif
    1007210074
    … …  
    1017210174                AssertContinue(idxLabel != UINT32_MAX);
    1017310175                iemNativeLabelDefine(pReNative, idxLabel, off);
    10174                 off = iemNativeEmitTbExit(pReNative, off, (IEMNATIVEEXITREASON)enmLabel);
     10176                off = iemNativeEmitTbExit(pReNative, off, enmLabel);
    1017510177            } while (fTailLabels);
    1017610178        }
    … …  
    1029210294    {
    1029310295        Assert(paTbExitFixups[i].off < off);
    10294         Assert(   paTbExitFixups[i].enmExitReason < kIemNativeExitReason_Max
    10295                && paTbExitFixups[i].enmExitReason > kIemNativeExitReason_Invalid);
     10296        Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(paTbExitFixups[i].enmExitReason));
    1029610297        RTPTRUNION const Ptr = { &paFinalInstrBuf[paTbExitFixups[i].off] };
    1029710298

  • trunk/src/VBox/VMM/include/IEMN8veRecompiler.h

    r105261 r105271  
    450450
    451451
    452 /** TB exit reasons. */
    453 typedef enum
    454 {
    455     kIemNativeExitReason_Invalid = 0,
    456     kIemNativeExitReason_RaiseDe,                /**< Raise (throw) X86_XCPT_DE (00h). */
    457     kIemNativeExitReason_RaiseUd,                /**< Raise (throw) X86_XCPT_UD (06h). */
    458     kIemNativeExitReason_RaiseSseRelated,        /**< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to cr0 & cr4. */
    459     kIemNativeExitReason_RaiseAvxRelated,        /**< Raise (throw) X86_XCPT_UD or X86_XCPT_NM according to xcr0, cr0 & cr4. */
    460     kIemNativeExitReason_RaiseSseAvxFpRelated,   /**< Raise (throw) X86_XCPT_UD or X86_XCPT_XF according to c4. */
    461     kIemNativeExitReason_RaiseNm,                /**< Raise (throw) X86_XCPT_NM (07h). */
    462     kIemNativeExitReason_RaiseGp0,               /**< Raise (throw) X86_XCPT_GP (0dh) w/ errcd=0. */
    463     kIemNativeExitReason_RaiseMf,                /**< Raise (throw) X86_XCPT_MF (10h). */
    464     kIemNativeExitReason_RaiseXf,                /**< Raise (throw) X86_XCPT_XF (13h). */
    465     kIemNativeExitReason_ObsoleteTb,
    466     kIemNativeExitReason_NeedCsLimChecking,
    467     kIemNativeExitReason_CheckBranchMiss,
    468     kIemNativeExitReason_ReturnBreak,
    469     kIemNativeExitReason_ReturnBreakFF,
    470     kIemNativeExitReason_ReturnBreakViaLookup,
    471     kIemNativeExitReason_ReturnBreakViaLookupWithIrq,
    472     kIemNativeExitReason_ReturnBreakViaLookupWithTlb,
    473     kIemNativeExitReason_ReturnBreakViaLookupWithTlbAndIrq,
    474     kIemNativeExitReason_ReturnWithFlags,
    475     kIemNativeExitReason_NonZeroRetOrPassUp,
    476     kIemNativeExitReason_Return,                /**< This is a little bit special, but needs to be included here. */
    477     kIemNativeExitReason_Max
    478 } IEMNATIVEEXITREASON;
    479 
    480 
    481452/** Native code generator label types. */
    482453typedef enum
    … …  
    484455    kIemNativeLabelType_Invalid = 0,
    485456    /*
    486      * Labels w/o data, only once instance per TB.
     457     * Labels w/o data, only once instance per TB - aka exit reasons.
    487458     *
    488459     * Note! Jumps to these requires instructions that are capable of spanning
    … …  
    536507} IEMNATIVELABELTYPE;
    537508
    538 /** Temporary kludge until all jumps to TB exit labels are converted to the new TB exiting style,
    539  * see @bugref{10677}.
    540  * @note update bird: This won't happen, unfortunately, since we'll keep using
    541  *       the local labels on arm64 so we can avoid inverting branch conditions
    542  *       and inserting extra of unconditional branches in order to reach the
    543  *       common code.  Instead we'll have everyone jump to the same tail lable
    544  *       which then jumps to the common (per chunk) code. */
    545 #define IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(a_Reason) \
    546     ((int)kIemNativeLabelType_ ## a_Reason == (int)kIemNativeExitReason_ ## a_Reason)
    547 AssertCompile(   IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseDe)
    548               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseUd)
    549               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseSseRelated)
    550               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseAvxRelated)
    551               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseSseAvxFpRelated)
    552               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseNm)
    553               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseGp0)
    554               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseMf)
    555               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(RaiseXf)
    556               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ObsoleteTb)
    557               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(NeedCsLimChecking)
    558               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(CheckBranchMiss)
    559               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreak)
    560               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreakFF)
    561               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreakViaLookup)
    562               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreakViaLookupWithIrq)
    563               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreakViaLookupWithTlb)
    564               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnBreakViaLookupWithTlbAndIrq)
    565               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(ReturnWithFlags)
    566               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(NonZeroRetOrPassUp)
    567               && IEM_N8VE_RECOMP_LABELTYPE_EQ_EXITREASON(Return));
    568 AssertCompile((int)kIemNativeExitReason_Max == (int)kIemNativeLabelType_LastTbExit + 1);
     509#define IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmLabel) \
     510    ((a_enmLabel) <= kIemNativeLabelType_LastTbExit && (a_enmLabel) > kIemNativeLabelType_Invalid)
    569511
    570512
    … …  
    623565{
    624566    /** Code offset of the fixup location. */
    625     uint32_t    off;
    626     /** The exit reason (IEMNATIVEEXITREASON). */
    627     uint32_t    enmExitReason;
     567    uint32_t            off;
     568    /** The exit reason. */
     569    IEMNATIVELABELTYPE  enmExitReason;
    628570} IEMNATIVEEXITFIXUP;
    629571/** Pointer to a native code generator TB exit fixup. */
    … …  
    636578{
    637579    /** Pointers to the exit labels */
    638     PIEMNATIVEINSTR apExitLabels[kIemNativeExitReason_Max];
     580    PIEMNATIVEINSTR apExitLabels[kIemNativeLabelType_LastTbExit + 1];
    639581} IEMNATIVEPERCHUNKCTX;
    640582/** Pointer to per-chunk recompiler context. */
    … …  
    16871629                                              IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);
    16881630#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
    1689 DECL_HIDDEN_THROW(void)     iemNativeAddTbExitFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, IEMNATIVEEXITREASON enmExitReason);
     1631DECL_HIDDEN_THROW(void)     iemNativeAddTbExitFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, IEMNATIVELABELTYPE enmExitReason);
    16901632#endif
    16911633DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);

  • trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h

    r105261 r105271  
    80108010DECL_FORCE_INLINE_THROW(uint32_t)
    80118011iemNativeEmitJccTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
    8012                          IEMNATIVEEXITREASON enmExitReason, IEMNATIVEINSTRCOND enmCond)
    8013 {
     8012                         IEMNATIVELABELTYPE enmExitReason, IEMNATIVEINSTRCOND enmCond)
     8013{
     8014    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
    80148015#if defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) && defined(RT_ARCH_AMD64)
    80158016    /* jcc rel32 */
    … …  
    80258026    /* ARM64 doesn't have the necessary jump range, so we jump via local label
    80268027       just like when we keep everything local. */
    8027     uint32_t const idxLabel = iemNativeLabelCreate(pReNative, (IEMNATIVELABELTYPE)enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     8028    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
    80288029    off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabel, enmCond);
    80298030#endif
    … …  
    80368037 */
    80378038DECL_INLINE_THROW(uint32_t)
    8038 iemNativeEmitJccTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    8039                        IEMNATIVEEXITREASON enmExitReason, IEMNATIVEINSTRCOND enmCond)
    8040 {
     8039iemNativeEmitJccTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmExitReason, IEMNATIVEINSTRCOND enmCond)
     8040{
     8041    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
    80418042#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
    80428043# ifdef RT_ARCH_AMD64
    8043     PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
     8044    off = iemNativeEmitJccTbExitEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 6), off, enmExitReason, enmCond);
    80448045# elif defined(RT_ARCH_ARM64)
    8045     PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
     8046    off = iemNativeEmitJccTbExitEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 2), off, enmExitReason, enmCond);
    80468047# else
    80478048#  error "Port me!"
    80488049# endif
    8049     return iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, enmExitReason, enmCond);
    8050 #else
    8051     return iemNativeEmitJccToNewLabel(pReNative, off, (IEMNATIVELABELTYPE)enmExitReason, 0 /*uData*/, enmCond);
     8050    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     8051    return off;
     8052#else
     8053    return iemNativeEmitJccToNewLabel(pReNative, off, enmExitReason, 0 /*uData*/, enmCond);
    80528054#endif
    80538055}
    … …  
    80578059 * Emits a JNZ/JNE rel32 / B.NE imm19 to the TB exit routine with the given reason.
    80588060 */
    8059 DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    8060                                                    IEMNATIVEEXITREASON enmExitReason)
     8061DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmExitReason)
    80618062{
    80628063#ifdef RT_ARCH_AMD64
    … …  
    80738074 * Emits a JZ/JE rel32 / B.EQ imm19 to the TB exit routine with the given reason.
    80748075 */
    8075 DECL_INLINE_THROW(uint32_t) iemNativeEmitJzTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    8076                                                    IEMNATIVEEXITREASON enmExitReason)
     8076DECL_INLINE_THROW(uint32_t) iemNativeEmitJzTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmExitReason)
    80778077{
    80788078#ifdef RT_ARCH_AMD64
    … …  
    80898089 * Emits a JA/JNBE rel32 / B.HI imm19 to the TB exit.
    80908090 */
    8091 DECL_INLINE_THROW(uint32_t) iemNativeEmitJaTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    8092                                                   IEMNATIVEEXITREASON enmExitReason)
     8091DECL_INLINE_THROW(uint32_t) iemNativeEmitJaTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmExitReason)
    80938092{
    80948093#ifdef RT_ARCH_AMD64
    … …  
    81058104 * Emits a JL/JNGE rel32 / B.LT imm19 to the TB exit with the given reason.
    81068105 */
    8107 DECL_INLINE_THROW(uint32_t) iemNativeEmitJlTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    8108                                                   IEMNATIVEEXITREASON enmExitReason)
     8106DECL_INLINE_THROW(uint32_t) iemNativeEmitJlTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmExitReason)
    81098107{
    81108108#ifdef RT_ARCH_AMD64
    … …  
    81198117
    81208118DECL_INLINE_THROW(uint32_t)
    8121 iemNativeEmitTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off, IEMNATIVEEXITREASON enmExitReason)
    8122 {
     8119iemNativeEmitTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off, IEMNATIVELABELTYPE enmExitReason)
     8120{
     8121    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
    81238122#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
    81248123# ifdef RT_ARCH_AMD64
    … …  
    81418140
    81428141#else
    8143     uint32_t const idxLabel = iemNativeLabelCreate(pReNative, (IEMNATIVELABELTYPE)enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     8142    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
    81448143    return iemNativeEmitJmpToLabelEx(pReNative, pCodeBuf, off, idxLabel);
    81458144#endif
    … …  
    81478146
    81488147
    8149 DECL_INLINE_THROW(uint32_t)
    8150 iemNativeEmitTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEEXITREASON enmExitReason)
    8151 {
     8148DECL_INLINE_THROW(uint32_t) iemNativeEmitTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmExitReason)
     8149{
     8150    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
    81528151#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
    81538152# ifdef RT_ARCH_AMD64
    … …  
    81748173
    81758174#else
    8176     return iemNativeEmitJmpToNewLabel(pReNative, off, (IEMNATIVELABELTYPE)enmExitReason);
     8175    return iemNativeEmitJmpToNewLabel(pReNative, off, enmExitReason);
    81778176#endif
    81788177}
    … …  
    81858184DECL_INLINE_THROW(uint32_t)
    81868185iemNativeEmitTestAnyBitsInGprAndTbExitIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    8187                                                uint8_t iGprSrc, uint64_t fBits, IEMNATIVEEXITREASON enmExitReason)
     8186                                               uint8_t iGprSrc, uint64_t fBits, IEMNATIVELABELTYPE enmExitReason)
    81888187{
    81898188    Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
    … …  
    82008199DECL_INLINE_THROW(uint32_t)
    82018200iemNativeEmitTestAnyBitsInGprAndTbExitIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    8202                                                 uint8_t iGprSrc, uint64_t fBits, IEMNATIVEEXITREASON enmExitReason)
     8201                                                uint8_t iGprSrc, uint64_t fBits, IEMNATIVELABELTYPE enmExitReason)
    82038202{
    82048203    Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
    … …  
    82158214DECL_INLINE_THROW(uint32_t)
    82168215iemNativeEmitTestIfGprNotEqualGprAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    8217                                            uint8_t iGprLeft, uint8_t iGprRight, IEMNATIVEEXITREASON enmExitReason)
     8216                                           uint8_t iGprLeft, uint8_t iGprRight, IEMNATIVELABELTYPE enmExitReason)
    82188217{
    82198218    off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);
    … …  
    82278226 * @a uImm.
    82288227 */
    8229 DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32NotEqualImmAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    8230                                                                          uint8_t iGprSrc, uint32_t uImm, IEMNATIVEEXITREASON enmExitReason)
     8228DECL_INLINE_THROW(uint32_t)
     8229iemNativeEmitTestIfGpr32NotEqualImmAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     8230                                             uint8_t iGprSrc, uint32_t uImm, IEMNATIVELABELTYPE enmExitReason)
    82318231{
    82328232    off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
    … …  
    82418241DECL_INLINE_THROW(uint32_t)
    82428242iemNativeEmitTestIfGprNotEqualImmAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    8243                                            uint8_t iGprSrc, uint64_t uImm, IEMNATIVEEXITREASON enmExitReason)
     8243                                           uint8_t iGprSrc, uint64_t uImm, IEMNATIVELABELTYPE enmExitReason)
    82448244{
    82458245    off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);
    … …  
    82548254DECL_INLINE_THROW(uint32_t)
    82558255iemNativeEmitTestIfGpr32EqualsImmAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    8256                                            uint8_t iGprSrc, uint32_t uImm, IEMNATIVEEXITREASON enmExitReason)
     8256                                           uint8_t iGprSrc, uint32_t uImm, IEMNATIVELABELTYPE enmExitReason)
    82578257{
    82588258    off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
    … …  
    82708270DECL_INLINE_THROW(uint32_t)
    82718271iemNativeEmitTestBitInGprAndTbExitIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    8272                                         uint8_t iGprSrc, uint8_t iBitNo, IEMNATIVEEXITREASON enmExitReason)
    8273 {
     8272                                        uint8_t iGprSrc, uint8_t iBitNo, IEMNATIVELABELTYPE enmExitReason)
     8273{
     8274    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
    82748275#if defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) && defined(RT_ARCH_AMD64)
    82758276    Assert(iBitNo < 64);
    … …  
    83048305    /* ARM64 doesn't have the necessary jump range, so we jump via local label
    83058306       just like when we keep everything local. */
    8306     uint32_t const idxLabel = iemNativeLabelCreate(pReNative, (IEMNATIVELABELTYPE)enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     8307    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
    83078308    return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /*fJmpIfSet*/);
    83088309#endif
    … …  
    83178318DECL_FORCE_INLINE_THROW(uint32_t)
    83188319iemNativeEmitTestIfGprIsNotZeroAndTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
    8319                                            uint8_t iGprSrc, bool f64Bit, IEMNATIVEEXITREASON enmExitReason)
    8320 {
     8320                                           uint8_t iGprSrc, bool f64Bit, IEMNATIVELABELTYPE enmExitReason)
     8321{
     8322    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
    83218323#if defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) && defined(RT_ARCH_AMD64)
    83228324    /* test reg32,reg32  / test reg64,reg64 */
    … …  
    83348336    /* ARM64 doesn't have the necessary jump range, so we jump via local label
    83358337       just like when we keep everything local. */
    8336     uint32_t const idxLabel = iemNativeLabelCreate(pReNative, (IEMNATIVELABELTYPE)enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     8338    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
    83378339    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
    83388340                                                                f64Bit, true /*fJmpIfNotZero*/, idxLabel);
    … …  
    83468348 * The operand size is given by @a f64Bit.
    83478349 */
    8348 DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsNotZeroAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    8349                                                                      uint8_t iGprSrc, bool f64Bit, IEMNATIVEEXITREASON enmExitReason)
     8350DECL_INLINE_THROW(uint32_t)
     8351iemNativeEmitTestIfGprIsNotZeroAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     8352                                         uint8_t iGprSrc, bool f64Bit, IEMNATIVELABELTYPE enmExitReason)
    83508353{
    83518354#if defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) && defined(RT_ARCH_AMD64)
    … …  
    83558358    return off;
    83568359#else
    8357     uint32_t const idxLabel = iemNativeLabelCreate(pReNative, (IEMNATIVELABELTYPE)enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     8360    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
    83588361    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, true /*fJmpIfNotZero*/, idxLabel);
    83598362#endif
Note: See TracChangeset for help on using the changeset viewer.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette