VirtualBox

Changeset 106123 in vbox for trunk


Ignore:
Timestamp:
Sep 23, 2024 10:04:30 PM (2 months ago)
Author:
vboxsync
Message:

VMM/IEM: More work on recompilation-time checks of skipped & postponed EFLAGS - fixed incorrect skipping for variants that clears EFLAGS when advancing RIP. The latter had slipped thru and this change reduces the effectiveness of the skipping stuff. :-( bugref:10720

Location:
trunk/src/VBox/VMM
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8vePython.py

    r106061 r106123  
    595595                oNewStmt = copy.deepcopy(oStmt);
    596596                oNewStmt.sName = 'IEM_MC_BEGIN_EX';
    597                 fWithoutFlags = (    self.oVariation.isWithFlagsCheckingAndClearingVariation()
    598                                  and self.oVariation.oParent.hasWithFlagsCheckingAndClearingVariation());
    599                 if fWithoutFlags or self.oVariation.oParent.dsCImplFlags:
     597                fWithFlags    = self.oVariation.isWithFlagsCheckingAndClearingVariation();
     598                fWithoutFlags = not fWithFlags and self.oVariation.oParent.hasWithFlagsCheckingAndClearingVariation();
     599                if fWithFlags or fWithoutFlags or self.oVariation.oParent.dsCImplFlags:
    600600                    if fWithoutFlags:
    601601                        oNewStmt.asParams[0] = ' | '.join(sorted(  list(self.oVariation.oParent.oMcBlock.dsMcFlags.keys())
    602602                                                                 + ['IEM_MC_F_WITHOUT_FLAGS',] ));
     603                    else:
     604                        oNewStmt.asParams[0] = ' | '.join(sorted(  list(self.oVariation.oParent.oMcBlock.dsMcFlags.keys())
     605                                                                 + ['IEM_MC_F_WITH_FLAGS',] ));
    603606                    if self.oVariation.oParent.dsCImplFlags:
    604607                        oNewStmt.asParams[1] = ' | '.join(sorted(self.oVariation.oParent.dsCImplFlags.keys()));

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompFuncs.h

    r106117 r106123  
    34013401iemNativeEmitIfEflagAnysBitsSet(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fBitsInEfl, uint64_t fLivenessEflBits)
    34023402{
     3403    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(pReNative,  fBitsInEfl);
    34033404    IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(pReNative, off, fBitsInEfl);
    34043405    PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative);
    … …  
    34283429iemNativeEmitIfEflagNoBitsSet(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t fBitsInEfl, uint64_t fLivenessEflBits)
    34293430{
     3431    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(pReNative,  fBitsInEfl);
    34303432    IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(pReNative, off, fBitsInEfl);
    34313433    PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative);
    … …  
    34563458iemNativeEmitIfEflagsBitSet(PIEMRECOMPILERSTATE pReNative, uint32_t off, unsigned iBitNo, uint64_t fLivenessEflBit)
    34573459{
     3460    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(pReNative,  RT_BIT_32(iBitNo));
    34583461    IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(pReNative, off, RT_BIT_32(iBitNo));
    34593462    PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative);
    … …  
    34843487iemNativeEmitIfEflagsBitNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off, unsigned iBitNo, uint64_t fLivenessEflBit)
    34853488{
     3489    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(pReNative,  RT_BIT_32(iBitNo));
    34863490    IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(pReNative, off, RT_BIT_32(iBitNo));
    34873491    PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative);
    … …  
    35233527{
    35243528    Assert(iBitNo1 != iBitNo2);
     3529    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(pReNative,  RT_BIT_32(iBitNo1) | RT_BIT_32(iBitNo2));
    35253530    IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(pReNative, off, RT_BIT_32(iBitNo1) | RT_BIT_32(iBitNo2));
    35263531    PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative);
    … …  
    36003605    Assert(iBitNo2 != iBitNo);
    36013606    Assert(iBitNo2 != iBitNo1);
     3607    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(pReNative,  RT_BIT_32(iBitNo) | RT_BIT_32(iBitNo1) | RT_BIT_32(iBitNo2));
    36023608    IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(pReNative, off, RT_BIT_32(iBitNo) | RT_BIT_32(iBitNo1) | RT_BIT_32(iBitNo2));
    36033609    PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative);
    … …  
    37773783                                          bool fCheckIfSet, unsigned iBitNo, uint64_t fLivenessEflBit)
    37783784{
     3785    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(pReNative,  RT_BIT_32(iBitNo));
    37793786    IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(pReNative, off, RT_BIT_32(iBitNo));
    37803787    PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative);
    … …  
    38453852
    38463853{
     3854    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(pReNative,  RT_BIT_32(iBitNo));
    38473855    IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(pReNative, off, RT_BIT_32(iBitNo));
    38483856    PIEMNATIVECOND const pEntry = iemNativeCondPushIf(pReNative);
    … …  
    41194127
    41204128{
     4129    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(pReNative,  X86_EFL_STATUS_BITS);
    41214130    IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(pReNative, off, X86_EFL_STATUS_BITS);
    41224131
    … …  
    59305939#endif
    59315940
     5941    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(pReNative,  a_fEflInput);
    59325942    IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(pReNative, off, a_fEflInput);
    59335943
    … …  
    62466256
    62476257#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
    6248     Assert(!(pReNative->fSkippingEFlags & fEflInput));
     6258    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(pReNative,  fEflInput);
     6259    IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(pReNative, off, fEflInput);
    62496260    pReNative->fSkippingEFlags &= ~fEflOutput;
    6250     IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(pReNative, off, fEflInput);
    62516261# ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
    62526262

  • trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

    r106117 r106123  
    65636563iemNativeEmitThreadedCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
    65646564{
     6565    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(pReNative,  X86_EFL_STATUS_BITS);
    65656566    IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(pReNative, off, X86_EFL_STATUS_BITS);
    65666567
    … …  
    69226923    pReNative->Core.bmHstRegs |= RT_BIT_32(IEMNATIVE_CALL_RET_GREG); /* HACK: For IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK (return register is already set to status code). */
    69236924
     6925    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(pReNative,  X86_EFL_STATUS_BITS);
    69246926    IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(pReNative, off, X86_EFL_STATUS_BITS);
    69256927
    … …  
    1030710309            STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatNativeFullyRecompiledTbs);
    1030810310
    10309 #ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
    10310         Assert(pReNative->fSkippingEFlags == 0);
    10311 #endif
    10312 #ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
    10313         Assert(pReNative->fPostponingEFlags == 0);
    10314 #endif
     10311        IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(pReNative, UINT32_MAX);
    1031510312
    1031610313#ifdef VBOX_WITH_STATISTICS

  • trunk/src/VBox/VMM/VMMAll/target-x86/IEMAllN8veEmit-x86.h

    r106117 r106123  
    214214     */
    215215    PCIEMLIVENESSENTRY const pLivenessEntry = &pReNative->paLivenessEntries[pReNative->idxCurCall];
    216     if (IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(pLivenessEntry))
     216    if (   IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(pLivenessEntry)
     217        && !(pReNative->fMc & IEM_MC_F_WITH_FLAGS))
    217218    {
    218219        STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativeEflSkippedLogical);
    … …  
    322323     */
    323324    PCIEMLIVENESSENTRY const pLivenessEntry = &pReNative->paLivenessEntries[pReNative->idxCurCall];
    324     if (IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(pLivenessEntry))
     325    if (   IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(pLivenessEntry)
     326        && !(pReNative->fMc & IEM_MC_F_WITH_FLAGS))
    325327    {
    326328        STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativeEflSkippedArithmetic);
    … …  
    15771579     */
    15781580    PCIEMLIVENESSENTRY const pLivenessEntry = &pReNative->paLivenessEntries[pReNative->idxCurCall];
    1579     if (IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(pLivenessEntry))
     1581    if (   IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(pLivenessEntry)
     1582        && !(pReNative->fMc & IEM_MC_F_WITH_FLAGS))
    15801583    {
    15811584        STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativeEflSkippedShift);

  • trunk/src/VBox/VMM/include/IEMInternal.h

    r106101 r106123  
    931931#define IEM_MC_F_64BIT              RT_BIT_32(6)
    932932#define IEM_MC_F_NOT_64BIT          RT_BIT_32(7)
     933/** This is set by IEMAllN8vePython.py to indicate a variation with the
     934 * flags-clearing-and-checking. */
     935#define IEM_MC_F_WITH_FLAGS         RT_BIT_32(8)
    933936/** This is set by IEMAllN8vePython.py to indicate a variation without the
    934937 * flags-clearing-and-checking, when there is also a variation with that.
    935  * @note Do not use this manully, it's only for python and for testing in
     938 * @note Do not set this manully, it's only for python and for testing in
    936939 *       the native recompiler! */
    937 #define IEM_MC_F_WITHOUT_FLAGS      RT_BIT_32(8)
     940#define IEM_MC_F_WITHOUT_FLAGS      RT_BIT_32(9)
    938941/** @} */
    939942

  • trunk/src/VBox/VMM/include/IEMN8veRecompiler.h

    r106117 r106123  
    10461046/** @} */
    10471047
     1048/** @def IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY
     1049 * Debug assertion that the required flags are available and not incorrectly skipped.
     1050 */
     1051#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
     1052# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(a_pReNative, a_fEflNeeded) \
     1053    AssertMsg(!((a_pReNative)->fSkippingEFlags & (a_fEflNeeded)), \
     1054              ("%#x & %#x -> %#x\n", (a_pReNative)->fSkippingEFlags, \
     1055               a_fEflNeeded, (a_pReNative)->fSkippingEFlags & (a_fEflNeeded) ))
     1056#else
     1057# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(a_pReNative, a_fEflNeeded) ((void)0)
     1058#endif
     1059
     1060/** @def IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY
     1061 * Debug assertion that the required flags are available and not incorrectly postponed.
     1062 */
     1063#ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
     1064# define IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(a_pReNative, a_fEflNeeded) \
     1065    AssertMsg(!((a_pReNative)->fPostponingEFlags & (a_fEflNeeded)), \
     1066              ("%#x & %#x -> %#x\n", (a_pReNative)->fPostponingEFlags, \
     1067               a_fEflNeeded, (a_pReNative)->fPostponingEFlags & (a_fEflNeeded) ))
     1068#else
     1069# define IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(a_pReNative, a_fEflNeeded) ((void)0)
     1070#endif
     1071
     1072/** @def IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING
     1073 * Debug assertion that the required flags are available and not incorrectly
     1074 * skipped or postponed.
     1075 */
     1076#if defined(IEMNATIVE_WITH_EFLAGS_SKIPPING) && defined(IEMNATIVE_WITH_EFLAGS_POSTPONING)
     1077# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) \
     1078    AssertMsg(!(((a_pReNative)->fSkippingEFlags | (a_pReNative)->fPostponingEFlags) & (a_fEflNeeded)), \
     1079              ("(%#x | %#x) & %#x -> %#x\n", (a_pReNative)->fSkippingEFlags, (a_pReNative)->fPostponingEFlags, \
     1080               a_fEflNeeded, ((a_pReNative)->fSkippingEFlags | (a_pReNative)->fPostponingEFlags) & (a_fEflNeeded) ))
     1081#elif defined(IEMNATIVE_WITH_EFLAGS_SKIPPING)
     1082# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) \
     1083    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(a_pReNative, a_fEflNeeded)
     1084#elif defined(IEMNATIVE_WITH_EFLAGS_POSTPONING) \
     1085# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) \
     1086    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(a_pReNative, a_fEflNeeded)
     1087#else
     1088# define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) ((void)0)
     1089#endif
     1090
    10481091/** @def IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK
    10491092 * Checks that the EFLAGS bits specified by @a a_fEflNeeded are actually
    … …  
    10541097#ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
    10551098# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { \
    1056         AssertMsg(!((a_pReNative)->fSkippingEFlags & (a_fEflNeeded)), \
    1057                   ("%#x & %#x -> %#x; off=%#x\n", (a_pReNative)->fSkippingEFlags, a_fEflNeeded, \
    1058                   ((a_pReNative)->fSkippingEFlags & (a_fEflNeeded)), a_off)); \
    10591099        (a_off) = iemNativeEmitEFlagsSkippingCheck(a_pReNative, a_off, a_fEflNeeded); \
    10601100    } while (0)
    10611101#else
    1062 # define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) \
    1063     AssertMsg(!((a_pReNative)->fSkippingEFlags & (a_fEflNeeded)), \
    1064               ("%#x & %#x -> %#x; off=%#x\n", (a_pReNative)->fSkippingEFlags, a_fEflNeeded, \
    1065               ((a_pReNative)->fSkippingEFlags & (a_fEflNeeded)), a_off))
     1102# define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
    10661103#endif
    10671104

  • trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h

    r106061 r106123  
    82788278
    82798279
     8280
    82808281/*********************************************************************************************************************************
    82818282*   TB exiting helpers.                                                                                                          *
    … …  
    83058306                         IEMNATIVELABELTYPE enmExitReason, IEMNATIVEINSTRCOND enmCond)
    83068307{
     8308    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
     8309    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
    83078310    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
    83088311
    … …  
    83338336iemNativeEmitJccTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmExitReason, IEMNATIVEINSTRCOND enmCond)
    83348337{
     8338    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
     8339    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
    83358340    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
     8341
    83368342#ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
    83378343# ifdef RT_ARCH_AMD64
    … …  
    84138419iemNativeEmitTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off, IEMNATIVELABELTYPE enmExitReason)
    84148420{
     8421    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
     8422    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here. */
    84158423    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
    84168424
    … …  
    84478455                    bool fActuallyExitingTb = true)
    84488456{
     8457    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
     8458    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here. */
    84498459    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
    84508460
    … …  
    85768586{
    85778587    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
     8588
    85788589#if defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) && defined(RT_ARCH_AMD64)
    85798590    Assert(iBitNo < 64);
    … …  
    86088619    /* ARM64 doesn't have the necessary jump range, so we jump via local label
    86098620       just like when we keep everything local. */
     8621    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
     8622    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
    86108623    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
    86118624    return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /*fJmpIfSet*/);
    … …  
    86228635iemNativeEmitTestIfGprIsNotZeroAndTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
    86238636                                           uint8_t iGprSrc, bool f64Bit, IEMNATIVELABELTYPE enmExitReason)
     8637{
     8638    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
     8639
     8640#if defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) && defined(RT_ARCH_AMD64)
     8641    /* test reg32,reg32  / test reg64,reg64 */
     8642    if (f64Bit)
     8643        pCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_R | X86_OP_REX_B);
     8644    else if (iGprSrc >= 8)
     8645        pCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
     8646    pCodeBuf[off++] = 0x85;
     8647    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
     8648
     8649    /* jnz idxLabel  */
     8650    return iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, enmExitReason, kIemNativeInstrCond_ne);
     8651
     8652#else
     8653    /* ARM64 doesn't have the necessary jump range, so we jump via local label
     8654       just like when we keep everything local. */
     8655    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
     8656    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
     8657    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     8658    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
     8659                                                                f64Bit, true /*fJmpIfNotZero*/, idxLabel);
     8660#endif
     8661}
     8662
     8663
     8664/**
     8665 * Emits code to exit the current TB with the given reason @a enmExitReason if @a iGprSrc is not zero.
     8666 *
     8667 * The operand size is given by @a f64Bit.
     8668 */
     8669DECL_INLINE_THROW(uint32_t)
     8670iemNativeEmitTestIfGprIsNotZeroAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
     8671                                         uint8_t iGprSrc, bool f64Bit, IEMNATIVELABELTYPE enmExitReason)
     8672{
     8673#if defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) && defined(RT_ARCH_AMD64)
     8674    off = iemNativeEmitTestIfGprIsNotZeroAndTbExitEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 3 + 6),
     8675                                                     off, iGprSrc, f64Bit, enmExitReason);
     8676    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     8677    return off;
     8678#else
     8679    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
     8680    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
     8681    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     8682    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, true /*fJmpIfNotZero*/, idxLabel);
     8683#endif
     8684}
     8685
     8686
     8687/**
     8688 * Emits code that exits the current TB with @a enmExitReason if @a iGprSrc is zero.
     8689 *
     8690 * The operand size is given by @a f64Bit.
     8691 */
     8692DECL_FORCE_INLINE_THROW(uint32_t)
     8693iemNativeEmitTestIfGprIsZeroAndTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
     8694                                        uint8_t iGprSrc, bool f64Bit, IEMNATIVELABELTYPE enmExitReason)
    86248695{
    86258696    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
    … …  
    86348705
    86358706    /* jnz idxLabel  */
    8636     return iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, enmExitReason, kIemNativeInstrCond_ne);
     8707    return iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, enmExitReason, kIemNativeInstrCond_e);
    86378708
    86388709#else
    86398710    /* ARM64 doesn't have the necessary jump range, so we jump via local label
    86408711       just like when we keep everything local. */
    8641     uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
    8642     return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
    8643                                                                 f64Bit, true /*fJmpIfNotZero*/, idxLabel);
    8644 #endif
    8645 }
    8646 
    8647 
    8648 /**
    8649  * Emits code to exit the current TB with the given reason @a enmExitReason if @a iGprSrc is not zero.
    8650  *
    8651  * The operand size is given by @a f64Bit.
    8652  */
    8653 DECL_INLINE_THROW(uint32_t)
    8654 iemNativeEmitTestIfGprIsNotZeroAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
    8655                                          uint8_t iGprSrc, bool f64Bit, IEMNATIVELABELTYPE enmExitReason)
    8656 {
    8657 #if defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) && defined(RT_ARCH_AMD64)
    8658     off = iemNativeEmitTestIfGprIsNotZeroAndTbExitEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 3 + 6),
    8659                                                      off, iGprSrc, f64Bit, enmExitReason);
    8660     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
    8661     return off;
    8662 #else
    8663     uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
    8664     return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, true /*fJmpIfNotZero*/, idxLabel);
    8665 #endif
    8666 }
    8667 
    8668 
    8669 /**
    8670  * Emits code that exits the current TB with @a enmExitReason if @a iGprSrc is zero.
    8671  *
    8672  * The operand size is given by @a f64Bit.
    8673  */
    8674 DECL_FORCE_INLINE_THROW(uint32_t)
    8675 iemNativeEmitTestIfGprIsZeroAndTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
    8676                                         uint8_t iGprSrc, bool f64Bit, IEMNATIVELABELTYPE enmExitReason)
    8677 {
    8678     Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
    8679 #if defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) && defined(RT_ARCH_AMD64)
    8680     /* test reg32,reg32  / test reg64,reg64 */
    8681     if (f64Bit)
    8682         pCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_R | X86_OP_REX_B);
    8683     else if (iGprSrc >= 8)
    8684         pCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
    8685     pCodeBuf[off++] = 0x85;
    8686     pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
    8687 
    8688     /* jnz idxLabel  */
    8689     return iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, enmExitReason, kIemNativeInstrCond_e);
    8690 
    8691 #else
    8692     /* ARM64 doesn't have the necessary jump range, so we jump via local label
    8693        just like when we keep everything local. */
     8712    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
     8713    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
    86948714    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
    86958715    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
    … …  
    87148734    return off;
    87158735#else
     8736    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
     8737    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
    87168738    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
    87178739    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, false /*fJmpIfNotZero*/, idxLabel);
Note: See TracChangeset for help on using the changeset viewer.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette