Changeset 105673 in vbox for trunk/src/VBox/VMM/include

Timestamp:

Aug 14, 2024 1:57:57 PM (6 months ago)

Author:

vboxsync

Message:

VMM/IEM,TM: Do full-TB looping. Redid timer polling in the recompiler. Rewrote the Blt_CheckIrq code, eliminating a conditional. Fixed some TLB related assertions. Moved some IEMCPU members around in hope of better cache-locality. bugref:10656

Location:

trunk/src/VBox/VMM/include

Files:

• IEMInternal.h (modified) (18 diffs)
• IEMN8veRecompiler.h (modified) (3 diffs)
• IEMN8veRecompilerEmit.h (modified) (9 diffs)

trunk/src/VBox/VMM/include/IEMInternal.h

@@ -92 +92 @@
 # define IEM_WITH_THROW_CATCH
 #endif /*ASM-NOINC-END*/
+
+/** @def IEM_WITH_INTRA_TB_JUMPS
+ * Enables loop-jumps within a TB (currently only to the first call).
+ */
+#if defined(DOXYGEN_RUNNING) || 1
+# define IEM_WITH_INTRA_TB_JUMPS
+#endif
 
 /** @def IEMNATIVE_WITH_DELAYED_PC_UPDATING
@@ -1245 +1252 @@
     uint8_t     uTbLookup;
 
-    /** Unused atm. */
-    uint8_t     uUnused0;
+    /** Flags - IEMTHREADEDCALLENTRY_F_XXX. */
+    uint8_t     fFlags;
 
     /** Generic parameters. */
@@ -1270 +1277 @@
 /** Make a IEMTHRDEDCALLENTRY::uTbLookup value. */
 #define IEM_TB_LOOKUP_TAB_MAKE(a_idxTable, a_fLarge)    ((a_idxTable) | ((a_fLarge) ? 0x80 : 0))
+
+
+/** The call entry is a jump target. */
+#define IEMTHREADEDCALLENTRY_F_JUMP_TARGET              UINT8_C(0x01)
+
 
 /**
@@ -2129 +2141 @@
     uint8_t                 abAlignment9[42];
 
-    /** @name Recompilation
+
+    /** @name Recompiled Exection
      * @{ */
     /** Pointer to the current translation block.
@@ -2156 +2169 @@
     uint64_t                u64Unused;
 #endif
-    /** Fixed TB used for threaded recompilation.
-     * This is allocated once with maxed-out sizes and re-used afterwards. */
-    R3PTRTYPE(PIEMTB)       pThrdCompileTbR3;
     /** Pointer to the ring-3 TB cache for this EMT. */
     R3PTRTYPE(PIEMTBCACHE)  pTbCacheR3;
@@ -2165 +2175 @@
      * entry, thus it can always safely be used w/o NULL checking. */
     R3PTRTYPE(PIEMTB *)     ppTbLookupEntryR3;
+#if 0 /* unused */
     /** The PC (RIP) at the start of pCurTbR3/pCurTbR0.
      * The TBs are based on physical addresses, so this is needed to correleated
      * RIP to opcode bytes stored in the TB (AMD-V / VT-x). */
     uint64_t                uCurTbStartPc;
+#endif
+
     /** Number of threaded TBs executed. */
     uint64_t                cTbExecThreaded;
     /** Number of native TBs executed. */
     uint64_t                cTbExecNative;
+
+    /** The number of IRQ/FF checks till the next timer poll call. */
+    uint32_t                cIrqChecksTillNextPoll;
+    /** The virtual sync time at the last timer poll call in milliseconds. */
+    uint32_t                msRecompilerPollNow;
+    /** The virtual sync time at the last timer poll call in nanoseconds. */
+    uint64_t                nsRecompilerPollNow;
+    /** The previous cIrqChecksTillNextPoll value. */
+    uint32_t                cIrqChecksTillNextPollPrev;
+    /** The ideal nanosecond interval between two timer polls.
+     * @todo make this adaptive?  */
+    uint32_t                cNsIdealPollInterval;
+
+    /** The current instruction number in a native TB.
+     * This is set by code that may trigger an unexpected TB exit (throw/longjmp)
+     * and will be picked up by the TB execution loop. Only used when
+     * IEMNATIVE_WITH_INSTRUCTION_COUNTING is defined. */
+    uint8_t                 idxTbCurInstr;
+    /** @} */
+
+    /** @name Recompilation
+     * @{ */
     /** Whether we need to check the opcode bytes for the current instruction.
      * This is set by a previous instruction if it modified memory or similar.  */
@@ -2182 +2217 @@
     /** Whether to end the current TB. */
     bool                    fEndTb;
+    /** Indicates that the current instruction is an STI.  This is set by the
+     * iemCImpl_sti code and subsequently cleared by the recompiler. */
+    bool                    fTbCurInstrIsSti;
+    /** Spaced reserved for recompiler data / alignment. */
+    bool                    afRecompilerStuff1[1];
     /** Number of instructions before we need emit an IRQ check call again.
      * This helps making sure we don't execute too long w/o checking for
@@ -2189 +2229 @@
      * fTbCurInstrIsSti. */
     uint8_t                 cInstrTillIrqCheck;
-    /** Indicates that the current instruction is an STI.  This is set by the
-     * iemCImpl_sti code and subsequently cleared by the recompiler. */
-    bool                    fTbCurInstrIsSti;
+    /** The index of the last CheckIrq call during threaded recompilation. */
+    uint16_t                idxLastCheckIrqCallNo;
     /** The size of the IEMTB::pabOpcodes allocation in pThrdCompileTbR3. */
     uint16_t                cbOpcodesAllocated;
-    /** The current instruction number in a native TB.
-     * This is set by code that may trigger an unexpected TB exit (throw/longjmp)
-     * and will be picked up by the TB execution loop. Only used when
-     * IEMNATIVE_WITH_INSTRUCTION_COUNTING is defined. */
-    uint8_t                 idxTbCurInstr;
-    /** Spaced reserved for recompiler data / alignment. */
-    bool                    afRecompilerStuff1[3];
-    /** The virtual sync time at the last timer poll call. */
-    uint32_t                msRecompilerPollNow;
     /** The IEMTB::cUsed value when to attempt native recompilation of a TB. */
     uint32_t                uTbNativeRecompileAtUsedCount;
@@ -2212 +2242 @@
      *  currently not up to date in EFLAGS. */
     uint32_t                fSkippingEFlags;
+    /** Spaced reserved for recompiler data / alignment. */
+    uint32_t                u32RecompilerStuff2;
+#if 0  /* unused */
     /** Previous GCPhysInstrBuf value - only valid if fTbCrossedPage is set.   */
     RTGCPHYS                GCPhysInstrBufPrev;
+#endif
+
+    /** Fixed TB used for threaded recompilation.
+     * This is allocated once with maxed-out sizes and re-used afterwards. */
+    R3PTRTYPE(PIEMTB)       pThrdCompileTbR3;
     /** Pointer to the ring-3 TB allocator for this EMT. */
     R3PTRTYPE(PIEMTBALLOCATOR) pTbAllocatorR3;
@@ -2222 +2260 @@
     /** Dummy entry for ppTbLookupEntryR3. */
     R3PTRTYPE(PIEMTB)       pTbLookupEntryDummyR3;
+    /** @} */
 
     /** Dummy TLB entry used for accesses to pages with databreakpoints. */
@@ -2230 +2269 @@
     /** Statistics: Times BltIn_CheckIrq breaks out of the TB. */
     STAMCOUNTER             StatCheckIrqBreaks;
+    /** Statistics: Times BltIn_CheckTimers breaks direct linking TBs. */
+    STAMCOUNTER             StatCheckTimersBreaks;
     /** Statistics: Times BltIn_CheckMode breaks out of the TB. */
     STAMCOUNTER             StatCheckModeBreaks;
@@ -2240 +2281 @@
     /** Statistics: Times a jump or page crossing required a TB with CS.LIM checking. */
     STAMCOUNTER             StatCheckNeedCsLimChecking;
-    /** Statistics: Times a loop was detected within a TB.. */
+    /** Statistics: Times a loop was detected within a TB. */
     STAMCOUNTER             StatTbLoopInTbDetected;
+    /** Statistics: Times a loop back to the start of the TB was detected. */
+    STAMCOUNTER             StatTbLoopFullTbDetected;
     /** Exec memory allocator statistics: Number of times allocaintg executable memory failed. */
     STAMCOUNTER             StatNativeExecMemInstrBufAllocFailed;
@@ -2421 +2464 @@
     STAMCOUNTER             StatNativeTbExitObsoleteTb;
 
+    /** Native recompiler: Number of full TB loops (jumps from end to start). */
+    STAMCOUNTER             StatNativeTbExitLoopFullTb;
+
     /** Native recompiler: Failure situations with direct linking scenario \#1.
      * Counter with StatNativeTbExitReturnBreak. Not in release builds.
@@ -2448 +2494 @@
 
 #ifdef IEM_WITH_TLB_TRACE
-    uint64_t                au64Padding[2];
+    uint64_t                au64Padding[6];
 #else
-    uint64_t                au64Padding[4];
-#endif
-    /** @} */
+    //uint64_t                au64Padding[1];
+#endif
 
 #ifdef IEM_WITH_TLB_TRACE
@@ -2491 +2536 @@
 AssertCompileMemberAlignment(IEMCPU, aMemMappingLocks, 16);
 AssertCompileMemberAlignment(IEMCPU, aBounceBuffers, 64);
+AssertCompileMemberAlignment(IEMCPU, pCurTbR3, 64);
 AssertCompileMemberAlignment(IEMCPU, DataTlb, 64);
 AssertCompileMemberAlignment(IEMCPU, CodeTlb, 64);
@@ -6742 +6788 @@
 extern const PFNIEMOP g_apfnIemThreadedRecompilerVecMap3[1024];
 
+DECLHIDDEN(int)     iemPollTimers(PVMCC pVM, PVMCPUCC pVCpu) RT_NOEXCEPT;
+
 DECLCALLBACK(int)   iemTbInit(PVMCC pVM, uint32_t cInitialTbs, uint32_t cMaxTbs,
                               uint64_t cbInitialExec, uint64_t cbMaxExec, uint32_t cbChunkExec);
@@ -6777 +6825 @@
 
 IEM_DECL_IEMTHREADEDFUNC_PROTO(iemThreadedFunc_BltIn_CheckIrq);
+IEM_DECL_IEMTHREADEDFUNC_PROTO(iemThreadedFunc_BltIn_CheckTimers);
+IEM_DECL_IEMTHREADEDFUNC_PROTO(iemThreadedFunc_BltIn_CheckTimersAndIrq);
 IEM_DECL_IEMTHREADEDFUNC_PROTO(iemThreadedFunc_BltIn_CheckMode);
 IEM_DECL_IEMTHREADEDFUNC_PROTO(iemThreadedFunc_BltIn_CheckHwInstrBps);
@@ -6806 +6856 @@
 IEM_DECL_IEMTHREADEDFUNC_PROTO(iemThreadedFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb);
 IEM_DECL_IEMTHREADEDFUNC_PROTO(iemThreadedFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim);
+
+IEM_DECL_IEMTHREADEDFUNC_PROTO(iemThreadedFunc_BltIn_Jump);
 
 bool iemThreadedCompileEmitIrqCheckBefore(PVMCPUCC pVCpu, PIEMTB pTb);

trunk/src/VBox/VMM/include/IEMN8veRecompiler.h

@@ -491 +491 @@
     kIemNativeLabelType_LastTbExit        = kIemNativeLabelType_Return,
 
+    /** Loop-jump target. */
+    kIemNativeLabelType_LoopJumpTarget,
+
     /*
      * Labels with data, potentially multiple instances per TB:
@@ -1449 +1452 @@
     /** Condition sequence number (for generating unique labels). */
     uint16_t                    uCondSeqNo;
-    /** Check IRQ seqeunce number (for generating unique labels). */
+    /** Check IRQ sequence number (for generating unique labels). */
     uint16_t                    uCheckIrqSeqNo;
     /** TLB load sequence number (for generating unique labels). */
@@ -1632 +1635 @@
                                                  uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);
 DECL_HIDDEN_THROW(void)     iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);
+DECLHIDDEN(uint32_t)        iemNativeLabelFind(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
+                                               uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
 DECL_HIDDEN_THROW(void)     iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
                                               IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);

trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h

@@ -503 +503 @@
 }
 
+/**
+ * Special variant of iemNativeEmitGprByVCpuDisp for accessing the VM structure.
+ */
+DECL_FORCE_INLINE(uint32_t)
+iemNativeEmitGprByVCpuSignedDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offVCpu)
+{
+    Assert(offVCpu < 0);
+    if (offVCpu < 128 && offVCpu >= -128)
+    {
+        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
+        pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
+    }
+    else
+    {
+        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
+        pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
+        pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
+        pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
+        pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
+    }
+    return off;
+}
+
 #elif defined(RT_ARCH_ARM64)
 
@@ -585 +608 @@
 }
 
+
+/**
+ * Special variant of iemNativeEmitGprByVCpuLdStEx for accessing the VM
+ * structure.
+ *
+ * @note Loads can use @a iGprReg for large offsets, stores requires a temporary
+ *       registers (@a iGprTmp).
+ * @note DON'T try this with prefetch.
+ */
+DECL_FORCE_INLINE_THROW(uint32_t)
+iemNativeEmitGprBySignedVCpuLdStEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offVCpu,
+                                   ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData, uint8_t iGprTmp = UINT8_MAX)
+{
+    Assert(offVCpu < 0);
+    Assert((uint32_t)-offVCpu < RT_BIT_32(28)); /* we should be way out of range for problematic sign extending issues. */
+    Assert(!((uint32_t)-offVCpu & (cbData - 1)));
+
+   /*
+     * For negative offsets we need to use put the displacement in a register
+     * as the two variants with signed immediates will either post or pre
+     * increment the base address register.
+     */
+    if (!ARMV8A64INSTRLDSTTYPE_IS_STORE(enmOperation) || iGprTmp != UINT8_MAX)
+    {
+        uint8_t const idxIndexReg = !ARMV8A64INSTRLDSTTYPE_IS_STORE(enmOperation) ? iGprReg : IEMNATIVE_REG_FIXED_TMP0;
+        off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, idxIndexReg, offVCpu / (int32_t)cbData);
+        pCodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, idxIndexReg,
+                                                    kArmv8A64InstrLdStExtend_Sxtw, cbData > 1 /*fShifted*/);
+    }
+    else
+# ifdef IEM_WITH_THROW_CATCH
+        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
+# else
+        AssertReleaseFailedStmt(off = UINT32_MAX);
+# endif
+
+    return off;
+}
+
+/**
+ * Special variant of iemNativeEmitGprByVCpuLdSt for accessing the VM structure.
+ */
+DECL_FORCE_INLINE_THROW(uint32_t)
+iemNativeEmitGprBySignedVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
+                                 int32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
+{
+    off = iemNativeEmitGprBySignedVCpuLdStEx(iemNativeInstrBufEnsure(pReNative, off, 2 + 1), off, iGprReg,
+                                             offVCpu, enmOperation, cbData, IEMNATIVE_REG_FIXED_TMP0);
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+}
+
 #endif /* RT_ARCH_ARM64 */
 
@@ -601 +676 @@
         pCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
     pCodeBuf[off++] = 0x8b;
-    off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off,iGpr, offVCpu);
+    off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, iGpr, offVCpu);
 
 #elif defined(RT_ARCH_ARM64)
@@ -773 +848 @@
 #endif
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+}
+
+
+/**
+ * Emits a store of a GPR value to a 32-bit VCpu field.
+ *
+ * @note Limited range on ARM64.
+ */
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitStoreGprToVCpuU32Ex(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
+{
+#ifdef RT_ARCH_AMD64
+    /* mov mem32, reg32 */
+    if (iGpr >= 8)
+        pCodeBuf[off++] = X86_OP_REX_R;
+    pCodeBuf[off++] = 0x89;
+    off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, iGpr, offVCpu);
+
+#elif defined(RT_ARCH_ARM64)
+    off = iemNativeEmitGprByVCpuLdStEx(pCodeBuf, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
+
+#else
+# error "port me"
+#endif
     return off;
 }
@@ -5436 +5536 @@
 {
 #if defined(RT_ARCH_AMD64)
-    /* and Ev, imm */
+    /* xor Ev, imm */
     if (iGprDst >= 8)
         pCodeBuf[off++] = X86_OP_REX_B;
@@ -6130 +6230 @@
  */
 DECL_INLINE_THROW(uint32_t)
+iemNativeEmitCmpGprWithImmEx(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGprLeft,
+                             uint64_t uImm, uint8_t idxTmpReg = UINT8_MAX)
+{
+#ifdef RT_ARCH_AMD64
+    if ((int8_t)uImm == (int64_t)uImm)
+    {
+        /* cmp Ev, Ib */
+        pCodeBuf[off++] = X86_OP_REX_W | (iGprLeft >= 8 ? X86_OP_REX_B : 0);
+        pCodeBuf[off++] = 0x83;
+        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
+        pCodeBuf[off++] = (uint8_t)uImm;
+        return off;
+    }
+    if ((int32_t)uImm == (int64_t)uImm)
+    {
+        /* cmp Ev, imm */
+        pCodeBuf[off++] = X86_OP_REX_W | (iGprLeft >= 8 ? X86_OP_REX_B : 0);
+        pCodeBuf[off++] = 0x81;
+        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);
+        pCodeBuf[off++] = RT_BYTE1(uImm);
+        pCodeBuf[off++] = RT_BYTE2(uImm);
+        pCodeBuf[off++] = RT_BYTE3(uImm);
+        pCodeBuf[off++] = RT_BYTE4(uImm);
+        return off;
+    }
+
+#elif defined(RT_ARCH_ARM64)
+    if (uImm < _4K)
+    {
+        pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,
+                                                      true /*64Bit*/, true /*fSetFlags*/);
+        return off;
+    }
+    if ((uImm & ~(uint64_t)0xfff000) == 0)
+    {
+        pCodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm >> 12,
+                                                      true /*64Bit*/, true /*fSetFlags*/, true /*fShift12*/);
+        return off;
+    }
+
+#else
+# error "Port me!"
+#endif
+
+    if (idxTmpReg != UINT8_MAX)
+    {
+        /* Use temporary register for the immediate. */
+        off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, idxTmpReg, uImm);
+        off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, iGprLeft, idxTmpReg);
+    }
+    else
+# ifdef IEM_WITH_THROW_CATCH
+        AssertFailedStmt(IEMNATIVE_DO_LONGJMP(NULL, VERR_IEM_IPE_9));
+# else
+        AssertReleaseFailedStmt(off = UINT32_MAX);
+# endif
+
+    return off;
+}
+
+
+/**
+ * Emits a compare of a 64-bit GPR with a constant value, settings status
+ * flags/whatever for use with conditional instruction.
+ */
+DECL_INLINE_THROW(uint32_t)
 iemNativeEmitCmpGprWithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint64_t uImm)
 {
 #ifdef RT_ARCH_AMD64
-    if (uImm <= UINT32_C(0xff))
+    if ((int8_t)uImm == (int64_t)uImm)
     {
         /* cmp Ev, Ib */
@@ -6142 +6308 @@
         pbCodeBuf[off++] = (uint8_t)uImm;
     }
-    else if ((int64_t)uImm == (int32_t)uImm)
+    else if ((int32_t)uImm == (int64_t)uImm)
     {
         /* cmp Ev, imm */
@@ -6974 +7140 @@
 
 #elif defined(RT_ARCH_ARM64)
+    int32_t const    offDisp     = offTarget - offFixup;
     uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;
     if ((pu32CodeBuf[offFixup] & UINT32_C(0xff000000)) == UINT32_C(0x54000000))
     {
         /* B.COND + BC.COND */
-        int32_t const offDisp = offTarget - offFixup;
         Assert(offDisp >= -262144 && offDisp < 262144);
         pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xff00001f))
                               | (((uint32_t)offDisp    & UINT32_C(0x0007ffff)) << 5);
     }
-    else
+    else if ((pu32CodeBuf[offFixup] & UINT32_C(0xfc000000)) == UINT32_C(0x14000000))
     {
         /* B imm26 */
-        Assert((pu32CodeBuf[offFixup] & UINT32_C(0xfc000000)) == UINT32_C(0x14000000));
-        int32_t const offDisp = offTarget - offFixup;
         Assert(offDisp >= -33554432 && offDisp < 33554432);
         pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xfc000000))
                               | ((uint32_t)offDisp     & UINT32_C(0x03ffffff));
+    }
+    else
+    {
+        /* CBZ / CBNZ reg, imm19 */
+        Assert((pu32CodeBuf[offFixup] & UINT32_C(0x7e000000)) == UINT32_C(0x34000000));
+        Assert(offDisp >= -1048576 && offDisp < 1048576);
+        pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup]    & UINT32_C(0xff00001f))
+                              | (((uint32_t)offDisp << 5) & UINT32_C(0x00ffffe0));
+
     }
 
@@ -8451 +8624 @@
 
 
+/**
+ * Emits code that exits the current TB with @a enmExitReason if @a iGprSrc is zero.
+ *
+ * The operand size is given by @a f64Bit.
+ */
+DECL_FORCE_INLINE_THROW(uint32_t)
+iemNativeEmitTestIfGprIsZeroAndTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
+                                        uint8_t iGprSrc, bool f64Bit, IEMNATIVELABELTYPE enmExitReason)
+{
+    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
+#if defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) && defined(RT_ARCH_AMD64)
+    /* test reg32,reg32  / test reg64,reg64 */
+    if (f64Bit)
+        pCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_R | X86_OP_REX_B);
+    else if (iGprSrc >= 8)
+        pCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
+    pCodeBuf[off++] = 0x85;
+    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);
+
+    /* jnz idxLabel  */
+    return iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, enmExitReason, kIemNativeInstrCond_e);
+
+#else
+    /* ARM64 doesn't have the necessary jump range, so we jump via local label
+       just like when we keep everything local. */
+    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
+    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
+                                                                f64Bit, false /*fJmpIfNotZero*/, idxLabel);
+#endif
+}
+
+
+/**
+ * Emits code to exit the current TB with the given reason @a enmExitReason if @a iGprSrc is zero.
+ *
+ * The operand size is given by @a f64Bit.
+ */
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTestIfGprIsZeroAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
+                                      uint8_t iGprSrc, bool f64Bit, IEMNATIVELABELTYPE enmExitReason)
+{
+#if defined(IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE) && defined(RT_ARCH_AMD64)
+    off = iemNativeEmitTestIfGprIsZeroAndTbExitEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 3 + 6),
+                                                  off, iGprSrc, f64Bit, enmExitReason);
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+#else
+    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
+    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, false /*fJmpIfNotZero*/, idxLabel);
+#endif
+}
+
+
 #ifdef IEMNATIVE_WITH_SIMD_REG_ALLOCATOR
 /*********************************************************************************************************************************