Changeset 106180 in vbox for trunk/src/VBox/VMM/include

Timestamp:

Sep 30, 2024 1:51:48 PM (5 months ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

164964

Message:

VMM/IEM: Refactoring TbExits and other early eflags calculation postponement changes. bugref:10720

Location:

trunk/src/VBox/VMM/include

Files:

• IEMN8veRecompiler.h (modified) (14 diffs)
• IEMN8veRecompilerEmit.h (modified) (19 diffs)

trunk/src/VBox/VMM/include/IEMN8veRecompiler.h

@@ -80 +80 @@
 # if 0 || defined(DOXYGEN_RUNNING)
 #  define IEMNATIVE_WITH_EFLAGS_POSTPONING
+# endif
+#endif
+#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+# ifndef IEMNATIVE_WITH_EFLAGS_SKIPPING
+#  error "IEMNATIVE_WITH_EFLAGS_POSTPONING requires IEMNATIVE_WITH_EFLAGS_SKIPPING at present"
 # endif
 #endif
@@ -314 +319 @@
  * Mask of registers the callee will not save and may trash. */
 #ifdef RT_ARCH_AMD64
-# define IEMNATIVE_CALL_RET_GREG             X86_GREG_xAX
+# define IEMNATIVE_CALL_RET_GREG            X86_GREG_xAX
 
 # ifdef RT_OS_WINDOWS
@@ -435 +440 @@
 # define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK    IEMNATIVE_CALL_VOLATILE_GREG_MASK
 #endif
+
+/** @def IEMNATIVE_CALL_NONVOLATILE_GREG_MASK
+ * The allocatable non-volatile general purpose register set.  */
+#define IEMNATIVE_CALL_NONVOLATILE_GREG_MASK \
+    (~IEMNATIVE_CALL_VOLATILE_GREG_MASK & ~IEMNATIVE_REG_FIXED_MASK & IEMNATIVE_HST_GREG_MASK)
 /** @} */
 
@@ -480 +490 @@
 {
     kIemNativeLabelType_Invalid = 0,
-    /*
-     * Labels w/o data, only once instance per TB - aka exit reasons.
+    /** @name Exit reasons - Labels w/o data, only once instance per TB.
      *
-     * Note! Jumps to these requires instructions that are capable of spanning
-     *       the max TB length.
+     * The labels requiring register inputs are documented.
+     *
+     * @note Jumps to these requires instructions that are capable of spanning the
+     *       max TB length.
+     * @{
      */
     /* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
@@ -496 +508 @@
     kIemNativeLabelType_RaiseMf,                /**< Raise (throw) X86_XCPT_MF (10h). */
     kIemNativeLabelType_RaiseXf,                /**< Raise (throw) X86_XCPT_XF (13h). */
-    kIemNativeLabelType_ObsoleteTb,
-    kIemNativeLabelType_NeedCsLimChecking,
-    kIemNativeLabelType_CheckBranchMiss,
+    kIemNativeLabelType_ObsoleteTb,             /**< Calls iemNativeHlpObsoleteTb (no inputs). */
+    kIemNativeLabelType_NeedCsLimChecking,      /**< Calls iemNativeHlpNeedCsLimChecking (no inputs). */
+    kIemNativeLabelType_CheckBranchMiss,        /**< Calls iemNativeHlpCheckBranchMiss (no inputs). */
     kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
-    /* Manually defined labels. */
+
+    /* Manually defined labels: */
+    /**< Returns with VINF_SUCCESS, no inputs. */
+    kIemNativeLabelType_ReturnSuccess,
+    /** Returns with VINF_IEM_REEXEC_FINISH_WITH_FLAGS, no inputs. */
+    kIemNativeLabelType_ReturnWithFlags,
+    /** Returns with VINF_IEM_REEXEC_BREAK, no inputs. */
     kIemNativeLabelType_ReturnBreak,
+    /** Returns with VINF_IEM_REEXEC_BREAK_FF, no inputs. */
     kIemNativeLabelType_ReturnBreakFF,
+    /** The last TB exit label that doesn't have any input registers. */
+    kIemNativeLabelType_LastTbExitWithoutInputs = kIemNativeLabelType_ReturnBreakFF,
+
+    /** Argument registers 1, 2 & 3 are set up.  */
     kIemNativeLabelType_ReturnBreakViaLookup,
+    /** Argument registers 1, 2 & 3 are set up.  */
     kIemNativeLabelType_ReturnBreakViaLookupWithIrq,
+    /** Argument registers 1 & 2 are set up.  */
     kIemNativeLabelType_ReturnBreakViaLookupWithTlb,
+    /** Argument registers 1 & 2 are set up.  */
     kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq,
-    kIemNativeLabelType_ReturnWithFlags,
+    /** Return register holds the RC and the instruction number is in CL/RCX
+     * on amd64 and the 2rd argument register elsewhere. */
     kIemNativeLabelType_NonZeroRetOrPassUp,
-    kIemNativeLabelType_ReturnSuccess,          /**< Sets eax/w0 to zero and returns. */
+
     /** The last fixup for branches that can span almost the whole TB length.
      * @note Whether kIemNativeLabelType_Return needs to be one of these is
      *       a bit questionable, since nobody jumps to it except other tail code. */
-    kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_ReturnSuccess,
+    kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_NonZeroRetOrPassUp,
     /** The last fixup for branches that exits the TB. */
-    kIemNativeLabelType_LastTbExit        = kIemNativeLabelType_ReturnSuccess,
+    kIemNativeLabelType_LastTbExit        = kIemNativeLabelType_NonZeroRetOrPassUp,
+    /** @} */
 
     /** Loop-jump target. */
@@ -538 +566 @@
 #define IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmLabel) \
     ((a_enmLabel) <= kIemNativeLabelType_LastTbExit && (a_enmLabel) > kIemNativeLabelType_Invalid)
+
+#define IEMNATIVELABELTYPE_IS_EXIT_WITHOUT_INPUTS(a_enmLabel) \
+    ((a_enmLabel) <= kIemNativeLabelType_LastTbExitWithoutInputs && (a_enmLabel) > kIemNativeLabelType_Invalid)
+
+/**
+ * Get the mask of input registers for an TB exit label.
+ * This will return zero for any non-exit lable.
+ */
+#ifdef RT_ARCH_AMD64
+# define IEMNATIVELABELTYPE_GET_INPUT_REG_MASK(a_enmLabel) \
+    (     (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookup \
+       || (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithIrq \
+     ? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) | RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
+     :    (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlb \
+       || (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq \
+     ? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
+     : (a_enmLabel) == kIemNativeLabelType_NonZeroRetOrPassUp \
+     ? RT_BIT_32(IEMNATIVE_CALL_RET_GREG)  | RT_BIT_32(X86_GREG_xCX) /* <-- the difference */ \
+     : 0)
+# else
+# define IEMNATIVELABELTYPE_GET_INPUT_REG_MASK(a_enmLabel) \
+    (     (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookup \
+       || (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithIrq \
+     ? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) | RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
+     :    (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlb \
+       || (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq \
+     ? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
+     : (a_enmLabel) == kIemNativeLabelType_NonZeroRetOrPassUp \
+     ? RT_BIT_32(IEMNATIVE_CALL_RET_GREG)  | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
+     : 0)
+#endif
 
 
@@ -660 +719 @@
 #define IEMLIVENESSBIT_IDX_EFL_SF       ((unsigned)kIemNativeGstReg_EFlags + 5)
 #define IEMLIVENESSBIT_IDX_EFL_OF       ((unsigned)kIemNativeGstReg_EFlags + 6)
+#define IEMLIVENESSBIT_IDX_EFL_COUNT    7
 
 
@@ -819 +879 @@
 # define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
     ( (((a_pCurEntry)->Bit0.bm64 | (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
+
+/***
+ * Construct a mask of what will be clobbered and never used.
+ *
+ * This is mainly used with IEMLIVENESSBIT_STATUS_EFL_MASK to avoid
+ * unnecessary EFLAGS calculations.
+ *
+ * @param a_pCurEntry  The current liveness entry.
+ * @note  Used by actual code.
+ */
+# define IEMLIVENESS_STATE_GET_WILL_BE_CLOBBERED_SET(a_pCurEntry) \
+    ( ~((a_pCurEntry)->Bit0.bm64 | (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_MASK )
 
 /** Construct a mask of the guest registers in the UNUSED and XCPT_OR_CALL
@@ -864 +936 @@
  * @note  Used by actual code. */
 # define IEMLIVENESS_STATE_GET_CAN_BE_FREED_SET(a_pCurEntry) \
-            (  ~(a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
-             & ~(a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 \
-             & IEMLIVENESSBIT_MASK )
-
+    (  ~(a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
+     & ~(a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 \
+     & IEMLIVENESSBIT_MASK )
+
+/***
+ * Construct a mask of what will be clobbered and never used.
+ *
+ * This is mainly used with IEMLIVENESSBIT_STATUS_EFL_MASK to avoid
+ * unnecessary EFLAGS calculations.
+ *
+ * @param a_pCurEntry  The current liveness entry.
+ * @note  Used by actual code.
+ */
+# define IEMLIVENESS_STATE_GET_WILL_BE_CLOBBERED_SET(a_pCurEntry) \
+    (  (a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 \
+     & ~(  (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \
+         | (a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
+         | (a_pCurEntry)->aBits[IEMLIVENESS_BIT_CALL].bm64) )
+
+/**
+ * Construct a mask of what (EFLAGS) which can be postponed.
+ *
+ * The postponement is for the avoiding EFLAGS status bits calculations in the
+ * primary code stream whenever possible, and instead only do these in the TLB
+ * load and TB exit code paths which shouldn't be traveled quite as often.
+ * A requirement, though, is that the status bits will be clobbered later in the
+ * TB.
+ *
+ * User need to apply IEMLIVENESSBIT_STATUS_EFL_MASK if appropriate/necessary.
+ *
+ * @param a_pCurEntry  The current liveness entry.
+ * @note  Used by actual code.
+ */
+# define IEMLIVENESS_STATE_GET_CAN_BE_POSTPONED_SET(a_pCurEntry) \
+    (  (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \
+     & (a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 \
+     & ~(  (a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
+         | (a_pCurEntry)->aBits[IEMLIVENESS_BIT_CALL].bm64) )
 
 #endif /* IEMLIVENESS_EXTENDED_LAYOUT */
@@ -1039 +1145 @@
 #ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
 # define IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(a_pReNative, a_fEflNeeded) \
-    AssertMsg(!((a_pReNative)->fPostponingEFlags & (a_fEflNeeded)), \
-              ("%#x & %#x -> %#x\n", (a_pReNative)->fPostponingEFlags, \
-               a_fEflNeeded, (a_pReNative)->fPostponingEFlags & (a_fEflNeeded) ))
+    AssertMsg(!((a_pReNative)->PostponedEfl.fEFlags & (a_fEflNeeded)), \
+              ("%#x & %#x -> %#x\n", (a_pReNative)->PostponedEfl.fEFlags, \
+               a_fEflNeeded, (a_pReNative)->PostponedEfl.fEFlags & (a_fEflNeeded) ))
 #else
 # define IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(a_pReNative, a_fEflNeeded) ((void)0)
@@ -1052 +1158 @@
 #if defined(IEMNATIVE_WITH_EFLAGS_SKIPPING) && defined(IEMNATIVE_WITH_EFLAGS_POSTPONING)
 # define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) \
-    AssertMsg(!(((a_pReNative)->fSkippingEFlags | (a_pReNative)->fPostponingEFlags) & (a_fEflNeeded)), \
-              ("(%#x | %#x) & %#x -> %#x\n", (a_pReNative)->fSkippingEFlags, (a_pReNative)->fPostponingEFlags, \
-               a_fEflNeeded, ((a_pReNative)->fSkippingEFlags | (a_pReNative)->fPostponingEFlags) & (a_fEflNeeded) ))
+    AssertMsg(!(((a_pReNative)->fSkippingEFlags | (a_pReNative)->PostponedEfl.fEFlags) & (a_fEflNeeded)), \
+              ("(%#x | %#x) & %#x -> %#x\n", (a_pReNative)->fSkippingEFlags, (a_pReNative)->PostponedEfl.fEFlags, \
+               a_fEflNeeded, ((a_pReNative)->fSkippingEFlags | (a_pReNative)->PostponedEfl.fEFlags) & (a_fEflNeeded) ))
 #elif defined(IEMNATIVE_WITH_EFLAGS_SKIPPING)
 # define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) \
@@ -1077 +1183 @@
 #else
 # define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
+#endif
+
+
+/** @def IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS
+ * Number of extra instructions to allocate for each TB exit to account for
+ * postponed EFLAGS calculations.
+ */
+#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+# ifdef RT_ARCH_AMD64
+#  define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS   32
+# elif defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING)
+#  define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS   32
+# else
+#  error "port me"
+# endif
+#else
+# define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS    0
+#endif
+
+/** @def IEMNATIVE_CLEAR_POSTPONED_EFLAGS
+ * Helper macro function for calling iemNativeClearPostponedEFlags() when
+ * IEMNATIVE_WITH_EFLAGS_POSTPONING is enabled.
+ */
+#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+# define IEMNATIVE_CLEAR_POSTPONED_EFLAGS(a_pReNative, a_fEflClobbered) iemNativeClearPostponedEFlags<a_fEflClobbered>(a_pReNative)
+#else
+# define IEMNATIVE_CLEAR_POSTPONED_EFLAGS(a_pReNative, a_fEflClobbered) ((void)0)
 #endif
 
@@ -1533 +1666 @@
 
 
+#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+typedef enum IEMNATIVE_POSTPONED_EFL_OP_T : uint8_t
+{
+    kIemNativePostponedEflOp_Invalid = 0,
+    /** Logical operation.
+     * Operands: result register.
+     * @note This clears OF, CF and (undefined) AF, thus no need for inputs. */
+    kIemNativePostponedEflOp_Logical,
+    kIemNativePostponedEflOp_End
+} IEMNATIVE_POSTPONED_EFL_OP_T;
+#endif /* IEMNATIVE_WITH_EFLAGS_POSTPONING */
+
 /**
  * Conditional stack entry.
@@ -1670 +1815 @@
 #endif
 #ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
-    uint32_t                    fPostponingEFlags;
+    struct
+    {
+        /** EFLAGS status bits that we're currently postponing the calculcation of. */
+        uint32_t                        fEFlags;
+        /** The postponed EFLAGS status bits calculation operation. */
+        IEMNATIVE_POSTPONED_EFL_OP_T    enmOp;
+        /** The bit-width of the postponed EFLAGS calculation. */
+        uint8_t                         cOpBits;
+        /** Host register holding result or first source for the delayed operation,
+         *  UINT8_MAX if not in use. */
+        uint8_t                         idxReg1;
+        /** Host register holding second source for the delayed operation,
+         *  UINT8_MAX if not in use. */
+        uint8_t                         idxReg2;
+    } PostponedEfl;
 #endif
 

trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h

@@ -234 +234 @@
         pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32 & UINT32_C(0xffff), 0, false /*f64Bit*/);
         pCodeBuf[off++] = Armv8A64MkInstrMovK(iGpr, uImm32 >> 16,              1, false /*f64Bit*/);
+    }
+
+#else
+# error "port me"
+#endif
+    return off;
+}
+
+
+/**
+ * Variant of iemNativeEmitLoadGpr32Imm where the caller ensures sufficent
+ * buffer space.
+ *
+ * Max buffer consumption:
+ *      - AMD64: 6 instruction bytes.
+ *      - ARM64: 2 instruction words (8 bytes).
+ *
+ * @note The top 32 bits will be cleared.
+ */
+template<uint32_t const a_uImm32>
+DECL_FORCE_INLINE(uint32_t) iemNativeEmitLoadGpr32ImmExT(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr)
+{
+#ifdef RT_ARCH_AMD64
+    if (a_uImm32 == 0)
+    {
+        /* xor gpr, gpr */
+        if (iGpr >= 8)
+            pCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
+        pCodeBuf[off++] = 0x33;
+        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
+    }
+    else
+    {
+        /* mov gpr, imm32 */
+        if (iGpr >= 8)
+            pCodeBuf[off++] = X86_OP_REX_B;
+        pCodeBuf[off++] = 0xb8 + (iGpr & 7);
+        pCodeBuf[off++] = RT_BYTE1(a_uImm32);
+        pCodeBuf[off++] = RT_BYTE2(a_uImm32);
+        pCodeBuf[off++] = RT_BYTE3(a_uImm32);
+        pCodeBuf[off++] = RT_BYTE4(a_uImm32);
+    }
+
+#elif defined(RT_ARCH_ARM64)
+    if RT_CONSTEXPR((a_uImm32 >> 16) == 0)
+        /* movz gpr, imm16 */
+        pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, a_uImm32,                    0, false /*f64Bit*/);
+    else if RT_CONSTEXPR((a_uImm32 & UINT32_C(0xffff)) == 0)
+        /* movz gpr, imm16, lsl #16 */
+        pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, a_uImm32 >> 16,              1, false /*f64Bit*/);
+    else if RT_CONSTEXPR((a_uImm32 & UINT32_C(0xffff)) == UINT32_C(0xffff))
+        /* movn gpr, imm16, lsl #16 */
+        pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~a_uImm32 >> 16,             1, false /*f64Bit*/);
+    else if RT_CONSTEXPR((a_uImm32 >> 16) == UINT32_C(0xffff))
+        /* movn gpr, imm16 */
+        pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~a_uImm32,                   0, false /*f64Bit*/);
+    else
+    {
+        pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, a_uImm32 & UINT32_C(0xffff), 0, false /*f64Bit*/);
+        pCodeBuf[off++] = Armv8A64MkInstrMovK(iGpr, a_uImm32 >> 16,              1, false /*f64Bit*/);
     }
 
@@ -8156 +8216 @@
  * Emits a call to a 64-bit address.
  */
+template<bool const a_fSkipEflChecks = false>
 DECL_INLINE_THROW(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
 {
+    if RT_CONSTEXPR(!a_fSkipEflChecks)
+    {
+        IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS);
+        IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(  pReNative, X86_EFL_STATUS_BITS);
+    }
+
 #ifdef RT_ARCH_AMD64
     off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
@@ -8281 +8348 @@
 *********************************************************************************************************************************/
 
+#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+/* IEMAllN8veEmit-x86.h: */
+template<uint32_t const a_bmInputRegs>
+DECL_FORCE_INLINE_THROW(uint32_t)
+iemNativeDoPostponedEFlagsAtTbExitEx(PIEMRECOMPILERSTATE pReNative, uint32_t off, PIEMNATIVEINSTR pCodeBuf);
+
+template<uint32_t const a_bmInputRegs>
+DECL_FORCE_INLINE_THROW(uint32_t)
+iemNativeDoPostponedEFlagsAtTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off);
+#endif
+
+
 /**
  * Helper for marking the current conditional branch as exiting the TB.
@@ -8298 +8377 @@
 
 /**
+ * Unconditionally exits the translation block via a branch instructions.
+ *
+ * @note In case a delayed EFLAGS calculation is pending, this may emit an
+ *       additional IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS instructions.
+ */
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fActuallyExitingTb = true, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t) iemNativeEmitTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off)
+{
+    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
+    AssertCompile(IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmExitReason));
+
+    if RT_CONSTEXPR(a_fActuallyExitingTb)
+        iemNativeMarkCurCondBranchAsExiting(pReNative);
+
+#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+    if RT_CONSTEXPR(a_fPostponedEfl)
+        off = iemNativeDoPostponedEFlagsAtTbExitEx<IEMNATIVELABELTYPE_GET_INPUT_REG_MASK(a_enmExitReason)>(pReNative, off,
+                                                                                                           pCodeBuf);
+#endif
+
+#ifdef RT_ARCH_AMD64
+    /* jmp rel32 */
+    pCodeBuf[off++] = 0xe9;
+    iemNativeAddTbExitFixup(pReNative, off, a_enmExitReason);
+    pCodeBuf[off++] = 0xfe;
+    pCodeBuf[off++] = 0xff;
+    pCodeBuf[off++] = 0xff;
+    pCodeBuf[off++] = 0xff;
+
+#elif defined(RT_ARCH_ARM64)
+    iemNativeAddTbExitFixup(pReNative, off, a_enmExitReason);
+    pCodeBuf[off++] = Armv8A64MkInstrB(-1);
+
+#else
+# error "Port me!"
+#endif
+    return off;
+}
+
+
+/**
+ * Unconditionally exits the translation block via a branch instructions.
+ *
+ * @note In case a delayed EFLAGS calculation is pending, this may emit an
+ *       additional IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS instructions.
+ */
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fActuallyExitingTb = true, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t) iemNativeEmitTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off)
+{
+    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
+    AssertCompile(IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmExitReason));
+
+    if RT_CONSTEXPR(a_fActuallyExitingTb)
+        iemNativeMarkCurCondBranchAsExiting(pReNative);
+
+#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+    if RT_CONSTEXPR(a_fPostponedEfl)
+        off = iemNativeDoPostponedEFlagsAtTbExit<IEMNATIVELABELTYPE_GET_INPUT_REG_MASK(a_enmExitReason)>(pReNative, off);
+#endif
+
+#ifdef RT_ARCH_AMD64
+    PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
+
+    /* jmp rel32 */
+    pCodeBuf[off++] = 0xe9;
+    iemNativeAddTbExitFixup(pReNative, off, a_enmExitReason);
+    pCodeBuf[off++] = 0xfe;
+    pCodeBuf[off++] = 0xff;
+    pCodeBuf[off++] = 0xff;
+    pCodeBuf[off++] = 0xff;
+
+#elif defined(RT_ARCH_ARM64)
+    PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+    iemNativeAddTbExitFixup(pReNative, off, a_enmExitReason);
+    pCodeBuf[off++] = Armv8A64MkInstrB(-1);
+
+#else
+# error "Port me!"
+#endif
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+}
+
+
+/**
  * Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
- */
+ *
+ * @note In case a delayed EFLAGS calculation is pending, this may emit an
+ *       additional IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS instructions.
+ */
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
 DECL_FORCE_INLINE_THROW(uint32_t)
-iemNativeEmitJccTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
-                         IEMNATIVELABELTYPE enmExitReason, IEMNATIVEINSTRCOND enmCond)
+iemNativeEmitTbExitJccEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off, IEMNATIVEINSTRCOND enmCond)
 {
     IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
-    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
-    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
+    AssertCompile(IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmExitReason));
+
+#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+    if RT_CONSTEXPR(a_fPostponedEfl)
+        if (pReNative->PostponedEfl.fEFlags)
+        {
+            /* Jcc l_NonPrimaryCodeStreamTarget */
+            uint32_t const offFixup1 = off;
+            off = iemNativeEmitJccToFixedEx(pCodeBuf, off, off + 1, enmCond);
+
+            /* JMP l_PrimaryCodeStreamResume */
+            uint32_t const offFixup2 = off;
+            off = iemNativeEmitJmpToFixedEx(pCodeBuf, off, off + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
+
+            /* l_NonPrimaryCodeStreamTarget: */
+            iemNativeFixupFixedJump(pReNative, offFixup1, off);
+            off = iemNativeEmitTbExitEx<a_enmExitReason, false /*a_fActuallyExitingTb*/, true>(pReNative, pCodeBuf, off);
+
+            /* l_PrimaryCodeStreamResume: */
+            iemNativeFixupFixedJump(pReNative, offFixup2, off);
+            IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+            return off;
+        }
+#endif
 
 #if defined(RT_ARCH_AMD64)
@@ -8312 +8501 @@
     pCodeBuf[off++] = 0x0f;
     pCodeBuf[off++] = (uint8_t)enmCond | 0x80;
-    iemNativeAddTbExitFixup(pReNative, off, enmExitReason);
+    iemNativeAddTbExitFixup(pReNative, off, a_enmExitReason);
     pCodeBuf[off++] = 0x00;
     pCodeBuf[off++] = 0x00;
@@ -8321 +8510 @@
     /* ARM64 doesn't have the necessary jump range, so we jump via local label
        just like when we keep everything local. */
-    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
+    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, a_enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabel, enmCond);
 #endif
@@ -8331 +8520 @@
  * Emits a Jcc rel32 / B.cc imm19 to the epilog.
  */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitJccTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmExitReason, IEMNATIVEINSTRCOND enmCond)
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t) iemNativeEmitTbExitJcc(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEINSTRCOND enmCond)
 {
     IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
-    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
-    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
-
-#ifdef RT_ARCH_AMD64
-    off = iemNativeEmitJccTbExitEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 6), off, enmExitReason, enmCond);
-#elif defined(RT_ARCH_ARM64)
-    off = iemNativeEmitJccTbExitEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 2), off, enmExitReason, enmCond);
+    AssertCompile(IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmExitReason));
+
+#ifdef RT_ARCH_AMD64
+    PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS + 5);
+#elif defined(RT_ARCH_ARM64)
+    PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS + 1);
 #else
 # error "Port me!"
 #endif
+    off = iemNativeEmitTbExitJccEx<a_enmExitReason, a_fPostponedEfl>(pReNative, pCodeBuf, off, enmCond);
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     return off;
@@ -8353 +8542 @@
  * Emits a JNZ/JNE rel32 / B.NE imm19 to the TB exit routine with the given reason.
  */
-DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmExitReason)
-{
-#ifdef RT_ARCH_AMD64
-    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kIemNativeInstrCond_ne);
-#elif defined(RT_ARCH_ARM64)
-    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kArmv8InstrCond_Ne);
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t) iemNativeEmitTbExitJnz(PIEMRECOMPILERSTATE pReNative, uint32_t off)
+{
+#ifdef RT_ARCH_AMD64
+    return iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kIemNativeInstrCond_ne);
+#elif defined(RT_ARCH_ARM64)
+    return iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kArmv8InstrCond_Ne);
 #else
 # error "Port me!"
@@ -8368 +8558 @@
  * Emits a JZ/JE rel32 / B.EQ imm19 to the TB exit routine with the given reason.
  */
-DECL_INLINE_THROW(uint32_t) iemNativeEmitJzTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmExitReason)
-{
-#ifdef RT_ARCH_AMD64
-    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kIemNativeInstrCond_e);
-#elif defined(RT_ARCH_ARM64)
-    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kArmv8InstrCond_Eq);
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t) iemNativeEmitTbExitJz(PIEMRECOMPILERSTATE pReNative, uint32_t off)
+{
+#ifdef RT_ARCH_AMD64
+    return iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kIemNativeInstrCond_e);
+#elif defined(RT_ARCH_ARM64)
+    return iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kArmv8InstrCond_Eq);
 #else
 # error "Port me!"
@@ -8383 +8574 @@
  * Emits a JA/JNBE rel32 / B.HI imm19 to the TB exit.
  */
-DECL_INLINE_THROW(uint32_t) iemNativeEmitJaTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmExitReason)
-{
-#ifdef RT_ARCH_AMD64
-    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kIemNativeInstrCond_nbe);
-#elif defined(RT_ARCH_ARM64)
-    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kArmv8InstrCond_Hi);
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t) iemNativeEmitTbExitJa(PIEMRECOMPILERSTATE pReNative, uint32_t off)
+{
+#ifdef RT_ARCH_AMD64
+    return iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kIemNativeInstrCond_nbe);
+#elif defined(RT_ARCH_ARM64)
+    return iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kArmv8InstrCond_Hi);
 #else
 # error "Port me!"
@@ -8398 +8590 @@
  * Emits a JL/JNGE rel32 / B.LT imm19 to the TB exit with the given reason.
  */
-DECL_INLINE_THROW(uint32_t) iemNativeEmitJlTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmExitReason)
-{
-#ifdef RT_ARCH_AMD64
-    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kIemNativeInstrCond_l);
-#elif defined(RT_ARCH_ARM64)
-    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kArmv8InstrCond_Lt);
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t) iemNativeEmitTbExitJl(PIEMRECOMPILERSTATE pReNative, uint32_t off)
+{
+#ifdef RT_ARCH_AMD64
+    return iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kIemNativeInstrCond_l);
+#elif defined(RT_ARCH_ARM64)
+    return iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kArmv8InstrCond_Lt);
 #else
 # error "Port me!"
@@ -8410 +8603 @@
 
 
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off, IEMNATIVELABELTYPE enmExitReason)
-{
-    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
-    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here. */
-    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
-
-    iemNativeMarkCurCondBranchAsExiting(pReNative);
-
-#ifdef RT_ARCH_AMD64
-    /* jmp rel32 */
-    pCodeBuf[off++] = 0xe9;
-    iemNativeAddTbExitFixup(pReNative, off, enmExitReason);
-    pCodeBuf[off++] = 0xfe;
-    pCodeBuf[off++] = 0xff;
-    pCodeBuf[off++] = 0xff;
-    pCodeBuf[off++] = 0xff;
-
-#elif defined(RT_ARCH_ARM64)
-    iemNativeAddTbExitFixup(pReNative, off, enmExitReason);
-    pCodeBuf[off++] = Armv8A64MkInstrB(-1);
-
-#else
-# error "Port me!"
-#endif
-    return off;
-}
-
-
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmExitReason,
-                    bool fActuallyExitingTb = true)
-{
-    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
-    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here. */
-    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
-
-    if (fActuallyExitingTb)
-        iemNativeMarkCurCondBranchAsExiting(pReNative);
-
-#ifdef RT_ARCH_AMD64
-    PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
-
-    /* jmp rel32 */
-    pCodeBuf[off++] = 0xe9;
-    iemNativeAddTbExitFixup(pReNative, off, enmExitReason);
-    pCodeBuf[off++] = 0xfe;
-    pCodeBuf[off++] = 0xff;
-    pCodeBuf[off++] = 0xff;
-    pCodeBuf[off++] = 0xff;
-
-#elif defined(RT_ARCH_ARM64)
-    PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
-    iemNativeAddTbExitFixup(pReNative, off, enmExitReason);
-    pCodeBuf[off++] = Armv8A64MkInstrB(-1);
-
-#else
-# error "Port me!"
-#endif
-    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
-    return off;
-}
-
-
-/**
- * Emits a jump to the TB exit with @a enmExitReason on the condition _any_ of the bits in @a fBits
- * are set in @a iGprSrc.
- */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTestAnyBitsInGprAndTbExitIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
-                                               uint8_t iGprSrc, uint64_t fBits, IEMNATIVELABELTYPE enmExitReason)
+/**
+ * Emits a jump to the TB exit with @a a_enmExitReason on the condition _any_ of
+ * the bits in @a fBits are set in @a iGprSrc.
+ */
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTbExitIfAnyBitsSetInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
 {
     Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
 
     off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
-    return iemNativeEmitJnzTbExit(pReNative, off, enmExitReason);
-}
-
-
+    return iemNativeEmitTbExitJnz<a_enmExitReason, a_fPostponedEfl>(pReNative, off);
+}
+
+
+#if 0 /* unused */
 /**
  * Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
  * are set in @a iGprSrc.
  */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTestAnyBitsInGprAndTbExitIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
-                                                uint8_t iGprSrc, uint64_t fBits, IEMNATIVELABELTYPE enmExitReason)
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTbExitIfNoBitsSetInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
 {
     Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
 
     off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
-    return iemNativeEmitJzTbExit(pReNative, off, enmExitReason);
-}
-
-
+    return iemNativeEmitJzTbExit<a_enmExitReason, a_fPostponedEfl>(pReNative, off);
+}
+#endif
+
+
+#if 0 /* unused */
 /**
  * Emits code that exits the TB with the given reason if @a iGprLeft and @a iGprRight
  * differs.
  */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTestIfGprNotEqualGprAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
-                                           uint8_t iGprLeft, uint8_t iGprRight, IEMNATIVELABELTYPE enmExitReason)
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTbExitIfGprNotEqualGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
 {
     off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);
-    off = iemNativeEmitJnzTbExit(pReNative, off, enmExitReason);
-    return off;
-}
+    off = iemNativeEmitJnzTbExit<a_enmExitReason, a_fPostponedEfl>(pReNative, off);
+    return off;
+}
+#endif
 
 
@@ -8522 +8655 @@
  * @a uImm.
  */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTestIfGpr32NotEqualImmAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
-                                             uint8_t iGprSrc, uint32_t uImm, IEMNATIVELABELTYPE enmExitReason)
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTbExitIfGpr32NotEqualImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint32_t uImm)
 {
     off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
-    off = iemNativeEmitJnzTbExit(pReNative, off, enmExitReason);
+    off = iemNativeEmitTbExitJnz<a_enmExitReason, a_fPostponedEfl>(pReNative, off);
     return off;
 }
@@ -8535 +8668 @@
  * Emits code that exits the current TB if @a iGprSrc differs from @a uImm.
  */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTestIfGprNotEqualImmAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
-                                           uint8_t iGprSrc, uint64_t uImm, IEMNATIVELABELTYPE enmExitReason)
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTbExitIfGprNotEqualImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t uImm)
 {
     off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);
-    off = iemNativeEmitJnzTbExit(pReNative, off, enmExitReason);
+    off = iemNativeEmitTbExitJnz<a_enmExitReason, a_fPostponedEfl>(pReNative, off);
     return off;
 }
@@ -8548 +8681 @@
  * Emits code that exits the current TB with the given reason if 32-bit @a iGprSrc equals @a uImm.
  */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTestIfGpr32EqualsImmAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
-                                           uint8_t iGprSrc, uint32_t uImm, IEMNATIVELABELTYPE enmExitReason)
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTbExitIfGpr32EqualsImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint32_t uImm)
 {
     off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
-    off = iemNativeEmitJzTbExit(pReNative, off, enmExitReason);
-    return off;
-}
-
-
-/**
- * Emits code to exit the current TB with the reason @a enmExitReason on the condition that bit @a iBitNo _is_ _set_ in
- * @a iGprSrc.
+    off = iemNativeEmitTbExitJz<a_enmExitReason, a_fPostponedEfl>(pReNative, off);
+    return off;
+}
+
+
+/**
+ * Emits code to exit the current TB with the reason @a a_enmExitReason on the
+ * condition that bit @a iBitNo _is_ _set_ in @a iGprSrc.
  *
  * @note On ARM64 the range is only +/-8191 instructions.
  */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTestBitInGprAndTbExitIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
-                                        uint8_t iGprSrc, uint8_t iBitNo, IEMNATIVELABELTYPE enmExitReason)
-{
-    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTbExitIfBitSetInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint8_t iBitNo)
+{
+    AssertCompile(IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmExitReason));
 
 #if defined(RT_ARCH_AMD64)
@@ -8581 +8714 @@
         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
         pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
-        off = iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kIemNativeInstrCond_ne);
+        IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+        off = iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kIemNativeInstrCond_ne);
     }
     else
@@ -8594 +8728 @@
         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
         pbCodeBuf[off++] = iBitNo;
-        off = iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kIemNativeInstrCond_c);
-    }
-    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
-    return off;
-
-#else
-    /* ARM64 doesn't have the necessary jump range, so we jump via local label
-       just like when we keep everything local. */
+        IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+        off = iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kIemNativeInstrCond_c);
+    }
+    return off;
+
+#elif defined(RT_ARCH_ARM64)
     IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
-    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
-    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
+    /** @todo Perhaps we should always apply the PostponedEfl code pattern here,
+     *        it's the same number of instructions as the TST + B.CC stuff? */
+# ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+    if RT_CONSTEXPR(a_fPostponedEfl)
+        if (pReNative->PostponedEfl.fEFlags)
+        {
+            PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off,
+                                                                     3 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
+            pCodeBuf[off++] = Armv8A64MkInstrTbnz(1 /*l_NonPrimaryCodeStreamTarget*/, iGprSrc, iBitNo);
+            uint32_t const offFixup = off;
+            pCodeBuf[off++] = Armv8A64MkInstrB(0 /*l_PrimaryCodeStreamResume*/);
+            /* l_NonPrimaryCodeStreamTarget: */
+            off = iemNativeEmitTbExitEx<a_enmExitReason, false /*a_fActuallyExitingTb*/, true>(pReNative, pCodeBuf, off);
+            /* l_PrimaryCodeStreamResume: */
+            iemNativeFixupFixedJump(pReNative, offFixup, off);
+            IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+            return off;
+        }
+# endif
+    /* ARM64 doesn't have the necessary range to reach the per-chunk code, so
+       we go via a local trampoline. */
+    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, a_enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /*fJmpIfSet*/);
-#endif
-}
-
-
-/**
- * Emits code that exits the current TB with @a enmExitReason if @a iGprSrc is not zero.
+#else
+# error "port me"
+#endif
+}
+
+
+/**
+ * Emits code that exits the current TB with @a a_enmExitReason if @a iGprSrc is
+ * not zero.
  *
  * The operand size is given by @a f64Bit.
  */
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
 DECL_FORCE_INLINE_THROW(uint32_t)
-iemNativeEmitTestIfGprIsNotZeroAndTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
-                                           uint8_t iGprSrc, bool f64Bit, IEMNATIVELABELTYPE enmExitReason)
-{
-    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
+iemNativeEmitTbExitIfGprIsNotZeroEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
+                                    uint8_t iGprSrc, bool f64Bit)
+{
+    AssertCompile(IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmExitReason));
 
 #if defined(RT_ARCH_AMD64)
@@ -8631 +8787 @@
 
     /* jnz idxLabel  */
-    return iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, enmExitReason, kIemNativeInstrCond_ne);
-
-#else
-    /* ARM64 doesn't have the necessary jump range, so we jump via local label
-       just like when we keep everything local. */
+    return iemNativeEmitTbExitJccEx<a_enmExitReason, a_fPostponedEfl>(pReNative, pCodeBuf, off, kIemNativeInstrCond_ne);
+
+#elif defined(RT_ARCH_ARM64)
     IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
-    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
-    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
+# ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+    if RT_CONSTEXPR(a_fPostponedEfl)
+        if (pReNative->PostponedEfl.fEFlags)
+        {
+            pCodeBuf[off++] = Armv8A64MkInstrCbnz(1 /*l_NonPrimaryCodeStreamTarget*/, iGprSrc, f64Bit);
+            uint32_t const offFixup = off;
+            pCodeBuf[off++] = Armv8A64MkInstrB(0 /*l_PrimaryCodeStreamResume*/);
+            /* l_NonPrimaryCodeStreamTarget: */
+            off = iemNativeEmitTbExitEx<a_enmExitReason, false /*a_fActuallyExitingTb*/, true>(pReNative, pCodeBuf, off);
+            /* l_PrimaryCodeStreamResume: */
+            iemNativeFixupFixedJump(pReNative, offFixup, off);
+            IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+            return off;
+        }
+# endif
+    /* ARM64 doesn't have the necessary range to reach the per-chunk code, so
+       we go via a local trampoline. */
+    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, a_enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
                                                                 f64Bit, true /*fJmpIfNotZero*/, idxLabel);
-#endif
-}
-
-
-/**
- * Emits code to exit the current TB with the given reason @a enmExitReason if @a iGprSrc is not zero.
+#else
+# error "port me"
+#endif
+}
+
+
+/**
+ * Emits code to exit the current TB with the given reason @a a_enmExitReason if
+ * @a iGprSrc is not zero.
  *
  * The operand size is given by @a f64Bit.
  */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTestIfGprIsNotZeroAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
-                                         uint8_t iGprSrc, bool f64Bit, IEMNATIVELABELTYPE enmExitReason)
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTbExitIfGprIsNotZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit)
 {
 #if defined(RT_ARCH_AMD64)
-    off = iemNativeEmitTestIfGprIsNotZeroAndTbExitEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 3 + 6),
-                                                     off, iGprSrc, f64Bit, enmExitReason);
-    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
-    return off;
-#else
-    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
-    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
-    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
-    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, true /*fJmpIfNotZero*/, idxLabel);
-#endif
-}
-
-
-/**
- * Emits code that exits the current TB with @a enmExitReason if @a iGprSrc is zero.
+    PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3 + 6 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
+
+#else
+    PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
+#endif
+    off = iemNativeEmitTbExitIfGprIsNotZeroEx<a_enmExitReason, a_fPostponedEfl>(pReNative, pCodeBuf, off, iGprSrc, f64Bit);
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+}
+
+
+/**
+ * Emits code that exits the current TB with @a a_enmExitReason if @a iGprSrc is
+ * zero.
  *
  * The operand size is given by @a f64Bit.
  */
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
 DECL_FORCE_INLINE_THROW(uint32_t)
-iemNativeEmitTestIfGprIsZeroAndTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
-                                        uint8_t iGprSrc, bool f64Bit, IEMNATIVELABELTYPE enmExitReason)
-{
-    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
+iemNativeEmitTbExitIfGprIsZeroEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
+                                 uint8_t iGprSrc, bool f64Bit)
+{
+    AssertCompile(IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmExitReason));
+
 #if defined(RT_ARCH_AMD64)
     /* test reg32,reg32  / test reg64,reg64 */
@@ -8688 +8862 @@
 
     /* jnz idxLabel  */
-    return iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, enmExitReason, kIemNativeInstrCond_e);
-
-#else
-    /* ARM64 doesn't have the necessary jump range, so we jump via local label
-       just like when we keep everything local. */
+    return iemNativeEmitTbExitJccEx<a_enmExitReason, a_fPostponedEfl>(pReNative, pCodeBuf, off, kIemNativeInstrCond_e);
+
+#elif defined(RT_ARCH_ARM64)
     IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
-    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
-    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
+# ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+    if RT_CONSTEXPR(a_fPostponedEfl)
+        if (pReNative->PostponedEfl.fEFlags)
+        {
+            pCodeBuf[off++] = Armv8A64MkInstrCbz(1 /*l_NonPrimaryCodeStreamTarget*/, iGprSrc, f64Bit);
+            uint32_t const offFixup = off;
+            pCodeBuf[off++] = Armv8A64MkInstrB(0 /*l_PrimaryCodeStreamResume*/);
+            /* l_NonPrimaryCodeStreamTarget: */
+            off = iemNativeEmitTbExitEx<a_enmExitReason, false /*a_fActuallyExitingTb*/, true>(pReNative, pCodeBuf, off);
+            /* l_PrimaryCodeStreamResume: */
+            iemNativeFixupFixedJump(pReNative, offFixup, off);
+            IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+            return off;
+        }
+# endif
+    /* ARM64 doesn't have the necessary range to reach the per-chunk code, so
+       we go via a local trampoline. */
+    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, a_enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
                                                                 f64Bit, false /*fJmpIfNotZero*/, idxLabel);
-#endif
-}
-
-
-/**
- * Emits code to exit the current TB with the given reason @a enmExitReason if @a iGprSrc is zero.
+#else
+# error "port me"
+#endif
+}
+
+
+/**
+ * Emits code to exit the current TB with the given reason @a a_enmExitReason if @a iGprSrc is zero.
  *
  * The operand size is given by @a f64Bit.
  */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTestIfGprIsZeroAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
-                                      uint8_t iGprSrc, bool f64Bit, IEMNATIVELABELTYPE enmExitReason)
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTbExitIfGprIsZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit)
 {
 #if defined(RT_ARCH_AMD64)
-    off = iemNativeEmitTestIfGprIsZeroAndTbExitEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 3 + 6),
-                                                  off, iGprSrc, f64Bit, enmExitReason);
-    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
-    return off;
-#else
-    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
-    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
-    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
-    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, false /*fJmpIfNotZero*/, idxLabel);
-#endif
+    PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3 + 6 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
+
+#else
+    PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
+#endif
+    off = iemNativeEmitTbExitIfGprIsZeroEx<a_enmExitReason, a_fPostponedEfl>(pReNative, pCodeBuf, off, iGprSrc, f64Bit);
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
 }