Changeset 106180 in vbox

Timestamp:

Sep 30, 2024 1:51:48 PM (5 months ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

164964

Message:

VMM/IEM: Refactoring TbExits and other early eflags calculation postponement changes. bugref:10720

Location:

trunk/src/VBox/VMM

Files:

• VMMAll/IEMAllInstPython.py (modified) (1 diff)
• VMMAll/IEMAllInstVexMap2.cpp.h (modified) (2 diffs)
• VMMAll/IEMAllN8veRecompBltIn.cpp (modified) (22 diffs)
• VMMAll/IEMAllN8veRecompFuncs.h (modified) (32 diffs)
• VMMAll/IEMAllN8veRecompiler.cpp (modified) (9 diffs)
• VMMAll/target-x86/IEMAllN8veEmit-x86.h (modified) (15 diffs)
• VMMR3/IEMR3.cpp (modified) (1 diff)
• include/IEMN8veRecompiler.h (modified) (14 diffs)
• include/IEMN8veRecompilerEmit.h (modified) (19 diffs)

trunk/src/VBox/VMM/VMMAll/IEMAllInstPython.py

@@ -2944 +2944 @@
         if sRet:
             asRet.append(sRet);
+
+        ## @todo Check that IEM_MC_REF_EFLAGS isn't used before memory fetches and does
+        # not have any stores or conditional raises afterwards.
 
         return asRet;

trunk/src/VBox/VMM/VMMAll/IEMAllInstVexMap2.cpp.h

@@ -313 +313 @@
             IEM_MC_REF_XREG_U128_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); \
             IEM_MC_REF_EFLAGS(pEFlags); \
-                IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_ ## a_Instr ## _u128, \
-                                                                           iemAImpl_ ## a_Instr ## _u128_fallback), \
-                                         puSrc1, puSrc2, pEFlags); \
+            IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_ ## a_Instr ## _u128, \
+                                                                       iemAImpl_ ## a_Instr ## _u128_fallback), \
+                                     puSrc1, puSrc2, pEFlags); \
             IEM_MC_ADVANCE_RIP_AND_FINISH(); \
             IEM_MC_END(); \
@@ -362 +362 @@
             IEM_MC_REF_XREG_U128_CONST(puSrc1,  IEM_GET_MODRM_REG(pVCpu, bRm)); \
             IEM_MC_REF_EFLAGS(pEFlags); \
-                IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_ ## a_Instr ## _u128, \
-                                                                           iemAImpl_ ## a_Instr ## _u128_fallback), \
-                                         puSrc1, puSrc2, pEFlags); \
+            IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_ ## a_Instr ## _u128, \
+                                                                       iemAImpl_ ## a_Instr ## _u128_fallback), \
+                                     puSrc1, puSrc2, pEFlags); \
             IEM_MC_ADVANCE_RIP_AND_FINISH(); \
             IEM_MC_END(); \

trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompBltIn.cpp

@@ -56 +56 @@
 #include "IEMN8veRecompilerEmit.h"
 #include "IEMN8veRecompilerTlbLookup.h"
+#include "target-x86/IEMAllN8veEmit-x86.h"
 
 
@@ -213 +214 @@
     uint8_t const         idxTmpReg1 = iemNativeRegAllocTmp(pReNative, &off);
     uint8_t const         idxTmpReg2 = a_fCheckIrqs ? iemNativeRegAllocTmp(pReNative, &off) : UINT8_MAX;
-    PIEMNATIVEINSTR const pCodeBuf   = iemNativeInstrBufEnsure(pReNative, off, RT_ARCH_VAL == RT_ARCH_VAL_AMD64 ? 72 : 32);
+    PIEMNATIVEINSTR const pCodeBuf   = iemNativeInstrBufEnsure(pReNative, off,
+                                                                 (RT_ARCH_VAL == RT_ARCH_VAL_AMD64 ? 72 : 32)
+                                                               + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS * 3);
 
     /*
@@ -226 +229 @@
 
         /* jz   ReturnBreakFF */
-        off = iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, kIemNativeLabelType_ReturnBreakFF, kIemNativeInstrCond_e);
+        off = iemNativeEmitTbExitJccEx<kIemNativeLabelType_ReturnBreakFF>(pReNative, pCodeBuf, off, kIemNativeInstrCond_e);
 
 # elif defined(RT_ARCH_ARM64)
@@ -235 +238 @@
 
         /* cbz reg1, ReturnBreakFF */
-        off = iemNativeEmitTestIfGprIsZeroAndTbExitEx(pReNative, pCodeBuf, off, idxTmpReg1, false /*f64Bit*/,
-                                                      kIemNativeLabelType_ReturnBreakFF);
+        off = iemNativeEmitTbExitIfGprIsZeroEx<kIemNativeLabelType_ReturnBreakFF>(pReNative, pCodeBuf, off,
+                                                                                  idxTmpReg1, false /*f64Bit*/);
 
 # else
@@ -341 +344 @@
         off = iemNativeEmitCmpGprWithImmEx(pCodeBuf, off, idxTmpReg1, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC);
         /* ja ReturnBreakFF */
-        off = iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, kIemNativeLabelType_ReturnBreakFF, kIemNativeInstrCond_nbe);
+        off = iemNativeEmitTbExitJccEx<kIemNativeLabelType_ReturnBreakFF>(pReNative, pCodeBuf, off, kIemNativeInstrCond_nbe);
 
         /*
@@ -357 +360 @@
 # ifdef RT_ARCH_AMD64
         /* jz   ReturnBreakFF */
-        off = iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, kIemNativeLabelType_ReturnBreakFF, kIemNativeInstrCond_e);
+        off = iemNativeEmitTbExitJccEx<kIemNativeLabelType_ReturnBreakFF>(pReNative, pCodeBuf, off, kIemNativeInstrCond_e);
 
 # elif defined(RT_ARCH_ARM64)
         /* cbz  reg1, ReturnBreakFF */
-        off = iemNativeEmitTestIfGprIsZeroAndTbExitEx(pReNative, pCodeBuf, off, idxTmpReg1, false /*f64Bit*/,
-                                                      kIemNativeLabelType_ReturnBreakFF);
+        off = iemNativeEmitTbExitIfGprIsZeroEx<kIemNativeLabelType_ReturnBreakFF>(pReNative, pCodeBuf, off,
+                                                                                  idxTmpReg1, false /*f64Bit*/);
 # else
 #  error "port me"
@@ -475 +478 @@
     off = iemNativeEmitLoadGprFromVCpuU32(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, iem.s.fExec));
     off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, IEMTB_F_KEY_MASK);
-    off = iemNativeEmitTestIfGpr32NotEqualImmAndTbExit(pReNative, off, idxTmpReg, fExpectedExec & IEMTB_F_KEY_MASK,
-                                                       kIemNativeLabelType_ReturnBreak);
+    off = iemNativeEmitTbExitIfGpr32NotEqualImm<kIemNativeLabelType_ReturnBreak>(pReNative, off, idxTmpReg,
+                                                                                 fExpectedExec & IEMTB_F_KEY_MASK);
     iemNativeRegFreeTmp(pReNative, idxTmpReg);
 
@@ -606 +609 @@
 
     /* 3. Jump if greater. */
-    off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeLabelType_RaiseGp0);
+    off = iemNativeEmitTbExitJa<kIemNativeLabelType_RaiseGp0>(pReNative, off);
 
     iemNativeRegFreeTmp(pReNative, idxRegCsLim);
@@ -694 +697 @@
     /* Compare the two and jump out if we're too close to the limit. */
     off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegLeft, idxRegRight);
-    off = iemNativeEmitJlTbExit(pReNative, off, kIemNativeLabelType_NeedCsLimChecking);
+    off = iemNativeEmitTbExitJl<kIemNativeLabelType_NeedCsLimChecking>(pReNative, off);
 
     iemNativeRegFreeTmp(pReNative, idxRegRight);
@@ -754 +757 @@
         } while (0)
 
-# define NEAR_JMP_SIZE 5
-
 # define CHECK_OPCODES_CMP_JMP() /* cost: 7 bytes first time, then 2 bytes */ do { \
             if (offConsolidatedJump != UINT32_MAX) \
@@ -766 +767 @@
             else \
             { \
-                pbCodeBuf[off++] = 0x74; /* jz near +NEAR_JMP_SIZE */ \
-                pbCodeBuf[off++] = NEAR_JMP_SIZE + BP_ON_OBSOLETION; \
-                offConsolidatedJump = off; \
+                pbCodeBuf[off++] = 0x74; /* jz near +5 */ \
+                offConsolidatedJump = ++off; \
                 if (BP_ON_OBSOLETION) pbCodeBuf[off++] = 0xcc; \
-                off = iemNativeEmitTbExitEx(pReNative, pbCodeBuf, off, kIemNativeLabelType_ObsoleteTb); \
+                off = iemNativeEmitTbExitEx<kIemNativeLabelType_ObsoleteTb, false /*a_fActuallyExitingTb*/>(pReNative, \
+                                                                                                            pbCodeBuf, off); \
+                pbCodeBuf[offConsolidatedJump - 1]  = off - offConsolidatedJump; \
             } \
         } while (0)
@@ -836 +838 @@
         }
 
-        uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6 + 14 + 54 + 8 + 6 + BP_ON_OBSOLETION /* = 88 */);
+        uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6 + 14 + 54 + 8 + 6 + BP_ON_OBSOLETION /* = 88 */
+                                                            + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
 
         if (cbLeft > 8)
@@ -880 +883 @@
         uint8_t const idxRegCx = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xCX));
 
-        uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6 + 10 + 5 + 5 + 3 + 4 + 3 + BP_ON_OBSOLETION /*= 36*/);
+        uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6 + 10 + 5 + 5 + 3 + 4 + 3 + BP_ON_OBSOLETION /*= 36*/
+                                                            + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
 
         /** @todo profile and optimize this further.  Maybe an idea to align by
@@ -943 +947 @@
     uint8_t const idxRegSrc1Val = iemNativeRegAllocTmp(pReNative, &off);
 
-    uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
+    uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS * 2);
 
     /* One byte compare can be done with the opcode byte as an immediate. We'll
@@ -1014 +1018 @@
                 if (fPendingJmp)
                 {
-                    off = iemNativeEmitJccTbExitEx(pReNative, pu32CodeBuf, off, kIemNativeLabelType_ObsoleteTb,
-                                                   kArmv8InstrCond_Ne);
+                    off = iemNativeEmitTbExitJccEx<kIemNativeLabelType_ObsoleteTb>(pReNative, pu32CodeBuf, off,
+                                                                                   kArmv8InstrCond_Ne);
                     fPendingJmp = false;
                 }
@@ -1051 +1055 @@
                                                             ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
 
-                off = iemNativeEmitJccTbExitEx(pReNative, pu32CodeBuf, off, kIemNativeLabelType_ObsoleteTb,
-                                               kArmv8InstrCond_Ne);
+                off = iemNativeEmitTbExitJccEx<kIemNativeLabelType_ObsoleteTb>(pReNative, pu32CodeBuf, off, kArmv8InstrCond_Ne);
 
                 /* Advance and loop. */
@@ -1185 +1188 @@
      */
     if (fPendingJmp)
-        off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_ObsoleteTb);
+        off = iemNativeEmitTbExitJnz<kIemNativeLabelType_ObsoleteTb>(pReNative, off);
 
     RT_NOREF(pu32CodeBuf, cbLeft, offPage, pbOpcodes, offConsolidatedJump);
@@ -1346 +1349 @@
     /* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal. */
     off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
-    off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeLabelType_CheckBranchMiss);
+    off = iemNativeEmitTbExitJa<kIemNativeLabelType_CheckBranchMiss>(pReNative, off);
 
     /* 4. Add iem.s.GCPhysInstrBuf and compare with GCPhysRangePageWithOffset. */
@@ -1377 +1380 @@
                                                 | pTb->aRanges[idxRange].offPhysPage)
                                              + offRange;
-    off = iemNativeEmitTestIfGprNotEqualImmAndTbExit(pReNative, off, idxRegTmp, GCPhysRangePageWithOffset,
-                                                     kIemNativeLabelType_CheckBranchMiss);
+    off = iemNativeEmitTbExitIfGprNotEqualImm<kIemNativeLabelType_CheckBranchMiss>(pReNative, off, idxRegTmp,
+                                                                                   GCPhysRangePageWithOffset);
 
     iemNativeRegFreeTmp(pReNative, idxRegTmp);
@@ -1487 +1490 @@
      */
     RTGCPHYS const GCPhysNewPage = iemTbGetRangePhysPageAddr(pTb, idxRange);
-    off = iemNativeEmitTestIfGprNotEqualImmAndTbExit(pReNative, off, idxRegGCPhys, GCPhysNewPage,
-                                                     kIemNativeLabelType_ObsoleteTb);
+    off = iemNativeEmitTbExitIfGprNotEqualImm<kIemNativeLabelType_ObsoleteTb>(pReNative, off, idxRegGCPhys, GCPhysNewPage);
 
     iemNativeRegFreeTmp(pReNative, idxRegGCPhys);
@@ -1699 +1701 @@
     off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegTmp2, GCPhysRangePageWithOffset);
     off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegTmp, idxRegTmp2);
-    off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_CheckBranchMiss);
+    off = iemNativeEmitTbExitJnz<kIemNativeLabelType_CheckBranchMiss>(pReNative, off);
     uint32_t const offFixedJumpToEnd = off;
     off = iemNativeEmitJmpToFixed(pReNative, off, off + 512 /* force rel32 */);
@@ -1711 +1713 @@
 
     /* Check that we haven't been here before. */
-    off = iemNativeEmitTestIfGprIsNotZeroAndTbExit(pReNative, off, idxRegTmp2,  false /*f64Bit*/,
-                                                   kIemNativeLabelType_CheckBranchMiss);
+    off = iemNativeEmitTbExitIfGprIsNotZero<kIemNativeLabelType_CheckBranchMiss>(pReNative, off, idxRegTmp2, false /*f64Bit*/);
 
     /* Jump to the TLB lookup code. */

trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompFuncs.h

@@ -60 +60 @@
 #include "IEMN8veRecompilerTlbLookup.h"
 #include "IEMNativeFunctions.h"
+#include "target-x86/IEMAllN8veEmit-x86.h"
 
 
@@ -442 +443 @@
                                                                  RT_BIT_64(IEMLIVENESSBIT_IDX_EFL_OTHER),
                                                                  RT_BIT_64(IEMLIVENESSBIT_IDX_EFL_OTHER));
-    off = iemNativeEmitTestAnyBitsInGprAndTbExitIfAnySet(pReNative, off, idxEflReg,
-                                                         X86_EFL_TF | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK,
-                                                         kIemNativeLabelType_ReturnWithFlags);
+    off = iemNativeEmitTbExitIfAnyBitsSetInGpr<kIemNativeLabelType_ReturnWithFlags>(pReNative, off, idxEflReg,
+                                                                                      X86_EFL_TF
+                                                                                    | CPUMCTX_DBG_HIT_DRX_MASK
+                                                                                    | CPUMCTX_DBG_DBGF_MASK);
     off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxEflReg, ~(uint32_t)(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW));
     off = iemNativeEmitStoreGprToVCpuU32(pReNative, off, idxEflReg, RT_UOFFSETOF(VMCPU, cpum.GstCtx.eflags));
@@ -556 +558 @@
 
                 if (pReNative->idxLastCheckIrqCallNo != UINT32_MAX)
-                    return iemNativeEmitTbExit(pReNative, off, kIemNativeLabelType_ReturnBreakViaLookup);
-                return iemNativeEmitTbExit(pReNative, off, kIemNativeLabelType_ReturnBreakViaLookupWithIrq);
+                    return iemNativeEmitTbExit<kIemNativeLabelType_ReturnBreakViaLookup>(pReNative, off);
+                return iemNativeEmitTbExit<kIemNativeLabelType_ReturnBreakViaLookupWithIrq>(pReNative, off);
             }
         }
         if (pReNative->idxLastCheckIrqCallNo != UINT32_MAX)
-            return iemNativeEmitTbExit(pReNative, off, kIemNativeLabelType_ReturnBreakViaLookupWithTlb);
-        return iemNativeEmitTbExit(pReNative, off, kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq);
+            return iemNativeEmitTbExit<kIemNativeLabelType_ReturnBreakViaLookupWithTlb>(pReNative, off);
+        return iemNativeEmitTbExit<kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq>(pReNative, off);
 #endif
     }
@@ -749 +751 @@
     off = iemNativeEmitAddGpr32Imm(pReNative, off, iTmpReg, (int32_t)0x8000);
     off = iemNativeEmitShiftGprRight(pReNative, off, iTmpReg, 16);
-    off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_RaiseGp0);
+    off = iemNativeEmitTbExitJnz<kIemNativeLabelType_RaiseGp0>(pReNative, off);
 
     iemNativeRegFreeTmp(pReNative, iTmpReg);
@@ -768 +770 @@
     off = iemNativeEmitAddTwoGprs(pReNative, off, iTmpReg, idxAddrReg);
     off = iemNativeEmitCmpArm64(pReNative, off, ARMV8_A64_REG_XZR, iTmpReg, true /*f64Bit*/, 48 /*cShift*/, kArmv8A64InstrShift_Lsr);
-    off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_RaiseGp0);
+    off = iemNativeEmitTbExitJnz<kIemNativeLabelType_RaiseGp0>(pReNative, off);
 
     iemNativeRegFreeTmp(pReNative, iTmpReg);
@@ -854 +856 @@
     off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxAddrReg, RT_UOFFSETOF(VMCPU, cpum.GstCtx.rip));
 
-    off = iemNativeEmitTbExit(pReNative, off, kIemNativeLabelType_RaiseGp0, false /*fActuallyExitingTb*/);
+    off = iemNativeEmitTbExit<kIemNativeLabelType_RaiseGp0, false /*a_fActuallyExitingTb*/>(pReNative, off);
 
     /* .Lnoexcept: */
@@ -950 +952 @@
         off = iemNativeEmitStoreGprToVCpuU64(pReNative, off, idxOldPcReg, RT_UOFFSETOF(VMCPU, cpum.GstCtx.rip));
 
-        off = iemNativeEmitTbExit(pReNative, off, kIemNativeLabelType_RaiseGp0, false /*fActuallyExitingTb*/);
+        off = iemNativeEmitTbExit<kIemNativeLabelType_RaiseGp0, false /*a_fActuallyExitingTb*/>(pReNative, off);
         iemNativeFixupFixedJump(pReNative, offFixup, off);
     }
     else
 #endif
-        off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_RaiseGp0);
+        off = iemNativeEmitTbExitJnz<kIemNativeLabelType_RaiseGp0>(pReNative, off);
 
     iemNativeRegFreeTmp(pReNative, iTmpReg);
@@ -995 +997 @@
 
     off = iemNativeEmitCmpGpr32WithGpr(pReNative, off, idxAddrReg, idxRegCsLim);
-    off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeLabelType_RaiseGp0);
+    off = iemNativeEmitTbExitJa<kIemNativeLabelType_RaiseGp0>(pReNative, off);
 
     iemNativeRegFreeTmp(pReNative, idxRegCsLim);
@@ -1058 +1060 @@
         off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr));
 # endif
-        off = iemNativeEmitTbExit(pReNative, off, kIemNativeLabelType_RaiseGp0, false /*fActuallyExitingTb*/);
+        off = iemNativeEmitTbExit<kIemNativeLabelType_RaiseGp0, false /*a_fActuallyExitingTb*/>(pReNative, off);
         iemNativeFixupFixedJump(pReNative, offFixup, off);
     }
     else
 #endif
-        off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeLabelType_RaiseGp0);
+        off = iemNativeEmitTbExitJa<kIemNativeLabelType_RaiseGp0>(pReNative, off);
 
     iemNativeRegFreeTmp(pReNative, idxRegCsLim);
@@ -2599 +2601 @@
          */
         /* Test and jump. */
-        off = iemNativeEmitTestAnyBitsInGprAndTbExitIfAnySet(pReNative, off, idxCr0Reg, X86_CR0_EM | X86_CR0_TS,
-                                                             kIemNativeLabelType_RaiseNm);
+        off = iemNativeEmitTbExitIfAnyBitsSetInGpr<kIemNativeLabelType_RaiseNm>(pReNative, off, idxCr0Reg,
+                                                                                X86_CR0_EM | X86_CR0_TS);
 
         /* Free but don't flush the CR0 register. */
@@ -2659 +2661 @@
         off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxCr0Reg, X86_CR0_MP | X86_CR0_TS);
         /* Test and jump. */
-        off = iemNativeEmitTestIfGpr32EqualsImmAndTbExit(pReNative, off, idxCr0Reg, X86_CR0_MP | X86_CR0_TS,
-                                                         kIemNativeLabelType_RaiseNm);
+        off = iemNativeEmitTbExitIfGpr32EqualsImm<kIemNativeLabelType_RaiseNm>(pReNative, off, idxCr0Reg, X86_CR0_MP | X86_CR0_TS);
 
         /* Free the CR0 register. */
@@ -2712 +2713 @@
      */
     /* Test and jump. */
-    off = iemNativeEmitTestBitInGprAndTbExitIfSet(pReNative, off, idxFpuFswReg, X86_FSW_ES_BIT, kIemNativeLabelType_RaiseMf);
+    off = iemNativeEmitTbExitIfBitSetInGpr<kIemNativeLabelType_RaiseMf>(pReNative, off, idxFpuFswReg, X86_FSW_ES_BIT);
 
     /* Free but don't flush the FSW register. */
@@ -2769 +2770 @@
          * be a safe assumption.
          */
-        PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1+6+3+3+7+7+6);
+        PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off,
+                                                                 1+6+3+3+7+7+6 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
         //pCodeBuf[off++] = 0xcc;
         off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off,    idxTmpReg, X86_CR4_OSFXSR); /* Isolate CR4.OSFXSR as CR4.TSD and */
@@ -2776 +2778 @@
         off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off,   idxTmpReg, X86_CR0_EM | X86_CR0_TS | X86_CR4_OSFXSR);
         off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off,   idxTmpReg, X86_CR4_OSFXSR);
-        off = iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, kIemNativeLabelType_RaiseSseRelated, kIemNativeInstrCond_ne);
+        off = iemNativeEmitTbExitJccEx<kIemNativeLabelType_RaiseSseRelated>(pReNative, pCodeBuf, off, kIemNativeInstrCond_ne);
 
 #elif defined(RT_ARCH_ARM64)
@@ -2784 +2786 @@
          *  else                                                                                { goto RaiseSseRelated; }
          */
-        PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1+5);
+        PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off,
+                                                                 1+5 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
         //pCodeBuf[off++] = Armv8A64MkInstrBrk(0x1111);
         Assert(Armv8A64ConvertImmRImmS2Mask32(1, 32 - X86_CR0_EM_BIT) == (X86_CR0_EM | X86_CR0_TS));
@@ -2793 +2796 @@
         pCodeBuf[off++] = Armv8A64MkInstrEorImm(idxTmpReg, idxTmpReg, 0, 0, false /*f64Bit*/);
         /* -> idxTmpReg[0]=~OSFXSR; idxTmpReg[2]=EM; idxTmpReg[3]=TS; (the rest is zero) */
-        off = iemNativeEmitTestIfGprIsNotZeroAndTbExitEx(pReNative, pCodeBuf, off, idxTmpReg, false /*f64Bit*/,
-                                                         kIemNativeLabelType_RaiseSseRelated);
+        off = iemNativeEmitTbExitIfGprIsNotZeroEx<kIemNativeLabelType_RaiseSseRelated>(pReNative, pCodeBuf, off,
+                                                                                       idxTmpReg, false /*f64Bit*/);
 
 #else
@@ -2870 +2873 @@
                   ^ 0x1a) ) { likely }
             else            { goto RaiseAvxRelated; } */
-        PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1+6+3+5+3+5+3+7+6);
+        PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off,
+                                                                 1+6+3+5+3+5+3+7+6 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
         //pCodeBuf[off++] = 0xcc;
         off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off,                 idxTmpReg, XSAVE_C_YMM | XSAVE_C_SSE);
@@ -2882 +2886 @@
         off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off,                idxTmpReg, ((XSAVE_C_YMM | XSAVE_C_SSE) << 2) | 2);
         /* -> idxTmpReg[0]=CR0.TS idxTmpReg[1]=~CR4.OSXSAVE; idxTmpReg[2]=0; idxTmpReg[3]=~SSE; idxTmpReg[4]=~YMM; */
-        off = iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, kIemNativeLabelType_RaiseAvxRelated, kIemNativeInstrCond_ne);
+        off = iemNativeEmitTbExitJccEx<kIemNativeLabelType_RaiseAvxRelated>(pReNative, pCodeBuf, off, kIemNativeInstrCond_ne);
 
 #elif defined(RT_ARCH_ARM64)
@@ -2888 +2892 @@
                   | ((cr0 >> X86_CR0_TS_BIT) & 1) ) { likely }
             else                                    { goto RaiseAvxRelated; } */
-        PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1+6);
+        PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off,
+                                                                 1+6 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
         //pCodeBuf[off++] = Armv8A64MkInstrBrk(0x1111);
         Assert(Armv8A64ConvertImmRImmS2Mask32(1, 32 - XSAVE_C_SSE_BIT) == (XSAVE_C_YMM | XSAVE_C_SSE));
@@ -2900 +2905 @@
         pCodeBuf[off++] = Armv8A64MkInstrBfxil(idxTmpReg, idxCr0Reg, X86_CR0_TS_BIT, 1, false /*f64Bit*/);
         /* -> idxTmpReg[0]=CR0.TS; idxTmpReg[1]=~CR4.OSXSAVE; idxTmpReg[2]=~SSE; idxTmpReg[3]=~YMM; (the rest is zero) */
-        off = iemNativeEmitTestIfGprIsNotZeroAndTbExitEx(pReNative, pCodeBuf, off, idxTmpReg, false /*f64Bit*/,
-                                                         kIemNativeLabelType_RaiseAvxRelated);
+        off = iemNativeEmitTbExitIfGprIsNotZeroEx<kIemNativeLabelType_RaiseAvxRelated>(pReNative, pCodeBuf, off,
+                                                                                       idxTmpReg, false /*f64Bit*/);
 
 #else
@@ -2952 +2957 @@
     /* Do the job we're here for. */
     uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxVar, &off);
-    off = iemNativeEmitTestIfGprIsZeroAndTbExit(pReNative, off, idxVarReg, false /*f64Bit*/, kIemNativeLabelType_RaiseDe);
+    off = iemNativeEmitTbExitIfGprIsZero<kIemNativeLabelType_RaiseDe>(pReNative, off, idxVarReg, false /*f64Bit*/);
     iemNativeVarRegisterRelease(pReNative, idxVar);
 
@@ -2991 +2996 @@
 
     uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxVarEffAddr, &off);
-    off = iemNativeEmitTestAnyBitsInGprAndTbExitIfAnySet(pReNative, off, idxVarReg, cbAlign - 1,
-                                                         kIemNativeLabelType_RaiseGp0);
+    off = iemNativeEmitTbExitIfAnyBitsSetInGpr<kIemNativeLabelType_RaiseGp0>(pReNative, off, idxVarReg, cbAlign - 1);
     iemNativeVarRegisterRelease(pReNative, idxVarEffAddr);
 
@@ -4335 +4339 @@
      * Make the call and update the return code variable if we've got one.
      */
-    off = iemNativeEmitCallImm(pReNative, off, pfnAImpl);
+    off = iemNativeEmitCallImm<true /*a_fSkipEflChecks*/>(pReNative, off, pfnAImpl);
     if (idxVarRc != UINT8_MAX)
         iemNativeVarRegisterSet(pReNative, idxVarRc, IEMNATIVE_CALL_RET_GREG, off, false /*fAllocated*/);
@@ -5865 +5869 @@
 # else
         PCIEMLIVENESSENTRY const pLivenessEntry       = &pReNative->paLivenessEntries[pReNative->idxCurCall];
-        IEMLIVENESSBIT const     LivenessClobbered    =
-        {
-              pLivenessEntry->aBits[IEMLIVENESS_BIT_WRITE].bm64
-            & ~(  pLivenessEntry->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64
-                | pLivenessEntry->aBits[IEMLIVENESS_BIT_READ].bm64
-                | pLivenessEntry->aBits[IEMLIVENESS_BIT_CALL].bm64)
-        };
-        IEMLIVENESSBIT const     LivenessDelayable =
-        {
-              pLivenessEntry->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64
-            & pLivenessEntry->aBits[IEMLIVENESS_BIT_WRITE].bm64
-            & ~(  pLivenessEntry->aBits[IEMLIVENESS_BIT_READ].bm64
-                | pLivenessEntry->aBits[IEMLIVENESS_BIT_CALL].bm64)
-        };
+        IEMLIVENESSBIT const     LivenessClobbered    = { IEMLIVENESS_STATE_GET_WILL_BE_CLOBBERED_SET(pLivenessEntry) };
+        IEMLIVENESSBIT const     LivenessDelayable    = { IEMLIVENESS_STATE_GET_CAN_BE_POSTPONED_SET(pLivenessEntry)  };
 #  define CHECK_FLAG_AND_UPDATE_STATS(a_fEfl, a_fLivenessMember, a_CoreStatName) \
             if (fEflOutput & (a_fEfl)) \
@@ -5989 +5981 @@
 #define IEM_MC_COMMIT_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput) \
     IEMNATIVE_EFLAGS_OPTIMIZATION_STATS(a_fEflInput, a_fEflOutput); \
-    off = iemNativeEmitCommitEFlags<true /*fUpdateSkipping*/, a_fEflOutput, \
+    off = iemNativeEmitCommitEFlags<true /*a_fUpdateSkippingAndPostponing*/, a_fEflOutput, \
                                     iemNativeEflagsToLivenessMask<a_fEflInput>(), \
                                     iemNativeEflagsToLivenessMask<a_fEflOutput>()>(pReNative, off, a_EFlags, a_fEflInput)
@@ -5996 +5988 @@
 #define IEM_MC_COMMIT_EFLAGS_OPT_EX(a_EFlags, a_fEflInput, a_fEflOutput) \
     IEMNATIVE_EFLAGS_OPTIMIZATION_STATS(a_fEflInput, a_fEflOutput); \
-    off = iemNativeEmitCommitEFlags<false /*fUpdateSkipping*/, a_fEflOutput, \
+    off = iemNativeEmitCommitEFlags<false /*a_fUpdateSkippingAndPostponing*/, a_fEflOutput, \
                                     iemNativeEflagsToLivenessMask<a_fEflInput>(), \
                                     iemNativeEflagsToLivenessMask<a_fEflOutput>()>(pReNative, off, a_EFlags, a_fEflInput)
 
 /** Handles IEM_MC_COMMIT_EFLAGS_EX. */
-template<bool const a_fUpdateSkipping, uint32_t const a_fEflOutput,
+template<bool const a_fUpdateSkippingAndPostponing, uint32_t const a_fEflOutput,
          uint64_t const a_fLivenessEflInputBits, uint64_t const a_fLivenessEflOutputBits>
 DECL_INLINE_THROW(uint32_t)
@@ -6052 +6044 @@
 
 #ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
-    if RT_CONSTEXPR_IF(a_fUpdateSkipping)
+    if RT_CONSTEXPR_IF(a_fUpdateSkippingAndPostponing)
     {
         Assert(!(pReNative->fSkippingEFlags & fElfInput)); RT_NOREF(fElfInput);
@@ -6066 +6058 @@
                                                  RT_UOFFSETOF(VMCPU, iem.s.fSkippingEFlags));
 # endif
+        IEMNATIVE_CLEAR_POSTPONED_EFLAGS(pReNative, a_fEflOutput);
     }
 #endif
@@ -6252 +6245 @@
 #define IEM_MC_REF_EFLAGS_EX(a_pEFlags, a_fEflInput, a_fEflOutput) \
     IEMNATIVE_EFLAGS_OPTIMIZATION_STATS(a_fEflInput, a_fEflOutput); \
-    off = iemNativeEmitRefEFlags(pReNative, off, a_pEFlags, a_fEflInput, a_fEflOutput)
+    off = iemNativeEmitRefEFlags<a_fEflOutput>(pReNative, off, a_pEFlags, a_fEflInput)
 
 /** Handles IEM_MC_REF_EFLAGS. */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitRefEFlags(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarRef, uint32_t fEflInput, uint32_t fEflOutput)
+template<uint32_t const a_fEflOutput>
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitRefEFlags(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarRef, uint32_t fEflInput)
 {
     iemNativeVarSetKindToGstRegRef(pReNative, idxVarRef, kIemNativeGstRegRef_EFlags, 0);
@@ -6264 +6258 @@
     IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(pReNative,  fEflInput);
     IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(pReNative, off, fEflInput);
-    pReNative->fSkippingEFlags &= ~fEflOutput;
+    pReNative->fSkippingEFlags &= ~a_fEflOutput;
 # ifdef IEMNATIVE_STRICT_EFLAGS_SKIPPING
 
     /* Updating the skipping according to the outputs is a little early, but
        we don't have any other hooks for references atm. */
-    if ((fEflOutput & X86_EFL_STATUS_BITS) == X86_EFL_STATUS_BITS)
+    if RT_CONSTEXPR((a_fEflOutput & X86_EFL_STATUS_BITS) == X86_EFL_STATUS_BITS)
         off = iemNativeEmitStoreImmToVCpuU32(pReNative, off, 0, RT_UOFFSETOF(VMCPU, iem.s.fSkippingEFlags));
-    else if (fEflOutput & X86_EFL_STATUS_BITS)
-        off = iemNativeEmitAndImmIntoVCpuU32(pReNative, off, ~(fEflOutput & X86_EFL_STATUS_BITS),
+    else if RT_CONSTEXPR((a_fEflOutput & X86_EFL_STATUS_BITS) != 0)
+        off = iemNativeEmitAndImmIntoVCpuU32(pReNative, off, ~(a_fEflOutput & X86_EFL_STATUS_BITS),
                                              RT_UOFFSETOF(VMCPU, iem.s.fSkippingEFlags));
 # endif
-#endif
-    RT_NOREF(fEflInput, fEflOutput);
+
+    /* This ASSUMES that EFLAGS references are not taken before use. */
+    IEMNATIVE_CLEAR_POSTPONED_EFLAGS(pReNative, a_fEflOutput);
+
+#endif
+    RT_NOREF(fEflInput);
 
     /* If we've delayed writing back the register value, flush it now. */
@@ -11055 +11053 @@
      * Make the call.
      */
-    off = iemNativeEmitCallImm(pReNative, off, pfnAImpl);
+    off = iemNativeEmitCallImm<true /*a_fSkipEflChecks*/>(pReNative, off, pfnAImpl);
 
     /*
@@ -11097 +11095 @@
     /* tmp &= mxcsr */
     off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, idxRegTmp, idxRegTmp2);
-    off = iemNativeEmitTestAnyBitsInGprAndTbExitIfAnySet(pReNative, off, idxRegTmp, X86_MXCSR_XCPT_FLAGS,
-                                                         kIemNativeLabelType_RaiseSseAvxFpRelated);
+    off = iemNativeEmitTbExitIfAnyBitsSetInGpr<kIemNativeLabelType_RaiseSseAvxFpRelated>(pReNative, off, idxRegTmp,
+                                                                                         X86_MXCSR_XCPT_FLAGS);
 
     iemNativeRegFreeTmp(pReNative, idxRegTmp2);

trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

@@ -9 +9 @@
  *      - Level 3  (Log3) : Disassemble native code after recompiling.
  *      - Level 4  (Log4) : Delayed PC updating.
- *      - Level 5  (Log5) : ...
+ *      - Level 5  (Log5) : Postponed and skipped EFLAGS calculations.
  *      - Level 6  (Log6) : ...
  *      - Level 7  (Log7) : ...
@@ -78 +78 @@
 #include "IEMN8veRecompilerTlbLookup.h"
 #include "IEMNativeFunctions.h"
+#include "target-x86/IEMAllN8veEmit-x86.h"
 
 
@@ -2070 +2071 @@
 #endif
 #ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
-    pReNative->fPostponingEFlags           = 0;
+    pReNative->PostponedEfl.fEFlags        = 0;
+    pReNative->PostponedEfl.enmOp          = kIemNativePostponedEflOp_Invalid;
+    pReNative->PostponedEfl.cOpBits        = 0;
+    pReNative->PostponedEfl.idxReg1        = UINT8_MAX;
+    pReNative->PostponedEfl.idxReg2        = UINT8_MAX;
 #endif
 
@@ -6442 +6447 @@
 
     /* Jump to non-zero status return path. */
-    off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_NonZeroRetOrPassUp);
+    off = iemNativeEmitTbExitJnz<kIemNativeLabelType_NonZeroRetOrPassUp>(pReNative, off);
 
     /* done. */
@@ -6450 +6455 @@
      * ARM64: w0 = call status code.
      */
+    PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1+3+3 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
+
 # ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
-    off = iemNativeEmitLoadGprImm64(pReNative, off, ARMV8_A64_REG_X2, idxInstr);
+    AssertCompile(ARMV8_A64_REG_X2 == IEMNATIVE_CALL_ARG2_GREG);
+    off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, ARMV8_A64_REG_X2, idxInstr);
 # endif
-    off = iemNativeEmitLoadGprFromVCpuU32(pReNative, off, ARMV8_A64_REG_X3, RT_UOFFSETOF(VMCPUCC, iem.s.rcPassUp));
-
-    uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
-
-    pu32CodeBuf[off++] = Armv8A64MkInstrOrr(ARMV8_A64_REG_X4, ARMV8_A64_REG_X3, ARMV8_A64_REG_X0, false /*f64Bit*/);
-
-    off = iemNativeEmitTestIfGprIsNotZeroAndTbExitEx(pReNative, pu32CodeBuf, off, ARMV8_A64_REG_X4, true /*f64Bit*/,
-                                                     kIemNativeLabelType_NonZeroRetOrPassUp);
+    off = iemNativeEmitLoadGprFromVCpuU32Ex(pCodeBuf, off, ARMV8_A64_REG_X3, RT_UOFFSETOF(VMCPUCC, iem.s.rcPassUp));
+
+    pCodeBuf[off++] = Armv8A64MkInstrOrr(ARMV8_A64_REG_X4, ARMV8_A64_REG_X3, ARMV8_A64_REG_X0, false /*f64Bit*/);
+
+    off = iemNativeEmitTbExitIfGprIsNotZeroEx<kIemNativeLabelType_NonZeroRetOrPassUp>(pReNative, pCodeBuf, off,
+                                                                                      ARMV8_A64_REG_X4, true /*f64Bit*/);
 
 #else
@@ -9990 +9996 @@
                     = szSegSel[X86_SREG_COUNT] = '\0';
 
-                char szEFlags[8];
-                for (unsigned i = 0; i < 7; i++)
+                char szEFlags[IEMLIVENESSBIT_IDX_EFL_COUNT + 1];
+                for (unsigned i = 0; i < IEMLIVENESSBIT_IDX_EFL_COUNT; i++)
                     szEFlags[i] = s_achState[iemNativeLivenessGetStateByGstRegEx(pLivenessEntry, i + kIemNativeGstReg_EFlags)];
                 szEFlags[7] = '\0';
 
-                Log2(("liveness: grp=%s segbase=%s segattr=%s seglim=%s segsel=%s efl=%s\n",
+                Log2(("liveness: gpr=%s segbase=%s segattr=%s seglim=%s segsel=%s efl=%s\n",
                       szGpr, szSegBase, szSegAttrib, szSegLimit, szSegSel, szEFlags));
             }
@@ -10027 +10033 @@
          */
         //off = iemNativeEmitBrk(pReNative, off, 0x1227);
-        off = iemNativeEmitTbExit(pReNative, off, kIemNativeLabelType_ReturnSuccess);
+        off = iemNativeEmitTbExit<kIemNativeLabelType_ReturnSuccess, true, false>(pReNative, off);
 
         /*
@@ -10039 +10045 @@
         if (fTailLabels)
         {
+            PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, kIemNativeLabelType_LastTbExit + 1);
             do
             {
@@ -10047 +10054 @@
                 AssertContinue(idxLabel != UINT32_MAX);
                 iemNativeLabelDefine(pReNative, idxLabel, off);
-                off = iemNativeEmitTbExit(pReNative, off, enmLabel);
+
+                iemNativeAddTbExitFixup(pReNative, off, enmLabel);
+# ifdef RT_ARCH_ARM64
+                pCodeBuf[off++] = Armv8A64MkInstrB(-1);
+# else
+#  error "port me"
+# endif
             } while (fTailLabels);
+            IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
         }
 #else

trunk/src/VBox/VMM/VMMAll/target-x86/IEMAllN8veEmit-x86.h

@@ -195 +195 @@
 #endif /* RT_ARCH_AMD64 */
 
+
+
+/*********************************************************************************************************************************
+*   EFLAGS                                                                                                                       *
+*********************************************************************************************************************************/
+
+#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+
+/** @def IEMNATIVE_POSTPONING_REG_MASK
+ * Register suitable for keeping the inputs or result for a postponed EFLAGS
+ * calculation.
+ *
+ * We use non-volatile register here so we don't have to save & restore them
+ * accross callouts (i.e. TLB loads).
+ *
+ * @note  On x86 we cannot use RDI and RSI because these are used by the
+ *        opcode checking code.  The usual joy of the x86 instruction set.
+ */
+# ifdef RT_ARCH_AMD64
+#  define IEMNATIVE_POSTPONING_REG_MASK \
+    (IEMNATIVE_CALL_NONVOLATILE_GREG_MASK & ~(RT_BIT_32(X86_GREG_xDI) | RT_BIT_32(X86_GREG_xSI)))
+# else
+#  define IEMNATIVE_POSTPONING_REG_MASK     IEMNATIVE_CALL_NONVOLATILE_GREG_MASK
+# endif
+
+/**
+ * This is normally invoked via IEMNATIVE_CLEAR_POSTPONED_EFLAGS().
+ */
+template<uint32_t const a_fEflClobbered>
+DECL_FORCE_INLINE(void) iemNativeClearPostponedEFlags(PIEMRECOMPILERSTATE pReNative)
+{
+    AssertCompile(!(a_fEflClobbered & ~X86_EFL_STATUS_BITS));
+    uint32_t fEFlags = pReNative->PostponedEfl.fEFlags;
+    if (fEFlags)
+    {
+        if RT_CONSTEXPR(a_fEflClobbered != X86_EFL_STATUS_BITS)
+        {
+            fEFlags &= ~a_fEflClobbered;
+            if (!fEFlags)
+            { /* likely */ }
+            else
+            {
+                Log5(("iemNativeClearPostponedEFlags: Clobbering %#x: %#x -> %#x (op=%d bits=%u)\n", a_fEflClobbered,
+                      pReNative->PostponedEfl.fEFlags, fEFlags, pReNative->PostponedEfl.enmOp, pReNative->PostponedEfl.cOpBits));
+                pReNative->PostponedEfl.fEFlags = fEFlags;
+                return;
+            }
+        }
+
+        /* Do cleanup.  */
+        Log5(("iemNativeClearPostponedEFlags: Cleanup of op=%u bits=%u efl=%#x upon clobbering %#x\n",
+              pReNative->PostponedEfl.enmOp, pReNative->PostponedEfl.cOpBits, pReNative->PostponedEfl.fEFlags, a_fEflClobbered));
+        pReNative->PostponedEfl.fEFlags = 0;
+        pReNative->PostponedEfl.enmOp   = kIemNativePostponedEflOp_Invalid;
+        pReNative->PostponedEfl.cOpBits = 0;
+        iemNativeRegFreeTmp(pReNative, pReNative->PostponedEfl.idxReg1);
+        if (pReNative->PostponedEfl.idxReg2 != UINT8_MAX)
+            iemNativeRegFreeTmp(pReNative, pReNative->PostponedEfl.idxReg2);
+        pReNative->PostponedEfl.idxReg1 = UINT8_MAX;
+        pReNative->PostponedEfl.idxReg2 = UINT8_MAX;
+    }
+}
+
+DECL_INLINE_THROW(uint32_t) iemNativeEmitPostponedEFlagsCalcLogical(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t cOpBits,
+                                                                    uint8_t idxRegResult, uint8_t idxRegEfl, uint8_t idxRegTmp)
+{
+#ifdef RT_ARCH_AMD64
+    /*
+     * Do an AND and collect flags and merge them with eflags.
+     */
+    /* Do TEST idxRegResult, idxRegResult to set flags. */
+    off = iemNativeEmitAmd64OneByteModRmInstrRREx(pCodeBuf, off, 0x84, 0x85, cOpBits, idxRegResult, idxRegResult);
+
+    if (idxRegTmp == X86_GREG_xAX)
+    {
+        /* sahf ; AH = EFLAGS */
+        pCodeBuf[off++] = 0x9e;
+        if (idxRegEfl <= X86_GREG_xBX)
+        {
+            /* mov [CDB]L, AH */
+            pCodeBuf[off++] = 0x88;
+            pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4 /*AH*/, idxRegEfl);
+        }
+        else
+        {
+            /* mov   AL, AH */
+            pCodeBuf[off++] = 0x88;
+            pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4 /*AH*/, 0 /*AL*/);
+            /* mov   xxL, AL */
+            pCodeBuf[off++] = idxRegEfl >= 8 ? X86_OP_REX_B : X86_OP_REX;
+            pCodeBuf[off++] = 0x88;
+            pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0 /*AL*/, idxRegEfl & 7);
+        }
+    }
+    else if (idxRegEfl != X86_GREG_xAX)
+    {
+        /* pushf */
+        pCodeBuf[off++] = 0x9c;
+        /* pop  tmp */
+        if (idxRegTmp >= 8)
+            pCodeBuf[off++] = X86_OP_REX_B;
+        pCodeBuf[off++] = 0x58 + (idxRegTmp & 7);
+        /* mov   byte(efl), byte(tmp) */
+        pCodeBuf[off++] = (idxRegEfl >= 8 ? X86_OP_REX_B : X86_OP_REX)
+                        | (idxRegTmp >= 8 ? X86_OP_REX_R : 0);
+        pCodeBuf[off++] = 0x88;
+        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxRegTmp & 7, idxRegEfl & 7);
+    }
+    else
+    {
+        /* xchg al, ah  */
+        pCodeBuf[off++] = 0x86;
+        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4 /*AH*/, 0 /*AL*/);
+        /* sahf ; AH = EFLAGS */
+        pCodeBuf[off++] = 0x9e;
+        /* xchg al, ah  */
+        pCodeBuf[off++] = 0x86;
+        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4 /*AH*/, 0 /*AL*/);
+    }
+    /* BTC idxEfl, 11; Clear OF */
+    if (idxRegEfl >= 8)
+        pCodeBuf[off++] = X86_OP_REX_B;
+    pCodeBuf[off++] = 0xf;
+    pCodeBuf[off++] = 0xba;
+    pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, idxRegEfl & 7);
+    pCodeBuf[off++] = X86_EFL_OF_BIT;
+
+#elif defined(RT_ARCH_ARM64)
+    /*
+     * Calculate flags.
+     */
+    /* Clear the status bits. ~0x8D5 (or ~0x8FD) can't be AND immediate, so use idxRegTmp for constant. */
+    off = iemNativeEmitLoadGpr32ImmExT<~X86_EFL_STATUS_BITS>(pCodeBuf, off, idxRegTmp);
+    off = iemNativeEmitAndGpr32ByGpr32Ex(pCodeBuf, off, idxRegEfl, idxRegTmp);
+
+    /* N,Z -> SF,ZF */
+    if (cOpBits < 32)
+        pCodeBuf[off++] = Armv8A64MkInstrSetF8SetF16(idxRegResult, cOpBits > 8); /* sets NZ */
+    else
+        pCodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, idxRegResult, idxRegResult, cOpBits > 32 /*f64Bit*/);
+    pCodeBuf[off++] = Armv8A64MkInstrMrs(idxRegTmp, ARMV8_AARCH64_SYSREG_NZCV); /* Bits: 31=N; 30=Z; 29=C; 28=V; */
+    pCodeBuf[off++] = Armv8A64MkInstrLsrImm(idxRegTmp, idxRegTmp, 30);
+    pCodeBuf[off++] = Armv8A64MkInstrBfi(idxRegEfl, idxRegTmp, X86_EFL_ZF_BIT, 2, false /*f64Bit*/);
+    AssertCompile(X86_EFL_ZF_BIT + 1 == X86_EFL_SF_BIT);
+
+    /* Calculate 8-bit parity of the result. */
+    pCodeBuf[off++] = Armv8A64MkInstrEor(idxRegTmp, idxRegResult, idxRegResult, false /*f64Bit*/,
+                                         4 /*offShift6*/, kArmv8A64InstrShift_Lsr);
+    pCodeBuf[off++] = Armv8A64MkInstrEor(idxRegTmp, idxRegTmp,    idxRegTmp,    false /*f64Bit*/,
+                                         2 /*offShift6*/, kArmv8A64InstrShift_Lsr);
+    pCodeBuf[off++] = Armv8A64MkInstrEor(idxRegTmp, idxRegTmp,    idxRegTmp,    false /*f64Bit*/,
+                                         1 /*offShift6*/, kArmv8A64InstrShift_Lsr);
+    Assert(Armv8A64ConvertImmRImmS2Mask32(0, 0) == 1);
+    pCodeBuf[off++] = Armv8A64MkInstrEorImm(idxRegTmp, idxRegTmp, 0, 0, false /*f64Bit*/);
+    pCodeBuf[off++] = Armv8A64MkInstrBfi(idxRegEfl, idxRegTmp, X86_EFL_PF_BIT, 1, false /*f64Bit*/);
+
+#else
+# error "port me"
+#endif
+    return off;
+}
+
+
+template<uint32_t const a_bmInputRegs>
+static uint32_t iemNativeDoPostponedEFlagsAtTbExitInternal(PIEMRECOMPILERSTATE pReNative, uint32_t off, PIEMNATIVEINSTR pCodeBuf)
+{
+    /*
+     * We can't do regular register allocations here, but since we're in an exit
+     * path where all pending writes has been flushed and we have a known set of
+     * registers with input for the exit label, we do our own simple stuff here.
+     *
+     * Note! On x86 we prefer using RAX as the first TMP register, so we can
+     *       make use of LAHF which is typically faster than PUSHF/POP.  This
+     *       is why the idxRegTmp allocation is first when there is no EFLAG
+     *       shadow, since RAX is represented by bit 0 in the mask.
+     */
+    uint32_t bmAvailableRegs = ~(a_bmInputRegs | IEMNATIVE_REG_FIXED_MASK) & IEMNATIVE_HST_GREG_MASK;
+    if (pReNative->PostponedEfl.idxReg2 != UINT8_MAX)
+        bmAvailableRegs &= ~(RT_BIT_32(pReNative->PostponedEfl.idxReg1) | RT_BIT_32(pReNative->PostponedEfl.idxReg2));
+    else
+        bmAvailableRegs &= ~RT_BIT_32(pReNative->PostponedEfl.idxReg1);
+
+    /* Use existing EFLAGS shadow if available. */
+    uint8_t idxRegEfl, idxRegTmp;
+    if (pReNative->Core.bmGstRegShadowDirty & RT_BIT_64(kIemNativeGstReg_EFlags))
+    {
+        idxRegEfl = pReNative->Core.aidxGstRegShadows[kIemNativeGstReg_EFlags];
+        Assert(idxRegEfl < IEMNATIVE_HST_GREG_COUNT && (bmAvailableRegs & RT_BIT_32(idxRegEfl)));
+        bmAvailableRegs &= ~RT_BIT_32(idxRegEfl);
+#ifdef VBOX_STRICT
+        /** @todo check shadow register content. */
+#endif
+
+        idxRegTmp = ASMBitFirstSetU32(bmAvailableRegs) - 1;
+        bmAvailableRegs &= ~RT_BIT_32(idxRegTmp);
+    }
+    else
+    {
+        idxRegTmp = ASMBitFirstSetU32(bmAvailableRegs) - 1; /* allocate the temp register first to prioritize EAX on x86. */
+        bmAvailableRegs &= ~RT_BIT_32(idxRegTmp);
+
+        idxRegEfl = ASMBitFirstSetU32(bmAvailableRegs) - 1;
+        bmAvailableRegs &= ~RT_BIT_32(idxRegTmp);
+        off = iemNativeEmitLoadGprFromVCpuU32Ex(pCodeBuf, off, idxRegEfl, RT_UOFFSETOF(VMCPU, cpum.GstCtx.eflags));
+    }
+    Assert(bmAvailableRegs != 0);
+
+    /*
+     * Do the actual EFLAGS calculation.
+     */
+    switch (pReNative->PostponedEfl.enmOp)
+    {
+        case kIemNativePostponedEflOp_Logical:
+            Assert(pReNative->PostponedEfl.idxReg2 == UINT8_MAX);
+            off = iemNativeEmitPostponedEFlagsCalcLogical(pCodeBuf, off, pReNative->PostponedEfl.cOpBits,
+                                                          pReNative->PostponedEfl.idxReg1, idxRegEfl, idxRegTmp);
+            break;
+
+        default:
+            AssertFailedBreak();
+    }
+
+    /*
+     * Store EFLAGS.
+     */
+    off = iemNativeEmitStoreGprToVCpuU32Ex(pCodeBuf, off, idxRegEfl, RT_UOFFSETOF(VMCPU, cpum.GstCtx.eflags));
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+
+    return off;
+}
+
+
+
+template<uint32_t const a_bmInputRegs>
+DECL_FORCE_INLINE_THROW(uint32_t)
+iemNativeDoPostponedEFlagsAtTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off)
+{
+    if (pReNative->PostponedEfl.fEFlags)
+    {
+        PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
+        return iemNativeDoPostponedEFlagsAtTbExitInternal<a_bmInputRegs>(pReNative, off, pCodeBuf);
+    }
+    return off;
+}
+
+
+template<uint32_t const a_bmInputRegs>
+DECL_FORCE_INLINE_THROW(uint32_t)
+iemNativeDoPostponedEFlagsAtTbExitEx(PIEMRECOMPILERSTATE pReNative, uint32_t off, PIEMNATIVEINSTR pCodeBuf)
+{
+    if (pReNative->PostponedEfl.fEFlags)
+        return iemNativeDoPostponedEFlagsAtTbExitInternal<a_bmInputRegs>(pReNative, off, pCodeBuf);
+    return off;
+}
+
+
+#endif /* IEMNATIVE_WITH_EFLAGS_POSTPONING */
+
+
 /**
  * This is an implementation of IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGICAL.
@@ -201 +460 @@
  */
 DECL_INLINE_THROW(uint32_t)
-iemNativeEmitEFlagsForLogical(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarEfl
+iemNativeEmitEFlagsForLogical(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarEfl,
+                              uint8_t cOpBits, uint8_t idxRegResult
 #ifndef RT_ARCH_AMD64
-                              , uint8_t cOpBits, uint8_t idxRegResult, bool fNativeFlags = false
+                              , bool fNativeFlags = false
 #endif
                               )
 {
     STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativeEflTotalLogical);
+    IEMNATIVE_CLEAR_POSTPONED_EFLAGS(pReNative, X86_EFL_STATUS_BITS);
+    RT_NOREF(cOpBits, idxRegResult);
 
 #ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
@@ -214 +476 @@
      */
     PCIEMLIVENESSENTRY const pLivenessEntry = &pReNative->paLivenessEntries[pReNative->idxCurCall];
-    if (   IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(pLivenessEntry)
+    uint64_t const           fEflClobbered  = IEMLIVENESS_STATE_GET_WILL_BE_CLOBBERED_SET(pLivenessEntry)
+                                            & IEMLIVENESSBIT_STATUS_EFL_MASK;
+# ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+    uint64_t                 fEflPostponing;
+# endif
+    if (   fEflClobbered == IEMLIVENESSBIT_STATUS_EFL_MASK
         && !(pReNative->fMc & IEM_MC_F_WITH_FLAGS))
     {
@@ -222 +489 @@
         off = iemNativeEmitOrImmIntoVCpuU32(pReNative, off, X86_EFL_STATUS_BITS, RT_UOFFSETOF(VMCPU, iem.s.fSkippingEFlags));
 # endif
-    }
+        Log5(("iemNativeEmitEFlagsForLogical: Skipping %#x\n", X86_EFL_STATUS_BITS));
+    }
+# ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+    else if (      (  (fEflPostponing = IEMLIVENESS_STATE_GET_CAN_BE_POSTPONED_SET(pLivenessEntry) & IEMLIVENESSBIT_STATUS_EFL_MASK)
+                    | fEflClobbered)
+                == IEMLIVENESSBIT_STATUS_EFL_MASK
+             && idxRegResult != UINT8_MAX)
+    {
+        STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativeEflPostponedLogical);
+        pReNative->fSkippingEFlags       = 0;
+        pReNative->PostponedEfl.fEFlags  = X86_EFL_STATUS_BITS;
+        pReNative->PostponedEfl.enmOp    = kIemNativePostponedEflOp_Logical;
+        pReNative->PostponedEfl.cOpBits  = cOpBits;
+        pReNative->PostponedEfl.idxReg1  = iemNativeRegAllocTmpEx(pReNative, &off, IEMNATIVE_POSTPONING_REG_MASK, false);
+        /** @todo it would normally be possible to use idxRegResult, iff it is
+         *        already a non-volatile register and we can be user the caller
+         *        doesn't modify it.  That'll save a register move and allocation. */
+        off = iemNativeEmitLoadGprFromGpr(pReNative, off, pReNative->PostponedEfl.idxReg1, idxRegResult);
+        Log5(("iemNativeEmitEFlagsForLogical: Postponing %#x op=%u bits=%u reg1=%u\n", X86_EFL_STATUS_BITS,
+              kIemNativePostponedEflOp_Logical, cOpBits, pReNative->PostponedEfl.idxReg1));
+    }
+# endif
     else
 #endif
@@ -317 +605 @@
 {
     STAM_COUNTER_INC(&pReNative->pVCpu->iem.s.StatNativeEflTotalArithmetic);
+    IEMNATIVE_CLEAR_POSTPONED_EFLAGS(pReNative, X86_EFL_STATUS_BITS);
 
 #ifdef IEMNATIVE_WITH_EFLAGS_SKIPPING
@@ -504 +793 @@
 
 
+
+/*********************************************************************************************************************************
+*   Bitwise Logical Operations                                                                                                   *
+*********************************************************************************************************************************/
+
 /**
  * The AND instruction will clear OF, CF and AF (latter is undefined) and
@@ -521 +815 @@
     iemNativeVarRegisterRelease(pReNative, idxVarSrc);
 
-    off = iemNativeEmitEFlagsForLogical(pReNative, off, idxVarEfl);
+    off = iemNativeEmitEFlagsForLogical(pReNative, off, idxVarEfl, cOpBits, idxRegDst);
 
 #elif defined(RT_ARCH_ARM64)
@@ -553 +847 @@
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
 
-    off = iemNativeEmitEFlagsForLogical(pReNative, off, idxVarEfl);
+    off = iemNativeEmitEFlagsForLogical(pReNative, off, idxVarEfl, cOpBits, idxRegDst);
 
 #elif defined(RT_ARCH_ARM64)
@@ -628 +922 @@
 
 #ifdef RT_ARCH_AMD64
-    off = iemNativeEmitEFlagsForLogical(pReNative, off, idxVarEfl);
+    off = iemNativeEmitEFlagsForLogical(pReNative, off, idxVarEfl, cOpBits, UINT8_MAX);
 #else
     off = iemNativeEmitEFlagsForLogical(pReNative, off, idxVarEfl, cOpBits, idxRegResult, cOpBits >= 32 /*fNativeFlags*/);
@@ -652 +946 @@
     iemNativeVarRegisterRelease(pReNative, idxVarDst);
 
-    off = iemNativeEmitEFlagsForLogical(pReNative, off, idxVarEfl);
+    off = iemNativeEmitEFlagsForLogical(pReNative, off, idxVarEfl, cOpBits, UINT8_MAX);
 
 #elif defined(RT_ARCH_ARM64)
@@ -712 +1006 @@
     iemNativeVarRegisterRelease(pReNative, idxVarSrc);
 
-    off = iemNativeEmitEFlagsForLogical(pReNative, off, idxVarEfl);
+    off = iemNativeEmitEFlagsForLogical(pReNative, off, idxVarEfl, cOpBits, idxRegDst);
 
 #elif defined(RT_ARCH_ARM64)
@@ -745 +1039 @@
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
 
-    off = iemNativeEmitEFlagsForLogical(pReNative, off, idxVarEfl);
+    off = iemNativeEmitEFlagsForLogical(pReNative, off, idxVarEfl, cOpBits, idxRegDst);
 
 #elif defined(RT_ARCH_ARM64)
@@ -795 +1089 @@
     iemNativeVarRegisterRelease(pReNative, idxVarSrc);
 
-    off = iemNativeEmitEFlagsForLogical(pReNative, off, idxVarEfl);
+    off = iemNativeEmitEFlagsForLogical(pReNative, off, idxVarEfl, cOpBits, idxRegDst);
 
 #elif defined(RT_ARCH_ARM64)
@@ -828 +1122 @@
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
 
-    off = iemNativeEmitEFlagsForLogical(pReNative, off, idxVarEfl);
+    off = iemNativeEmitEFlagsForLogical(pReNative, off, idxVarEfl, cOpBits, idxRegDst);
 
 #elif defined(RT_ARCH_ARM64)
@@ -2733 +3027 @@
     /* tmp &= mxcsr */
     off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, idxRegMxCsrXcptFlags, idxRegTmp);
-    off = iemNativeEmitTestAnyBitsInGprAndTbExitIfAnySet(pReNative, off, idxRegMxCsrXcptFlags, X86_MXCSR_XCPT_FLAGS,
-                                                         kIemNativeLabelType_RaiseSseAvxFpRelated);
+    off = iemNativeEmitTbExitIfAnyBitsSetInGpr<kIemNativeLabelType_RaiseSseAvxFpRelated>(pReNative, off, idxRegMxCsrXcptFlags,
+                                                                                         X86_MXCSR_XCPT_FLAGS);
 
     uint8_t const idxSimdRegDst = iemNativeSimdRegAllocTmpForGuestSimdReg(pReNative, &off, IEMNATIVEGSTSIMDREG_SIMD(idxSimdGstRegDst),

trunk/src/VBox/VMM/VMMR3/IEMR3.cpp

@@ -731 +731 @@
                         "/IEM/CPU%u/re/NativeRegFindFreeLivenessHelped", idCpu);
 
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.StatNativeEflSkippedArithmetic,      STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "Skipped all status flag updating, arithmetic instructions",
-                        "/IEM/CPU%u/re/NativeEFlags/ArithmeticSkipped", idCpu);
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.StatNativeEflTotalArithmetic,        STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "Total number of arithmetic intructions with status flag updating",
-                        "/IEM/CPU%u/re/NativeEFlags/ArithmeticTotal", idCpu);
-        RTStrPrintf(szPat, sizeof(szPat), "/IEM/CPU%u/re/NativeEFlags/ArithmeticTotal", idCpu);
-        RTStrPrintf(szVal, sizeof(szVal), "/IEM/CPU%u/re/NativeEFlags/ArithmeticSkipped", idCpu);
-        STAMR3RegisterPctOfSum(pVM->pUVM, STAMVISIBILITY_ALWAYS, STAMUNIT_PCT, szVal, false, szPat,
-                               "Skipped all status flag updating, arithmetic instructions, percentage",
-                               "/IEM/CPU%u/re/NativeEFlags/ArithmeticSkippedPct", idCpu);
-
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.StatNativeEflSkippedLogical,         STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "Skipped all status flag updating, logical instructions",
-                        "/IEM/CPU%u/re/NativeEFlags/LogicalSkipped", idCpu);
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.StatNativeEflTotalLogical,           STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "Total number of logical intructions with status flag updating",
-                        "/IEM/CPU%u/re/NativeEFlags/LogicalTotal", idCpu);
-        RTStrPrintf(szPat, sizeof(szPat), "/IEM/CPU%u/re/NativeEFlags/LogicalTotal", idCpu);
-        RTStrPrintf(szVal, sizeof(szVal), "/IEM/CPU%u/re/NativeEFlags/LogicalSkipped", idCpu);
-        STAMR3RegisterPctOfSum(pVM->pUVM, STAMVISIBILITY_ALWAYS, STAMUNIT_PCT, szVal, false, szPat,
-                               "Skipped all status flag updating, logical instructions, percentage",
-                               "/IEM/CPU%u/re/NativeEFlags/LogicalSkippedPct", idCpu);
-
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.StatNativeEflSkippedShift,           STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "Skipped all status flag updating, shift instructions",
-                        "/IEM/CPU%u/re/NativeEFlags/ShiftSkipped", idCpu);
-        STAMR3RegisterF(pVM, &pVCpu->iem.s.StatNativeEflTotalShift,             STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "Total number of shift intructions with status flag updating",
-                        "/IEM/CPU%u/re/NativeEFlags/ShiftTotal", idCpu);
-        RTStrPrintf(szPat, sizeof(szPat), "/IEM/CPU%u/re/NativeEFlags/ShiftTotal", idCpu);
-        RTStrPrintf(szVal, sizeof(szVal), "/IEM/CPU%u/re/NativeEFlags/ShiftSkipped", idCpu);
-        STAMR3RegisterPctOfSum(pVM->pUVM, STAMVISIBILITY_ALWAYS, STAMUNIT_PCT, szVal, false, szPat,
-                               "Skipped all status flag updating, shift instructions, percentage",
-                               "/IEM/CPU%u/re/NativeEFlags/ShiftSkippedPct", idCpu);
+#   define REG_NATIVE_EFL_GROUP(a_Lower, a_Camel) do { \
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.StatNativeEflPostponed ## a_Camel, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, \
+                        "Postponed all status flag updating, " #a_Lower " instructions", \
+                        "/IEM/CPU%u/re/NativeEFlags/" #a_Camel "Postponed", idCpu); \
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.StatNativeEflSkipped ## a_Camel,   STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, \
+                        "Skipped all status flag updating, " #a_Lower " instructions", \
+                        "/IEM/CPU%u/re/NativeEFlags/" #a_Camel "Skipped", idCpu); \
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.StatNativeEflTotal ## a_Camel,     STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, \
+                        "Total number of " #a_Lower " intructions with status flag updating", \
+                        "/IEM/CPU%u/re/NativeEFlags/" #a_Camel "Total", idCpu); \
+        RTStrPrintf(szPat, sizeof(szPat), "/IEM/CPU%u/re/NativeEFlags/" #a_Camel "Total", idCpu); \
+        RTStrPrintf(szVal, sizeof(szVal), "/IEM/CPU%u/re/NativeEFlags/" #a_Camel "Postponed", idCpu); \
+        STAMR3RegisterPctOfSum(pVM->pUVM, STAMVISIBILITY_ALWAYS, STAMUNIT_PCT, szVal, false, szPat, \
+                               "Postponed all status flag updating, " #a_Lower " instructions, percentage", \
+                               "/IEM/CPU%u/re/NativeEFlags/" #a_Camel "PostponedPct", idCpu); \
+        RTStrPrintf(szVal, sizeof(szVal), "/IEM/CPU%u/re/NativeEFlags/" #a_Camel "Skipped", idCpu); \
+        STAMR3RegisterPctOfSum(pVM->pUVM, STAMVISIBILITY_ALWAYS, STAMUNIT_PCT, szVal, false, szPat, \
+                               "Skipped all status flag updating, " #a_Lower " instructions, percentage", \
+                               "/IEM/CPU%u/re/NativeEFlags/" #a_Camel "SkippedPct", idCpu); \
+    } while (0)
+        REG_NATIVE_EFL_GROUP(arithmetic, Arithmetic);
+        REG_NATIVE_EFL_GROUP(logical,    Logical);
+        REG_NATIVE_EFL_GROUP(shift,      Shift);
+#   undef REG_NATIVE_EFL_GROUP
 
         STAMR3RegisterF(pVM, &pVCpu->iem.s.StatNativeLivenessEflCfSkippable,    STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Skippable EFLAGS.CF updating",       "/IEM/CPU%u/re/NativeLivenessEFlags/CfSkippable", idCpu);

trunk/src/VBox/VMM/include/IEMN8veRecompiler.h

@@ -80 +80 @@
 # if 0 || defined(DOXYGEN_RUNNING)
 #  define IEMNATIVE_WITH_EFLAGS_POSTPONING
+# endif
+#endif
+#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+# ifndef IEMNATIVE_WITH_EFLAGS_SKIPPING
+#  error "IEMNATIVE_WITH_EFLAGS_POSTPONING requires IEMNATIVE_WITH_EFLAGS_SKIPPING at present"
 # endif
 #endif
@@ -314 +319 @@
  * Mask of registers the callee will not save and may trash. */
 #ifdef RT_ARCH_AMD64
-# define IEMNATIVE_CALL_RET_GREG             X86_GREG_xAX
+# define IEMNATIVE_CALL_RET_GREG            X86_GREG_xAX
 
 # ifdef RT_OS_WINDOWS
@@ -435 +440 @@
 # define IEMNATIVE_CALL_VOLATILE_NOTMP_GREG_MASK    IEMNATIVE_CALL_VOLATILE_GREG_MASK
 #endif
+
+/** @def IEMNATIVE_CALL_NONVOLATILE_GREG_MASK
+ * The allocatable non-volatile general purpose register set.  */
+#define IEMNATIVE_CALL_NONVOLATILE_GREG_MASK \
+    (~IEMNATIVE_CALL_VOLATILE_GREG_MASK & ~IEMNATIVE_REG_FIXED_MASK & IEMNATIVE_HST_GREG_MASK)
 /** @} */
 
@@ -480 +490 @@
 {
     kIemNativeLabelType_Invalid = 0,
-    /*
-     * Labels w/o data, only once instance per TB - aka exit reasons.
+    /** @name Exit reasons - Labels w/o data, only once instance per TB.
      *
-     * Note! Jumps to these requires instructions that are capable of spanning
-     *       the max TB length.
+     * The labels requiring register inputs are documented.
+     *
+     * @note Jumps to these requires instructions that are capable of spanning the
+     *       max TB length.
+     * @{
      */
     /* Simple labels comes first for indexing reasons. RaiseXx is order by the exception's numerical value(s). */
@@ -496 +508 @@
     kIemNativeLabelType_RaiseMf,                /**< Raise (throw) X86_XCPT_MF (10h). */
     kIemNativeLabelType_RaiseXf,                /**< Raise (throw) X86_XCPT_XF (13h). */
-    kIemNativeLabelType_ObsoleteTb,
-    kIemNativeLabelType_NeedCsLimChecking,
-    kIemNativeLabelType_CheckBranchMiss,
+    kIemNativeLabelType_ObsoleteTb,             /**< Calls iemNativeHlpObsoleteTb (no inputs). */
+    kIemNativeLabelType_NeedCsLimChecking,      /**< Calls iemNativeHlpNeedCsLimChecking (no inputs). */
+    kIemNativeLabelType_CheckBranchMiss,        /**< Calls iemNativeHlpCheckBranchMiss (no inputs). */
     kIemNativeLabelType_LastSimple = kIemNativeLabelType_CheckBranchMiss,
-    /* Manually defined labels. */
+
+    /* Manually defined labels: */
+    /**< Returns with VINF_SUCCESS, no inputs. */
+    kIemNativeLabelType_ReturnSuccess,
+    /** Returns with VINF_IEM_REEXEC_FINISH_WITH_FLAGS, no inputs. */
+    kIemNativeLabelType_ReturnWithFlags,
+    /** Returns with VINF_IEM_REEXEC_BREAK, no inputs. */
     kIemNativeLabelType_ReturnBreak,
+    /** Returns with VINF_IEM_REEXEC_BREAK_FF, no inputs. */
     kIemNativeLabelType_ReturnBreakFF,
+    /** The last TB exit label that doesn't have any input registers. */
+    kIemNativeLabelType_LastTbExitWithoutInputs = kIemNativeLabelType_ReturnBreakFF,
+
+    /** Argument registers 1, 2 & 3 are set up.  */
     kIemNativeLabelType_ReturnBreakViaLookup,
+    /** Argument registers 1, 2 & 3 are set up.  */
     kIemNativeLabelType_ReturnBreakViaLookupWithIrq,
+    /** Argument registers 1 & 2 are set up.  */
     kIemNativeLabelType_ReturnBreakViaLookupWithTlb,
+    /** Argument registers 1 & 2 are set up.  */
     kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq,
-    kIemNativeLabelType_ReturnWithFlags,
+    /** Return register holds the RC and the instruction number is in CL/RCX
+     * on amd64 and the 2rd argument register elsewhere. */
     kIemNativeLabelType_NonZeroRetOrPassUp,
-    kIemNativeLabelType_ReturnSuccess,          /**< Sets eax/w0 to zero and returns. */
+
     /** The last fixup for branches that can span almost the whole TB length.
      * @note Whether kIemNativeLabelType_Return needs to be one of these is
      *       a bit questionable, since nobody jumps to it except other tail code. */
-    kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_ReturnSuccess,
+    kIemNativeLabelType_LastWholeTbBranch = kIemNativeLabelType_NonZeroRetOrPassUp,
     /** The last fixup for branches that exits the TB. */
-    kIemNativeLabelType_LastTbExit        = kIemNativeLabelType_ReturnSuccess,
+    kIemNativeLabelType_LastTbExit        = kIemNativeLabelType_NonZeroRetOrPassUp,
+    /** @} */
 
     /** Loop-jump target. */
@@ -538 +566 @@
 #define IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmLabel) \
     ((a_enmLabel) <= kIemNativeLabelType_LastTbExit && (a_enmLabel) > kIemNativeLabelType_Invalid)
+
+#define IEMNATIVELABELTYPE_IS_EXIT_WITHOUT_INPUTS(a_enmLabel) \
+    ((a_enmLabel) <= kIemNativeLabelType_LastTbExitWithoutInputs && (a_enmLabel) > kIemNativeLabelType_Invalid)
+
+/**
+ * Get the mask of input registers for an TB exit label.
+ * This will return zero for any non-exit lable.
+ */
+#ifdef RT_ARCH_AMD64
+# define IEMNATIVELABELTYPE_GET_INPUT_REG_MASK(a_enmLabel) \
+    (     (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookup \
+       || (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithIrq \
+     ? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) | RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
+     :    (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlb \
+       || (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq \
+     ? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
+     : (a_enmLabel) == kIemNativeLabelType_NonZeroRetOrPassUp \
+     ? RT_BIT_32(IEMNATIVE_CALL_RET_GREG)  | RT_BIT_32(X86_GREG_xCX) /* <-- the difference */ \
+     : 0)
+# else
+# define IEMNATIVELABELTYPE_GET_INPUT_REG_MASK(a_enmLabel) \
+    (     (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookup \
+       || (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithIrq \
+     ? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) | RT_BIT_32(IEMNATIVE_CALL_ARG3_GREG) \
+     :    (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlb \
+       || (a_enmLabel) == kIemNativeLabelType_ReturnBreakViaLookupWithTlbAndIrq \
+     ? RT_BIT_32(IEMNATIVE_CALL_ARG1_GREG) | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
+     : (a_enmLabel) == kIemNativeLabelType_NonZeroRetOrPassUp \
+     ? RT_BIT_32(IEMNATIVE_CALL_RET_GREG)  | RT_BIT_32(IEMNATIVE_CALL_ARG2_GREG) \
+     : 0)
+#endif
 
 
@@ -660 +719 @@
 #define IEMLIVENESSBIT_IDX_EFL_SF       ((unsigned)kIemNativeGstReg_EFlags + 5)
 #define IEMLIVENESSBIT_IDX_EFL_OF       ((unsigned)kIemNativeGstReg_EFlags + 6)
+#define IEMLIVENESSBIT_IDX_EFL_COUNT    7
 
 
@@ -819 +879 @@
 # define IEMLIVENESS_STATE_ARE_STATUS_EFL_TO_BE_CLOBBERED(a_pCurEntry) \
     ( (((a_pCurEntry)->Bit0.bm64 | (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_STATUS_EFL_MASK) == 0 )
+
+/***
+ * Construct a mask of what will be clobbered and never used.
+ *
+ * This is mainly used with IEMLIVENESSBIT_STATUS_EFL_MASK to avoid
+ * unnecessary EFLAGS calculations.
+ *
+ * @param a_pCurEntry  The current liveness entry.
+ * @note  Used by actual code.
+ */
+# define IEMLIVENESS_STATE_GET_WILL_BE_CLOBBERED_SET(a_pCurEntry) \
+    ( ~((a_pCurEntry)->Bit0.bm64 | (a_pCurEntry)->Bit1.bm64) & IEMLIVENESSBIT_MASK )
 
 /** Construct a mask of the guest registers in the UNUSED and XCPT_OR_CALL
@@ -864 +936 @@
  * @note  Used by actual code. */
 # define IEMLIVENESS_STATE_GET_CAN_BE_FREED_SET(a_pCurEntry) \
-            (  ~(a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
-             & ~(a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 \
-             & IEMLIVENESSBIT_MASK )
-
+    (  ~(a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
+     & ~(a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 \
+     & IEMLIVENESSBIT_MASK )
+
+/***
+ * Construct a mask of what will be clobbered and never used.
+ *
+ * This is mainly used with IEMLIVENESSBIT_STATUS_EFL_MASK to avoid
+ * unnecessary EFLAGS calculations.
+ *
+ * @param a_pCurEntry  The current liveness entry.
+ * @note  Used by actual code.
+ */
+# define IEMLIVENESS_STATE_GET_WILL_BE_CLOBBERED_SET(a_pCurEntry) \
+    (  (a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 \
+     & ~(  (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \
+         | (a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
+         | (a_pCurEntry)->aBits[IEMLIVENESS_BIT_CALL].bm64) )
+
+/**
+ * Construct a mask of what (EFLAGS) which can be postponed.
+ *
+ * The postponement is for the avoiding EFLAGS status bits calculations in the
+ * primary code stream whenever possible, and instead only do these in the TLB
+ * load and TB exit code paths which shouldn't be traveled quite as often.
+ * A requirement, though, is that the status bits will be clobbered later in the
+ * TB.
+ *
+ * User need to apply IEMLIVENESSBIT_STATUS_EFL_MASK if appropriate/necessary.
+ *
+ * @param a_pCurEntry  The current liveness entry.
+ * @note  Used by actual code.
+ */
+# define IEMLIVENESS_STATE_GET_CAN_BE_POSTPONED_SET(a_pCurEntry) \
+    (  (a_pCurEntry)->aBits[IEMLIVENESS_BIT_POTENTIAL_CALL].bm64 \
+     & (a_pCurEntry)->aBits[IEMLIVENESS_BIT_WRITE].bm64 \
+     & ~(  (a_pCurEntry)->aBits[IEMLIVENESS_BIT_READ].bm64 \
+         | (a_pCurEntry)->aBits[IEMLIVENESS_BIT_CALL].bm64) )
 
 #endif /* IEMLIVENESS_EXTENDED_LAYOUT */
@@ -1039 +1145 @@
 #ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
 # define IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(a_pReNative, a_fEflNeeded) \
-    AssertMsg(!((a_pReNative)->fPostponingEFlags & (a_fEflNeeded)), \
-              ("%#x & %#x -> %#x\n", (a_pReNative)->fPostponingEFlags, \
-               a_fEflNeeded, (a_pReNative)->fPostponingEFlags & (a_fEflNeeded) ))
+    AssertMsg(!((a_pReNative)->PostponedEfl.fEFlags & (a_fEflNeeded)), \
+              ("%#x & %#x -> %#x\n", (a_pReNative)->PostponedEfl.fEFlags, \
+               a_fEflNeeded, (a_pReNative)->PostponedEfl.fEFlags & (a_fEflNeeded) ))
 #else
 # define IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(a_pReNative, a_fEflNeeded) ((void)0)
@@ -1052 +1158 @@
 #if defined(IEMNATIVE_WITH_EFLAGS_SKIPPING) && defined(IEMNATIVE_WITH_EFLAGS_POSTPONING)
 # define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) \
-    AssertMsg(!(((a_pReNative)->fSkippingEFlags | (a_pReNative)->fPostponingEFlags) & (a_fEflNeeded)), \
-              ("(%#x | %#x) & %#x -> %#x\n", (a_pReNative)->fSkippingEFlags, (a_pReNative)->fPostponingEFlags, \
-               a_fEflNeeded, ((a_pReNative)->fSkippingEFlags | (a_pReNative)->fPostponingEFlags) & (a_fEflNeeded) ))
+    AssertMsg(!(((a_pReNative)->fSkippingEFlags | (a_pReNative)->PostponedEfl.fEFlags) & (a_fEflNeeded)), \
+              ("(%#x | %#x) & %#x -> %#x\n", (a_pReNative)->fSkippingEFlags, (a_pReNative)->PostponedEfl.fEFlags, \
+               a_fEflNeeded, ((a_pReNative)->fSkippingEFlags | (a_pReNative)->PostponedEfl.fEFlags) & (a_fEflNeeded) ))
 #elif defined(IEMNATIVE_WITH_EFLAGS_SKIPPING)
 # define IEMNATIVE_ASSERT_EFLAGS_SKIPPING_AND_POSTPONING(a_pReNative, a_fEflNeeded) \
@@ -1077 +1183 @@
 #else
 # define IEMNATIVE_STRICT_EFLAGS_SKIPPING_EMIT_CHECK(a_pReNative, a_off, a_fEflNeeded) do { } while (0)
+#endif
+
+
+/** @def IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS
+ * Number of extra instructions to allocate for each TB exit to account for
+ * postponed EFLAGS calculations.
+ */
+#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+# ifdef RT_ARCH_AMD64
+#  define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS   32
+# elif defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING)
+#  define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS   32
+# else
+#  error "port me"
+# endif
+#else
+# define IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS    0
+#endif
+
+/** @def IEMNATIVE_CLEAR_POSTPONED_EFLAGS
+ * Helper macro function for calling iemNativeClearPostponedEFlags() when
+ * IEMNATIVE_WITH_EFLAGS_POSTPONING is enabled.
+ */
+#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+# define IEMNATIVE_CLEAR_POSTPONED_EFLAGS(a_pReNative, a_fEflClobbered) iemNativeClearPostponedEFlags<a_fEflClobbered>(a_pReNative)
+#else
+# define IEMNATIVE_CLEAR_POSTPONED_EFLAGS(a_pReNative, a_fEflClobbered) ((void)0)
 #endif
 
@@ -1533 +1666 @@
 
 
+#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+typedef enum IEMNATIVE_POSTPONED_EFL_OP_T : uint8_t
+{
+    kIemNativePostponedEflOp_Invalid = 0,
+    /** Logical operation.
+     * Operands: result register.
+     * @note This clears OF, CF and (undefined) AF, thus no need for inputs. */
+    kIemNativePostponedEflOp_Logical,
+    kIemNativePostponedEflOp_End
+} IEMNATIVE_POSTPONED_EFL_OP_T;
+#endif /* IEMNATIVE_WITH_EFLAGS_POSTPONING */
+
 /**
  * Conditional stack entry.
@@ -1670 +1815 @@
 #endif
 #ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
-    uint32_t                    fPostponingEFlags;
+    struct
+    {
+        /** EFLAGS status bits that we're currently postponing the calculcation of. */
+        uint32_t                        fEFlags;
+        /** The postponed EFLAGS status bits calculation operation. */
+        IEMNATIVE_POSTPONED_EFL_OP_T    enmOp;
+        /** The bit-width of the postponed EFLAGS calculation. */
+        uint8_t                         cOpBits;
+        /** Host register holding result or first source for the delayed operation,
+         *  UINT8_MAX if not in use. */
+        uint8_t                         idxReg1;
+        /** Host register holding second source for the delayed operation,
+         *  UINT8_MAX if not in use. */
+        uint8_t                         idxReg2;
+    } PostponedEfl;
 #endif
 

trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h

@@ -234 +234 @@
         pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, uImm32 & UINT32_C(0xffff), 0, false /*f64Bit*/);
         pCodeBuf[off++] = Armv8A64MkInstrMovK(iGpr, uImm32 >> 16,              1, false /*f64Bit*/);
+    }
+
+#else
+# error "port me"
+#endif
+    return off;
+}
+
+
+/**
+ * Variant of iemNativeEmitLoadGpr32Imm where the caller ensures sufficent
+ * buffer space.
+ *
+ * Max buffer consumption:
+ *      - AMD64: 6 instruction bytes.
+ *      - ARM64: 2 instruction words (8 bytes).
+ *
+ * @note The top 32 bits will be cleared.
+ */
+template<uint32_t const a_uImm32>
+DECL_FORCE_INLINE(uint32_t) iemNativeEmitLoadGpr32ImmExT(PIEMNATIVEINSTR pCodeBuf, uint32_t off, uint8_t iGpr)
+{
+#ifdef RT_ARCH_AMD64
+    if (a_uImm32 == 0)
+    {
+        /* xor gpr, gpr */
+        if (iGpr >= 8)
+            pCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
+        pCodeBuf[off++] = 0x33;
+        pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
+    }
+    else
+    {
+        /* mov gpr, imm32 */
+        if (iGpr >= 8)
+            pCodeBuf[off++] = X86_OP_REX_B;
+        pCodeBuf[off++] = 0xb8 + (iGpr & 7);
+        pCodeBuf[off++] = RT_BYTE1(a_uImm32);
+        pCodeBuf[off++] = RT_BYTE2(a_uImm32);
+        pCodeBuf[off++] = RT_BYTE3(a_uImm32);
+        pCodeBuf[off++] = RT_BYTE4(a_uImm32);
+    }
+
+#elif defined(RT_ARCH_ARM64)
+    if RT_CONSTEXPR((a_uImm32 >> 16) == 0)
+        /* movz gpr, imm16 */
+        pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, a_uImm32,                    0, false /*f64Bit*/);
+    else if RT_CONSTEXPR((a_uImm32 & UINT32_C(0xffff)) == 0)
+        /* movz gpr, imm16, lsl #16 */
+        pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, a_uImm32 >> 16,              1, false /*f64Bit*/);
+    else if RT_CONSTEXPR((a_uImm32 & UINT32_C(0xffff)) == UINT32_C(0xffff))
+        /* movn gpr, imm16, lsl #16 */
+        pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~a_uImm32 >> 16,             1, false /*f64Bit*/);
+    else if RT_CONSTEXPR((a_uImm32 >> 16) == UINT32_C(0xffff))
+        /* movn gpr, imm16 */
+        pCodeBuf[off++] = Armv8A64MkInstrMovN(iGpr, ~a_uImm32,                   0, false /*f64Bit*/);
+    else
+    {
+        pCodeBuf[off++] = Armv8A64MkInstrMovZ(iGpr, a_uImm32 & UINT32_C(0xffff), 0, false /*f64Bit*/);
+        pCodeBuf[off++] = Armv8A64MkInstrMovK(iGpr, a_uImm32 >> 16,              1, false /*f64Bit*/);
     }
 
@@ -8156 +8216 @@
  * Emits a call to a 64-bit address.
  */
+template<bool const a_fSkipEflChecks = false>
 DECL_INLINE_THROW(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)
 {
+    if RT_CONSTEXPR(!a_fSkipEflChecks)
+    {
+        IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS);
+        IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(  pReNative, X86_EFL_STATUS_BITS);
+    }
+
 #ifdef RT_ARCH_AMD64
     off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);
@@ -8281 +8348 @@
 *********************************************************************************************************************************/
 
+#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+/* IEMAllN8veEmit-x86.h: */
+template<uint32_t const a_bmInputRegs>
+DECL_FORCE_INLINE_THROW(uint32_t)
+iemNativeDoPostponedEFlagsAtTbExitEx(PIEMRECOMPILERSTATE pReNative, uint32_t off, PIEMNATIVEINSTR pCodeBuf);
+
+template<uint32_t const a_bmInputRegs>
+DECL_FORCE_INLINE_THROW(uint32_t)
+iemNativeDoPostponedEFlagsAtTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off);
+#endif
+
+
 /**
  * Helper for marking the current conditional branch as exiting the TB.
@@ -8298 +8377 @@
 
 /**
+ * Unconditionally exits the translation block via a branch instructions.
+ *
+ * @note In case a delayed EFLAGS calculation is pending, this may emit an
+ *       additional IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS instructions.
+ */
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fActuallyExitingTb = true, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t) iemNativeEmitTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off)
+{
+    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
+    AssertCompile(IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmExitReason));
+
+    if RT_CONSTEXPR(a_fActuallyExitingTb)
+        iemNativeMarkCurCondBranchAsExiting(pReNative);
+
+#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+    if RT_CONSTEXPR(a_fPostponedEfl)
+        off = iemNativeDoPostponedEFlagsAtTbExitEx<IEMNATIVELABELTYPE_GET_INPUT_REG_MASK(a_enmExitReason)>(pReNative, off,
+                                                                                                           pCodeBuf);
+#endif
+
+#ifdef RT_ARCH_AMD64
+    /* jmp rel32 */
+    pCodeBuf[off++] = 0xe9;
+    iemNativeAddTbExitFixup(pReNative, off, a_enmExitReason);
+    pCodeBuf[off++] = 0xfe;
+    pCodeBuf[off++] = 0xff;
+    pCodeBuf[off++] = 0xff;
+    pCodeBuf[off++] = 0xff;
+
+#elif defined(RT_ARCH_ARM64)
+    iemNativeAddTbExitFixup(pReNative, off, a_enmExitReason);
+    pCodeBuf[off++] = Armv8A64MkInstrB(-1);
+
+#else
+# error "Port me!"
+#endif
+    return off;
+}
+
+
+/**
+ * Unconditionally exits the translation block via a branch instructions.
+ *
+ * @note In case a delayed EFLAGS calculation is pending, this may emit an
+ *       additional IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS instructions.
+ */
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fActuallyExitingTb = true, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t) iemNativeEmitTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off)
+{
+    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
+    AssertCompile(IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmExitReason));
+
+    if RT_CONSTEXPR(a_fActuallyExitingTb)
+        iemNativeMarkCurCondBranchAsExiting(pReNative);
+
+#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+    if RT_CONSTEXPR(a_fPostponedEfl)
+        off = iemNativeDoPostponedEFlagsAtTbExit<IEMNATIVELABELTYPE_GET_INPUT_REG_MASK(a_enmExitReason)>(pReNative, off);
+#endif
+
+#ifdef RT_ARCH_AMD64
+    PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
+
+    /* jmp rel32 */
+    pCodeBuf[off++] = 0xe9;
+    iemNativeAddTbExitFixup(pReNative, off, a_enmExitReason);
+    pCodeBuf[off++] = 0xfe;
+    pCodeBuf[off++] = 0xff;
+    pCodeBuf[off++] = 0xff;
+    pCodeBuf[off++] = 0xff;
+
+#elif defined(RT_ARCH_ARM64)
+    PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+    iemNativeAddTbExitFixup(pReNative, off, a_enmExitReason);
+    pCodeBuf[off++] = Armv8A64MkInstrB(-1);
+
+#else
+# error "Port me!"
+#endif
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+}
+
+
+/**
  * Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).
- */
+ *
+ * @note In case a delayed EFLAGS calculation is pending, this may emit an
+ *       additional IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS instructions.
+ */
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
 DECL_FORCE_INLINE_THROW(uint32_t)
-iemNativeEmitJccTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
-                         IEMNATIVELABELTYPE enmExitReason, IEMNATIVEINSTRCOND enmCond)
+iemNativeEmitTbExitJccEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off, IEMNATIVEINSTRCOND enmCond)
 {
     IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
-    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
-    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
+    AssertCompile(IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmExitReason));
+
+#ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+    if RT_CONSTEXPR(a_fPostponedEfl)
+        if (pReNative->PostponedEfl.fEFlags)
+        {
+            /* Jcc l_NonPrimaryCodeStreamTarget */
+            uint32_t const offFixup1 = off;
+            off = iemNativeEmitJccToFixedEx(pCodeBuf, off, off + 1, enmCond);
+
+            /* JMP l_PrimaryCodeStreamResume */
+            uint32_t const offFixup2 = off;
+            off = iemNativeEmitJmpToFixedEx(pCodeBuf, off, off + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
+
+            /* l_NonPrimaryCodeStreamTarget: */
+            iemNativeFixupFixedJump(pReNative, offFixup1, off);
+            off = iemNativeEmitTbExitEx<a_enmExitReason, false /*a_fActuallyExitingTb*/, true>(pReNative, pCodeBuf, off);
+
+            /* l_PrimaryCodeStreamResume: */
+            iemNativeFixupFixedJump(pReNative, offFixup2, off);
+            IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+            return off;
+        }
+#endif
 
 #if defined(RT_ARCH_AMD64)
@@ -8312 +8501 @@
     pCodeBuf[off++] = 0x0f;
     pCodeBuf[off++] = (uint8_t)enmCond | 0x80;
-    iemNativeAddTbExitFixup(pReNative, off, enmExitReason);
+    iemNativeAddTbExitFixup(pReNative, off, a_enmExitReason);
     pCodeBuf[off++] = 0x00;
     pCodeBuf[off++] = 0x00;
@@ -8321 +8510 @@
     /* ARM64 doesn't have the necessary jump range, so we jump via local label
        just like when we keep everything local. */
-    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
+    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, a_enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabel, enmCond);
 #endif
@@ -8331 +8520 @@
  * Emits a Jcc rel32 / B.cc imm19 to the epilog.
  */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitJccTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmExitReason, IEMNATIVEINSTRCOND enmCond)
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t) iemNativeEmitTbExitJcc(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEINSTRCOND enmCond)
 {
     IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
-    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
-    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
-
-#ifdef RT_ARCH_AMD64
-    off = iemNativeEmitJccTbExitEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 6), off, enmExitReason, enmCond);
-#elif defined(RT_ARCH_ARM64)
-    off = iemNativeEmitJccTbExitEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 2), off, enmExitReason, enmCond);
+    AssertCompile(IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmExitReason));
+
+#ifdef RT_ARCH_AMD64
+    PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS + 5);
+#elif defined(RT_ARCH_ARM64)
+    PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS + 1);
 #else
 # error "Port me!"
 #endif
+    off = iemNativeEmitTbExitJccEx<a_enmExitReason, a_fPostponedEfl>(pReNative, pCodeBuf, off, enmCond);
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
     return off;
@@ -8353 +8542 @@
  * Emits a JNZ/JNE rel32 / B.NE imm19 to the TB exit routine with the given reason.
  */
-DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmExitReason)
-{
-#ifdef RT_ARCH_AMD64
-    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kIemNativeInstrCond_ne);
-#elif defined(RT_ARCH_ARM64)
-    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kArmv8InstrCond_Ne);
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t) iemNativeEmitTbExitJnz(PIEMRECOMPILERSTATE pReNative, uint32_t off)
+{
+#ifdef RT_ARCH_AMD64
+    return iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kIemNativeInstrCond_ne);
+#elif defined(RT_ARCH_ARM64)
+    return iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kArmv8InstrCond_Ne);
 #else
 # error "Port me!"
@@ -8368 +8558 @@
  * Emits a JZ/JE rel32 / B.EQ imm19 to the TB exit routine with the given reason.
  */
-DECL_INLINE_THROW(uint32_t) iemNativeEmitJzTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmExitReason)
-{
-#ifdef RT_ARCH_AMD64
-    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kIemNativeInstrCond_e);
-#elif defined(RT_ARCH_ARM64)
-    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kArmv8InstrCond_Eq);
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t) iemNativeEmitTbExitJz(PIEMRECOMPILERSTATE pReNative, uint32_t off)
+{
+#ifdef RT_ARCH_AMD64
+    return iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kIemNativeInstrCond_e);
+#elif defined(RT_ARCH_ARM64)
+    return iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kArmv8InstrCond_Eq);
 #else
 # error "Port me!"
@@ -8383 +8574 @@
  * Emits a JA/JNBE rel32 / B.HI imm19 to the TB exit.
  */
-DECL_INLINE_THROW(uint32_t) iemNativeEmitJaTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmExitReason)
-{
-#ifdef RT_ARCH_AMD64
-    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kIemNativeInstrCond_nbe);
-#elif defined(RT_ARCH_ARM64)
-    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kArmv8InstrCond_Hi);
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t) iemNativeEmitTbExitJa(PIEMRECOMPILERSTATE pReNative, uint32_t off)
+{
+#ifdef RT_ARCH_AMD64
+    return iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kIemNativeInstrCond_nbe);
+#elif defined(RT_ARCH_ARM64)
+    return iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kArmv8InstrCond_Hi);
 #else
 # error "Port me!"
@@ -8398 +8590 @@
  * Emits a JL/JNGE rel32 / B.LT imm19 to the TB exit with the given reason.
  */
-DECL_INLINE_THROW(uint32_t) iemNativeEmitJlTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmExitReason)
-{
-#ifdef RT_ARCH_AMD64
-    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kIemNativeInstrCond_l);
-#elif defined(RT_ARCH_ARM64)
-    return iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kArmv8InstrCond_Lt);
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t) iemNativeEmitTbExitJl(PIEMRECOMPILERSTATE pReNative, uint32_t off)
+{
+#ifdef RT_ARCH_AMD64
+    return iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kIemNativeInstrCond_l);
+#elif defined(RT_ARCH_ARM64)
+    return iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kArmv8InstrCond_Lt);
 #else
 # error "Port me!"
@@ -8410 +8603 @@
 
 
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off, IEMNATIVELABELTYPE enmExitReason)
-{
-    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
-    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here. */
-    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
-
-    iemNativeMarkCurCondBranchAsExiting(pReNative);
-
-#ifdef RT_ARCH_AMD64
-    /* jmp rel32 */
-    pCodeBuf[off++] = 0xe9;
-    iemNativeAddTbExitFixup(pReNative, off, enmExitReason);
-    pCodeBuf[off++] = 0xfe;
-    pCodeBuf[off++] = 0xff;
-    pCodeBuf[off++] = 0xff;
-    pCodeBuf[off++] = 0xff;
-
-#elif defined(RT_ARCH_ARM64)
-    iemNativeAddTbExitFixup(pReNative, off, enmExitReason);
-    pCodeBuf[off++] = Armv8A64MkInstrB(-1);
-
-#else
-# error "Port me!"
-#endif
-    return off;
-}
-
-
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmExitReason,
-                    bool fActuallyExitingTb = true)
-{
-    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
-    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here. */
-    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
-
-    if (fActuallyExitingTb)
-        iemNativeMarkCurCondBranchAsExiting(pReNative);
-
-#ifdef RT_ARCH_AMD64
-    PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
-
-    /* jmp rel32 */
-    pCodeBuf[off++] = 0xe9;
-    iemNativeAddTbExitFixup(pReNative, off, enmExitReason);
-    pCodeBuf[off++] = 0xfe;
-    pCodeBuf[off++] = 0xff;
-    pCodeBuf[off++] = 0xff;
-    pCodeBuf[off++] = 0xff;
-
-#elif defined(RT_ARCH_ARM64)
-    PIEMNATIVEINSTR pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
-    iemNativeAddTbExitFixup(pReNative, off, enmExitReason);
-    pCodeBuf[off++] = Armv8A64MkInstrB(-1);
-
-#else
-# error "Port me!"
-#endif
-    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
-    return off;
-}
-
-
-/**
- * Emits a jump to the TB exit with @a enmExitReason on the condition _any_ of the bits in @a fBits
- * are set in @a iGprSrc.
- */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTestAnyBitsInGprAndTbExitIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
-                                               uint8_t iGprSrc, uint64_t fBits, IEMNATIVELABELTYPE enmExitReason)
+/**
+ * Emits a jump to the TB exit with @a a_enmExitReason on the condition _any_ of
+ * the bits in @a fBits are set in @a iGprSrc.
+ */
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTbExitIfAnyBitsSetInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
 {
     Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
 
     off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
-    return iemNativeEmitJnzTbExit(pReNative, off, enmExitReason);
-}
-
-
+    return iemNativeEmitTbExitJnz<a_enmExitReason, a_fPostponedEfl>(pReNative, off);
+}
+
+
+#if 0 /* unused */
 /**
  * Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits
  * are set in @a iGprSrc.
  */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTestAnyBitsInGprAndTbExitIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
-                                                uint8_t iGprSrc, uint64_t fBits, IEMNATIVELABELTYPE enmExitReason)
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTbExitIfNoBitsSetInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)
 {
     Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));
 
     off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);
-    return iemNativeEmitJzTbExit(pReNative, off, enmExitReason);
-}
-
-
+    return iemNativeEmitJzTbExit<a_enmExitReason, a_fPostponedEfl>(pReNative, off);
+}
+#endif
+
+
+#if 0 /* unused */
 /**
  * Emits code that exits the TB with the given reason if @a iGprLeft and @a iGprRight
  * differs.
  */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTestIfGprNotEqualGprAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
-                                           uint8_t iGprLeft, uint8_t iGprRight, IEMNATIVELABELTYPE enmExitReason)
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTbExitIfGprNotEqualGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)
 {
     off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);
-    off = iemNativeEmitJnzTbExit(pReNative, off, enmExitReason);
-    return off;
-}
+    off = iemNativeEmitJnzTbExit<a_enmExitReason, a_fPostponedEfl>(pReNative, off);
+    return off;
+}
+#endif
 
 
@@ -8522 +8655 @@
  * @a uImm.
  */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTestIfGpr32NotEqualImmAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
-                                             uint8_t iGprSrc, uint32_t uImm, IEMNATIVELABELTYPE enmExitReason)
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTbExitIfGpr32NotEqualImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint32_t uImm)
 {
     off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
-    off = iemNativeEmitJnzTbExit(pReNative, off, enmExitReason);
+    off = iemNativeEmitTbExitJnz<a_enmExitReason, a_fPostponedEfl>(pReNative, off);
     return off;
 }
@@ -8535 +8668 @@
  * Emits code that exits the current TB if @a iGprSrc differs from @a uImm.
  */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTestIfGprNotEqualImmAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
-                                           uint8_t iGprSrc, uint64_t uImm, IEMNATIVELABELTYPE enmExitReason)
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTbExitIfGprNotEqualImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t uImm)
 {
     off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);
-    off = iemNativeEmitJnzTbExit(pReNative, off, enmExitReason);
+    off = iemNativeEmitTbExitJnz<a_enmExitReason, a_fPostponedEfl>(pReNative, off);
     return off;
 }
@@ -8548 +8681 @@
  * Emits code that exits the current TB with the given reason if 32-bit @a iGprSrc equals @a uImm.
  */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTestIfGpr32EqualsImmAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
-                                           uint8_t iGprSrc, uint32_t uImm, IEMNATIVELABELTYPE enmExitReason)
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTbExitIfGpr32EqualsImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint32_t uImm)
 {
     off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);
-    off = iemNativeEmitJzTbExit(pReNative, off, enmExitReason);
-    return off;
-}
-
-
-/**
- * Emits code to exit the current TB with the reason @a enmExitReason on the condition that bit @a iBitNo _is_ _set_ in
- * @a iGprSrc.
+    off = iemNativeEmitTbExitJz<a_enmExitReason, a_fPostponedEfl>(pReNative, off);
+    return off;
+}
+
+
+/**
+ * Emits code to exit the current TB with the reason @a a_enmExitReason on the
+ * condition that bit @a iBitNo _is_ _set_ in @a iGprSrc.
  *
  * @note On ARM64 the range is only +/-8191 instructions.
  */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTestBitInGprAndTbExitIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,
-                                        uint8_t iGprSrc, uint8_t iBitNo, IEMNATIVELABELTYPE enmExitReason)
-{
-    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTbExitIfBitSetInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint8_t iBitNo)
+{
+    AssertCompile(IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmExitReason));
 
 #if defined(RT_ARCH_AMD64)
@@ -8581 +8714 @@
         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
         pbCodeBuf[off++] = (uint8_t)1 << iBitNo;
-        off = iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kIemNativeInstrCond_ne);
+        IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+        off = iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kIemNativeInstrCond_ne);
     }
     else
@@ -8594 +8728 @@
         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);
         pbCodeBuf[off++] = iBitNo;
-        off = iemNativeEmitJccTbExit(pReNative, off, enmExitReason, kIemNativeInstrCond_c);
-    }
-    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
-    return off;
-
-#else
-    /* ARM64 doesn't have the necessary jump range, so we jump via local label
-       just like when we keep everything local. */
+        IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+        off = iemNativeEmitTbExitJcc<a_enmExitReason, a_fPostponedEfl>(pReNative, off, kIemNativeInstrCond_c);
+    }
+    return off;
+
+#elif defined(RT_ARCH_ARM64)
     IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
-    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
-    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
+    /** @todo Perhaps we should always apply the PostponedEfl code pattern here,
+     *        it's the same number of instructions as the TST + B.CC stuff? */
+# ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+    if RT_CONSTEXPR(a_fPostponedEfl)
+        if (pReNative->PostponedEfl.fEFlags)
+        {
+            PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off,
+                                                                     3 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
+            pCodeBuf[off++] = Armv8A64MkInstrTbnz(1 /*l_NonPrimaryCodeStreamTarget*/, iGprSrc, iBitNo);
+            uint32_t const offFixup = off;
+            pCodeBuf[off++] = Armv8A64MkInstrB(0 /*l_PrimaryCodeStreamResume*/);
+            /* l_NonPrimaryCodeStreamTarget: */
+            off = iemNativeEmitTbExitEx<a_enmExitReason, false /*a_fActuallyExitingTb*/, true>(pReNative, pCodeBuf, off);
+            /* l_PrimaryCodeStreamResume: */
+            iemNativeFixupFixedJump(pReNative, offFixup, off);
+            IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+            return off;
+        }
+# endif
+    /* ARM64 doesn't have the necessary range to reach the per-chunk code, so
+       we go via a local trampoline. */
+    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, a_enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /*fJmpIfSet*/);
-#endif
-}
-
-
-/**
- * Emits code that exits the current TB with @a enmExitReason if @a iGprSrc is not zero.
+#else
+# error "port me"
+#endif
+}
+
+
+/**
+ * Emits code that exits the current TB with @a a_enmExitReason if @a iGprSrc is
+ * not zero.
  *
  * The operand size is given by @a f64Bit.
  */
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
 DECL_FORCE_INLINE_THROW(uint32_t)
-iemNativeEmitTestIfGprIsNotZeroAndTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
-                                           uint8_t iGprSrc, bool f64Bit, IEMNATIVELABELTYPE enmExitReason)
-{
-    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
+iemNativeEmitTbExitIfGprIsNotZeroEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
+                                    uint8_t iGprSrc, bool f64Bit)
+{
+    AssertCompile(IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmExitReason));
 
 #if defined(RT_ARCH_AMD64)
@@ -8631 +8787 @@
 
     /* jnz idxLabel  */
-    return iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, enmExitReason, kIemNativeInstrCond_ne);
-
-#else
-    /* ARM64 doesn't have the necessary jump range, so we jump via local label
-       just like when we keep everything local. */
+    return iemNativeEmitTbExitJccEx<a_enmExitReason, a_fPostponedEfl>(pReNative, pCodeBuf, off, kIemNativeInstrCond_ne);
+
+#elif defined(RT_ARCH_ARM64)
     IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
-    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
-    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
+# ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+    if RT_CONSTEXPR(a_fPostponedEfl)
+        if (pReNative->PostponedEfl.fEFlags)
+        {
+            pCodeBuf[off++] = Armv8A64MkInstrCbnz(1 /*l_NonPrimaryCodeStreamTarget*/, iGprSrc, f64Bit);
+            uint32_t const offFixup = off;
+            pCodeBuf[off++] = Armv8A64MkInstrB(0 /*l_PrimaryCodeStreamResume*/);
+            /* l_NonPrimaryCodeStreamTarget: */
+            off = iemNativeEmitTbExitEx<a_enmExitReason, false /*a_fActuallyExitingTb*/, true>(pReNative, pCodeBuf, off);
+            /* l_PrimaryCodeStreamResume: */
+            iemNativeFixupFixedJump(pReNative, offFixup, off);
+            IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+            return off;
+        }
+# endif
+    /* ARM64 doesn't have the necessary range to reach the per-chunk code, so
+       we go via a local trampoline. */
+    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, a_enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
                                                                 f64Bit, true /*fJmpIfNotZero*/, idxLabel);
-#endif
-}
-
-
-/**
- * Emits code to exit the current TB with the given reason @a enmExitReason if @a iGprSrc is not zero.
+#else
+# error "port me"
+#endif
+}
+
+
+/**
+ * Emits code to exit the current TB with the given reason @a a_enmExitReason if
+ * @a iGprSrc is not zero.
  *
  * The operand size is given by @a f64Bit.
  */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTestIfGprIsNotZeroAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
-                                         uint8_t iGprSrc, bool f64Bit, IEMNATIVELABELTYPE enmExitReason)
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTbExitIfGprIsNotZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit)
 {
 #if defined(RT_ARCH_AMD64)
-    off = iemNativeEmitTestIfGprIsNotZeroAndTbExitEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 3 + 6),
-                                                     off, iGprSrc, f64Bit, enmExitReason);
-    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
-    return off;
-#else
-    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
-    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
-    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
-    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, true /*fJmpIfNotZero*/, idxLabel);
-#endif
-}
-
-
-/**
- * Emits code that exits the current TB with @a enmExitReason if @a iGprSrc is zero.
+    PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3 + 6 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
+
+#else
+    PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
+#endif
+    off = iemNativeEmitTbExitIfGprIsNotZeroEx<a_enmExitReason, a_fPostponedEfl>(pReNative, pCodeBuf, off, iGprSrc, f64Bit);
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+}
+
+
+/**
+ * Emits code that exits the current TB with @a a_enmExitReason if @a iGprSrc is
+ * zero.
  *
  * The operand size is given by @a f64Bit.
  */
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
 DECL_FORCE_INLINE_THROW(uint32_t)
-iemNativeEmitTestIfGprIsZeroAndTbExitEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
-                                        uint8_t iGprSrc, bool f64Bit, IEMNATIVELABELTYPE enmExitReason)
-{
-    Assert(IEMNATIVELABELTYPE_IS_EXIT_REASON(enmExitReason));
+iemNativeEmitTbExitIfGprIsZeroEx(PIEMRECOMPILERSTATE pReNative, PIEMNATIVEINSTR pCodeBuf, uint32_t off,
+                                 uint8_t iGprSrc, bool f64Bit)
+{
+    AssertCompile(IEMNATIVELABELTYPE_IS_EXIT_REASON(a_enmExitReason));
+
 #if defined(RT_ARCH_AMD64)
     /* test reg32,reg32  / test reg64,reg64 */
@@ -8688 +8862 @@
 
     /* jnz idxLabel  */
-    return iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, enmExitReason, kIemNativeInstrCond_e);
-
-#else
-    /* ARM64 doesn't have the necessary jump range, so we jump via local label
-       just like when we keep everything local. */
+    return iemNativeEmitTbExitJccEx<a_enmExitReason, a_fPostponedEfl>(pReNative, pCodeBuf, off, kIemNativeInstrCond_e);
+
+#elif defined(RT_ARCH_ARM64)
     IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
-    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
-    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
+# ifdef IEMNATIVE_WITH_EFLAGS_POSTPONING
+    if RT_CONSTEXPR(a_fPostponedEfl)
+        if (pReNative->PostponedEfl.fEFlags)
+        {
+            pCodeBuf[off++] = Armv8A64MkInstrCbz(1 /*l_NonPrimaryCodeStreamTarget*/, iGprSrc, f64Bit);
+            uint32_t const offFixup = off;
+            pCodeBuf[off++] = Armv8A64MkInstrB(0 /*l_PrimaryCodeStreamResume*/);
+            /* l_NonPrimaryCodeStreamTarget: */
+            off = iemNativeEmitTbExitEx<a_enmExitReason, false /*a_fActuallyExitingTb*/, true>(pReNative, pCodeBuf, off);
+            /* l_PrimaryCodeStreamResume: */
+            iemNativeFixupFixedJump(pReNative, offFixup, off);
+            IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+            return off;
+        }
+# endif
+    /* ARM64 doesn't have the necessary range to reach the per-chunk code, so
+       we go via a local trampoline. */
+    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, a_enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
     return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, iGprSrc,
                                                                 f64Bit, false /*fJmpIfNotZero*/, idxLabel);
-#endif
-}
-
-
-/**
- * Emits code to exit the current TB with the given reason @a enmExitReason if @a iGprSrc is zero.
+#else
+# error "port me"
+#endif
+}
+
+
+/**
+ * Emits code to exit the current TB with the given reason @a a_enmExitReason if @a iGprSrc is zero.
  *
  * The operand size is given by @a f64Bit.
  */
-DECL_INLINE_THROW(uint32_t)
-iemNativeEmitTestIfGprIsZeroAndTbExit(PIEMRECOMPILERSTATE pReNative, uint32_t off,
-                                      uint8_t iGprSrc, bool f64Bit, IEMNATIVELABELTYPE enmExitReason)
+template<IEMNATIVELABELTYPE const a_enmExitReason, bool const a_fPostponedEfl = true>
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTbExitIfGprIsZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit)
 {
 #if defined(RT_ARCH_AMD64)
-    off = iemNativeEmitTestIfGprIsZeroAndTbExitEx(pReNative, iemNativeInstrBufEnsure(pReNative, off, 3 + 6),
-                                                  off, iGprSrc, f64Bit, enmExitReason);
-    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
-    return off;
-#else
-    IEMNATIVE_ASSERT_EFLAGS_SKIPPING_ONLY(pReNative, X86_EFL_STATUS_BITS);
-    IEMNATIVE_ASSERT_EFLAGS_POSTPONING_ONLY(pReNative, X86_EFL_STATUS_BITS); /** @todo emit postponed stuff here and invert the condition. */
-    uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmExitReason, UINT32_MAX /*offWhere*/, 0 /*uData*/);
-    return iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, false /*fJmpIfNotZero*/, idxLabel);
-#endif
+    PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3 + 6 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
+
+#else
+    PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3 + IEMNATIVE_MAX_POSTPONED_EFLAGS_INSTRUCTIONS);
+#endif
+    off = iemNativeEmitTbExitIfGprIsZeroEx<a_enmExitReason, a_fPostponedEfl>(pReNative, pCodeBuf, off, iGprSrc, f64Bit);
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
 }