Changeset 101640 in vbox for trunk/src

Timestamp:

Oct 28, 2023 1:01:28 AM (16 months ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

159729

Message:

VMM/IEM: Emit native code for BltIn_CheckIrq. bugref:10371

Location:

trunk/src/VBox/VMM

Files:

• VMMAll/IEMAllN8veRecompiler.cpp (modified) (14 diffs)
• VMMAll/IEMAllThrdFuncsBltIn.cpp (modified) (1 diff)
• VMMAll/IEMAllThrdPython.py (modified) (1 diff)
• include/IEMInternal.h (modified) (1 diff)
• include/IEMN8veRecompiler.h (modified) (15 diffs)

trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

@@ -1588 +1588 @@
     pReNative->cCondDepth                  = 0;
     pReNative->uCondSeqNo                  = 0;
+    pReNative->uCheckIrqSeqNo              = 0;
 
     pReNative->Core.bmHstRegs              = IEMNATIVE_REG_FIXED_MASK
@@ -1994 +1995 @@
  * Debug Info: Record info about a threaded call.
  */
-static bool iemNativeDbgInfoAddThreadedCall(PIEMRECOMPILERSTATE pReNative, IEMTHREADEDFUNCS enmCall) RT_NOEXCEPT
+static bool iemNativeDbgInfoAddThreadedCall(PIEMRECOMPILERSTATE pReNative, IEMTHREADEDFUNCS enmCall, bool fRecompiled) RT_NOEXCEPT
 {
     PIEMTBDBGENTRY const pEntry = iemNativeDbgInfoAddNewEntry(pReNative, pReNative->pDbgInfo);
     AssertReturn(pEntry, false);
 
-    pEntry->ThreadedCall.uType   = kIemTbDbgEntryType_ThreadedCall;
-    pEntry->ThreadedCall.uUnused = 0;
-    pEntry->ThreadedCall.enmCall = (uint16_t)enmCall;
+    pEntry->ThreadedCall.uType       = kIemTbDbgEntryType_ThreadedCall;
+    pEntry->ThreadedCall.fRecompiled = fRecompiled;
+    pEntry->ThreadedCall.uUnused     = 0;
+    pEntry->ThreadedCall.enmCall     = (uint16_t)enmCall;
 
     return true;
@@ -2526 +2528 @@
 
 
-
-/**
- * Intended use statement for iemNativeRegAllocTmpForGuestReg().
- */
-typedef enum IEMNATIVEGSTREGUSE
-{
-    /** The usage is read-only, the register holding the guest register
-     * shadow copy will not be modified by the caller. */
-    kIemNativeGstRegUse_ReadOnly = 0,
-    /** The caller will update the guest register (think: PC += cbInstr).
-     * The guest shadow copy will follow the returned register. */
-    kIemNativeGstRegUse_ForUpdate,
-    /** The caller will use the guest register value as input in a calculation
-     * and the host register will be modified.
-     * This means that the returned host register will not be marked as a shadow
-     * copy of the guest register. */
-    kIemNativeGstRegUse_Calculation
-} IEMNATIVEGSTREGUSE;
-
-/**
- * Allocates a temporary host general purpose register for updating a guest
+/**
+ * Allocates a temporary host general purpose register for keeping a guest
  * register value.
  *
@@ -2561 +2544 @@
  * @param   enmGstReg       The guest register that will is to be updated.
  * @param   enmIntendedUse  How the caller will be using the host register.
+ * @sa      iemNativeRegAllocTmpForGuestRegIfAlreadyPresent
  */
 DECLHIDDEN(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
@@ -2678 +2662 @@
 
     return idxRegNew;
+}
+
+
+/**
+ * Allocates a temporary host general purpose register that already holds the
+ * given guest register value.
+ *
+ * The use case for this function is places where the shadowing state cannot be
+ * modified due to branching and such.  This will fail if the we don't have a
+ * current shadow copy handy or if it's incompatible.  The only code that will
+ * be emitted here is value checking code in strict builds.
+ *
+ * The intended use can only be readonly!
+ *
+ * @returns The host register number, UINT8_MAX on failure.
+ * @param   pReNative       The native recompile state.
+ * @param   poff            Pointer to the instruction buffer offset.
+ *                          Will be updated in strict builds if a register is
+ *                          found.
+ * @param   enmGstReg       The guest register that will is to be updated.
+ * @sa iemNativeRegAllocTmpForGuestReg
+ */
+DECLHIDDEN(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
+                                                                    IEMNATIVEGSTREG enmGstReg) RT_NOEXCEPT
+{
+    Assert(enmGstReg < kIemNativeGstReg_End && g_aGstShadowInfo[enmGstReg].cb != 0);
+
+    /*
+     * First check if the guest register value is already in a host register.
+     */
+    if (pReNative->Core.bmGstRegShadows & RT_BIT_64(enmGstReg))
+    {
+        uint8_t idxReg = pReNative->Core.aidxGstRegShadows[enmGstReg];
+        Assert(idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs));
+        Assert(pReNative->Core.aHstRegs[idxReg].fGstRegShadows & RT_BIT_64(enmGstReg));
+        Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxReg));
+
+        if (!(pReNative->Core.bmHstRegs & RT_BIT_32(idxReg)))
+        {
+            /*
+             * We only do readonly use here, so easy compared to the other
+             * variant of this code.
+             */
+            pReNative->Core.bmHstRegs |= RT_BIT_32(idxReg);
+            pReNative->Core.aHstRegs[idxReg].enmWhat = kIemNativeWhat_Tmp;
+            pReNative->Core.aHstRegs[idxReg].idxVar  = UINT8_MAX;
+            Log12(("iemNativeRegAllocTmpForGuestRegIfAlreadyPresent: Reusing %s for guest %s readonly\n",
+                   g_apszIemNativeHstRegNames[idxReg], g_aGstShadowInfo[enmGstReg].pszName));
+
+#ifdef VBOX_STRICT
+            /* Strict builds: Check that the value is correct. */
+            uint32_t off = *poff;
+            *poff = off = iemNativeEmitGuestRegValueCheck(pReNative, off, idxReg, enmGstReg);
+            AssertReturn(off != UINT32_MAX, UINT8_MAX);
+#else
+            RT_NOREF(poff);
+#endif
+            return idxReg;
+        }
+    }
+
+    return UINT8_MAX;
 }
 
@@ -4883 +4929 @@
 
 /**
+ * Built-in function that checks for pending interrupts that can be delivered or
+ * forced action flags.
+ *
+ * This triggers after the completion of an instruction, so EIP is already at
+ * the next instruction.  If an IRQ or important FF is pending, this will return
+ * a non-zero status that stops TB execution.
+ */
+static IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckIrq)
+{
+    RT_NOREF(pCallEntry);
+//pReNative->pInstrBuf[off++] = 0xcc;
+
+    /* It's too convenient to use iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet below
+       and I'm too lazy to create a 'Fixed' version of that one. */
+    uint32_t const idxLabelVmCheck = iemNativeLabelCreate(pReNative, kIemNativeLabelType_CheckIrq,
+                                                          UINT32_MAX, pReNative->uCheckIrqSeqNo++);
+    AssertReturn(idxLabelVmCheck != UINT32_MAX, UINT32_MAX);
+
+    uint32_t const idxLabelReturnBreak = iemNativeLabelCreate(pReNative, kIemNativeLabelType_ReturnBreak);
+    AssertReturn(idxLabelReturnBreak != UINT32_MAX, UINT32_MAX);
+
+    /* Again, we need to load the extended EFLAGS before we actually need them
+       in case we jump.  We couldn't use iemNativeRegAllocTmpForGuestReg if we
+       loaded them inside the check, as the shadow state would not be correct
+       when the code branches before the load.  Ditto PC. */
+    uint8_t const idxEflReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
+                                                              kIemNativeGstRegUse_ReadOnly);
+    AssertReturn(idxEflReg != UINT8_MAX, UINT32_MAX);
+
+    uint8_t const idxPcReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
+                                                             kIemNativeGstRegUse_ReadOnly);
+    AssertReturn(idxPcReg != UINT8_MAX, UINT32_MAX);
+
+    uint8_t idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
+    AssertReturn(idxTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
+
+    /*
+     * Start by checking the local forced actions of the EMT we're on for IRQs
+     * and other FFs that needs servicing.
+     */
+    /** @todo this isn't even close to the NMI and interrupt conditions in EM! */
+    /* Load FFs in to idxTmpReg and AND with all relevant flags. */
+    off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, fLocalForcedActions));
+    off = iemNativeEmitAndGprByImm(pReNative, off, idxTmpReg,
+                                   VMCPU_FF_ALL_MASK & ~(  VMCPU_FF_PGM_SYNC_CR3
+                                                         | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
+                                                         | VMCPU_FF_TLB_FLUSH
+                                                         | VMCPU_FF_UNHALT ),
+                                   true /*fSetFlags*/);
+    /* If we end up with ZERO in idxTmpReg there is nothing to do.*/
+    uint32_t const offFixupJumpToVmCheck1 = off;
+    off = iemNativeEmitJzToFixed(pReNative, off, 0);
+
+    /* Some relevant FFs are set, but if's only APIC or/and PIC being set,
+       these may be supressed by EFLAGS.IF or CPUMIsInInterruptShadow. */
+    off = iemNativeEmitAndGprByImm(pReNative, off, idxTmpReg,
+                                   ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC), true /*fSetFlags*/);
+    /* Return VINF_IEM_REEXEC_BREAK if other FFs are set. */
+    off = iemNativeEmitJnzToLabel(pReNative, off, idxLabelReturnBreak);
+
+    /* So, it's only interrupt releated FFs and we need to see if IRQs are being
+       suppressed by the CPU or not. */
+    off = iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(pReNative, off, idxEflReg, X86_EFL_IF_BIT, idxLabelVmCheck);
+    off = iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(pReNative, off, idxEflReg, CPUMCTX_INHIBIT_SHADOW,
+                                                              idxLabelReturnBreak);
+
+    /* We've got shadow flags set, so we must check that the PC they are valid
+       for matches our current PC value. */
+    /** @todo AMD64 can do this more efficiently w/o loading uRipInhibitInt into
+     *        a register. */
+    off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.uRipInhibitInt));
+    off = iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, idxTmpReg, idxPcReg, idxLabelReturnBreak);
+
+    /*
+     * Now check the force flags of the VM.
+     */
+    iemNativeLabelDefine(pReNative, idxLabelVmCheck, off);
+    iemNativeFixupFixedJump(pReNative, offFixupJumpToVmCheck1, off);
+    off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, CTX_SUFF(pVM))); /* idxTmpReg = pVM */
+    off = iemNativeEmitLoadGpr32ByGpr(pReNative, off, idxTmpReg, idxTmpReg, RT_UOFFSETOF(VMCC, fGlobalForcedActions));
+    off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, VM_FF_ALL_MASK, true /*fSetFlags*/);
+    off = iemNativeEmitJnzToLabel(pReNative, off, idxLabelReturnBreak);
+
+    /** @todo STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatCheckIrqBreaks); */
+
+    /*
+     * We're good, no IRQs or FFs pending.
+     */
+    iemNativeRegFreeTmp(pReNative, idxTmpReg);
+    iemNativeRegFreeTmp(pReNative, idxEflReg);
+    iemNativeRegFreeTmp(pReNative, idxPcReg);
+
+    return off;
+}
+
+
+/**
  * Built-in function checks if IEMCPU::fExec has the expected value.
  */
@@ -4888 +5031 @@
 {
     uint32_t const fExpectedExec = (uint32_t)pCallEntry->auParams[0];
-
     uint8_t idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
+
     AssertReturn(idxTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
     off = iemNativeEmitLoadGprFromVCpuU32(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, iem.s.fExec));
@@ -5167 +5310 @@
                         case kIemTbDbgEntryType_ThreadedCall:
                             pHlp->pfnPrintf(pHlp,
-                                            "  Call #%u to %s (%u args)\n",
+                                            "  Call #%u to %s (%u args)%s\n",
                                             idxThreadedCall,
                                             g_apszIemThreadedFunctions[pDbgInfo->aEntries[iDbgEntry].ThreadedCall.enmCall],
-                                            g_acIemThreadedFunctionUsedArgs[pDbgInfo->aEntries[iDbgEntry].ThreadedCall.enmCall]);
+                                            g_acIemThreadedFunctionUsedArgs[pDbgInfo->aEntries[iDbgEntry].ThreadedCall.enmCall],
+                                            pDbgInfo->aEntries[iDbgEntry].ThreadedCall.fRecompiled ? " - recompiled" : "");
                             idxThreadedCall++;
                             continue;
@@ -5204 +5348 @@
                                     pszName = "ReturnBreak";
                                     break;
+                                case kIemNativeLabelType_NonZeroRetOrPassUp:
+                                    pszName = "NonZeroRetOrPassUp";
+                                    break;
+                                case kIemNativeLabelType_RaiseGp0:
+                                    pszName = "RaiseGp0";
+                                    break;
                                 case kIemNativeLabelType_If:
                                     pszName = "If";
@@ -5217 +5367 @@
                                     fNumbered = true;
                                     break;
-                                case kIemNativeLabelType_NonZeroRetOrPassUp:
-                                    pszName = "NonZeroRetOrPassUp";
-                                    break;
-                                case kIemNativeLabelType_RaiseGp0:
-                                    pszName = "RaiseGp0";
+                                case kIemNativeLabelType_CheckIrq:
+                                    pszName = "CheckIrq_CheckVM";
+                                    fNumbered = true;
                                     break;
                                 case kIemNativeLabelType_Invalid:
@@ -5259 +5407 @@
                     uint32_t const uInfo = *(uint32_t const *)&Dis.Instr.ab[3];
                     if (RT_HIWORD(uInfo) < kIemThreadedFunc_End)
-                        pHlp->pfnPrintf(pHlp, "    %p: nop ; marker: call #%u to %s (%u args)\n",
-                                        &paNative[offNative], RT_LOWORD(uInfo), g_apszIemThreadedFunctions[RT_HIWORD(uInfo)],
-                                        g_acIemThreadedFunctionUsedArgs[RT_HIWORD(uInfo)]);
+                        pHlp->pfnPrintf(pHlp, "    %p: nop ; marker: call #%u to %s (%u args)%s\n",
+                                        &paNative[offNative], uInfo & 0x7fff, g_apszIemThreadedFunctions[RT_HIWORD(uInfo)],
+                                        g_acIemThreadedFunctionUsedArgs[RT_HIWORD(uInfo)],
+                                        uInfo & 0x8000 ? " - recompiled" : "");
                     else
                         pHlp->pfnPrintf(pHlp, "    %p: nop ; unknown marker: %#x (%d)\n", &paNative[offNative], uInfo, uInfo);
@@ -5346 +5495 @@
                     uint32_t const uInfo = *(uint32_t const *)&Dis.Instr.ab[3];
                     if (RT_HIWORD(uInfo) < kIemThreadedFunc_End)
-                        pHlp->pfnPrintf(pHlp, "\n    %p: nop ; marker: call #%u to %s (%u args)\n",
-                                        &paNative[offNative], RT_LOWORD(uInfo), g_apszIemThreadedFunctions[RT_HIWORD(uInfo)],
-                                        g_acIemThreadedFunctionUsedArgs[RT_HIWORD(uInfo)]);
+                        pHlp->pfnPrintf(pHlp, "\n    %p: nop ; marker: call #%u to %s (%u args)%s\n",
+                                        &paNative[offNative], uInfo & 0x7fff, g_apszIemThreadedFunctions[RT_HIWORD(uInfo)],
+                                        g_acIemThreadedFunctionUsedArgs[RT_HIWORD(uInfo)],
+                                        uInfo & 0x8000 ? " - recompiled" : "");
                     else
                         pHlp->pfnPrintf(pHlp, "    %p: nop ; unknown marker: %#x (%d)\n", &paNative[offNative], uInfo, uInfo);
@@ -5426 +5576 @@
     while (cCallsLeft-- > 0)
     {
+        PFNIEMNATIVERECOMPFUNC const pfnRecom = g_apfnIemNativeRecompileFunctions[pCallEntry->enmFunction];
+
         /*
          * Debug info and assembly markup.
@@ -5441 +5593 @@
             iGstInstr = pCallEntry->idxInstr;
         }
-        iemNativeDbgInfoAddThreadedCall(pReNative, (IEMTHREADEDFUNCS)pCallEntry->enmFunction);
+        iemNativeDbgInfoAddThreadedCall(pReNative, (IEMTHREADEDFUNCS)pCallEntry->enmFunction, pfnRecom != NULL);
 #endif
 
 #ifdef VBOX_STRICT
-        off = iemNativeEmitMarker(pReNative, off, RT_MAKE_U32(pTb->Thrd.cCalls - cCallsLeft - 1, pCallEntry->enmFunction));
+        off = iemNativeEmitMarker(pReNative, off,
+                                  RT_MAKE_U32((pTb->Thrd.cCalls - cCallsLeft - 1) | (pfnRecom ? 0x8000 : 0),
+                                              pCallEntry->enmFunction));
         AssertReturn(off != UINT32_MAX, pTb);
 #endif
+
         /*
          * Actual work.
          */
-        PFNIEMNATIVERECOMPFUNC const pfnRecom = g_apfnIemNativeRecompileFunctions[pCallEntry->enmFunction];
         if (pfnRecom) /** @todo stats on this.   */
         {

trunk/src/VBox/VMM/VMMAll/IEMAllThrdFuncsBltIn.cpp

@@ -128 +128 @@
     return VINF_IEM_REEXEC_BREAK;
 }
-
 
 

trunk/src/VBox/VMM/VMMAll/IEMAllThrdPython.py

@@ -1877 +1877 @@
     katBltIns = (
         ( 'DeferToCImpl0',                                      2, True  ),
-        ( 'CheckIrq',                                           0, False ),
+        ( 'CheckIrq',                                           0, True  ),
         ( 'CheckMode',                                          1, True  ),
         ( 'CheckHwInstrBps',                                    0, False ),

trunk/src/VBox/VMM/include/IEMInternal.h

@@ -929 +929 @@
     {
         /* kIemTbDbgEntryType_ThreadedCall. */
-        uint32_t    uType      : 4;
-        uint32_t    uUnused    : 12;
+        uint32_t    uType       : 4;
+        /** Set if the call was recompiled to native code, clear if just calling
+         *  threaded function. */
+        uint32_t    fRecompiled : 1;
+        uint32_t    uUnused     : 11;
         /** The threaded call number (IEMTHREADEDFUNCS). */
-        uint32_t    enmCall    : 16;
+        uint32_t    enmCall     : 16;
     } ThreadedCall;
 

trunk/src/VBox/VMM/include/IEMN8veRecompiler.h

@@ -263 +263 @@
 {
     kIemNativeLabelType_Invalid = 0,
+    /* Labels w/o data, only once instance per TB: */
     kIemNativeLabelType_Return,
     kIemNativeLabelType_ReturnBreak,
+    kIemNativeLabelType_NonZeroRetOrPassUp,
+    kIemNativeLabelType_RaiseGp0,
+    /* Labels with data, potentially multiple instances per TB: */
     kIemNativeLabelType_If,
     kIemNativeLabelType_Else,
     kIemNativeLabelType_Endif,
-    kIemNativeLabelType_NonZeroRetOrPassUp,
-    kIemNativeLabelType_RaiseGp0,
+    kIemNativeLabelType_CheckIrq,
     kIemNativeLabelType_End
 } IEMNATIVELABELTYPE;
@@ -338 +341 @@
     kIemNativeGstReg_End
 } IEMNATIVEGSTREG;
+
+/**
+ * Intended use statement for iemNativeRegAllocTmpForGuestReg().
+ */
+typedef enum IEMNATIVEGSTREGUSE
+{
+    /** The usage is read-only, the register holding the guest register
+     * shadow copy will not be modified by the caller. */
+    kIemNativeGstRegUse_ReadOnly = 0,
+    /** The caller will update the guest register (think: PC += cbInstr).
+     * The guest shadow copy will follow the returned register. */
+    kIemNativeGstRegUse_ForUpdate,
+    /** The caller will use the guest register value as input in a calculation
+     * and the host register will be modified.
+     * This means that the returned host register will not be marked as a shadow
+     * copy of the guest register. */
+    kIemNativeGstRegUse_Calculation
+} IEMNATIVEGSTREGUSE;
 
 /**
@@ -591 +612 @@
     /** Condition sequence number (for generating unique labels). */
     uint16_t                    uCondSeqNo;
-    uint32_t                    uPadding3;
+    /** Check IRQ seqeunce number (for generating unique lables). */
+    uint16_t                    uCheckIrqSeqNo;
+    uint16_t                    uPadding3;
 
     /** Core state requiring care with branches. */
@@ -624 +647 @@
 #define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
 
-
 DECLHIDDEN(uint32_t)        iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
                                                  uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
@@ -637 +659 @@
 DECLHIDDEN(uint8_t)         iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
                                                     bool fPreferVolatile = true) RT_NOEXCEPT;
-DECLHIDDEN(uint8_t)         iemNativeRegAllocTmpForGuest(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
-                                                         IEMNATIVEGSTREG enmGstReg) RT_NOEXCEPT;
+DECLHIDDEN(uint8_t)         iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
+                                                            IEMNATIVEGSTREG enmGstReg,
+                                                            IEMNATIVEGSTREGUSE enmIntendedUse) RT_NOEXCEPT;
+DECLHIDDEN(uint8_t)         iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
+                                                                            IEMNATIVEGSTREG enmGstReg) RT_NOEXCEPT;
+
 DECLHIDDEN(uint8_t)         iemNativeRegAllocVar(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t idxVar) RT_NOEXCEPT;
 DECLHIDDEN(uint32_t)        iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs) RT_NOEXCEPT;
@@ -970 +996 @@
         pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
     pbCodeBuf[off++] = 0x8b;
-    off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
+    off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off,iGpr, offVCpu);
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
 
@@ -1263 +1289 @@
  * Emits a load effective address to a GRP with an BP relative source address.
  */
-DECLINLINE(uint32_t) iemNativeEmitLeaGrpByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
+DECLINLINE(uint32_t) iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
 {
     /* lea gprdst, [rbp + offDisp] */
@@ -1374 +1400 @@
 
 
+#ifdef RT_ARCH_AMD64
+/**
+ * Common bit of iemNativeEmitLoadGprByGpr and friends.
+ */
+DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByGprDisp(uint8_t *pbCodeBuf, uint32_t off,
+                                                      uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp)
+{
+    if (offDisp == 0 && (iGprBase & 7) != X86_GREG_xBP) /* Can use encoding w/o displacement field. */
+    {
+        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, iGprReg & 7, iGprBase & 7);
+        if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
+            pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
+    }
+    else if (offDisp == (int8_t)offDisp)
+    {
+        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
+        if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
+            pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
+        pbCodeBuf[off++] = (uint8_t)offDisp;
+    }
+    else
+    {
+        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
+        if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
+            pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
+        pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
+        pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
+        pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
+        pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
+    }
+    return off;
+}
+#elif RT_ARCH_ARM64
+/**
+ * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
+ */
+DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByGprLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
+                                                      uint8_t iGprBase, int32_t offDisp,
+                                                      ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
+{
+    /*
+     * There are a couple of ldr variants that takes an immediate offset, so
+     * try use those if we can, otherwise we have to use the temporary register
+     * help with the addressing.
+     */
+    if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
+    {
+        /* Use the unsigned variant of ldr Wt, [<Xn|SP>, #off]. */
+        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+        AssertReturn(pu32CodeBuf, UINT32_MAX);
+        pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
+    }
+    else
+    {
+        /* The offset is too large, so we must load it into a register and use
+           ldr Wt, [<Xn|SP>, (<Wm>|<Xm>)]. */
+        /** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
+        uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, off, (uint64)offDisp);
+        AssertReturn(idxTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
+
+        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+        AssertReturn(pu32CodeBuf, UINT32_MAX);
+        pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, idxTmpReg);
+
+        iemNativeRegFreeTmpImm(pReNative, idxTmpReg);
+    }
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+}
+#endif
+
+
+/**
+ * Emits a 64-bit GPR load via a GPR base address with a displacement.
+ */
+DECLINLINE(uint32_t) iemNativeEmitLoadGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off,
+                                               uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
+{
+#ifdef RT_ARCH_AMD64
+    /* mov reg64, mem64 */
+    uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
+    AssertReturn(pbCodeBuf, UINT32_MAX);
+    pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprBase < 8 ? 0 : X86_OP_REX_B);
+    pbCodeBuf[off++] = 0x8b;
+    off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+
+#elif RT_ARCH_ARM64
+    off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
+
+#else
+# error "port me"
+#endif
+    return off;
+}
+
+
+/**
+ * Emits a 32-bit GPR load via a GPR base address with a displacement.
+ * @note Bits 63 thru 32 in @a iGprDst will be cleared.
+ */
+DECLINLINE(uint32_t) iemNativeEmitLoadGpr32ByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off,
+                                                 uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
+{
+#ifdef RT_ARCH_AMD64
+    /* mov reg32, mem32 */
+    uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
+    AssertReturn(pbCodeBuf, UINT32_MAX);
+    if (iGprDst >= 8 || iGprBase >= 8)
+        pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprBase < 8 ? 0 : X86_OP_REX_B);
+    pbCodeBuf[off++] = 0x8b;
+    off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+
+#elif RT_ARCH_ARM64
+    off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
+
+#else
+# error "port me"
+#endif
+    return off;
+}
+
+
 /*********************************************************************************************************************************
 *   Subtraction and Additions                                                                                                    *
@@ -1683 +1833 @@
 /**
  * Emits code for AND'ing two 64-bit GPRs.
- */
-DECLINLINE(uint32_t ) iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
+ *
+ * @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
+ *       and ARM64 hosts.
+ */
+DECLINLINE(uint32_t ) iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc,
+                                               bool fSetFlags = false)
 {
 #if defined(RT_ARCH_AMD64)
@@ -1693 +1847 @@
     pbCodeBuf[off++] = 0x23;
     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
+    RT_NOREF(fSetFlags);
 
 #elif defined(RT_ARCH_ARM64)
     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
     AssertReturn(pu32CodeBuf, UINT32_MAX);
-    pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
+    if (!fSetFlags)
+        pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
+    else
+        pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
 
 #else
@@ -1735 +1893 @@
 
 /**
- * Emits code for AND'ing an 32-bit GPRs with a constant.
- */
-DECLINLINE(uint32_t ) iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm)
+ * Emits code for AND'ing a 64-bit GPRs with a constant.
+ *
+ * @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
+ *       and ARM64 hosts.
+ */
+DECLINLINE(uint32_t ) iemNativeEmitAndGprByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint64_t uImm,
+                                               bool fSetFlags = false)
 {
 #if defined(RT_ARCH_AMD64)
-    /* and Ev, imm */
-    uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
-    AssertReturn(pbCodeBuf, UINT32_MAX);
-    if (iGprDst >= 8)
-        pbCodeBuf[off++] = X86_OP_REX_R;
-    if (uImm < 128)
-    {
+    if ((int64_t)uImm == (int8_t)uImm)
+    {
+        /* and Ev, imm8 */
+        uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
+        AssertReturn(pbCodeBuf, UINT32_MAX);
+        pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R);
         pbCodeBuf[off++] = 0x83;
         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
         pbCodeBuf[off++] = (uint8_t)uImm;
     }
-    else
-    {
+    else if ((int64_t)uImm == (int32_t)uImm)
+    {
+        /* and Ev, imm32 */
+        uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
+        AssertReturn(pbCodeBuf, UINT32_MAX);
+        pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R);
         pbCodeBuf[off++] = 0x81;
         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
@@ -1760 +1925 @@
         pbCodeBuf[off++] = RT_BYTE4(uImm);
     }
+    else
+    {
+        /* Use temporary register for the 64-bit immediate. */
+        uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
+        AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
+        off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg);
+        iemNativeRegFreeTmpImm(pReNative, iTmpReg);
+    }
+    RT_NOREF(fSetFlags);
 
 #elif defined(RT_ARCH_ARM64)
@@ -1768 +1942 @@
         uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
         AssertReturn(pu32CodeBuf, UINT32_MAX);
-        pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /*f64Bit*/);
-    }
-    else
-    {
-        /* Use temporary register for the immediate. */
+        if (!fSetFlags)
+            pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR);
+        else
+            pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR);
+    }
+    else
+    {
+        /* Use temporary register for the 64-bit immediate. */
         uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
         AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
-
-        /* and gprdst, gprdst, tmpreg */
+        off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg, fSetFlags);
+        iemNativeRegFreeTmpImm(pReNative, iTmpReg);
+    }
+
+#else
+# error "Port me"
+#endif
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+}
+
+
+/**
+ * Emits code for AND'ing an 32-bit GPRs with a constant.
+ */
+DECLINLINE(uint32_t ) iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm,
+                                                 bool fSetFlags = false)
+{
+#if defined(RT_ARCH_AMD64)
+    /* and Ev, imm */
+    uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
+    AssertReturn(pbCodeBuf, UINT32_MAX);
+    if (iGprDst >= 8)
+        pbCodeBuf[off++] = X86_OP_REX_R;
+    if ((int32_t)uImm == (int8_t)uImm)
+    {
+        pbCodeBuf[off++] = 0x83;
+        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
+        pbCodeBuf[off++] = (uint8_t)uImm;
+    }
+    else
+    {
+        pbCodeBuf[off++] = 0x81;
+        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
+        pbCodeBuf[off++] = RT_BYTE1(uImm);
+        pbCodeBuf[off++] = RT_BYTE2(uImm);
+        pbCodeBuf[off++] = RT_BYTE3(uImm);
+        pbCodeBuf[off++] = RT_BYTE4(uImm);
+    }
+    RT_NOREF(fSetFlags);
+
+#elif defined(RT_ARCH_ARM64)
+    uint32_t uImmR     = 0;
+    uint32_t uImmNandS = 0;
+    if (Armv8A64ConvertMaskToImmRImmS(uImm, &uImmNandS, &uImmR))
+    {
         uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
         AssertReturn(pu32CodeBuf, UINT32_MAX);
-        pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iTmpReg, false /*f64Bit*/);
-
+        if (!fSetFlags)
+            pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /*f64Bit*/);
+        else
+            pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /*f64Bit*/);
+    }
+    else
+    {
+        /* Use temporary register for the 64-bit immediate. */
+        uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
+        AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
+        if (!fSetFlags)
+            off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg);
+        else
+            off = iemNativeEmitAndsGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg);
         iemNativeRegFreeTmpImm(pReNative, iTmpReg);
     }
@@ -2724 +2957 @@
         iemNativeRegFreeTmpImm(pReNative, iTmpReg);
     }
-    else
+    else if (fBits <= UINT32_MAX)
     {
         /* test Eb, imm8 or test Ev, imm32 */
@@ -2747 +2980 @@
         }
     }
+    /** @todo implement me. */
+    else
+        AssertFailedReturn(UINT32_MAX);
 
 #elif defined(RT_ARCH_ARM64)