Index: trunk/src/VBox/VMM/include/IEMInternal.h
===================================================================
--- trunk/src/VBox/VMM/include/IEMInternal.h	(revision 101626)
+++ trunk/src/VBox/VMM/include/IEMInternal.h	(revision 101640)
@@ -929,8 +929,11 @@
     {
         /* kIemTbDbgEntryType_ThreadedCall. */
-        uint32_t    uType      : 4;
-        uint32_t    uUnused    : 12;
+        uint32_t    uType       : 4;
+        /** Set if the call was recompiled to native code, clear if just calling
+         *  threaded function. */
+        uint32_t    fRecompiled : 1;
+        uint32_t    uUnused     : 11;
         /** The threaded call number (IEMTHREADEDFUNCS). */
-        uint32_t    enmCall    : 16;
+        uint32_t    enmCall     : 16;
     } ThreadedCall;
 
Index: trunk/src/VBox/VMM/include/IEMN8veRecompiler.h
===================================================================
--- trunk/src/VBox/VMM/include/IEMN8veRecompiler.h	(revision 101626)
+++ trunk/src/VBox/VMM/include/IEMN8veRecompiler.h	(revision 101640)
@@ -263,11 +263,14 @@
 {
     kIemNativeLabelType_Invalid = 0,
+    /* Labels w/o data, only once instance per TB: */
     kIemNativeLabelType_Return,
     kIemNativeLabelType_ReturnBreak,
+    kIemNativeLabelType_NonZeroRetOrPassUp,
+    kIemNativeLabelType_RaiseGp0,
+    /* Labels with data, potentially multiple instances per TB: */
     kIemNativeLabelType_If,
     kIemNativeLabelType_Else,
     kIemNativeLabelType_Endif,
-    kIemNativeLabelType_NonZeroRetOrPassUp,
-    kIemNativeLabelType_RaiseGp0,
+    kIemNativeLabelType_CheckIrq,
     kIemNativeLabelType_End
 } IEMNATIVELABELTYPE;
@@ -338,4 +341,22 @@
     kIemNativeGstReg_End
 } IEMNATIVEGSTREG;
+
+/**
+ * Intended use statement for iemNativeRegAllocTmpForGuestReg().
+ */
+typedef enum IEMNATIVEGSTREGUSE
+{
+    /** The usage is read-only, the register holding the guest register
+     * shadow copy will not be modified by the caller. */
+    kIemNativeGstRegUse_ReadOnly = 0,
+    /** The caller will update the guest register (think: PC += cbInstr).
+     * The guest shadow copy will follow the returned register. */
+    kIemNativeGstRegUse_ForUpdate,
+    /** The caller will use the guest register value as input in a calculation
+     * and the host register will be modified.
+     * This means that the returned host register will not be marked as a shadow
+     * copy of the guest register. */
+    kIemNativeGstRegUse_Calculation
+} IEMNATIVEGSTREGUSE;
 
 /**
@@ -591,5 +612,7 @@
     /** Condition sequence number (for generating unique labels). */
     uint16_t                    uCondSeqNo;
-    uint32_t                    uPadding3;
+    /** Check IRQ seqeunce number (for generating unique lables). */
+    uint16_t                    uCheckIrqSeqNo;
+    uint16_t                    uPadding3;
 
     /** Core state requiring care with branches. */
@@ -624,5 +647,4 @@
 #define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
 
-
 DECLHIDDEN(uint32_t)        iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
                                                  uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
@@ -637,6 +659,10 @@
 DECLHIDDEN(uint8_t)         iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,
                                                     bool fPreferVolatile = true) RT_NOEXCEPT;
-DECLHIDDEN(uint8_t)         iemNativeRegAllocTmpForGuest(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
-                                                         IEMNATIVEGSTREG enmGstReg) RT_NOEXCEPT;
+DECLHIDDEN(uint8_t)         iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
+                                                            IEMNATIVEGSTREG enmGstReg,
+                                                            IEMNATIVEGSTREGUSE enmIntendedUse) RT_NOEXCEPT;
+DECLHIDDEN(uint8_t)         iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
+                                                                            IEMNATIVEGSTREG enmGstReg) RT_NOEXCEPT;
+
 DECLHIDDEN(uint8_t)         iemNativeRegAllocVar(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t idxVar) RT_NOEXCEPT;
 DECLHIDDEN(uint32_t)        iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs) RT_NOEXCEPT;
@@ -970,5 +996,5 @@
         pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
     pbCodeBuf[off++] = 0x8b;
-    off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
+    off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off,iGpr, offVCpu);
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
 
@@ -1263,5 +1289,5 @@
  * Emits a load effective address to a GRP with an BP relative source address.
  */
-DECLINLINE(uint32_t) iemNativeEmitLeaGrpByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
+DECLINLINE(uint32_t) iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
 {
     /* lea gprdst, [rbp + offDisp] */
@@ -1374,4 +1400,128 @@
 
 
+#ifdef RT_ARCH_AMD64
+/**
+ * Common bit of iemNativeEmitLoadGprByGpr and friends.
+ */
+DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByGprDisp(uint8_t *pbCodeBuf, uint32_t off,
+                                                      uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp)
+{
+    if (offDisp == 0 && (iGprBase & 7) != X86_GREG_xBP) /* Can use encoding w/o displacement field. */
+    {
+        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, iGprReg & 7, iGprBase & 7);
+        if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
+            pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
+    }
+    else if (offDisp == (int8_t)offDisp)
+    {
+        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
+        if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
+            pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
+        pbCodeBuf[off++] = (uint8_t)offDisp;
+    }
+    else
+    {
+        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);
+        if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */
+            pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */
+        pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
+        pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
+        pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
+        pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
+    }
+    return off;
+}
+#elif RT_ARCH_ARM64
+/**
+ * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
+ */
+DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByGprLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
+                                                      uint8_t iGprBase, int32_t offDisp,
+                                                      ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
+{
+    /*
+     * There are a couple of ldr variants that takes an immediate offset, so
+     * try use those if we can, otherwise we have to use the temporary register
+     * help with the addressing.
+     */
+    if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))
+    {
+        /* Use the unsigned variant of ldr Wt, [<Xn|SP>, #off]. */
+        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+        AssertReturn(pu32CodeBuf, UINT32_MAX);
+        pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);
+    }
+    else
+    {
+        /* The offset is too large, so we must load it into a register and use
+           ldr Wt, [<Xn|SP>, (<Wm>|<Xm>)]. */
+        /** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
+        uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, off, (uint64)offDisp);
+        AssertReturn(idxTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
+
+        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+        AssertReturn(pu32CodeBuf, UINT32_MAX);
+        pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, idxTmpReg);
+
+        iemNativeRegFreeTmpImm(pReNative, idxTmpReg);
+    }
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+}
+#endif
+
+
+/**
+ * Emits a 64-bit GPR load via a GPR base address with a displacement.
+ */
+DECLINLINE(uint32_t) iemNativeEmitLoadGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off,
+                                               uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
+{
+#ifdef RT_ARCH_AMD64
+    /* mov reg64, mem64 */
+    uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
+    AssertReturn(pbCodeBuf, UINT32_MAX);
+    pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprBase < 8 ? 0 : X86_OP_REX_B);
+    pbCodeBuf[off++] = 0x8b;
+    off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+
+#elif RT_ARCH_ARM64
+    off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
+
+#else
+# error "port me"
+#endif
+    return off;
+}
+
+
+/**
+ * Emits a 32-bit GPR load via a GPR base address with a displacement.
+ * @note Bits 63 thru 32 in @a iGprDst will be cleared.
+ */
+DECLINLINE(uint32_t) iemNativeEmitLoadGpr32ByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off,
+                                                 uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)
+{
+#ifdef RT_ARCH_AMD64
+    /* mov reg32, mem32 */
+    uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
+    AssertReturn(pbCodeBuf, UINT32_MAX);
+    if (iGprDst >= 8 || iGprBase >= 8)
+        pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprBase < 8 ? 0 : X86_OP_REX_B);
+    pbCodeBuf[off++] = 0x8b;
+    off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+
+#elif RT_ARCH_ARM64
+    off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
+
+#else
+# error "port me"
+#endif
+    return off;
+}
+
+
 /*********************************************************************************************************************************
 *   Subtraction and Additions                                                                                                    *
@@ -1683,6 +1833,10 @@
 /**
  * Emits code for AND'ing two 64-bit GPRs.
- */
-DECLINLINE(uint32_t ) iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
+ *
+ * @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
+ *       and ARM64 hosts.
+ */
+DECLINLINE(uint32_t ) iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc,
+                                               bool fSetFlags = false)
 {
 #if defined(RT_ARCH_AMD64)
@@ -1693,9 +1847,13 @@
     pbCodeBuf[off++] = 0x23;
     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
+    RT_NOREF(fSetFlags);
 
 #elif defined(RT_ARCH_ARM64)
     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
     AssertReturn(pu32CodeBuf, UINT32_MAX);
-    pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
+    if (!fSetFlags)
+        pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);
+    else
+        pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);
 
 #else
@@ -1735,22 +1893,29 @@
 
 /**
- * Emits code for AND'ing an 32-bit GPRs with a constant.
- */
-DECLINLINE(uint32_t ) iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm)
+ * Emits code for AND'ing a 64-bit GPRs with a constant.
+ *
+ * @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD64
+ *       and ARM64 hosts.
+ */
+DECLINLINE(uint32_t ) iemNativeEmitAndGprByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint64_t uImm,
+                                               bool fSetFlags = false)
 {
 #if defined(RT_ARCH_AMD64)
-    /* and Ev, imm */
-    uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
-    AssertReturn(pbCodeBuf, UINT32_MAX);
-    if (iGprDst >= 8)
-        pbCodeBuf[off++] = X86_OP_REX_R;
-    if (uImm < 128)
-    {
+    if ((int64_t)uImm == (int8_t)uImm)
+    {
+        /* and Ev, imm8 */
+        uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
+        AssertReturn(pbCodeBuf, UINT32_MAX);
+        pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R);
         pbCodeBuf[off++] = 0x83;
         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
         pbCodeBuf[off++] = (uint8_t)uImm;
     }
-    else
-    {
+    else if ((int64_t)uImm == (int32_t)uImm)
+    {
+        /* and Ev, imm32 */
+        uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
+        AssertReturn(pbCodeBuf, UINT32_MAX);
+        pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R);
         pbCodeBuf[off++] = 0x81;
         pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
@@ -1760,4 +1925,13 @@
         pbCodeBuf[off++] = RT_BYTE4(uImm);
     }
+    else
+    {
+        /* Use temporary register for the 64-bit immediate. */
+        uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
+        AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
+        off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg);
+        iemNativeRegFreeTmpImm(pReNative, iTmpReg);
+    }
+    RT_NOREF(fSetFlags);
 
 #elif defined(RT_ARCH_ARM64)
@@ -1768,17 +1942,76 @@
         uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
         AssertReturn(pu32CodeBuf, UINT32_MAX);
-        pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /*f64Bit*/);
-    }
-    else
-    {
-        /* Use temporary register for the immediate. */
+        if (!fSetFlags)
+            pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR);
+        else
+            pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR);
+    }
+    else
+    {
+        /* Use temporary register for the 64-bit immediate. */
         uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
         AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
-
-        /* and gprdst, gprdst, tmpreg */
+        off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg, fSetFlags);
+        iemNativeRegFreeTmpImm(pReNative, iTmpReg);
+    }
+
+#else
+# error "Port me"
+#endif
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+}
+
+
+/**
+ * Emits code for AND'ing an 32-bit GPRs with a constant.
+ */
+DECLINLINE(uint32_t ) iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm,
+                                                 bool fSetFlags = false)
+{
+#if defined(RT_ARCH_AMD64)
+    /* and Ev, imm */
+    uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
+    AssertReturn(pbCodeBuf, UINT32_MAX);
+    if (iGprDst >= 8)
+        pbCodeBuf[off++] = X86_OP_REX_R;
+    if ((int32_t)uImm == (int8_t)uImm)
+    {
+        pbCodeBuf[off++] = 0x83;
+        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
+        pbCodeBuf[off++] = (uint8_t)uImm;
+    }
+    else
+    {
+        pbCodeBuf[off++] = 0x81;
+        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);
+        pbCodeBuf[off++] = RT_BYTE1(uImm);
+        pbCodeBuf[off++] = RT_BYTE2(uImm);
+        pbCodeBuf[off++] = RT_BYTE3(uImm);
+        pbCodeBuf[off++] = RT_BYTE4(uImm);
+    }
+    RT_NOREF(fSetFlags);
+
+#elif defined(RT_ARCH_ARM64)
+    uint32_t uImmR     = 0;
+    uint32_t uImmNandS = 0;
+    if (Armv8A64ConvertMaskToImmRImmS(uImm, &uImmNandS, &uImmR))
+    {
         uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
         AssertReturn(pu32CodeBuf, UINT32_MAX);
-        pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iTmpReg, false /*f64Bit*/);
-
+        if (!fSetFlags)
+            pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /*f64Bit*/);
+        else
+            pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /*f64Bit*/);
+    }
+    else
+    {
+        /* Use temporary register for the 64-bit immediate. */
+        uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
+        AssertReturn(iTmpReg < RT_ELEMENTS(pReNative->Core.aHstRegs), UINT32_MAX);
+        if (!fSetFlags)
+            off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg);
+        else
+            off = iemNativeEmitAndsGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg);
         iemNativeRegFreeTmpImm(pReNative, iTmpReg);
     }
@@ -2724,5 +2957,5 @@
         iemNativeRegFreeTmpImm(pReNative, iTmpReg);
     }
-    else
+    else if (fBits <= UINT32_MAX)
     {
         /* test Eb, imm8 or test Ev, imm32 */
@@ -2747,4 +2980,7 @@
         }
     }
+    /** @todo implement me. */
+    else
+        AssertFailedReturn(UINT32_MAX);
 
 #elif defined(RT_ARCH_ARM64)
