Index: trunk/src/VBox/VMM/include/IEMN8veRecompiler.h
===================================================================
--- trunk/src/VBox/VMM/include/IEMN8veRecompiler.h	(revision 102433)
+++ trunk/src/VBox/VMM/include/IEMN8veRecompiler.h	(revision 102436)
@@ -817,6 +817,9 @@
 DECLHIDDEN(void)            iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
 DECLHIDDEN(void)            iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
+DECLHIDDEN(void)            iemNativeRegFreeVar(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg, bool fFlushShadows) RT_NOEXCEPT;
 DECLHIDDEN(void)            iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
 DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off);
+
+DECL_HIDDEN_THROW(uint8_t)  iemNativeVarGetStackSlot(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar);
 
 DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,
Index: trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h
===================================================================
--- trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h	(revision 102433)
+++ trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h	(revision 102436)
@@ -65,5 +65,5 @@
         pbCodeBuf[off++] = RT_BYTE4(uInfo);
     }
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     /* nop */
     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
@@ -73,4 +73,26 @@
 #else
 # error "port me"
+#endif
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+}
+
+
+/**
+ * Emit a breakpoint instruction.
+ */
+DECL_INLINE_THROW(uint32_t) iemNativeEmitBrk(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)
+{
+#ifdef RT_ARCH_AMD64
+    uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+    pbCodeBuf[off++] = 0xcc;
+    RT_NOREF(uInfo);
+
+#elif defined(RT_ARCH_ARM64)
+    uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+    pu32CodeBuf[off++] = Armv8A64MkInstrBrk(uInfo & UINT32_C(0xffff));
+
+#else
+# error "error"
 #endif
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
@@ -97,5 +119,5 @@
     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     /* mov gpr, #0x0 */
     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
@@ -151,5 +173,5 @@
     }
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
 
@@ -223,5 +245,5 @@
     pbCodeBuf[off++] = RT_BYTE1(uImm8);
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     /* movz gpr, imm16, lsl #0 */
     uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
@@ -258,5 +280,5 @@
     return off;
 }
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
 /**
  * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
@@ -317,5 +339,5 @@
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
 
@@ -343,5 +365,5 @@
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
 
@@ -370,5 +392,5 @@
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
 
@@ -397,5 +419,5 @@
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
 
@@ -424,5 +446,5 @@
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
 
@@ -449,5 +471,5 @@
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
 
@@ -475,5 +497,5 @@
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
 
@@ -500,5 +522,5 @@
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
 
@@ -524,5 +546,5 @@
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     /* Cannot use IEMNATIVE_REG_FIXED_TMP0 for the immediate as that's used by iemNativeEmitGprByVCpuLdSt. */
     uint8_t const idxRegImm = iemNativeRegAllocTmpImm(pReNative, &off, bImm);
@@ -599,5 +621,5 @@
     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     /* mov dst, src;   alias for: orr dst, xzr, src */
     uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
@@ -629,5 +651,5 @@
     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     /* mov dst32, src32;   alias for: orr dst32, wzr, src32 */
     uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
@@ -660,5 +682,5 @@
     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     /* and gprdst, gprsrc, #0xffff */
     uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
@@ -699,5 +721,5 @@
     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     /* and gprdst, gprsrc, #0xff */
     uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
@@ -744,5 +766,5 @@
     pbCodeBuf[off++] = 8;
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     /* ubfx gprdst, gprsrc, #8, #8 - gprdst = gprsrc[15:8] */
     uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
@@ -790,5 +812,5 @@
     }
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     if ((uint32_t)iAddend < 4096)
     {
@@ -1151,5 +1173,5 @@
     return off;
 }
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
 /**
  * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
@@ -1202,5 +1224,5 @@
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
 
@@ -1228,5 +1250,5 @@
     IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
 
-#elif RT_ARCH_ARM64
+#elif defined(RT_ARCH_ARM64)
     off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
 
@@ -1582,5 +1604,5 @@
 #elif defined(RT_ARCH_ARM64)
     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
-    if (fSetFlags)
+    if (!fSetFlags)
         pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /*f64Bit*/);
     else
@@ -2674,4 +2696,49 @@
 
 /**
+ * Emits a test for any of the bits from @a fBits in the lower 8 bits of
+ * @a iGprSrc, setting CPU flags accordingly.
+ */
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitTestAnyBitsInGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint8_t fBits)
+{
+    Assert(fBits != 0);
+
+#ifdef RT_ARCH_AMD64
+    /* test Eb, imm8 */
+    uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
+    if (iGprSrc >= 4)
+        pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;
+    pbCodeBuf[off++] = 0xf6;
+    pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);
+    pbCodeBuf[off++] = fBits;
+
+#elif defined(RT_ARCH_ARM64)
+
+    /* ands xzr, src, [tmp|#imm] */
+    uint32_t uImmR     = 0;
+    uint32_t uImmNandS = 0;
+    if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))
+    {
+        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+        pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(ARMV8_A64_REG_XZR, iGprDst, uImmNandS, uImmR, false /*f64Bit*/);
+    }
+    else
+    {
+        /* Use temporary register for the 64-bit immediate. */
+        uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);
+        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+        pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg, false /*f64Bit*/);
+        iemNativeRegFreeTmpImm(pReNative, iTmpReg);
+    }
+
+#else
+# error "Port me!"
+#endif
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+}
+
+
+/**
  * Emits a jump to @a idxLabel on the condition _any_ of the bits in @a fBits
  * are set in @a iGprSrc.
@@ -2838,4 +2905,7 @@
 
 
+/*********************************************************************************************************************************
+*   Calls.                                                                                                                       *
+*********************************************************************************************************************************/
 
 /**
@@ -2866,4 +2936,86 @@
 
 
+/**
+ * Emits code to load a stack variable into an argument GPR.
+ * @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
+ */
+DECL_FORCE_INLINE_THROW(uint32_t)
+iemNativeEmitLoadArgGregFromStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
+                                     int32_t offAddend = 0)
+{
+    IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
+    AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
+               IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
+
+    uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
+    if (idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs))
+    {
+        Assert(!(RT_BIT_32(idxRegVar) & IEMNATIVE_CALL_VOLATILE_GREG_MASK));
+        if (!offAddend)
+            off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegArg, idxRegVar);
+        else
+            off = iemNativeEmitLoadGprFromGprWithAddend(pReNative, off, idxRegArg, idxRegVar, offAddend);
+    }
+    else
+    {
+        uint8_t const idxStackSlot = pReNative->Core.aVars[idxVar].idxStackSlot;
+        AssertStmt(idxStackSlot != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_NOT_INITIALIZED));
+        off = iemNativeEmitLoadGprByBp(pReNative, off, idxRegArg, iemNativeStackCalcBpDisp(idxStackSlot));
+        if (offAddend)
+            off = iemNativeEmitAddGprImm(pReNative, off, idxRegArg, offAddend);
+    }
+    return off;
+}
+
+
+/**
+ * Emits code to load a stack or immediate variable value into an argument GPR,
+ * optional with a addend.
+ * @throws VERR_IEM_VAR_NOT_INITIALIZED, VERR_IEM_VAR_UNEXPECTED_KIND
+ */
+DECL_FORCE_INLINE_THROW(uint32_t)
+iemNativeEmitLoadArgGregFromImmOrStackVar(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
+                                          int32_t offAddend = 0)
+{
+    IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
+    if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Immediate)
+        off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegArg, pReNative->Core.aVars[idxVar].u.uValue + offAddend);
+    else
+        off = iemNativeEmitLoadArgGregFromStackVar(pReNative, off, idxRegArg, idxVar, offAddend);
+    return off;
+}
+
+
+/**
+ * Emits code to load the variable address into an argument GRP.
+ *
+ * This only works for uninitialized and stack variables.
+ */
+DECL_FORCE_INLINE_THROW(uint32_t)
+iemNativeEmitLoadArgGregWithVarAddr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxRegArg, uint8_t idxVar,
+                                    bool fFlushShadows)
+{
+    IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
+    AssertStmt(   pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid
+               || pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
+               IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
+
+    uint8_t const idxStackSlot   = iemNativeVarGetStackSlot(pReNative, idxVar);
+    int32_t const offBpDisp      = iemNativeStackCalcBpDisp(idxStackSlot);
+
+    uint8_t const idxRegVar      = pReNative->Core.aVars[idxVar].idxReg;
+    if (idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs))
+    {
+        off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, idxRegVar);
+        iemNativeRegFreeVar(pReNative, idxRegVar, fFlushShadows);
+        Assert(pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
+    }
+    Assert(   pReNative->Core.aVars[idxVar].idxStackSlot != UINT8_MAX
+           && pReNative->Core.aVars[idxVar].idxReg       == UINT8_MAX);
+
+    return iemNativeEmitLeaGprByBp(pReNative, off, idxRegArg, offBpDisp);
+}
+
+
 /** @} */
 
