Index: trunk/src/VBox/VMM/VMMAll/IEMAllInstPython.py
===================================================================
--- trunk/src/VBox/VMM/VMMAll/IEMAllInstPython.py	(revision 101843)
+++ trunk/src/VBox/VMM/VMMAll/IEMAllInstPython.py	(revision 101844)
@@ -2798,5 +2798,5 @@
     'IEM_MC_FETCH_FCW':                                          (McBlock.parseMcGeneric,           False, False, ),
     'IEM_MC_FETCH_FSW':                                          (McBlock.parseMcGeneric,           False, False, ),
-    'IEM_MC_FETCH_GREG_U16':                                     (McBlock.parseMcGeneric,           False, False, ),
+    'IEM_MC_FETCH_GREG_U16':                                     (McBlock.parseMcGeneric,           False, True,  ),
     'IEM_MC_FETCH_GREG_U16_SX_U32':                              (McBlock.parseMcGeneric,           False, False, ),
     'IEM_MC_FETCH_GREG_U16_SX_U64':                              (McBlock.parseMcGeneric,           False, False, ),
Index: trunk/src/VBox/VMM/VMMAll/IEMAllN8vePython.py
===================================================================
--- trunk/src/VBox/VMM/VMMAll/IEMAllN8vePython.py	(revision 101843)
+++ trunk/src/VBox/VMM/VMMAll/IEMAllN8vePython.py	(revision 101844)
@@ -69,5 +69,5 @@
     'IEM_MC_CALC_RM_EFF_ADDR_THREADED_64':                       (None, False, False, ),
 
-    'IEM_MC_CALL_CIMPL_1_THREADED':                              (None, True,  False, ),
+    'IEM_MC_CALL_CIMPL_1_THREADED':                              (None, True,  True,  ),
     'IEM_MC_CALL_CIMPL_2_THREADED':                              (None, True,  False, ),
     'IEM_MC_CALL_CIMPL_3_THREADED':                              (None, True,  False, ),
Index: trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp
===================================================================
--- trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp	(revision 101843)
+++ trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp	(revision 101844)
@@ -1604,4 +1604,6 @@
     pReNative->Core.bmGstRegShadows        = 0;
     pReNative->Core.bmVars                 = 0;
+    pReNative->Core.bmStack                = 0;
+    AssertCompile(sizeof(pReNative->Core.bmStack) * 8 == IEMNATIVE_FRAME_VAR_SLOTS); /* Must set reserved slots to 1 otherwise. */
     pReNative->Core.u64ArgVars             = UINT64_MAX;
 
@@ -2092,4 +2094,27 @@
 #endif
 };
+
+#ifdef IEMNATIVE_FP_OFF_STACK_ARG0
+/**
+ * BP offset of the stack argument slots.
+ *
+ * This array is indexed by \#argument - IEMNATIVE_CALL_ARG_GREG_COUNT and has
+ * IEMNATIVE_FRAME_STACK_ARG_COUNT entries.
+ */
+DECL_HIDDEN_CONST(int32_t) const g_aoffIemNativeCallStackArgBpDisp[] =
+{
+    IEMNATIVE_FP_OFF_STACK_ARG0,
+# ifdef IEMNATIVE_FP_OFF_STACK_ARG1
+    IEMNATIVE_FP_OFF_STACK_ARG1,
+# endif
+# ifdef IEMNATIVE_FP_OFF_STACK_ARG2
+    IEMNATIVE_FP_OFF_STACK_ARG2,
+# endif
+# ifdef IEMNATIVE_FP_OFF_STACK_ARG3
+    IEMNATIVE_FP_OFF_STACK_ARG3,
+# endif
+};
+AssertCompile(RT_ELEMENTS(g_aoffIemNativeCallStackArgBpDisp) == IEMNATIVE_FRAME_STACK_ARG_COUNT);
+#endif /* IEMNATIVE_FP_OFF_STACK_ARG0 */
 
 /**
@@ -2182,21 +2207,14 @@
 
 /**
- * Locate a register, possibly freeing one up.
+ * Tries to locate a suitable register in the given register mask.
  *
  * This ASSUMES the caller has done the minimal/optimal allocation checks and
  * failed.
  *
- * @returns Host register number on success; throws VBox status code on failure, so no
- *          need to check the return value.
- */
-static uint8_t iemNativeRegAllocFindFree(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fAllowVolatile)
-{
-    uint32_t fRegMask = fAllowVolatile
-                      ? IEMNATIVE_HST_GREG_MASK & ~IEMNATIVE_REG_FIXED_MASK
-                      : IEMNATIVE_HST_GREG_MASK & ~(IEMNATIVE_REG_FIXED_MASK | IEMNATIVE_CALL_VOLATILE_GREG_MASK);
-
-    /*
-     * Try a freed register that's shadowing a guest register
-     */
+ * @returns Host register number on success, returns UINT8_MAX on failure.
+ */
+static uint8_t iemNativeRegTryAllocFree(PIEMRECOMPILERSTATE pReNative, uint32_t fRegMask)
+{
+    Assert(!(fRegMask & ~IEMNATIVE_HST_GREG_MASK));
     uint32_t fRegs = ~pReNative->Core.bmHstRegs & fRegMask;
     if (fRegs)
@@ -2204,4 +2222,50 @@
         /** @todo pick better here:    */
         unsigned const idxReg = ASMBitFirstSetU32(fRegs) - 1;
+
+        Assert(pReNative->Core.aHstRegs[idxReg].fGstRegShadows != 0);
+        Assert(   (pReNative->Core.aHstRegs[idxReg].fGstRegShadows & pReNative->Core.bmGstRegShadows)
+               == pReNative->Core.aHstRegs[idxReg].fGstRegShadows);
+        Assert(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxReg));
+
+        pReNative->Core.bmGstRegShadows        &= ~pReNative->Core.aHstRegs[idxReg].fGstRegShadows;
+        pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxReg);
+        pReNative->Core.aHstRegs[idxReg].fGstRegShadows = 0;
+        return idxReg;
+    }
+    return UINT8_MAX;
+}
+
+
+/**
+ * Locate a register, possibly freeing one up.
+ *
+ * This ASSUMES the caller has done the minimal/optimal allocation checks and
+ * failed.
+ *
+ * @returns Host register number on success. Returns UINT8_MAX if no registers
+ *          found, the caller is supposed to deal with this and raise a
+ *          allocation type specific status code (if desired).
+ *
+ * @throws  VBox status code if we're run into trouble spilling a variable of
+ *          recording debug info.  Does NOT throw anything if we're out of
+ *          registers, though.
+ */
+static uint8_t iemNativeRegAllocFindFree(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile,
+                                         uint32_t fRegMask = IEMNATIVE_HST_GREG_MASK & ~IEMNATIVE_REG_FIXED_MASK)
+{
+    Assert(!(fRegMask & ~IEMNATIVE_HST_GREG_MASK));
+    Assert(!(fRegMask & ~IEMNATIVE_REG_FIXED_MASK));
+
+    /*
+     * Try a freed register that's shadowing a guest register
+     */
+    uint32_t fRegs = ~pReNative->Core.bmHstRegs & fRegMask;
+    if (fRegs)
+    {
+        unsigned const idxReg = (fPreferVolatile
+                                 ? ASMBitFirstSetU32(fRegs)
+                                 : ASMBitLastSetU32(  fRegs & ~IEMNATIVE_CALL_VOLATILE_GREG_MASK
+                                                    ? fRegs & ~IEMNATIVE_CALL_VOLATILE_GREG_MASK: fRegs))
+                              - 1;
 
         Assert(pReNative->Core.aHstRegs[idxReg].fGstRegShadows != 0);
@@ -2261,5 +2325,5 @@
     }
 
-    AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_REG_OUT_OF_REGISTERS));
+    return UINT8_MAX;
 }
 
@@ -2391,5 +2455,5 @@
     else
     {
-        idxReg = iemNativeRegAllocFindFree(pReNative, poff, true /*fAllowVolatile*/);
+        idxReg = iemNativeRegAllocFindFree(pReNative, poff, fPreferVolatile);
         AssertStmt(idxReg != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_REG_ALLOCATOR_NO_FREE_TMP));
     }
@@ -2874,28 +2938,9 @@
  *                          been allocated as such already and won't need moving,
  *                          just freeing.
- * @param   fFreeArgVars    Whether to free argument variables for the call.
  */
 DECL_HIDDEN_THROW(uint32_t)
-iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, bool fFreeArgVars)
-{
-    /*
-     * Free argument variables first (simplified).
-     */
-    AssertStmt(cArgs <= RT_ELEMENTS(pReNative->Core.aidxArgVars), IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_LABEL_IPE_6));
-    if (fFreeArgVars && cArgs > 0)
-    {
-        for (uint32_t i = 0; i < cArgs; i++)
-        {
-            uint8_t idxVar = pReNative->Core.aidxArgVars[i];
-            if (idxVar < RT_ELEMENTS(pReNative->Core.aVars))
-            {
-                pReNative->Core.aidxArgVars[i] = UINT8_MAX;
-                pReNative->Core.bmVars        &= ~RT_BIT_32(idxVar);
-                Assert(   pReNative->Core.aVars[idxVar].idxReg
-                       == (i < RT_ELEMENTS(g_aidxIemNativeCallRegs) ? g_aidxIemNativeCallRegs[i] : UINT8_MAX));
-            }
-        }
-        Assert(pReNative->Core.u64ArgVars == UINT64_MAX);
-    }
+iemNativeRegMoveAndFreeAndFlushAtCall(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs)
+{
+    Assert(cArgs <= IEMNATIVE_CALL_MAX_ARG_COUNT);
 
     /*
@@ -3129,5 +3174,5 @@
             else
                 AssertStmt(g_aGstShadowInfo[enmGstReg].cb == sizeof(uint32_t),
-                           IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_LABEL_IPE_7));
+                           IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_LABEL_IPE_6));
             if (idxReg >= 8)
                 pbCodeBuf[off++] = X86_OP_REX_R;
@@ -3361,5 +3406,5 @@
  * idxSegReg, raising a \#GP(0) if it isn't.
  *
- * @returns New code buffer offset, UINT32_MAX on failure.
+ * @returns New code buffer offset; throws VBox status code on error.
  * @param   pReNative       The native recompile state.
  * @param   off             The code buffer offset.
@@ -3413,5 +3458,5 @@
 {
     iemNativeRegFlushGuestShadows(pReNative, UINT64_MAX); /** @todo optimize this */
-    off = iemNativeRegMoveAndFreeAndFlushAtCall(pReNative, off, 4, false /*fFreeArgVars*/);
+    off = iemNativeRegMoveAndFreeAndFlushAtCall(pReNative, off, 4);
 
     /*
@@ -3468,5 +3513,5 @@
 {
     iemNativeRegFlushGuestShadows(pReNative, UINT64_MAX); /** @todo optimize this */
-    off = iemNativeRegMoveAndFreeAndFlushAtCall(pReNative, off, 4, false /*fFreeArgVars*/);
+    off = iemNativeRegMoveAndFreeAndFlushAtCall(pReNative, off, 4);
     uint8_t const cParams = g_acIemThreadedFunctionUsedArgs[pCallEntry->enmFunction];
 
@@ -3840,10 +3885,14 @@
 
 
+
 /*********************************************************************************************************************************
-*   Emitters for IEM_MC_XXXX and the associated IEM_MC_XXXX recompiler definitions                                               *
+*   Emitters for IEM_MC_BEGIN and IEM_MC_END.                                                                                    *
 *********************************************************************************************************************************/
 
 #define IEM_MC_BEGIN(a_cArgs, a_cLocals, a_fMcFlags, a_fCImplFlags) \
-    {
+    { \
+        pReNative->fMc    = (a_fMcFlags); \
+        pReNative->fCImpl = (a_fCImplFlags); \
+        pReNative->cArgs  = ((a_cArgs) + iemNativeArgGetHiddenArgCount(pReNative))
 
 /** We have to get to the end in recompilation mode, as otherwise we won't
@@ -3853,13 +3902,18 @@
 
 
-/*
- * Standalone CImpl deferals.
- */
+
+/*********************************************************************************************************************************
+*   Emitters for standalone C-implementation deferals (IEM_MC_DEFER_TO_CIMPL_XXXX)                                               *
+*********************************************************************************************************************************/
 
 #define IEM_MC_DEFER_TO_CIMPL_0_RET_THREADED(a_cbInstr, a_fFlags, a_pfnCImpl) \
+    pReNative->fMc    = 0; \
+    pReNative->fCImpl = (a_fFlags); \
     return iemNativeEmitCImplCall0(pReNative, off, pCallEntry->idxInstr, (uintptr_t)a_pfnCImpl, a_cbInstr) /** @todo not used ... */
 
 
 #define IEM_MC_DEFER_TO_CIMPL_1_RET_THREADED(a_cbInstr, a_fFlags, a_pfnCImpl, a0) \
+    pReNative->fMc    = 0; \
+    pReNative->fCImpl = (a_fFlags); \
     return iemNativeEmitCImplCall1(pReNative, off, pCallEntry->idxInstr, (uintptr_t)a_pfnCImpl, a_cbInstr, a0)
 
@@ -3872,4 +3926,6 @@
 
 #define IEM_MC_DEFER_TO_CIMPL_2_RET_THREADED(a_cbInstr, a_fFlags, a_pfnCImpl, a0, a1) \
+    pReNative->fMc    = 0; \
+    pReNative->fCImpl = (a_fFlags); \
     return iemNativeEmitCImplCall2(pReNative, off, pCallEntry->idxInstr, (uintptr_t)a_pfnCImpl, a_cbInstr, a0, a1)
 
@@ -3882,4 +3938,6 @@
 
 #define IEM_MC_DEFER_TO_CIMPL_3_RET_THREADED(a_cbInstr, a_fFlags, a_pfnCImpl, a0, a1, a2) \
+    pReNative->fMc    = 0; \
+    pReNative->fCImpl = (a_fFlags); \
     return iemNativeEmitCImplCall3(pReNative, off, pCallEntry->idxInstr, (uintptr_t)a_pfnCImpl, a_cbInstr, a0, a1, a2)
 
@@ -3892,7 +3950,8 @@
 
 
-/*
- * Advancing PC/RIP/EIP/IP.
- */
+
+/*********************************************************************************************************************************
+*   Emitters for advancing PC/RIP/EIP/IP (IEM_MC_ADVANCE_RIP_AND_FINISH_XXX)                                                     *
+*********************************************************************************************************************************/
 
 /** Emits the flags check for IEM_MC_ADVANCE_RIP_AND_FINISH_THREADED_PC64_WITH_FLAGS
@@ -4007,7 +4066,8 @@
 
 
-/*
- * Changing PC/RIP/EIP/IP with a relative jump.
- */
+
+/*********************************************************************************************************************************
+*   Emitters for changing PC/RIP/EIP/IP with a relative jump (IEM_MC_REL_JMP_XXX_AND_FINISH_XXX).                                *
+*********************************************************************************************************************************/
 
 #define IEM_MC_REL_JMP_S8_AND_FINISH_THREADED_PC64(a_i8, a_cbInstr, a_enmEffOpSize) \
@@ -4176,7 +4236,8 @@
 
 
-/*
- * Conditionals.
- */
+
+/*********************************************************************************************************************************
+*   Emitters for conditionals (IEM_MC_IF_XXX, IEM_MC_ELSE, IEM_MC_ENDIF)                                                         *
+*********************************************************************************************************************************/
 
 /**
@@ -4791,7 +4852,831 @@
 
 
-/*
- * General purpose register stores.
- */
+
+/*********************************************************************************************************************************
+*   Emitters for IEM_MC_ARG_XXX, IEM_MC_LOCAL, IEM_MC_LOCAL_CONST, ++                                                            *
+*********************************************************************************************************************************/
+/** Number of hidden arguments for CIMPL calls.
+ * @note We're sufferning from the usual VBOXSTRICTRC fun on Windows. */
+#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
+# define IEM_CIMPL_HIDDEN_ARGS 3
+#else
+# define IEM_CIMPL_HIDDEN_ARGS 2
+#endif
+
+#define IEM_MC_ARG(a_Type, a_Name, a_iArg) \
+    uint8_t const a_Name = iemNativeArgAlloc(pReNative, (a_iArg), sizeof(a_Type))
+
+#define IEM_MC_ARG_CONST(a_Type, a_Name, a_iArg) \
+    uint8_t const a_Name = iemNativeArgAllocConst(pReNative, (a_iArg), sizeof(a_Type), (a_Value))
+
+#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_iArg) \
+    uint8_t const a_Name = iemNativeArgAllocLocalRef(pReNative, (a_iArg), (a_Local))
+
+#define IEM_MC_LOCAL(a_Type, a_Name) \
+    uint8_t const a_Name = iemNativeVarAlloc(pReNative, sizeof(a_Type))
+
+#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) \
+    uint8_t const a_Name = iemNativeVarAllocConst(pReNative, sizeof(a_Type), (a_Value))
+
+
+/**
+ * Gets the number of hidden arguments for an expected IEM_MC_CALL statement.
+ */
+DECLINLINE(uint8_t) iemNativeArgGetHiddenArgCount(PIEMRECOMPILERSTATE pReNative)
+{
+    if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_CIMPL)
+        return IEM_CIMPL_HIDDEN_ARGS;
+    if (pReNative->fCImpl & IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE)
+        return 1;
+    return 0;
+}
+
+
+/**
+ * Internal work that allocates a variable with kind set to
+ * kIemNativeVarKind_Invalid and no current stack allocation.
+ *
+ * The kind will either be set by the caller or later when the variable is first
+ * assigned a value.
+ */
+static uint8_t iemNativeVarAllocInt(PIEMRECOMPILERSTATE pReNative, uint8_t cbType)
+{
+    Assert(cbType > 0 && cbType <= 64);
+    unsigned const idxVar = ASMBitFirstSetU32(~pReNative->Core.bmVars);
+    AssertStmt(idxVar < RT_ELEMENTS(pReNative->Core.aVars), IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_EXHAUSTED));
+    pReNative->Core.bmVars |= RT_BIT_32(idxVar);
+    pReNative->Core.aVars[idxVar].enmKind        = kIemNativeVarKind_Invalid;
+    pReNative->Core.aVars[idxVar].cbVar          = cbType;
+    pReNative->Core.aVars[idxVar].idxStackSlot   = UINT8_MAX;
+    pReNative->Core.aVars[idxVar].idxReg         = UINT8_MAX;
+    pReNative->Core.aVars[idxVar].uArgNo         = UINT8_MAX;
+    pReNative->Core.aVars[idxVar].idxReferrerVar = UINT8_MAX;
+    pReNative->Core.aVars[idxVar].enmGstReg      = kIemNativeGstReg_End;
+    pReNative->Core.aVars[idxVar].u.uValue       = 0;
+    return idxVar;
+}
+
+
+/**
+ * Internal work that allocates an argument variable w/o setting enmKind.
+ */
+static uint8_t iemNativeArgAllocInt(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType)
+{
+    iArgNo += iemNativeArgGetHiddenArgCount(pReNative);
+    AssertStmt(iArgNo < RT_ELEMENTS(pReNative->Core.aidxArgVars), IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_1));
+    AssertStmt(pReNative->Core.aidxArgVars[iArgNo] == UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_DUP_ARG_NO));
+
+    uint8_t const idxVar = iemNativeVarAllocInt(pReNative, cbType);
+    pReNative->Core.aidxArgVars[iArgNo]  = idxVar;
+    pReNative->Core.aVars[idxVar].uArgNo = iArgNo;
+    return idxVar;
+}
+
+
+/**
+ * Changes the variable to a stack variable.
+ *
+ * Currently this is s only possible to do the first time the variable is used,
+ * switching later is can be implemented but not done.
+ *
+ * @param   pReNative   The recompiler state.
+ * @param   idxVar      The variable.
+ * @throws  VERR_IEM_VAR_OUT_OF_STACK_SLOTS, VERR_IEM_VAR_IPE_2
+ */
+static void iemNativeVarSetKindToStack(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
+{
+    Assert(idxVar < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxVar)));
+    if (pReNative->Core.aVars[idxVar].enmKind != kIemNativeVarKind_Stack)
+    {
+        /* We could in theory transition from immediate to stack as well, but it
+           would involve the caller doing work storing the value on the stack. So,
+           till that's required we only allow transition from invalid. */
+        AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid,
+                   IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_2));
+        pReNative->Core.aVars[idxVar].enmKind = kIemNativeVarKind_Stack;
+
+        if (pReNative->Core.aVars[idxVar].idxStackSlot == UINT8_MAX)
+        {
+            if (pReNative->Core.aVars[idxVar].cbVar <= sizeof(uint64_t))
+            {
+                unsigned const iSlot = ASMBitFirstSetU32(~pReNative->Core.bmStack);
+                AssertStmt(iSlot < IEMNATIVE_FRAME_VAR_SLOTS, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_OUT_OF_STACK_SLOTS));
+                pReNative->Core.bmStack |= RT_BIT_32(iSlot);
+                pReNative->Core.aVars[idxVar].idxStackSlot = iSlot;
+                return;
+            }
+            /* cbVar -> fBitAlignMask: 16 -> 1; 32 -> 3; 64 -> 7;*/
+            AssertCompile(RT_IS_POWER_OF_TWO(IEMNATIVE_FRAME_VAR_SLOTS)); /* If not we have to add an overflow check. */
+            Assert(pReNative->Core.aVars[idxVar].cbVar <= 64);
+            uint32_t const fBitAlignMask = RT_BIT_32(ASMBitLastSetU32(pReNative->Core.aVars[idxVar].cbVar) - 4) - 1;
+            uint32_t       fBitAllocMask = RT_BIT_32((pReNative->Core.aVars[idxVar].cbVar + 7) >> 3) - 1;
+            uint32_t       bmStack       = ~pReNative->Core.bmStack;
+            while (bmStack != UINT32_MAX)
+            {
+                unsigned const iSlot = ASMBitFirstSetU32(bmStack) - 1;
+                AssertStmt(iSlot < IEMNATIVE_FRAME_VAR_SLOTS, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_OUT_OF_STACK_SLOTS));
+                if (!(iSlot & fBitAlignMask))
+                {
+                    if ((bmStack & (fBitAllocMask << iSlot)) == (fBitAllocMask << iSlot))
+                    {
+                        pReNative->Core.bmStack |= (fBitAllocMask << iSlot);
+                        pReNative->Core.aVars[idxVar].idxStackSlot = iSlot;
+                        return;
+                    }
+                }
+                bmStack |= fBitAlignMask << (iSlot & ~fBitAlignMask);
+            }
+            AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_OUT_OF_STACK_SLOTS));
+        }
+    }
+}
+
+
+/**
+ * Changes it to a variable with a constant value.
+ *
+ * This does not require stack storage as we know the value and can always
+ * reload it, unless of course it's referenced.
+ *
+ * @param   pReNative   The recompiler state.
+ * @param   idxVar      The variable.
+ * @throws  VERR_IEM_VAR_OUT_OF_STACK_SLOTS, VERR_IEM_VAR_IPE_2
+ */
+static void iemNativeVarSetKindToConst(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint64_t uValue)
+{
+    Assert(idxVar < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxVar)));
+    if (pReNative->Core.aVars[idxVar].enmKind != kIemNativeVarKind_Immediate)
+    {
+        /* Only simple trasnsitions for now. */
+        AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid,
+                   IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_2));
+        pReNative->Core.aVars[idxVar].enmKind = kIemNativeVarKind_Immediate;
+    }
+    pReNative->Core.aVars[idxVar].u.uValue = uValue;
+}
+
+
+/**
+ * Changes the variable to a reference (pointer) to @a idxOtherVar.
+ *
+ * @param   pReNative   The recompiler state.
+ * @param   idxVar      The variable.
+ * @throws  VERR_IEM_VAR_OUT_OF_STACK_SLOTS, VERR_IEM_VAR_IPE_2
+ */
+static void iemNativeVarSetKindToLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint8_t idxOtherVar)
+{
+    Assert(idxVar      < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxVar)));
+    Assert(idxOtherVar < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxOtherVar)));
+
+    if (pReNative->Core.aVars[idxVar].enmKind != kIemNativeVarKind_VarRef)
+    {
+        /* Only simple trasnsitions for now. */
+        AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid,
+                   IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_2));
+        pReNative->Core.aVars[idxVar].enmKind = kIemNativeVarKind_Immediate;
+    }
+    pReNative->Core.aVars[idxVar].u.idxRefVar = idxOtherVar;
+
+    /* Update the other variable, ensure it's a stack variable. */
+    /** @todo handle variables with const values... that's go boom now. */
+    pReNative->Core.aVars[idxOtherVar].idxReferrerVar = idxVar;
+    iemNativeVarSetKindToStack(pReNative, idxOtherVar);
+}
+
+
+DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType)
+{
+    return iemNativeArgAllocInt(pReNative, iArgNo, cbType);
+}
+
+
+DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue)
+{
+    uint8_t const idxVar = iemNativeArgAllocInt(pReNative, iArgNo, cbType);
+    iemNativeVarSetKindToConst(pReNative, idxVar, uValue);
+    return idxVar;
+}
+
+
+DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar)
+{
+    AssertStmt(   idxOtherVar < RT_ELEMENTS(pReNative->Core.aVars)
+               && (pReNative->Core.bmVars & RT_BIT_32(idxOtherVar))
+               && pReNative->Core.aVars[idxOtherVar].uArgNo == UINT8_MAX,
+               IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_1));
+
+    uint8_t const idxArgVar = iemNativeArgAlloc(pReNative, iArgNo, sizeof(uintptr_t));
+    iemNativeVarSetKindToLocalRef(pReNative, idxArgVar, idxOtherVar);
+    return idxArgVar;
+}
+
+
+DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType)
+{
+    uint8_t const idxVar = iemNativeVarAllocInt(pReNative, cbType);
+    iemNativeVarSetKindToStack(pReNative, idxVar);
+    return idxVar;
+}
+
+
+DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue)
+{
+    uint8_t const idxVar = iemNativeVarAllocInt(pReNative, cbType);
+    iemNativeVarSetKindToConst(pReNative, idxVar, uValue);
+    return idxVar;
+}
+
+
+/**
+ * Makes sure variable @a idxVar has a register assigned to it.
+ *
+ * @returns The host register number.
+ * @param   pReNative   The recompiler state.
+ * @param   idxVar      The variable.
+ * @param   poff        Pointer to the instruction buffer offset.
+ *                      In case a register needs to be freed up.
+ */
+DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocRegister(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint32_t *poff)
+{
+    Assert(idxVar < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxVar)));
+
+    uint8_t idxReg = pReNative->Core.aVars[idxVar].idxReg;
+    if (idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs))
+        return idxReg;
+
+    /*
+     * We have to allocate a register for the variable, even if its a stack one
+     * as we don't know if there are modification being made to it before its
+     * finalized (todo: analyze and insert hints about that?).
+     *
+     * If we can, we try get the correct register for argument variables. This
+     * is assuming that most argument variables are fetched as close as possible
+     * to the actual call, so that there aren't any interfering hidden calls
+     * (memory accesses, etc) inbetween.
+     *
+     * If we cannot or it's a variable, we make sure no argument registers
+     * that will be used by this MC block will be allocated here, and we always
+     * prefer non-volatile registers to avoid needing to spill stuff for internal
+     * call.
+     */
+    /** @todo Detect too early argument value fetches and warn about hidden
+     * calls causing less optimal code to be generated in the python script. */
+
+    uint8_t const uArgNo = pReNative->Core.aVars[idxVar].uArgNo;
+    if (   uArgNo < RT_ELEMENTS(g_aidxIemNativeCallRegs)
+        && !(pReNative->Core.bmHstRegs & RT_BIT_32(g_aidxIemNativeCallRegs[uArgNo])))
+        idxReg = g_aidxIemNativeCallRegs[uArgNo];
+    else
+    {
+        uint32_t const fNotArgsMask = ~g_afIemNativeCallRegs[RT_MIN(pReNative->cArgs, IEMNATIVE_CALL_ARG_GREG_COUNT)];
+        uint32_t const fRegs        = ~pReNative->Core.bmHstRegs
+                                    & ~pReNative->Core.bmHstRegsWithGstShadow
+                                    & (~IEMNATIVE_REG_FIXED_MASK & IEMNATIVE_HST_GREG_MASK)
+                                    & fNotArgsMask;
+        if (fRegs)
+        {
+            /* Pick from the top as that both arm64 and amd64 have a block of non-volatile registers there. */
+            idxReg = (uint8_t)ASMBitLastSetU32(  fRegs & ~IEMNATIVE_CALL_VOLATILE_GREG_MASK
+                                               ? fRegs & ~IEMNATIVE_CALL_VOLATILE_GREG_MASK : fRegs) - 1;
+            Assert(pReNative->Core.aHstRegs[idxReg].fGstRegShadows == 0);
+            Assert(!(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxReg)));
+        }
+        else
+        {
+            idxReg = iemNativeRegAllocFindFree(pReNative, poff, false /*fPreferVolatile*/,
+                                               IEMNATIVE_HST_GREG_MASK & ~IEMNATIVE_REG_FIXED_MASK & fNotArgsMask);
+            AssertStmt(idxReg != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_REG_ALLOCATOR_NO_FREE_VAR));
+        }
+    }
+    iemNativeRegMarkAllocated(pReNative, idxReg, kIemNativeWhat_Var, idxVar);
+    pReNative->Core.aVars[idxVar].idxReg = idxReg;
+    return idxReg;
+}
+
+
+
+/*********************************************************************************************************************************
+*   Emitters for IEM_MC_CALL_CIMPL_XXX                                                                                           *
+*********************************************************************************************************************************/
+
+/**
+ * Emits code to load a reference to the given guest register into @a idxGprDst.
+  */
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitLeaGprByGstRegRef(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxGprDst,
+                               IEMNATIVEGSTREGREF enmClass, uint8_t idxRegInClass)
+{
+    /*
+     * Get the offset relative to the CPUMCTX structure.
+     */
+    uint32_t offCpumCtx;
+    switch (enmClass)
+    {
+        case kIemNativeGstRegRef_Gpr:
+            Assert(idxRegInClass < 16);
+            offCpumCtx = RT_UOFFSETOF_DYN(CPUMCTX, aGRegs[idxRegInClass]);
+            break;
+
+        case kIemNativeGstRegRef_GprHighByte:    /**< AH, CH, DH, BH*/
+            Assert(idxRegInClass < 4);
+            offCpumCtx = RT_UOFFSETOF_DYN(CPUMCTX, aGRegs[0].bHi) + idxRegInClass * sizeof(CPUMCTXGREG);
+            break;
+
+        case kIemNativeGstRegRef_EFlags:
+            Assert(idxRegInClass == 0);
+            offCpumCtx = RT_UOFFSETOF(CPUMCTX, eflags);
+            break;
+
+        case kIemNativeGstRegRef_MxCsr:
+            Assert(idxRegInClass == 0);
+            offCpumCtx = RT_UOFFSETOF(CPUMCTX, XState.x87.MXCSR);
+            break;
+
+        case kIemNativeGstRegRef_FpuReg:
+            Assert(idxRegInClass < 8);
+            AssertFailed(); /** @todo what kind of indexing? */
+            offCpumCtx = RT_UOFFSETOF_DYN(CPUMCTX, XState.x87.aRegs[idxRegInClass]);
+            break;
+
+        case kIemNativeGstRegRef_MReg:
+            Assert(idxRegInClass < 8);
+            AssertFailed(); /** @todo what kind of indexing? */
+            offCpumCtx = RT_UOFFSETOF_DYN(CPUMCTX, XState.x87.aRegs[idxRegInClass]);
+            break;
+
+        case kIemNativeGstRegRef_XReg:
+            Assert(idxRegInClass < 16);
+            offCpumCtx = RT_UOFFSETOF_DYN(CPUMCTX, XState.x87.aXMM[idxRegInClass]);
+            break;
+
+        default:
+            AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_5));
+    }
+
+    /*
+     * Load the value into the destination register.
+     */
+#ifdef RT_ARCH_AMD64
+    off = iemNativeEmitLeaGprByVCpu(pReNative, off, idxGprDst, offCpumCtx + RT_UOFFSETOF(VMCPUCC, cpum.GstCtx));
+
+#elif defined(RT_ARCH_ARM64)
+    uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
+    Assert(offCpumCtx < 4096);
+    pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, idxGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX, offCpumCtx);
+
+#else
+# error "Port me!"
+#endif
+
+    return off;
+}
+
+
+/**
+ * Common code for CIMPL and AIMPL calls.
+ *
+ * These are calls that uses argument variables and such.  They should not be
+ * confused with internal calls required to implement an MC operation,
+ * like a TLB load and similar.
+ *
+ * Upon return all that is left to do is to load any hidden arguments and
+ * perform the call. All argument variables are freed.
+ *
+ * @returns New code buffer offset; throws VBox status code on error.
+ * @param   pReNative       The native recompile state.
+ * @param   off             The code buffer offset.
+ * @param   cArgs           The total nubmer of arguments (includes hidden
+ *                          count).
+ * @param   cHiddenArgs     The number of hidden arguments.  The hidden
+ *                          arguments must not have any variable declared for
+ *                          them, whereas all the regular arguments must
+ *                          (tstIEMCheckMc ensures this).
+ */
+DECL_HIDDEN_THROW(uint32_t)
+iemNativeEmitCallCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs, uint8_t cHiddenArgs)
+{
+#ifdef VBOX_STRICT
+    /*
+     * Assert sanity.
+     */
+    Assert(cArgs <= IEMNATIVE_CALL_MAX_ARG_COUNT);
+    Assert(cHiddenArgs < IEMNATIVE_CALL_ARG_GREG_COUNT);
+    for (unsigned i = 0; i < cHiddenArgs; i++)
+        Assert(pReNative->Core.aidxArgVars[i] == UINT8_MAX);
+    for (unsigned i = cHiddenArgs; i < cArgs; i++)
+    {
+        Assert(pReNative->Core.aidxArgVars[i] != UINT8_MAX); /* checked by tstIEMCheckMc.cpp */
+        Assert(pReNative->Core.bmVars & RT_BIT_32(pReNative->Core.aidxArgVars[i]));
+    }
+#endif
+
+    uint8_t const cRegArgs = RT_MIN(cArgs, RT_ELEMENTS(g_aidxIemNativeCallRegs));
+
+    /*
+     * First, go over the host registers that will be used for arguments and make
+     * sure they either hold the desired argument or are free.
+     */
+    if (pReNative->Core.bmHstRegs & g_afIemNativeCallRegs[cRegArgs])
+        for (uint32_t i = 0; i < cRegArgs; i++)
+        {
+            uint8_t const idxArgReg = g_aidxIemNativeCallRegs[i];
+            if (pReNative->Core.bmHstRegs & RT_BIT_32(idxArgReg))
+            {
+                if (pReNative->Core.aHstRegs[idxArgReg].enmWhat == kIemNativeWhat_Var)
+                {
+                    uint8_t const idxVar = pReNative->Core.aHstRegs[idxArgReg].idxVar;
+                    Assert(idxVar < RT_ELEMENTS(pReNative->Core.aVars));
+                    Assert(pReNative->Core.aVars[idxVar].idxReg == idxArgReg);
+                    uint8_t const uArgNo = pReNative->Core.aVars[idxVar].uArgNo;
+                    if (uArgNo == i)
+                    { /* prefect */ }
+                    else
+                    {
+                        /* The variable allocator logic should make sure this is impossible. */
+                        AssertStmt(uArgNo == UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_REG_IPE_10));
+
+                        if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack)
+                            off = iemNativeRegMoveOrSpillStackVar(pReNative, off, idxVar);
+                        else
+                        {
+                            /* just free it, can be reloaded if used again */
+                            pReNative->Core.aVars[idxVar].idxReg = UINT8_MAX;
+                            pReNative->Core.bmHstRegs &= ~RT_BIT_32(idxArgReg);
+                            iemNativeRegClearGstRegShadowing(pReNative, idxArgReg, off);
+                        }
+                    }
+                }
+                else
+                    AssertStmt(pReNative->Core.aHstRegs[idxArgReg].enmWhat == kIemNativeWhat_Arg,
+                               IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_REG_IPE_8));
+            }
+        }
+
+    Assert(!(pReNative->Core.bmHstRegs & g_afIemNativeCallRegs[cHiddenArgs])); /* No variables for hidden arguments. */
+
+    /*
+     * Make sure the argument variables are loaded into their respective registers.
+     *
+     * We can optimize this by ASSUMING that any register allocations are for
+     * registeres that have already been loaded and are ready.  The previous step
+     * saw to that.
+     */
+    if (~pReNative->Core.bmHstRegs & (g_afIemNativeCallRegs[cRegArgs] & ~g_afIemNativeCallRegs[cHiddenArgs]))
+        for (unsigned i = cHiddenArgs; i < cRegArgs; i++)
+        {
+            uint8_t const idxArgReg = g_aidxIemNativeCallRegs[i];
+            if (pReNative->Core.bmHstRegs & RT_BIT_32(idxArgReg))
+                Assert(   pReNative->Core.aHstRegs[idxArgReg].idxVar == pReNative->Core.aidxArgVars[i]
+                       && pReNative->Core.aVars[pReNative->Core.aidxArgVars[i]].uArgNo == i
+                       && pReNative->Core.aVars[pReNative->Core.aidxArgVars[i]].idxReg == idxArgReg);
+            else
+            {
+                uint8_t const idxVar = pReNative->Core.aidxArgVars[i];
+                if (pReNative->Core.aVars[idxVar].uArgNo < RT_ELEMENTS(pReNative->Core.aHstRegs))
+                {
+                    Assert(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack);
+                    off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxArgReg, pReNative->Core.aVars[idxVar].idxReg);
+                }
+                else
+                {
+                    /* Use ARG0 as temp for stuff we need registers for. */
+                    switch (pReNative->Core.aVars[idxVar].enmKind)
+                    {
+                        case kIemNativeVarKind_Stack:
+                            AssertStmt(pReNative->Core.aVars[idxVar].idxStackSlot != UINT8_MAX,
+                                       IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_3));
+                            off = iemNativeEmitLoadGprByBp(pReNative, off, idxArgReg,
+                                                             IEMNATIVE_FP_OFF_STACK_VARS
+                                                           + pReNative->Core.aVars[idxVar].idxStackSlot * sizeof(uint64_t));
+                            continue;
+
+                        case kIemNativeVarKind_Immediate:
+                            off = iemNativeEmitLoadGprImm64(pReNative, off, idxArgReg, pReNative->Core.aVars[idxVar].u.uValue);
+                            continue;
+
+                        case kIemNativeVarKind_VarRef:
+                        {
+                            uint8_t const idxOtherVar = pReNative->Core.aVars[idxVar].u.idxRefVar;
+                            Assert(idxOtherVar < RT_ELEMENTS(pReNative->Core.aVars));
+                            AssertStmt(pReNative->Core.aVars[idxOtherVar].idxStackSlot != UINT8_MAX,
+                                       IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_4));
+                            off = iemNativeEmitLeaGprByBp(pReNative, off, idxArgReg,
+                                                            IEMNATIVE_FP_OFF_STACK_VARS
+                                                          + pReNative->Core.aVars[idxOtherVar].idxStackSlot * sizeof(uint64_t));
+                            continue;
+                        }
+
+                        case kIemNativeVarKind_GstRegRef:
+                            off = iemNativeEmitLeaGprByGstRegRef(pReNative, off, idxArgReg,
+                                                                 pReNative->Core.aVars[idxVar].u.GstRegRef.enmClass,
+                                                                 pReNative->Core.aVars[idxVar].u.GstRegRef.idx);
+                            continue;
+
+                        case kIemNativeVarKind_Invalid:
+                        case kIemNativeVarKind_End:
+                            break;
+                    }
+                    AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_3));
+                }
+            }
+        }
+#ifdef VBOX_STRICT
+    else
+        for (unsigned i = cHiddenArgs; i < cRegArgs; i++)
+        {
+            Assert(pReNative->Core.aVars[pReNative->Core.aidxArgVars[i]].uArgNo == i);
+            Assert(pReNative->Core.aVars[pReNative->Core.aidxArgVars[i]].idxReg == g_aidxIemNativeCallRegs[i]);
+        }
+#endif
+
+#ifdef IEMNATIVE_FP_OFF_STACK_ARG0
+    /*
+     * If there are any stack arguments, make sure they are in their place as well.
+     *
+     * We can use IEMNATIVE_CALL_ARG0_GREG as temporary register since it the
+     * caller will load it later and it must be free (see first loop).
+     */
+    if (cArgs > IEMNATIVE_CALL_ARG_GREG_COUNT)
+        for (unsigned i = IEMNATIVE_CALL_ARG_GREG_COUNT; i < cArgs; i++)
+        {
+            uint8_t const idxVar    = pReNative->Core.aidxArgVars[i];
+            int32_t const offBpDisp = g_aoffIemNativeCallStackArgBpDisp[i - IEMNATIVE_CALL_ARG_GREG_COUNT];
+            if (pReNative->Core.aVars[idxVar].uArgNo < RT_ELEMENTS(pReNative->Core.aHstRegs))
+            {
+                Assert(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack); /* Imm as well? */
+                off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, pReNative->Core.aVars[idxVar].uArgNo);
+            }
+            else
+            {
+                /* Use ARG0 as temp for stuff we need registers for. */
+                switch (pReNative->Core.aVars[idxVar].enmKind)
+                {
+                    case kIemNativeVarKind_Stack:
+                        AssertStmt(pReNative->Core.aVars[idxVar].idxStackSlot != UINT8_MAX,
+                                   IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_3));
+                        off = iemNativeEmitLoadGprByBp(pReNative, off, IEMNATIVE_CALL_ARG0_GREG /* is free */,
+                                                         IEMNATIVE_FP_OFF_STACK_VARS
+                                                       + pReNative->Core.aVars[idxVar].idxStackSlot * sizeof(uint64_t));
+                        off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, IEMNATIVE_CALL_ARG0_GREG);
+                        continue;
+
+                    case kIemNativeVarKind_Immediate:
+                        off = iemNativeEmitStoreImm64ByBp(pReNative, off, offBpDisp, pReNative->Core.aVars[idxVar].u.uValue);
+                        continue;
+
+                    case kIemNativeVarKind_VarRef:
+                    {
+                        uint8_t const idxOtherVar = pReNative->Core.aVars[idxVar].u.idxRefVar;
+                        Assert(idxOtherVar < RT_ELEMENTS(pReNative->Core.aVars));
+                        AssertStmt(pReNative->Core.aVars[idxOtherVar].idxStackSlot != UINT8_MAX,
+                                   IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_4));
+                        off = iemNativeEmitLeaGprByBp(pReNative, off, IEMNATIVE_CALL_ARG0_GREG,
+                                                        IEMNATIVE_FP_OFF_STACK_VARS
+                                                      + pReNative->Core.aVars[idxOtherVar].idxStackSlot * sizeof(uint64_t));
+                        off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, IEMNATIVE_CALL_ARG0_GREG);
+                        continue;
+                    }
+
+                    case kIemNativeVarKind_GstRegRef:
+                        off = iemNativeEmitLeaGprByGstRegRef(pReNative, off, IEMNATIVE_CALL_ARG0_GREG,
+                                                             pReNative->Core.aVars[idxVar].u.GstRegRef.enmClass,
+                                                             pReNative->Core.aVars[idxVar].u.GstRegRef.idx);
+                        off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, IEMNATIVE_CALL_ARG0_GREG);
+                        continue;
+
+                    case kIemNativeVarKind_Invalid:
+                    case kIemNativeVarKind_End:
+                        break;
+                }
+                AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_3));
+            }
+        }
+#else
+    AssertCompile(IEMNATIVE_CALL_MAX_ARG_COUNT <= IEMNATIVE_CALL_ARG_GREG_COUNT);
+#endif
+
+    /*
+     * Free all argument variables (simplified).
+     * Their lifetime always expires with the call they are for.
+     */
+    /** @todo Unfortunately we have some high 32-bit register clears using
+     *        register references held by argument vars. Almost all the
+     *        IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF uses are stupid!  However we need to
+     *        add checks to the python scripts to ensure argument vars aren't used
+     *        after a call. That's a bit of work. */
+    for (uint32_t i = cHiddenArgs; i < cArgs; i++)
+    {
+        uint8_t idxVar = pReNative->Core.aidxArgVars[i];
+        Assert(idxVar < RT_ELEMENTS(pReNative->Core.aVars));
+        pReNative->Core.aidxArgVars[i] = UINT8_MAX;
+        pReNative->Core.bmVars        &= ~RT_BIT_32(idxVar);
+        Assert(   pReNative->Core.aVars[idxVar].idxReg
+               == (i < RT_ELEMENTS(g_aidxIemNativeCallRegs) ? g_aidxIemNativeCallRegs[i] : UINT8_MAX));
+    }
+    Assert(pReNative->Core.u64ArgVars == UINT64_MAX);
+
+    /*
+     * Flush volatile registers as we make the call.
+     */
+    off = iemNativeRegMoveAndFreeAndFlushAtCall(pReNative, off, cRegArgs);
+
+    return off;
+}
+
+
+/** Common emit function for IEM_MC_CALL_CIMPL_XXXX. */
+DECL_HIDDEN_THROW(uint32_t)
+iemNativeEmitCallCImplCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr, uint8_t idxInstr,
+                             uintptr_t pfnCImpl, uint8_t cArgs)
+
+{
+//pReNative->pInstrBuf[off++] = 0xcc;
+    /*
+     * Do all the call setup and cleanup.
+     */
+    off = iemNativeEmitCallCommon(pReNative, off, cArgs + IEM_CIMPL_HIDDEN_ARGS, IEM_CIMPL_HIDDEN_ARGS);
+
+    /*
+     * Load the two hidden arguments.
+     */
+#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
+    off = iemNativeEmitLeaGprByBp(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_FP_OFF_IN_SHADOW_ARG0); /* rcStrict */
+    off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
+    off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG2_GREG, cbInstr);
+#else
+    off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
+    off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, cbInstr);
+#endif
+
+    /*
+     * Make the call and check the return code.
+     */
+    off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)pfnCImpl);
+#if defined(VBOXSTRICTRC_STRICT_ENABLED) && defined(RT_OS_WINDOWS) && defined(RT_ARCH_AMD64)
+    off = iemNativeEmitLoadGprByBpU32(pReNative, off, X86_GREG_xAX, IEMNATIVE_FP_OFF_IN_SHADOW_ARG0); /* rcStrict (see above) */
+#endif
+    return iemNativeEmitCheckCallRetAndPassUp(pReNative, off, idxInstr);
+}
+
+
+#define IEM_MC_CALL_CIMPL_1_THREADED(a_cbInstr, a_fFlags, a_pfnCImpl, a0) \
+    off = iemNativeEmitCallCImpl1(pReNative, off, a_cbInstr, pCallEntry->idxInstr, (uintptr_t)a_pfnCImpl, a0)
+
+/** Emits code for IEM_MC_CALL_CIMPL_1. */
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitCallCImpl1(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr, uint8_t idxInstr,
+                        uintptr_t pfnCImpl, uint8_t idxArg0)
+{
+    Assert(idxArg0 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg0)));
+    Assert(pReNative->Core.aVars[idxArg0].uArgNo == 0 + IEM_CIMPL_HIDDEN_ARGS);
+    RT_NOREF_PV(idxArg0);
+
+    return iemNativeEmitCallCImplCommon(pReNative, off, cbInstr, idxInstr, pfnCImpl, 1);
+}
+
+
+#define IEM_MC_CALL_CIMPL_2_THREADED(a_cbInstr, a_fFlags, a_pfnCImpl, a0, a1) \
+    off = iemNativeEmitCallCImpl2(pReNative, off, a_cbInstr, pCallEntry->idxInstr, (uintptr_t)a_pfnCImpl, a0, a1)
+
+/** Emits code for IEM_MC_CALL_CIMPL_2. */
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitCallCImpl2(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr, uint8_t idxInstr,
+                        uintptr_t pfnCImpl, uint8_t idxArg0, uint8_t idxArg1)
+{
+    Assert(idxArg0 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg0)));
+    Assert(pReNative->Core.aVars[idxArg0].uArgNo == 0 + IEM_CIMPL_HIDDEN_ARGS);
+    RT_NOREF_PV(idxArg0);
+
+    Assert(idxArg1 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg1)));
+    Assert(pReNative->Core.aVars[idxArg1].uArgNo == 1 + IEM_CIMPL_HIDDEN_ARGS);
+    RT_NOREF_PV(idxArg1);
+
+    return iemNativeEmitCallCImplCommon(pReNative, off, cbInstr, idxInstr, pfnCImpl, 2);
+}
+
+
+#define IEM_MC_CALL_CIMPL_3_THREADED(a_cbInstr, a_fFlags, a_pfnCImpl, a0, a1, a2) \
+    off = iemNativeEmitCallCImpl3(pReNative, off, a_cbInstr, pCallEntry->idxInstr, (uintptr_t)a_pfnCImpl, a0, a1, a2)
+
+/** Emits code for IEM_MC_CALL_CIMPL_3. */
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitCallCImpl3(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr, uint8_t idxInstr,
+                        uintptr_t pfnCImpl, uint8_t idxArg0, uint8_t idxArg1, uint8_t idxArg2)
+{
+    Assert(idxArg0 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg0)));
+    Assert(pReNative->Core.aVars[idxArg0].uArgNo == 0 + IEM_CIMPL_HIDDEN_ARGS);
+    RT_NOREF_PV(idxArg0);
+
+    Assert(idxArg1 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg1)));
+    Assert(pReNative->Core.aVars[idxArg1].uArgNo == 1 + IEM_CIMPL_HIDDEN_ARGS);
+    RT_NOREF_PV(idxArg1);
+
+    Assert(idxArg2 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg2)));
+    Assert(pReNative->Core.aVars[idxArg2].uArgNo == 2 + IEM_CIMPL_HIDDEN_ARGS);
+    RT_NOREF_PV(idxArg2);
+
+    return iemNativeEmitCallCImplCommon(pReNative, off, cbInstr, idxInstr, pfnCImpl, 3);
+}
+
+
+#define IEM_MC_CALL_CIMPL_4_THREADED(a_cbInstr, a_fFlags, a_pfnCImpl, a0, a1, a2, a3) \
+    off = iemNativeEmitCallCImpl4(pReNative, off, a_cbInstr, pCallEntry->idxInstr, (uintptr_t)a_pfnCImpl, a0, a1, a2, a3)
+
+/** Emits code for IEM_MC_CALL_CIMPL_4. */
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitCallCImpl4(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr, uint8_t idxInstr,
+                        uintptr_t pfnCImpl, uint8_t idxArg0, uint8_t idxArg1, uint8_t idxArg2, uint8_t idxArg3)
+{
+    Assert(idxArg0 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg0)));
+    Assert(pReNative->Core.aVars[idxArg0].uArgNo == 0 + IEM_CIMPL_HIDDEN_ARGS);
+    RT_NOREF_PV(idxArg0);
+
+    Assert(idxArg1 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg1)));
+    Assert(pReNative->Core.aVars[idxArg1].uArgNo == 1 + IEM_CIMPL_HIDDEN_ARGS);
+    RT_NOREF_PV(idxArg1);
+
+    Assert(idxArg2 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg2)));
+    Assert(pReNative->Core.aVars[idxArg2].uArgNo == 2 + IEM_CIMPL_HIDDEN_ARGS);
+    RT_NOREF_PV(idxArg2);
+
+    Assert(idxArg3 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg3)));
+    Assert(pReNative->Core.aVars[idxArg3].uArgNo == 3 + IEM_CIMPL_HIDDEN_ARGS);
+    RT_NOREF_PV(idxArg3);
+
+    return iemNativeEmitCallCImplCommon(pReNative, off, cbInstr, idxInstr, pfnCImpl, 4);
+}
+
+
+#define IEM_MC_CALL_CIMPL_5_THREADED(a_cbInstr, a_fFlags, a_pfnCImpl, a0, a1, a2, a3, a4) \
+    off = iemNativeEmitCallCImpl5(pReNative, off, a_cbInstr, pCallEntry->idxInstr, (uintptr_t)a_pfnCImpl, a0, a1, a2, a3, a4)
+
+/** Emits code for IEM_MC_CALL_CIMPL_4. */
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitCallCImpl5(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr, uint8_t idxInstr,
+                        uintptr_t pfnCImpl, uint8_t idxArg0, uint8_t idxArg1, uint8_t idxArg2, uint8_t idxArg3, uint8_t idxArg4)
+{
+    Assert(idxArg0 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg0)));
+    Assert(pReNative->Core.aVars[idxArg0].uArgNo == 0 + IEM_CIMPL_HIDDEN_ARGS);
+    RT_NOREF_PV(idxArg0);
+
+    Assert(idxArg1 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg1)));
+    Assert(pReNative->Core.aVars[idxArg1].uArgNo == 1 + IEM_CIMPL_HIDDEN_ARGS);
+    RT_NOREF_PV(idxArg1);
+
+    Assert(idxArg2 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg2)));
+    Assert(pReNative->Core.aVars[idxArg2].uArgNo == 2 + IEM_CIMPL_HIDDEN_ARGS);
+    RT_NOREF_PV(idxArg2);
+
+    Assert(idxArg3 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg3)));
+    Assert(pReNative->Core.aVars[idxArg3].uArgNo == 3 + IEM_CIMPL_HIDDEN_ARGS);
+    RT_NOREF_PV(idxArg3);
+
+    Assert(idxArg4 < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxArg4)));
+    Assert(pReNative->Core.aVars[idxArg4].uArgNo == 4 + IEM_CIMPL_HIDDEN_ARGS);
+    RT_NOREF_PV(idxArg4);
+
+    return iemNativeEmitCallCImplCommon(pReNative, off, cbInstr, idxInstr, pfnCImpl, 5);
+}
+
+
+
+/*********************************************************************************************************************************
+*   Emitters for general purpose register fetches (IEM_MC_FETCH_GREG_XXX).                                                       *
+*********************************************************************************************************************************/
+
+#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) \
+    off = iemNativeEmitFetchGregU16(pReNative, off, a_u16Dst, a_iGReg)
+
+/** Emits code for IEM_MC_FETCH_GREG_U16. */
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitFetchGregU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxDstVar, uint8_t iGReg)
+{
+    Assert(idxDstVar < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxDstVar)));
+    Assert(pReNative->Core.aVars[idxDstVar].cbVar == sizeof(uint16_t));
+
+    /*
+     * We can either just load the low 16-bit of the GPR into a host register
+     * for the variable, or we can do so via a shadow copy host register. The
+     * latter will avoid having to reload it if it's being stored later, but
+     * will waste a host register if it isn't touched again.  Since we don't
+     * know what going to happen, we choose the latter for now.
+     */
+    uint8_t const idxGstFullReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off,
+                                                                  (IEMNATIVEGSTREG)(kIemNativeGstReg_GprFirst + iGReg),
+                                                                  kIemNativeGstRegUse_ReadOnly);
+
+    iemNativeVarSetKindToStack(pReNative, idxDstVar);
+    uint8_t const idxVarReg = iemNativeVarAllocRegister(pReNative, idxDstVar, &off);
+    off = iemNativeEmitLoadGprFromGpr16(pReNative, off, idxVarReg, idxGstFullReg);
+
+    iemNativeRegFreeTmp(pReNative, idxGstFullReg);
+    return off;
+}
+
+
+
+/*********************************************************************************************************************************
+*   Emitters for general purpose register stores (IEM_MC_STORE_GREG_XXX).                                                        *
+*********************************************************************************************************************************/
 
 #define IEM_MC_STORE_GREG_U8_CONST_THREADED(a_iGRegEx, a_u8Value) \
@@ -4799,5 +5684,6 @@
 
 /** Emits code for IEM_MC_STORE_GREG_U8_CONST_THREADED. */
-DECLINLINE(uint32_t) iemNativeEmitStoreGregU8Const(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGRegEx, uint8_t u8Value)
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitStoreGregU8Const(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGRegEx, uint8_t u8Value)
 {
     uint8_t const idxGstTmpReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off,
@@ -4878,5 +5764,6 @@
 
 /** Emits code for IEM_MC_SUB_GREG_U16. */
-DECLINLINE(uint32_t) iemNativeEmitSubGregU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGReg, uint8_t uSubtrahend)
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitSubGregU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGReg, uint8_t uSubtrahend)
 {
     uint8_t const idxGstTmpReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off,
Index: trunk/src/VBox/VMM/include/IEMN8veRecompiler.h
===================================================================
--- trunk/src/VBox/VMM/include/IEMN8veRecompiler.h	(revision 101843)
+++ trunk/src/VBox/VMM/include/IEMN8veRecompiler.h	(revision 101844)
@@ -49,12 +49,22 @@
  * @{  */
 /** The size of the area for stack variables and spills and stuff.
- * @note This limit is duplicated in the python script(s). */
-#define IEMNATIVE_FRAME_VAR_SIZE            0xc0
-#ifdef RT_ARCH_AMD64
-/** Number of stack arguments slots for calls made from the frame. */
-# define IEMNATIVE_FRAME_STACK_ARG_COUNT    4
+ * @note This limit is duplicated in the python script(s).  We add 0x40 for
+ *       alignment padding. */
+#define IEMNATIVE_FRAME_VAR_SIZE            (0xc0 + 0x40)
+/** Number of 64-bit variable slots (0x100 / 8 = 32. */
+#define IEMNATIVE_FRAME_VAR_SLOTS           (IEMNATIVE_FRAME_VAR_SIZE / 8)
+AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);
+
+#ifdef RT_ARCH_AMD64
 /** An stack alignment adjustment (between non-volatile register pushes and
  *  the stack variable area, so the latter better aligned). */
 # define IEMNATIVE_FRAME_ALIGN_SIZE         8
+
+/** Number of stack arguments slots for calls made from the frame. */
+# ifdef RT_OS_WINDOWS
+#  define IEMNATIVE_FRAME_STACK_ARG_COUNT   4
+# else
+#  define IEMNATIVE_FRAME_STACK_ARG_COUNT   2
+# endif
 /** Number of any shadow arguments (spill area) for calls we make. */
 # ifdef RT_OS_WINDOWS
@@ -77,8 +87,10 @@
 /** Frame pointer (RBP) relative offset of the second stack argument for calls. */
 # define IEMNATIVE_FP_OFF_STACK_ARG1        (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
+# ifdef RT_OS_WINDOWS
 /** Frame pointer (RBP) relative offset of the third stack argument for calls. */
-# define IEMNATIVE_FP_OFF_STACK_ARG2        (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
+#  define IEMNATIVE_FP_OFF_STACK_ARG2       (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
 /** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
-# define IEMNATIVE_FP_OFF_STACK_ARG3        (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
+#  define IEMNATIVE_FP_OFF_STACK_ARG3       (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
+# endif
 
 # ifdef RT_OS_WINDOWS
@@ -94,5 +106,7 @@
 
 #elif RT_ARCH_ARM64
-/** No stack argument slots, enough got 8 registers for arguments.  */
+/** No alignment padding needed for arm64. */
+# define IEMNATIVE_FRAME_ALIGN_SIZE         0
+/** No stack argument slots, got 8 registers for arguments will suffice. */
 # define IEMNATIVE_FRAME_STACK_ARG_COUNT    0
 /** There are no argument spill area. */
@@ -239,4 +253,6 @@
 #endif
 
+/** This is the maximum argument count we'll ever be needing. */
+#define IEMNATIVE_CALL_MAX_ARG_COUNT        7
 /** @} */
 
@@ -374,5 +390,5 @@
     kIemNativeGstRegRef_MReg,
     kIemNativeGstRegRef_XReg,
-    kIemNativeGstRegRef_YReg,
+    //kIemNativeGstRegRef_YReg, - doesn't work.
     kIemNativeGstRegRef_End
 } IEMNATIVEGSTREGREF;
@@ -523,4 +539,6 @@
     };
 
+    /** Allocation bitmap for the stack. */
+    uint32_t                    bmStack;
     /** Allocation bitmap for aVars. */
     uint32_t                    bmVars;
@@ -615,5 +633,12 @@
     /** Check IRQ seqeunce number (for generating unique lables). */
     uint16_t                    uCheckIrqSeqNo;
-    uint16_t                    uPadding3;
+    uint8_t                     bPadding3;
+
+    /** The argument count + hidden regs from the IEM_MC_BEGIN statement. */
+    uint8_t                     cArgs;
+    /** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */
+    uint32_t                    fCImpl;
+    /** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */
+    uint32_t                    fMc;
 
     /** Core state requiring care with branches. */
@@ -1252,4 +1277,48 @@
 
 /**
+ * Emits a load effective address to a GRP of a VCpu field.
+ */
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitLeaGprByVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t offVCpu)
+{
+#ifdef RT_ARCH_AMD64
+    /* lea gprdst, [rbx + offDisp] */
+    uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
+    if (iGprDst < 8)
+        pbCodeBuf[off++] = X86_OP_REX_W;
+    else
+        pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
+    pbCodeBuf[off++] = 0x8d;
+    off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGprDst, offVCpu);
+
+#elif defined(RT_ARCH_ARM64)
+    if (offVCpu < (unsigned)_4K)
+    {
+        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+        pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu);
+    }
+    else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)_4K)
+    {
+        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+        pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX,
+                                                         offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx));
+    }
+    else
+    {
+        Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
+        off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offVCpu);
+        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+        pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX, iGprDst);
+    }
+
+#else
+# error "port me"
+#endif
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+}
+
+
+/**
  * Emits a gprdst = gprsrc load.
  */
@@ -1271,6 +1340,6 @@
 #elif RT_ARCH_ARM64
     /* mov dst, src;   alias for: orr dst, xzr, src */
-    uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
-    pu32CodeBuf[off++] = UINT32_C(0xaa000000) | ((uint32_t)iGprSrc << 16) | ((uint32_t)ARMV8_A64_REG_XZR << 5) | iGprDst;
+    uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+    pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_XZR, iGprSrc);
 
 #else
@@ -1280,4 +1349,142 @@
     return off;
 }
+
+
+/**
+ * Emits a gprdst = gprsrc[31:0] load.
+ * @note Bits 63 thru 32 are cleared.
+ */
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitLoadGprFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
+{
+#ifdef RT_ARCH_AMD64
+    /* mov gprdst, gprsrc */
+    uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
+    if ((iGprDst | iGprSrc) >= 8)
+        pbCodeBuf[off++] = iGprDst < 8  ? X86_OP_REX_B
+                         : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B
+                         :                X86_OP_REX_R;
+    pbCodeBuf[off++] = 0x8b;
+    pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
+
+#elif RT_ARCH_ARM64
+    /* mov dst32, src32;   alias for: orr dst32, wzr, src32 */
+    uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+    pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_WZR, iGprSrc, false /*f64bit*/);
+
+#else
+# error "port me"
+#endif
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+}
+
+
+/**
+ * Emits a gprdst = gprsrc[15:0] load.
+ * @note Bits 63 thru 15 are cleared.
+ */
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitLoadGprFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
+{
+#ifdef RT_ARCH_AMD64
+    /* movzx Gv,Ew */
+    uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
+    if ((iGprDst | iGprSrc) >= 8)
+        pbCodeBuf[off++] = iGprDst < 8  ? X86_OP_REX_B
+                         : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B
+                         :                X86_OP_REX_R;
+    pbCodeBuf[off++] = 0x0f;
+    pbCodeBuf[off++] = 0xb7;
+    pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
+
+#elif RT_ARCH_ARM64
+    /* and gprdst, gprsrc, #0xffff */
+    uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+# if 1
+    Assert(Armv8A64ConvertImmRImmS2Mask32(0x0f, 0) == UINT16_MAX);
+    pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x0f, 0, false /*f64Bit*/);
+# else
+    Assert(Armv8A64ConvertImmRImmS2Mask64(0x4f, 0) == UINT16_MAX);
+    pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x4f, 0);
+# endif
+
+#else
+# error "port me"
+#endif
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+}
+
+
+/**
+ * Emits a gprdst = gprsrc[7:0] load.
+ * @note Bits 63 thru 8 are cleared.
+ */
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitLoadGprFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
+{
+#ifdef RT_ARCH_AMD64
+    /* movzx Gv,Eb */
+    uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
+    if ((iGprDst | iGprSrc) >= 8)
+        pbCodeBuf[off++] = iGprDst < 8  ? X86_OP_REX_B
+                         : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B
+                         :                X86_OP_REX_R;
+    pbCodeBuf[off++] = 0x0f;
+    pbCodeBuf[off++] = 0xb6;
+    pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
+
+#elif RT_ARCH_ARM64
+    /* and gprdst, gprsrc, #0xff */
+    uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+# if 1
+    Assert(Armv8A64ConvertImmRImmS2Mask32(0x07, 0) == UINT8_MAX);
+    pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x07, 0, false /*f64Bit*/);
+# else
+    Assert(Armv8A64ConvertImmRImmS2Mask64(0x47, 0) == UINT8_MAX);
+    pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x47, 0);
+# endif
+
+#else
+# error "port me"
+#endif
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+}
+
+#if 0 /** @todo */
+/**
+ * Emits a gprdst = gprsrc[15:8] load (ah, ch, dh, bh).
+ * @note Bits 63 thru 8 are cleared.
+ */
+DECL_INLINE_THROW(uint32_t)
+iemNativeEmitLoadGprFromGpr8hi(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
+{
+#ifdef RT_ARCH_AMD64
+    /* movzx Gv,Eb */
+    /** @todo */
+    uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
+    if ((iGprDst | iGprSrc) >= 8)
+        pbCodeBuf[off++] = iGprDst < 8  ? X86_OP_REX_B
+                         : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B
+                         :                X86_OP_REX_R;
+    pbCodeBuf[off++] = 0x0f;
+    pbCodeBuf[off++] = 0xb6;
+    pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
+
+#elif RT_ARCH_ARM64
+    /* ubfx gprdst, gprsrc, #8, #8 */
+    uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+    Assert(Armv8A64ConvertImmRImmS2Mask64(0x47, 0) == UINT8_MAX);
+    pu32CodeBuf[off++] = /** @todo ubfx */;
+
+#else
+# error "port me"
+#endif
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+}
+#endif
 
 #ifdef RT_ARCH_AMD64
@@ -1343,5 +1550,4 @@
 
 
-#ifdef RT_ARCH_AMD64
 /**
  * Emits a load effective address to a GRP with an BP relative source address.
@@ -1350,4 +1556,5 @@
 iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
 {
+#ifdef RT_ARCH_AMD64
     /* lea gprdst, [rbp + offDisp] */
     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
@@ -1357,7 +1564,35 @@
         pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
     pbCodeBuf[off++] = 0x8d;
-    return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
-}
-#endif
+    off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);
+
+#elif defined(RT_ARCH_ARM64)
+    if ((uint32_t)offDisp < (unsigned)_4K)
+    {
+        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+        pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, ARMV8_A64_REG_SP, (uint32_t)offDisp);
+    }
+    else if ((uint32_t)-offDisp < (unsigned)_4K)
+    {
+        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+        pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, ARMV8_A64_REG_SP, (uint32_t)-offDisp);
+    }
+    else
+    {
+        Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);
+        off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offDisp >= 0 ? (uint32_t)offDisp : (uint32_t)-offDisp);
+        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+        if (offDisp >= 0)
+            pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, ARMV8_A64_REG_SP, iGprDst);
+        else
+            pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /*fSub*/, iGprDst, ARMV8_A64_REG_SP, iGprDst);
+    }
+
+#else
+# error "port me"
+#endif
+
+    IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
+    return off;
+}
 
 
@@ -1844,5 +2079,5 @@
 {
 #if defined(RT_ARCH_AMD64)
-    /* movzx reg32, reg16 */
+    /* movzx Gv,Ew */
     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
     if (iGprDst >= 8)
