Changeset 101247 in vbox for trunk

Timestamp:

Sep 22, 2023 11:48:24 PM (19 months ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

159230

Message:

VMM/IEM: Working on emitting native arm64 instructions... bugref:10370

Location:

trunk/src/VBox/VMM

Files:

• VMMAll/IEMAllN8veRecompiler.cpp (modified) (12 diffs)
• include/IEMN8veRecompiler.h (modified) (10 diffs)

trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

@@ -59 +59 @@
 #include <iprt/mem.h>
 #include <iprt/string.h>
+#if   defined(RT_ARCH_AMD64)
+# include <iprt/x86.h>
+#elif defined(RT_ARCH_ARM64)
+# include <iprt/armv8.h>
+#endif
 
 #ifdef RT_OS_WINDOWS
@@ -97 +102 @@
 # error The setjmp approach must be enabled for the recompiler.
 #endif
-
-
-/*********************************************************************************************************************************
-*   Defined Constants And Macros                                                                                                 *
-*********************************************************************************************************************************/
-/** @name Stack Frame Layout
- *
- * @{  */
-/** The size of the area for stack variables and spills and stuff. */
-#define IEMNATIVE_FRAME_VAR_SIZE            0x40
-#ifdef RT_ARCH_AMD64
-/** Number of stack arguments slots for calls made from the frame. */
-# define IEMNATIVE_FRAME_STACK_ARG_COUNT    4
-/** An stack alignment adjustment (between non-volatile register pushes and
- *  the stack variable area, so the latter better aligned). */
-# define IEMNATIVE_FRAME_ALIGN_SIZE         8
-/** Number of any shadow arguments (spill area) for calls we make. */
-# ifdef RT_OS_WINDOWS
-#  define IEMNATIVE_FRAME_SHADOW_ARG_COUNT  4
-# else
-#  define IEMNATIVE_FRAME_SHADOW_ARG_COUNT  0
-# endif
-
-/** Frame pointer (RBP) relative offset of the last push. */
-# ifdef RT_OS_WINDOWS
-#  define IEMNATIVE_FP_OFF_LAST_PUSH        (7 * -8)
-# else
-#  define IEMNATIVE_FP_OFF_LAST_PUSH        (5 * -8)
-# endif
-/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
- * address for it). */
-# define IEMNATIVE_FP_OFF_STACK_VARS        (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
-/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
-# define IEMNATIVE_FP_OFF_STACK_ARG0        (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
-/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
-# define IEMNATIVE_FP_OFF_STACK_ARG1        (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
-/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
-# define IEMNATIVE_FP_OFF_STACK_ARG2        (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
-/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
-# define IEMNATIVE_FP_OFF_STACK_ARG3        (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
-
-# ifdef RT_OS_WINDOWS
-/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
-#  define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0   (16)
-/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
-#  define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1   (24)
-/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
-#  define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2   (32)
-/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
-#  define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3   (40)
-# endif
-
-#elif RT_ARCH_ARM64
-
-#else
-# error "port me"
-#endif
-/** @} */
 
 
@@ -475 +422 @@
     Ptr = iemDwarfPutLeb128(Ptr, 1);                        /* Code alignment factor (LEB128 = 1). */
     Ptr = iemDwarfPutLeb128(Ptr, -8);                       /* Data alignment factor (LEB128 = -8). */
+#  ifdef RT_ARCH_AMD64
     Ptr = iemDwarfPutUleb128(Ptr, DWREG_AMD64_RA);          /* Return address column (ULEB128) */
+#  elif defined(RT_ARCH_ARM64)
+    Ptr = iemDwarfPutUleb128(Ptr, DWREG_ARM64_PC);          /* Return address column (ULEB128) */
+#  else
+#   error "port me"
+#  endif
     /* Initial instructions: */
+#  ifdef RT_ARCH_AMD64
     Ptr = iemDwarfPutCfaDefCfa(Ptr, DWREG_AMD64_RBP, 16);   /* CFA     = RBP + 0x10 - first stack parameter */
     Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_AMD64_RA,  1);    /* Ret RIP = [CFA + 1*-8] */
@@ -485 +439 @@
     Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_AMD64_R14, 6);    /* R14     = [CFA + 6*-8] */
     Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_AMD64_R15, 7);    /* R15     = [CFA + 7*-8] */
+#  elif defined(RT_ARCH_ARM64)
+    Ptr = iemDwarfPutCfaDefCfa(Ptr, DWREG_ARM64_BP,  0);    /* CFA     = BP + 0x00 - first stack parameter */
+    Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_ARM64_PC,  1);    /* Ret PC  = [CFA + 1*-8] */
+    Ptr = iemDwarfPutCfaOffset(Ptr, DWREG_ARM64_BP,  2);    /* Ret BP  = [CFA + 2*-8] */
+#  endif
     while ((Ptr.u - PtrCie.u) & 3)
         *Ptr.pb++ = DW_CFA_nop;
@@ -1147 +1106 @@
 # ifdef RT_OS_WINDOWS
 #  ifndef VBOXSTRICTRC_STRICT_ENABLED
-    off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xCX, X86_GREG_xBX);
+    off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xCX, IEMNATIVE_REG_FIXED_PVMCPU);
     AssertReturn(off != UINT32_MAX, UINT32_MAX);
     if (cParams > 0)
@@ -1165 +1124 @@
     }
 #  else  /* VBOXSTRICTRC: Returned via hidden parameter. Sigh. */
-    off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xDX, X86_GREG_xBX);
+    off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xDX, IEMNATIVE_REG_FIXED_PVMCPU);
     AssertReturn(off != UINT32_MAX, UINT32_MAX);
     if (cParams > 0)
@@ -1188 +1147 @@
 #  endif /* VBOXSTRICTRC_STRICT_ENABLED */
 # else
-    off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xDI, X86_GREG_xBX);
+    off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xDI, IEMNATIVE_REG_FIXED_PVMCPU);
     AssertReturn(off != UINT32_MAX, UINT32_MAX);
     if (cParams > 0)
@@ -1224 +1183 @@
 
 #elif RT_ARCH_ARM64
-    RT_NOREF(pReNative, pCallEntry);
+    RT_NOREF(pReNative, pCallEntry, cParams);
     off = UINT32_MAX;
 
@@ -1292 +1251 @@
         off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_x8,  X86_GREG_xCX); /* cl = instruction number */
         AssertReturn(off != UINT32_MAX, UINT32_MAX);
-        off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xCX, X86_GREG_xBX);
+        off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xCX, IEMNATIVE_REG_FIXED_PVMCPU);
         AssertReturn(off != UINT32_MAX, UINT32_MAX);
         off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xDX, X86_GREG_xAX);
         AssertReturn(off != UINT32_MAX, UINT32_MAX);
 # else
-        off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xDI, X86_GREG_xBX);
+        off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xDI, IEMNATIVE_REG_FIXED_PVMCPU);
         AssertReturn(off != UINT32_MAX, UINT32_MAX);
         off = iemNativeEmitLoadGprFromGpr(pReNative, off, X86_GREG_xSI, X86_GREG_xAX);
@@ -1343 +1302 @@
 
 
+typedef enum
+{
+    kArm64InstrStLdPairType_kPostIndex = 1,
+    kArm64InstrStLdPairType_kSigned    = 2,
+    kArm64InstrStLdPairType_kPreIndex  = 3
+} ARM64INSTRSTLDPAIRTYPE;
+
+DECL_FORCE_INLINE(uint32_t) Armv8A64MkInstrStLdPair(bool fLoad, uint32_t iOpc, ARM64INSTRSTLDPAIRTYPE enmType,
+                                                    uint32_t iReg1, uint32_t iReg2, uint32_t iBaseReg, int32_t iImm7 = 0)
+{
+    Assert(iOpc < 3); Assert(iReg1 <= 31); Assert(iReg2 <= 31); Assert(iBaseReg <= 31); Assert(iImm7 < 64 && iImm7 >= -64);
+    return (iOpc << 30)
+         | UINT32_C(0x28000000)
+         | ((uint32_t)enmType << 23)
+         | ((uint32_t)fLoad << 22)
+         | ((uint32_t)iImm7 << 15)
+         | (iReg2 << 10)
+         | (iBaseReg << 5)
+         | iReg1;
+}
+
+
+
 /**
  * Emits a standard prolog.
@@ -1364 +1346 @@
     pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, X86_GREG_xBP, X86_GREG_xSP);
     pbCodeBuf[off++] = 0x50 + X86_GREG_xBX;     /* push rbx */
+    AssertCompile(IEMNATIVE_REG_FIXED_PVMCPU == X86_GREG_xBX);
 # ifdef RT_OS_WINDOWS
     pbCodeBuf[off++] = X86_OP_REX_W;            /* mov rbx, rcx ; RBX = pVCpu */
@@ -1395 +1378 @@
 
 #elif RT_ARCH_ARM64
-    RT_NOREF(pReNative);
-    off = UINT32_MAX;
+    /*
+     * We set up a stack frame exactly like on x86, only we have to push the
+     * return address our selves here.  We save all non-volatile registers.
+     */
+    uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
+    AssertReturn(pu32CodeBuf, UINT32_MAX);
+    /* stp x19, x20, [sp, #-IEMNATIVE_FRAME_SAVE_REG_SIZE] ; Allocate space for saving registers and place x19+x20 at the bottom. */
+    AssertCompile(IEMNATIVE_FRAME_SAVE_REG_SIZE < 64*8);
+    pu32CodeBuf[off++] = Armv8A64MkInstrStLdPair(false /*fLoad*/, 2 /*64-bit*/, kArm64InstrStLdPairType_kPreIndex,
+                                                 ARMV8_A64_REG_X19, ARMV8_A64_REG_X20, ARMV8_A64_REG_SP,
+                                                 -IEMNATIVE_FRAME_SAVE_REG_SIZE / 8);
+    /* Save x21 thru x28 (SP remains unchanged in the kSigned variant). */
+    pu32CodeBuf[off++] = Armv8A64MkInstrStLdPair(false /*fLoad*/, 2 /*64-bit*/, kArm64InstrStLdPairType_kSigned,
+                                                 ARMV8_A64_REG_X21, ARMV8_A64_REG_X22, ARMV8_A64_REG_SP, 2);
+    pu32CodeBuf[off++] = Armv8A64MkInstrStLdPair(false /*fLoad*/, 2 /*64-bit*/, kArm64InstrStLdPairType_kSigned,
+                                                 ARMV8_A64_REG_X23, ARMV8_A64_REG_X24, ARMV8_A64_REG_SP, 4);
+    pu32CodeBuf[off++] = Armv8A64MkInstrStLdPair(false /*fLoad*/, 2 /*64-bit*/, kArm64InstrStLdPairType_kSigned,
+                                                 ARMV8_A64_REG_X25, ARMV8_A64_REG_X26, ARMV8_A64_REG_SP, 6);
+    pu32CodeBuf[off++] = Armv8A64MkInstrStLdPair(false /*fLoad*/, 2 /*64-bit*/, kArm64InstrStLdPairType_kSigned,
+                                                 ARMV8_A64_REG_X27, ARMV8_A64_REG_X28, ARMV8_A64_REG_SP, 8);
+    /* Save the BP and LR (ret address) registers at the top of the frame. */
+    pu32CodeBuf[off++] = Armv8A64MkInstrStLdPair(false /*fLoad*/, 2 /*64-bit*/, kArm64InstrStLdPairType_kSigned,
+                                                 ARMV8_A64_REG_BP, ARMV8_A64_REG_LR, ARMV8_A64_REG_SP, 10);
+    AssertCompile(IEMNATIVE_FRAME_SAVE_REG_SIZE / 8 == 12);
+    /* sub bp, sp, IEMNATIVE_FRAME_SAVE_REG_SIZE - 16 ; Set BP to point to the old BP stack address. */
+    AssertCompile(IEMNATIVE_FRAME_SAVE_REG_SIZE - 16 < 4096);
+    pu32CodeBuf[off++] = UINT32_C(0xd1000000) | ((IEMNATIVE_FRAME_SAVE_REG_SIZE - 16) << 10) | ARMV8_A64_REG_SP | ARMV8_A64_REG_BP;
+
+    /* sub sp, sp, IEMNATIVE_FRAME_VAR_SIZE ;  Allocate the variable area from SP. */
+    AssertCompile(IEMNATIVE_FRAME_VAR_SIZE < 4096);
+    pu32CodeBuf[off++] = UINT32_C(0xd1000000) | (IEMNATIVE_FRAME_VAR_SIZE << 10)             | ARMV8_A64_REG_SP | ARMV8_A64_REG_SP;
 
 #else

trunk/src/VBox/VMM/include/IEMN8veRecompiler.h

@@ -37 +37 @@
  * @{
  */
+
+/** @name Stack Frame Layout
+ *
+ * @{  */
+/** The size of the area for stack variables and spills and stuff. */
+#define IEMNATIVE_FRAME_VAR_SIZE            0x40
+#ifdef RT_ARCH_AMD64
+/** Number of stack arguments slots for calls made from the frame. */
+# define IEMNATIVE_FRAME_STACK_ARG_COUNT    4
+/** An stack alignment adjustment (between non-volatile register pushes and
+ *  the stack variable area, so the latter better aligned). */
+# define IEMNATIVE_FRAME_ALIGN_SIZE         8
+/** Number of any shadow arguments (spill area) for calls we make. */
+# ifdef RT_OS_WINDOWS
+#  define IEMNATIVE_FRAME_SHADOW_ARG_COUNT  4
+# else
+#  define IEMNATIVE_FRAME_SHADOW_ARG_COUNT  0
+# endif
+
+/** Frame pointer (RBP) relative offset of the last push. */
+# ifdef RT_OS_WINDOWS
+#  define IEMNATIVE_FP_OFF_LAST_PUSH        (7 * -8)
+# else
+#  define IEMNATIVE_FP_OFF_LAST_PUSH        (5 * -8)
+# endif
+/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
+ * address for it). */
+# define IEMNATIVE_FP_OFF_STACK_VARS        (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
+/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
+# define IEMNATIVE_FP_OFF_STACK_ARG0        (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
+/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
+# define IEMNATIVE_FP_OFF_STACK_ARG1        (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
+/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
+# define IEMNATIVE_FP_OFF_STACK_ARG2        (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
+/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
+# define IEMNATIVE_FP_OFF_STACK_ARG3        (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
+
+# ifdef RT_OS_WINDOWS
+/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
+#  define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0   (16)
+/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
+#  define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1   (24)
+/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
+#  define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2   (32)
+/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
+#  define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3   (40)
+# endif
+
+#elif RT_ARCH_ARM64
+/** No stack argument slots, enough got 8 registers for arguments.  */
+# define IEMNATIVE_FRAME_STACK_ARG_COUNT    0
+/** There are no argument spill area. */
+# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT   0
+
+/** Number of saved registers at the top of our stack frame.
+ * This includes the return address and old frame pointer, so x19 thru x30. */
+# define IEMNATIVE_FRAME_SAVE_REG_COUNT     (12)
+/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
+# define IEMNATIVE_FRAME_SAVE_REG_SIZE      (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
+
+/** Frame pointer (BP) relative offset of the last push. */
+# define IEMNATIVE_FP_OFF_LAST_PUSH         (7 * -8)
+
+/** Frame pointer (BP) relative offset of the stack variable area (the lowest
+ * address for it). */
+# define IEMNATIVE_FP_OFF_STACK_VARS        (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
+
+#else
+# error "port me"
+#endif
+/** @} */
+
+
+/** @name Fixed Register Allocation(s)
+ * @{ */
+/** @def IEMNATIVE_REG_FIXED_PVMCPU
+ * The register number hold in pVCpu pointer.  */
+#ifdef RT_ARCH_AMD64
+# define IEMNATIVE_REG_FIXED_PVMCPU         X86_GREG_xBX
+#elif RT_ARCH_ARM64
+# define IEMNATIVE_REG_FIXED_PVMCPU         ARMV8_A64_REG_X28
+/** Dedicated temporary register.
+ * @todo replace this by a register allocator and content tracker.  */
+# define IEMNATIVE_REG_FIXED_TMP0           ARMV8_A64_REG_X15
+#else
+# error "port me"
+#endif
+/** @} */
 
 /** Native code generator label types. */
@@ -139 +227 @@
  *          failure.
  * @param   pReNative   The native recompile state.
- * @param   off         Current instruction offset.
+ * @param   off         Current instruction offset.  Works safely for UINT32_MAX
+ *                      as well.
  * @param   cInstrReq   Number of instruction about to be added.  It's okay to
  *                      overestimate this a bit.
@@ -145 +234 @@
 DECL_FORCE_INLINE(PIEMNATIVEINSTR) iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
 {
-    if (RT_LIKELY(off + cInstrReq <= pReNative->cInstrBufAlloc))
+    if (RT_LIKELY(off + (uint64_t)cInstrReq <= pReNative->cInstrBufAlloc))
         return pReNative->pInstrBuf;
     return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
@@ -160 +249 @@
     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
     AssertReturn(pbCodeBuf, UINT32_MAX);
-    pbCodeBuf[off++] = 0x90;                    /* nop */
+    /* nop */
+    pbCodeBuf[off++] = 0x90;
 
 #elif RT_ARCH_ARM64
     uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
-    pu32CodeBuf[off++] = 0xe503201f;            /* nop? */
+    AssertReturn(pu32CodeBuf, UINT32_MAX);
+    /* nop */
+    pu32CodeBuf[off++] = 0xd503201f;
 
 #else
@@ -181 +273 @@
     uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
     AssertReturn(pbCodeBuf, UINT32_MAX);
-    if (iGpr >= 8)                          /* xor gpr32, gpr32 */
+    /* xor gpr32, gpr32 */
+    if (iGpr >= 8)
         pbCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
     pbCodeBuf[off++] = 0x33;
@@ -187 +280 @@
 
 #elif RT_ARCH_ARM64
-    RT_NOREF(pReNative, iGpr, uImm64);
-    off = UINT32_MAX;
+    uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+    AssertReturn(pu32CodeBuf, UINT32_MAX);
+    /* mov gpr, #0x0 */
+    pu32CodeBuf[off++] = UINT32_C(0xd2800000) | iGpr;
 
 #else
@@ -241 +336 @@
 
 #elif RT_ARCH_ARM64
-    RT_NOREF(pReNative, iGpr, uImm64);
-    off = UINT32_MAX;
+    uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
+    AssertReturn(pu32CodeBuf, UINT32_MAX);
+
+    /*
+     * We need to start this sequence with a 'mov grp, imm16, lsl #x' and
+     * supply remaining bits using 'movk grp, imm16, lsl #x'.
+     *
+     * The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
+     * while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
+     * is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
+     * after the first non-zero immediate component so we switch to movk for
+     * the remainder.
+     */
+    uint32_t fMovK = 0;
+    /* mov  gpr, imm16 */
+    uint32_t uImmPart = ((uint32_t)((uImm64 >>  0) & UINT32_C(0xffff)) << 5);
+    if (uImmPart)
+    {
+        pu32CodeBuf[off++] = UINT32_C(0xd2800000) |         (UINT32_C(0) << 21) | uImmPart | iGpr;
+        fMovK |= RT_BIT_32(29);
+    }
+    /* mov[k] gpr, imm16, lsl #16 */
+    uImmPart = ((uint32_t)((uImm64 >> 16) & UINT32_C(0xffff)) << 5);
+    if (uImmPart)
+    {
+        pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(1) << 21) | uImmPart | iGpr;
+        fMovK |= RT_BIT_32(29);
+    }
+    /* mov[k] gpr, imm16, lsl #32 */
+    uImmPart = ((uint32_t)((uImm64 >> 32) & UINT32_C(0xffff)) << 5);
+    if (uImmPart)
+    {
+        pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(2) << 21) | uImmPart | iGpr;
+        fMovK |= RT_BIT_32(29);
+    }
+    /* mov[k] gpr, imm16, lsl #48 */
+    uImmPart = ((uint32_t)((uImm64 >> 48) & UINT32_C(0xffff)) << 5);
+    if (uImmPart)
+        pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(3) << 21) | uImmPart | iGpr;
+
+    /** @todo there is an inverted mask variant we might want to explore if it
+     *        reduces the number of instructions... */
+    /** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
+     *        clang 12.x does that, only to use the 'x' version for the
+     *        addressing in the following ldr). */
 
 #else
@@ -266 +404 @@
     if (offVCpu < 128)
     {
-        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGpr & 7, X86_GREG_xBX);
+        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGpr & 7, IEMNATIVE_REG_FIXED_PVMCPU);
         pbCodeBuf[off++] = (uint8_t)offVCpu;
     }
     else
     {
-        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGpr & 7, X86_GREG_xBX);
+        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGpr & 7, IEMNATIVE_REG_FIXED_PVMCPU);
         pbCodeBuf[off++] = RT_BYTE1(offVCpu);
         pbCodeBuf[off++] = RT_BYTE2(offVCpu);
@@ -279 +417 @@
 
 #elif RT_ARCH_ARM64
-    RT_NOREF(pReNative, idxInstr);
-    off = UINT32_MAX;
+    /*
+     * There are a couple of ldr variants that takes an immediate offset, so
+     * try use those if we can, otherwise we have to use the temporary register
+     * help with the addressing.
+     */
+    if (offVCpu < _16K)
+    {
+        /* Use the unsigned variant of ldr Wt, [<Xn|SP>, #off]. */
+        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+        AssertReturn(pu32CodeBuf, UINT32_MAX);
+        pu32CodeBuf[off++] = UINT32_C(0xb9400000) | (offVCpu << 10) | (IEMNATIVE_REG_FIXED_PVMCPU << 5) | iGpr;
+    }
+    else
+    {
+        /* The offset is too large, so we must load it into a register and use
+           ldr Wt, [<Xn|SP>, (<Wm>|<Xm>). */
+        /** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
+        off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
+        uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+        AssertReturn(pu32CodeBuf, UINT32_MAX);
+        pu32CodeBuf[off++] = UINT32_C(0xb8600800) | ((uint32_t)IEMNATIVE_REG_FIXED_TMP0 << 16)
+                           | ((uint32_t)IEMNATIVE_REG_FIXED_PVMCPU << 5) | iGpr;
+    }
 
 #else
@@ -308 +467 @@
 
 #elif RT_ARCH_ARM64
-    RT_NOREF(pReNative, iGprDst, iGprSrc);
-    off = UINT32_MAX;
+    uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
+    AssertReturn(pu32CodeBuf, UINT32_MAX);
+    /* mov dst, src;   alias for: orr dst, xzr, src */
+    pu32CodeBuf[off++] = UINT32_C(0xaa000000) | ((uint32_t)iGprSrc << 16) | ((uint32_t)ARMV8_A64_REG_XZR << 5) | iGprDst;
 
 #else