Changeset 103622 in vbox for trunk/src/VBox

Timestamp:

Mar 1, 2024 12:42:36 AM (11 months ago)

Author:

vboxsync

Message:

VMM/IEM: Obfuscate most variable indexes we pass around in strict builds so we easily catch register/variable index mixups. bugref:10371

Location:

trunk/src/VBox/VMM

Files:

• VMMAll/IEMAllN8veRecompiler.cpp (modified) (112 diffs)
• include/IEMN8veRecompiler.h (modified) (11 diffs)
• include/IEMN8veRecompilerEmit.h (modified) (5 diffs)
• include/IEMN8veRecompilerTlbLookup.h (modified) (3 diffs)

trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

@@ -3737 +3737 @@
                 if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack)
                 {
-                    uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, idxVar);
+                    uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, IEMNATIVE_VAR_IDX_PACK(idxVar));
                     *poff = iemNativeEmitStoreGprByBp(pReNative, *poff, iemNativeStackCalcBpDisp(idxStackSlot), idxReg);
                 }
@@ -3771 +3771 @@
                                     uint8_t idxRegOld, uint8_t idxRegNew, const char *pszCaller)
 {
-    Assert(pReNative->Core.aVars[idxVar].idxReg == idxRegOld);
+    IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
+    Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].idxReg == idxRegOld);
     RT_NOREF(pszCaller);
 
@@ -3777 +3778 @@
 
     uint64_t fGstRegShadows = pReNative->Core.aHstRegs[idxRegOld].fGstRegShadows;
-    Log12(("%s: moving idxVar=%d from %s to %s (fGstRegShadows=%RX64)\n",
+    Log12(("%s: moving idxVar=%#x from %s to %s (fGstRegShadows=%RX64)\n",
            pszCaller, idxVar, g_apszIemNativeHstRegNames[idxRegOld], g_apszIemNativeHstRegNames[idxRegNew], fGstRegShadows));
     off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegNew, idxRegOld);
@@ -3798 +3799 @@
     }
 
-    pReNative->Core.aVars[idxVar].idxReg = (uint8_t)idxRegNew;
+    pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].idxReg = (uint8_t)idxRegNew;
     pReNative->Core.aHstRegs[idxRegOld].fGstRegShadows = 0;
     pReNative->Core.bmHstRegs = RT_BIT_32(idxRegNew) | (pReNative->Core.bmHstRegs & ~RT_BIT_32(idxRegOld));
@@ -3822 +3823 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
-    Assert(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack);
-    Assert(!pReNative->Core.aVars[idxVar].fRegAcquired);
-
-    uint8_t const idxRegOld = pReNative->Core.aVars[idxVar].idxReg;
+    PIEMNATIVEVAR const pVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)];
+    Assert(pVar->enmKind == kIemNativeVarKind_Stack);
+    Assert(!pVar->fRegAcquired);
+
+    uint8_t const idxRegOld = pVar->idxReg;
     Assert(idxRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs));
     Assert(pReNative->Core.bmHstRegs & RT_BIT_32(idxRegOld));
@@ -3861 +3863 @@
      */
     uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, idxVar);
-    Log12(("iemNativeRegMoveOrSpillStackVar: spilling idxVar=%d/idxReg=%d onto the stack (slot %#x bp+%d, off=%#x)\n",
+    Log12(("iemNativeRegMoveOrSpillStackVar: spilling idxVar=%#x/idxReg=%d onto the stack (slot %#x bp+%d, off=%#x)\n",
            idxVar, idxRegOld, idxStackSlot, iemNativeStackCalcBpDisp(idxStackSlot), off));
     off = iemNativeEmitStoreGprByBp(pReNative, off, iemNativeStackCalcBpDisp(idxStackSlot), idxRegOld);
 
-    pReNative->Core.aVars[idxVar].idxReg    = UINT8_MAX;
+    pVar->idxReg                            = UINT8_MAX;
     pReNative->Core.bmHstRegsWithGstShadow &= ~RT_BIT_32(idxRegOld);
     pReNative->Core.bmHstRegs              &= ~RT_BIT_32(idxRegOld);
@@ -4522 +4524 @@
 
 
-DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocVar(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t idxVar);
-
-
 /**
  * Allocates argument registers for a function call.
@@ -4571 +4570 @@
                     {
                         uint8_t const idxVar = pReNative->Core.aHstRegs[idxReg].idxVar;
-                        AssertStmt(idxVar < RT_ELEMENTS(pReNative->Core.aVars),
+                        IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
+                        AssertStmt(IEMNATIVE_VAR_IDX_UNPACK(idxVar) < RT_ELEMENTS(pReNative->Core.aVars),
                                    IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_REG_IPE_5));
-                        Assert(pReNative->Core.aVars[idxVar].idxReg == idxReg);
-                        Assert(pReNative->Core.bmVars & RT_BIT_32(idxVar));
-
-                        if (pReNative->Core.aVars[idxVar].enmKind != kIemNativeVarKind_Stack)
-                            pReNative->Core.aVars[idxVar].idxReg = UINT8_MAX;
+                        Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].idxReg == idxReg);
+
+                        if (pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].enmKind != kIemNativeVarKind_Stack)
+                            pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].idxReg = UINT8_MAX;
                         else
                         {
@@ -4694 +4693 @@
     Assert(pReNative->Core.aVars[idxVar].idxReg == idxHstReg);
 
-    pReNative->Core.aVars[idxVar].idxReg = UINT8_MAX;
+    pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].idxReg = UINT8_MAX;
     pReNative->Core.bmHstRegs &= ~RT_BIT_32(idxHstReg);
     if (!fFlushShadows)
-        Log12(("iemNativeRegFreeVar: %s (gst: %#RX64) idxVar=%d\n",
+        Log12(("iemNativeRegFreeVar: %s (gst: %#RX64) idxVar=%#x\n",
                g_apszIemNativeHstRegNames[idxHstReg], pReNative->Core.aHstRegs[idxHstReg].fGstRegShadows, idxVar));
     else
@@ -4714 +4713 @@
             pReNative->Core.aidxGstRegShadows[idxGstReg] = UINT8_MAX;
         }
-        Log12(("iemNativeRegFreeVar: %s (gst: %#RX64 -> 0) idxVar=%d\n",
+        Log12(("iemNativeRegFreeVar: %s (gst: %#RX64 -> 0) idxVar=%#x\n",
                g_apszIemNativeHstRegNames[idxHstReg], fGstRegShadowsOld, idxVar));
     }
@@ -4769 +4768 @@
                 case kIemNativeWhat_Var:
                 {
-                    uint8_t const idxVar = pReNative->Core.aHstRegs[idxReg].idxVar;
-                    Assert(idxVar < RT_ELEMENTS(pReNative->Core.aVars));
-                    Assert(pReNative->Core.bmVars & RT_BIT_32(idxVar));
-                    Assert(pReNative->Core.aVars[idxVar].idxReg == idxReg);
+                    uint8_t const       idxVar = pReNative->Core.aHstRegs[idxReg].idxVar;
+                    IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
+                    PIEMNATIVEVAR const pVar   = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)];
+                    Assert(pVar->idxReg == idxReg);
                     if (!(RT_BIT_32(idxVar) & fKeepVars))
                     {
-                        Log12(("iemNativeRegMoveAndFreeAndFlushAtCall: idxVar=%d enmKind=%d idxReg=%d\n",
-                               idxVar, pReNative->Core.aVars[idxVar].enmKind, pReNative->Core.aVars[idxVar].idxReg));
-                        if (pReNative->Core.aVars[idxVar].enmKind != kIemNativeVarKind_Stack)
-                            pReNative->Core.aVars[idxVar].idxReg = UINT8_MAX;
+                        Log12(("iemNativeRegMoveAndFreeAndFlushAtCall: idxVar=%#x enmKind=%d idxReg=%d\n",
+                               idxVar, pVar->enmKind, pVar->idxReg));
+                        if (pVar->enmKind != kIemNativeVarKind_Stack)
+                            pVar->idxReg = UINT8_MAX;
                         else
                             off = iemNativeRegMoveOrSpillStackVar(pReNative, off, idxVar);
@@ -6781 +6780 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarPc);
-    Assert(pReNative->Core.aVars[idxVarPc].cbVar == cbVar);
+    IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVarPc, cbVar);
 
     /* We speculatively modify PC and may raise #GP(0), so make sure the right values are in CPUMCTX. */
@@ -7071 +7070 @@
                             pReNative->Core.bmHstRegs &= ~RT_BIT_32(idxHstReg);
                             pReNative->Core.aVars[idxVar].idxReg = UINT8_MAX;
-                            Log12(("iemNativeEmitEndIf: Dropping hst reg %s for var #%u\n",
-                                   g_apszIemNativeHstRegNames[idxHstReg], idxVar));
+                            Log12(("iemNativeEmitEndIf: Dropping hst reg %s for var #%u/%#x\n",
+                                   g_apszIemNativeHstRegNames[idxHstReg], idxVar, IEMNATIVE_VAR_IDX_PACK(idxVar)));
                         }
                         continue;
@@ -7086 +7085 @@
                     pReNative->Core.bmHstRegs &= ~RT_BIT_32(idxHstReg);
                     pReNative->Core.aVars[idxVar].idxReg = UINT8_MAX;
-                    Log12(("iemNativeEmitEndIf: Dropping hst reg %s for var #%u (also dropped)\n",
-                           g_apszIemNativeHstRegNames[idxHstReg], idxVar));
+                    Log12(("iemNativeEmitEndIf: Dropping hst reg %s for var #%u/%#x (also dropped)\n",
+                           g_apszIemNativeHstRegNames[idxHstReg], idxVar, IEMNATIVE_VAR_IDX_PACK(idxVar)));
                 }
-                Log11(("iemNativeEmitEndIf: Freeing variable #%u\n", idxVar));
+                Log11(("iemNativeEmitEndIf: Freeing variable #%u/%#x\n", idxVar, IEMNATIVE_VAR_IDX_PACK(idxVar)));
                 pReNative->Core.bmVars &= ~RT_BIT_32(idxVar);
             } while (fVars);
@@ -7643 +7642 @@
  * The kind will either be set by the caller or later when the variable is first
  * assigned a value.
+ *
+ * @returns Unpacked index.
+ * @internal
  */
 static uint8_t iemNativeVarAllocInt(PIEMRECOMPILERSTATE pReNative, uint8_t cbType)
@@ -7665 +7667 @@
 /**
  * Internal work that allocates an argument variable w/o setting enmKind.
+ *
+ * @returns Unpacked index.
+ * @internal
  */
 static uint8_t iemNativeArgAllocInt(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType)
@@ -7673 +7678 @@
 
     uint8_t const idxVar = iemNativeVarAllocInt(pReNative, cbType);
-    pReNative->Core.aidxArgVars[iArgNo]  = idxVar;
+    pReNative->Core.aidxArgVars[iArgNo]  = idxVar; /* (unpacked) */
     pReNative->Core.aVars[idxVar].uArgNo = iArgNo;
     return idxVar;
@@ -7694 +7699 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
-    Assert(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack);
+    PIEMNATIVEVAR const pVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)];
+    Assert(pVar->enmKind == kIemNativeVarKind_Stack);
 
     /* Already got a slot? */
-    uint8_t const idxStackSlot = pReNative->Core.aVars[idxVar].idxStackSlot;
+    uint8_t const idxStackSlot = pVar->idxStackSlot;
     if (idxStackSlot != UINT8_MAX)
     {
@@ -7708 +7714 @@
      * Allocate them from the top end, closest to BP, to reduce the displacement.
      */
-    if (pReNative->Core.aVars[idxVar].cbVar <= sizeof(uint64_t))
+    if (pVar->cbVar <= sizeof(uint64_t))
     {
         unsigned const iSlot = ASMBitLastSetU32(~pReNative->Core.bmStack) - 1;
         AssertStmt(iSlot < IEMNATIVE_FRAME_VAR_SLOTS, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_OUT_OF_STACK_SLOTS));
         pReNative->Core.bmStack |= RT_BIT_32(iSlot);
-        pReNative->Core.aVars[idxVar].idxStackSlot = (uint8_t)iSlot;
-        Log11(("iemNativeVarSetKindToStack: idxVar=%d iSlot=%#x\n", idxVar, iSlot));
+        pVar->idxStackSlot       = (uint8_t)iSlot;
+        Log11(("iemNativeVarSetKindToStack: idxVar=%#x iSlot=%#x\n", idxVar, iSlot));
         return (uint8_t)iSlot;
     }
@@ -7724 +7730 @@
      */
     AssertCompile(RT_IS_POWER_OF_TWO(IEMNATIVE_FRAME_VAR_SLOTS)); /* If not we have to add an overflow check. */
-    Assert(pReNative->Core.aVars[idxVar].cbVar <= 64);
-    uint32_t const fBitAlignMask = RT_BIT_32(ASMBitLastSetU32(pReNative->Core.aVars[idxVar].cbVar) - 4) - 1;
-    uint32_t       fBitAllocMask = RT_BIT_32((pReNative->Core.aVars[idxVar].cbVar + 7) >> 3) - 1;
+    Assert(pVar->cbVar <= 64);
+    uint32_t const fBitAlignMask = RT_BIT_32(ASMBitLastSetU32(pVar->cbVar) - 4) - 1;
+    uint32_t       fBitAllocMask = RT_BIT_32((pVar->cbVar + 7) >> 3) - 1;
     uint32_t       bmStack       = ~pReNative->Core.bmStack;
     while (bmStack != UINT32_MAX)
@@ -7738 +7744 @@
             {
                 pReNative->Core.bmStack |= (fBitAllocMask << iSlot);
-                pReNative->Core.aVars[idxVar].idxStackSlot = (uint8_t)iSlot;
-                Log11(("iemNativeVarSetKindToStack: idxVar=%d iSlot=%#x/%#x (cbVar=%#x)\n",
-                       idxVar, iSlot, fBitAllocMask, pReNative->Core.aVars[idxVar].cbVar));
+                pVar->idxStackSlot       = (uint8_t)iSlot;
+                Log11(("iemNativeVarSetKindToStack: idxVar=%#x iSlot=%#x/%#x (cbVar=%#x)\n",
+                       idxVar, iSlot, fBitAllocMask, pVar->cbVar));
                 return (uint8_t)iSlot;
             }
@@ -7763 +7769 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
-    if (pReNative->Core.aVars[idxVar].enmKind != kIemNativeVarKind_Stack)
+    PIEMNATIVEVAR const pVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)];
+    if (pVar->enmKind != kIemNativeVarKind_Stack)
     {
         /* We could in theory transition from immediate to stack as well, but it
            would involve the caller doing work storing the value on the stack. So,
            till that's required we only allow transition from invalid. */
-        AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid,
-                   IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_2));
-        AssertStmt(pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_2));
-        pReNative->Core.aVars[idxVar].enmKind = kIemNativeVarKind_Stack;
+        AssertStmt(pVar->enmKind == kIemNativeVarKind_Invalid, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_2));
+        AssertStmt(pVar->idxReg  == UINT8_MAX,                 IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_2));
+        pVar->enmKind = kIemNativeVarKind_Stack;
 
         /* Note! We don't allocate a stack slot here, that's only done when a
@@ -7793 +7799 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
-    if (pReNative->Core.aVars[idxVar].enmKind != kIemNativeVarKind_Immediate)
+    PIEMNATIVEVAR const pVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)];
+    if (pVar->enmKind != kIemNativeVarKind_Immediate)
     {
         /* Only simple transitions for now. */
-        AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid,
-                   IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_2));
-        pReNative->Core.aVars[idxVar].enmKind = kIemNativeVarKind_Immediate;
-    }
-    AssertStmt(pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_2));
-
-    pReNative->Core.aVars[idxVar].u.uValue = uValue;
-    AssertMsg(   pReNative->Core.aVars[idxVar].cbVar >= sizeof(uint64_t)
-              || pReNative->Core.aVars[idxVar].u.uValue < RT_BIT_64(pReNative->Core.aVars[idxVar].cbVar * 8),
-              ("idxVar=%d cbVar=%u uValue=%#RX64\n", idxVar, pReNative->Core.aVars[idxVar].cbVar, uValue));
+        AssertStmt(pVar->enmKind == kIemNativeVarKind_Invalid, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_2));
+        pVar->enmKind = kIemNativeVarKind_Immediate;
+    }
+    AssertStmt(pVar->idxReg == UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_2));
+
+    pVar->u.uValue = uValue;
+    AssertMsg(   pVar->cbVar >= sizeof(uint64_t)
+              || pVar->u.uValue < RT_BIT_64(pVar->cbVar * 8),
+              ("idxVar=%d cbVar=%u uValue=%#RX64\n", idxVar, pVar->cbVar, uValue));
 }
 
@@ -7816 +7822 @@
  *
  * @param   pReNative   The recompiler state.
- * @param   idxVar      The variable.
- * @param   idxOtherVar The variable to take the (stack) address of.
+ * @param   idxVar      The variable. Unpacked.
+ * @param   idxOtherVar The variable to take the (stack) address of. Unpacked.
  *
  * @throws  VERR_IEM_VAR_OUT_OF_STACK_SLOTS, VERR_IEM_VAR_IPE_2
+ * @internal
  */
 static void iemNativeVarSetKindToLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar, uint8_t idxOtherVar)
@@ -7835 +7842 @@
     AssertStmt(pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_2));
 
-    pReNative->Core.aVars[idxVar].u.idxRefVar = idxOtherVar;
+    pReNative->Core.aVars[idxVar].u.idxRefVar = idxOtherVar; /* unpacked */
 
     /* Update the other variable, ensure it's a stack variable. */
     /** @todo handle variables with const values... that'll go boom now. */
     pReNative->Core.aVars[idxOtherVar].idxReferrerVar = idxVar;
-    iemNativeVarSetKindToStack(pReNative, idxOtherVar);
+    iemNativeVarSetKindToStack(pReNative, IEMNATIVE_VAR_IDX_PACK(idxOtherVar));
 }
 
@@ -7860 +7867 @@
                                            IEMNATIVEGSTREGREF enmRegClass, uint8_t idxReg)
 {
-    Assert(idxVar < RT_ELEMENTS(pReNative->Core.aVars) && (pReNative->Core.bmVars & RT_BIT_32(idxVar)));
-
-    if (pReNative->Core.aVars[idxVar].enmKind != kIemNativeVarKind_GstRegRef)
+    IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
+    PIEMNATIVEVAR const pVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)];
+
+    if (pVar->enmKind != kIemNativeVarKind_GstRegRef)
     {
         /* Only simple transitions for now. */
-        AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid,
-                   IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_2));
-        pReNative->Core.aVars[idxVar].enmKind = kIemNativeVarKind_GstRegRef;
-    }
-    AssertStmt(pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_2));
-
-    pReNative->Core.aVars[idxVar].u.GstRegRef.enmClass = enmRegClass;
-    pReNative->Core.aVars[idxVar].u.GstRegRef.idx      = idxReg;
+        AssertStmt(pVar->enmKind == kIemNativeVarKind_Invalid, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_2));
+        pVar->enmKind = kIemNativeVarKind_GstRegRef;
+    }
+    AssertStmt(pVar->idxReg == UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_2));
+
+    pVar->u.GstRegRef.enmClass = enmRegClass;
+    pVar->u.GstRegRef.idx      = idxReg;
 }
 
@@ -7878 +7885 @@
 DECL_HIDDEN_THROW(uint8_t) iemNativeArgAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType)
 {
-    return iemNativeArgAllocInt(pReNative, iArgNo, cbType);
+    return IEMNATIVE_VAR_IDX_PACK(iemNativeArgAllocInt(pReNative, iArgNo, cbType));
 }
 
@@ -7884 +7891 @@
 DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t cbType, uint64_t uValue)
 {
-    uint8_t const idxVar = iemNativeArgAllocInt(pReNative, iArgNo, cbType);
+    uint8_t const idxVar = IEMNATIVE_VAR_IDX_PACK(iemNativeArgAllocInt(pReNative, iArgNo, cbType));
 
     /* Since we're using a generic uint64_t value type, we must truncate it if
@@ -7908 +7915 @@
 DECL_HIDDEN_THROW(uint8_t) iemNativeArgAllocLocalRef(PIEMRECOMPILERSTATE pReNative, uint8_t iArgNo, uint8_t idxOtherVar)
 {
+    IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxOtherVar);
+    idxOtherVar = IEMNATIVE_VAR_IDX_UNPACK(idxOtherVar);
     AssertStmt(   idxOtherVar < RT_ELEMENTS(pReNative->Core.aVars)
                && (pReNative->Core.bmVars & RT_BIT_32(idxOtherVar))
@@ -7914 +7923 @@
 
     uint8_t const idxArgVar = iemNativeArgAlloc(pReNative, iArgNo, sizeof(uintptr_t));
-    iemNativeVarSetKindToLocalRef(pReNative, idxArgVar, idxOtherVar);
+    iemNativeVarSetKindToLocalRef(pReNative, IEMNATIVE_VAR_IDX_UNPACK(idxArgVar), idxOtherVar);
     return idxArgVar;
 }
@@ -7921 +7930 @@
 DECL_HIDDEN_THROW(uint8_t) iemNativeVarAlloc(PIEMRECOMPILERSTATE pReNative, uint8_t cbType)
 {
-    uint8_t const idxVar = iemNativeVarAllocInt(pReNative, cbType);
+    uint8_t const idxVar = IEMNATIVE_VAR_IDX_PACK(iemNativeVarAllocInt(pReNative, cbType));
     /* Don't set to stack now, leave that to the first use as for instance
        IEM_MC_CALC_RM_EFF_ADDR may produce a const/immediate result (esp. in DOS). */
@@ -7930 +7939 @@
 DECL_HIDDEN_THROW(uint8_t) iemNativeVarAllocConst(PIEMRECOMPILERSTATE pReNative, uint8_t cbType, uint64_t uValue)
 {
-    uint8_t const idxVar = iemNativeVarAllocInt(pReNative, cbType);
+    uint8_t const idxVar = IEMNATIVE_VAR_IDX_PACK(iemNativeVarAllocInt(pReNative, cbType));
 
     /* Since we're using a generic uint64_t value type, we must truncate it if
@@ -7965 +7974 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
-    Assert(pReNative->Core.aVars[idxVar].cbVar <= 8);
-    Assert(!pReNative->Core.aVars[idxVar].fRegAcquired);
-
-    uint8_t idxReg = pReNative->Core.aVars[idxVar].idxReg;
+    PIEMNATIVEVAR const pVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)];
+    Assert(pVar->cbVar <= 8);
+    Assert(!pVar->fRegAcquired);
+
+    uint8_t idxReg = pVar->idxReg;
     if (idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs))
     {
-        Assert(   pReNative->Core.aVars[idxVar].enmKind > kIemNativeVarKind_Invalid
-               && pReNative->Core.aVars[idxVar].enmKind < kIemNativeVarKind_End);
-        pReNative->Core.aVars[idxVar].fRegAcquired = true;
+        Assert(   pVar->enmKind > kIemNativeVarKind_Invalid
+               && pVar->enmKind < kIemNativeVarKind_End);
+        pVar->fRegAcquired = true;
         return idxReg;
     }
@@ -7980 +7990 @@
      * If the kind of variable has not yet been set, default to 'stack'.
      */
-    Assert(   pReNative->Core.aVars[idxVar].enmKind >= kIemNativeVarKind_Invalid
-           && pReNative->Core.aVars[idxVar].enmKind < kIemNativeVarKind_End);
-    if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid)
+    Assert(   pVar->enmKind >= kIemNativeVarKind_Invalid
+           && pVar->enmKind < kIemNativeVarKind_End);
+    if (pVar->enmKind == kIemNativeVarKind_Invalid)
         iemNativeVarSetKindToStack(pReNative, idxVar);
 
@@ -8003 +8013 @@
      * calls causing less optimal code to be generated in the python script. */
 
-    uint8_t const uArgNo = pReNative->Core.aVars[idxVar].uArgNo;
+    uint8_t const uArgNo = pVar->uArgNo;
     if (   uArgNo < RT_ELEMENTS(g_aidxIemNativeCallRegs)
         && !(pReNative->Core.bmHstRegs & RT_BIT_32(g_aidxIemNativeCallRegs[uArgNo])))
@@ -8009 +8019 @@
         idxReg = g_aidxIemNativeCallRegs[uArgNo];
         iemNativeRegClearGstRegShadowing(pReNative, idxReg, *poff);
-        Log11(("iemNativeVarRegisterAcquire: idxVar=%u idxReg=%u (matching arg %u)\n", idxVar, idxReg, uArgNo));
+        Log11(("iemNativeVarRegisterAcquire: idxVar=%#x idxReg=%u (matching arg %u)\n", idxVar, idxReg, uArgNo));
     }
     else if (   idxRegPref >= RT_ELEMENTS(pReNative->Core.aHstRegs)
@@ -8026 +8036 @@
             Assert(pReNative->Core.aHstRegs[idxReg].fGstRegShadows == 0);
             Assert(!(pReNative->Core.bmHstRegsWithGstShadow & RT_BIT_32(idxReg)));
-            Log11(("iemNativeVarRegisterAcquire: idxVar=%u idxReg=%u (uArgNo=%u)\n", idxVar, idxReg, uArgNo));
+            Log11(("iemNativeVarRegisterAcquire: idxVar=%#x idxReg=%u (uArgNo=%u)\n", idxVar, idxReg, uArgNo));
         }
         else
@@ -8033 +8043 @@
                                                IEMNATIVE_HST_GREG_MASK & ~IEMNATIVE_REG_FIXED_MASK & fNotArgsMask);
             AssertStmt(idxReg != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_REG_ALLOCATOR_NO_FREE_VAR));
-            Log11(("iemNativeVarRegisterAcquire: idxVar=%u idxReg=%u (slow, uArgNo=%u)\n", idxVar, idxReg, uArgNo));
+            Log11(("iemNativeVarRegisterAcquire: idxVar=%#x idxReg=%u (slow, uArgNo=%u)\n", idxVar, idxReg, uArgNo));
         }
     }
@@ -8040 +8050 @@
         idxReg = idxRegPref;
         iemNativeRegClearGstRegShadowing(pReNative, idxReg, *poff);
-        Log11(("iemNativeVarRegisterAcquire: idxVar=%u idxReg=%u (preferred)\n", idxVar, idxReg));
+        Log11(("iemNativeVarRegisterAcquire: idxVar=%#x idxReg=%u (preferred)\n", idxVar, idxReg));
     }
     iemNativeRegMarkAllocated(pReNative, idxReg, kIemNativeWhat_Var, idxVar);
-    pReNative->Core.aVars[idxVar].idxReg = idxReg;
+    pVar->idxReg = idxReg;
 
     /*
      * Load it off the stack if we've got a stack slot.
      */
-    uint8_t const idxStackSlot = pReNative->Core.aVars[idxVar].idxStackSlot;
+    uint8_t const idxStackSlot = pVar->idxStackSlot;
     if (idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS)
     {
         Assert(fInitialized);
         int32_t const offDispBp = iemNativeStackCalcBpDisp(idxStackSlot);
-        switch (pReNative->Core.aVars[idxVar].cbVar)
+        switch (pVar->cbVar)
         {
             case 1: *poff = iemNativeEmitLoadGprByBpU8( pReNative, *poff, idxReg, offDispBp); break;
@@ -8068 +8078 @@
         AssertStmt(!fInitialized, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_NOT_INITIALIZED));
     }
-    pReNative->Core.aVars[idxVar].fRegAcquired = true;
+    pVar->fRegAcquired = true;
     return idxReg;
 }
@@ -8097 +8107 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
-    Assert(!pReNative->Core.aVars[idxVar].fRegAcquired);
-    AssertMsgStmt(   pReNative->Core.aVars[idxVar].cbVar <= 8
-                  && (   pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Immediate
-                      || pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack),
-                  ("idxVar=%d cbVar=%d enmKind=%d enmGstReg=%s\n", idxVar, pReNative->Core.aVars[idxVar].cbVar,
-                   pReNative->Core.aVars[idxVar].enmKind, g_aGstShadowInfo[enmGstReg].pszName),
+    PIEMNATIVEVAR const pVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)];
+    Assert(!pVar->fRegAcquired);
+    AssertMsgStmt(   pVar->cbVar <= 8
+                  && (   pVar->enmKind == kIemNativeVarKind_Immediate
+                      || pVar->enmKind == kIemNativeVarKind_Stack),
+                  ("idxVar=%#x cbVar=%d enmKind=%d enmGstReg=%s\n", idxVar, pVar->cbVar,
+                   pVar->enmKind, g_aGstShadowInfo[enmGstReg].pszName),
                   IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_6));
 
@@ -8113 +8124 @@
      * ensure we get a register suitable for the intended argument number.
      */
-    AssertStmt(pReNative->Core.aVars[idxVar].uArgNo == UINT8_MAX, iemNativeVarRegisterAcquire(pReNative, idxVar, poff));
+    AssertStmt(pVar->uArgNo == UINT8_MAX, iemNativeVarRegisterAcquire(pReNative, idxVar, poff));
 
     /*
@@ -8119 +8130 @@
      * guest shadow copy assignment to it.
      */
-    uint8_t idxReg = pReNative->Core.aVars[idxVar].idxReg;
+    uint8_t idxReg = pVar->idxReg;
     if (idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs))
     {
@@ -8137 +8148 @@
         /** @todo figure this one out. We need some way of making sure the register isn't
          * modified after this point, just in case we start writing crappy MC code. */
-        pReNative->Core.aVars[idxVar].enmGstReg    = enmGstReg;
-        pReNative->Core.aVars[idxVar].fRegAcquired = true;
+        pVar->enmGstReg    = enmGstReg;
+        pVar->fRegAcquired = true;
         return idxReg;
     }
-    Assert(pReNative->Core.aVars[idxVar].uArgNo == UINT8_MAX);
+    Assert(pVar->uArgNo == UINT8_MAX);
 
     /*
@@ -8152 +8163 @@
     pReNative->Core.aHstRegs[idxReg].enmWhat = kIemNativeWhat_Var;
     pReNative->Core.aHstRegs[idxReg].idxVar  = idxVar;
-    pReNative->Core.aVars[idxVar].idxReg     = idxReg;
+    pVar->idxReg                             = idxReg;
 
     /*
      * Now we need to load the register value.
      */
-    if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Immediate)
-        *poff = iemNativeEmitLoadGprImm64(pReNative, *poff, idxReg, pReNative->Core.aVars[idxVar].u.uValue);
+    if (pVar->enmKind == kIemNativeVarKind_Immediate)
+        *poff = iemNativeEmitLoadGprImm64(pReNative, *poff, idxReg, pVar->u.uValue);
     else
     {
         uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, idxVar);
         int32_t const offDispBp    = iemNativeStackCalcBpDisp(idxStackSlot);
-        switch (pReNative->Core.aVars[idxVar].cbVar)
+        switch (pVar->cbVar)
         {
             case sizeof(uint64_t):
@@ -8182 +8193 @@
     }
 
-    pReNative->Core.aVars[idxVar].fRegAcquired = true;
+    pVar->fRegAcquired = true;
     return idxReg;
 }
@@ -8208 +8219 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
-    Assert(!pReNative->Core.aVars[idxVar].fRegAcquired);
+    PIEMNATIVEVAR const pVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)];
+    Assert(!pVar->fRegAcquired);
     Assert(idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs));
-    AssertStmt(pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_10));
+    AssertStmt(pVar->idxReg == UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_10));
     AssertStmt(!(pReNative->Core.bmHstRegs & RT_BIT_32(idxReg)), IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_11));
 
@@ -8217 +8229 @@
 
     iemNativeVarSetKindToStack(pReNative, idxVar);
-    pReNative->Core.aVars[idxVar].idxReg = idxReg;
+    pVar->idxReg = idxReg;
 
     return idxReg;
@@ -8230 +8242 @@
 {
     idxReg = iemNativeVarRegisterSet(pReNative, idxVar, idxReg, *poff);
-    pReNative->Core.aVars[idxVar].fRegAcquired = true;
+    pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = true;
     return idxReg;
 }
@@ -8262 +8274 @@
             {
                 uint8_t const idxVar = pReNative->Core.aHstRegs[idxHstReg].idxVar;
-                AssertStmt(   idxVar < RT_ELEMENTS(pReNative->Core.aVars)
-                           && (pReNative->Core.bmVars & RT_BIT_32(idxVar))
-                           && pReNative->Core.aVars[idxVar].idxReg == idxHstReg,
+                IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
+                AssertStmt(   IEMNATIVE_VAR_IDX_UNPACK(idxVar) < RT_ELEMENTS(pReNative->Core.aVars)
+                           && (pReNative->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(idxVar)))
+                           && pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].idxReg == idxHstReg,
                            IEMNATIVE_DO_LONGJMP(pReNative,  VERR_IEM_VAR_IPE_12));
-                switch (pReNative->Core.aVars[idxVar].enmKind)
+                switch (pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].enmKind)
                 {
                     case kIemNativeVarKind_Stack:
@@ -8272 +8285 @@
                         /* Temporarily spill the variable register. */
                         uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, idxVar);
-                        Log12(("iemNativeVarSaveVolatileRegsPreHlpCall: spilling idxVar=%d/idxReg=%d onto the stack (slot %#x bp+%d, off=%#x)\n",
+                        Log12(("iemNativeVarSaveVolatileRegsPreHlpCall: spilling idxVar=%#x/idxReg=%d onto the stack (slot %#x bp+%d, off=%#x)\n",
                                idxVar, idxHstReg, idxStackSlot, iemNativeStackCalcBpDisp(idxStackSlot), off));
                         off = iemNativeEmitStoreGprByBp(pReNative, off, iemNativeStackCalcBpDisp(idxStackSlot), idxHstReg);
@@ -8335 +8348 @@
             {
                 uint8_t const idxVar = pReNative->Core.aHstRegs[idxHstReg].idxVar;
-                AssertStmt(   idxVar < RT_ELEMENTS(pReNative->Core.aVars)
-                           && (pReNative->Core.bmVars & RT_BIT_32(idxVar))
-                           && pReNative->Core.aVars[idxVar].idxReg == idxHstReg,
+                IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
+                AssertStmt(   IEMNATIVE_VAR_IDX_UNPACK(idxVar) < RT_ELEMENTS(pReNative->Core.aVars)
+                           && (pReNative->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(idxVar)))
+                           && pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].idxReg == idxHstReg,
                            IEMNATIVE_DO_LONGJMP(pReNative,  VERR_IEM_VAR_IPE_12));
-                switch (pReNative->Core.aVars[idxVar].enmKind)
+                switch (pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].enmKind)
                 {
                     case kIemNativeVarKind_Stack:
@@ -8345 +8359 @@
                         /* Unspill the variable register. */
                         uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, idxVar);
-                        Log12(("iemNativeVarRestoreVolatileRegsPostHlpCall: unspilling idxVar=%d/idxReg=%d (slot %#x bp+%d, off=%#x)\n",
+                        Log12(("iemNativeVarRestoreVolatileRegsPostHlpCall: unspilling idxVar=%#x/idxReg=%d (slot %#x bp+%d, off=%#x)\n",
                                idxVar, idxHstReg, idxStackSlot, iemNativeStackCalcBpDisp(idxStackSlot), off));
                         off = iemNativeEmitLoadGprByBp(pReNative, off, idxHstReg, iemNativeStackCalcBpDisp(idxStackSlot));
@@ -8386 +8400 @@
  *
  * This is used both by iemNativeVarFreeOneWorker and iemNativeEmitCallCommon.
+ *
+ * ASSUMES that @a idxVar is valid and unpacked.
  */
 DECL_FORCE_INLINE(void) iemNativeVarFreeStackSlots(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
 {
+    Assert(idxVar < RT_ELEMENTS(pReNative->Core.aVars)); /* unpacked! */
     uint8_t const idxStackSlot = pReNative->Core.aVars[idxVar].idxStackSlot;
     if (idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS)
@@ -8397 +8414 @@
         Assert(cSlots > 0);
         Assert(((pReNative->Core.bmStack >> idxStackSlot) & fAllocMask) == fAllocMask);
-        Log11(("iemNativeVarFreeStackSlots: idxVar=%d iSlot=%#x/%#x (cbVar=%#x)\n", idxVar, idxStackSlot, fAllocMask, cbVar));
+        Log11(("iemNativeVarFreeStackSlots: idxVar=%d/%#x iSlot=%#x/%#x (cbVar=%#x)\n",
+               idxVar, IEMNATIVE_VAR_IDX_PACK(idxVar), idxStackSlot, fAllocMask, cbVar));
         pReNative->Core.bmStack &= ~(fAllocMask << idxStackSlot);
         pReNative->Core.aVars[idxVar].idxStackSlot = UINT8_MAX;
@@ -8409 +8427 @@
  * Worker that frees a single variable.
  *
- * ASSUMES that @a idxVar is valid.
+ * ASSUMES that @a idxVar is valid and unpacked.
  */
 DECLINLINE(void) iemNativeVarFreeOneWorker(PIEMRECOMPILERSTATE pReNative, uint8_t idxVar)
@@ -8421 +8439 @@
     if (idxHstReg < RT_ELEMENTS(pReNative->Core.aHstRegs))
     {
-        Assert(pReNative->Core.aHstRegs[idxHstReg].idxVar == idxVar);
+        Assert(pReNative->Core.aHstRegs[idxHstReg].idxVar == IEMNATIVE_VAR_IDX_PACK(idxVar));
         pReNative->Core.aHstRegs[idxHstReg].idxVar = UINT8_MAX;
         pReNative->Core.bmHstRegs &= ~RT_BIT_32(idxHstReg);
@@ -8457 +8475 @@
         if (idxHstReg < RT_ELEMENTS(pReNative->Core.aHstRegs))
         {
-            Assert(pReNative->Core.aHstRegs[idxHstReg].idxVar == idxVar);
+            Assert(pReNative->Core.aHstRegs[idxHstReg].idxVar == IEMNATIVE_VAR_IDX_PACK(idxVar));
             pReNative->Core.aHstRegs[idxHstReg].idxVar = UINT8_MAX;
             pReNative->Core.bmHstRegs &= ~RT_BIT_32(idxHstReg);
@@ -8492 +8510 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
-    Assert(pReNative->Core.aVars[idxVar].uArgNo == UINT8_MAX);
-    iemNativeVarFreeOneWorker(pReNative, idxVar);
+    Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].uArgNo == UINT8_MAX);
+    iemNativeVarFreeOneWorker(pReNative, IEMNATIVE_VAR_IDX_UNPACK(idxVar));
 }
 
@@ -8505 +8523 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
-    Assert(pReNative->Core.aVars[idxVar].uArgNo < RT_ELEMENTS(pReNative->Core.aidxArgVars));
-    iemNativeVarFreeOneWorker(pReNative, idxVar);
+    Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].uArgNo < RT_ELEMENTS(pReNative->Core.aidxArgVars));
+    iemNativeVarFreeOneWorker(pReNative, IEMNATIVE_VAR_IDX_UNPACK(idxVar));
 }
 
@@ -8519 +8537 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarDst);
-    AssertStmt(pReNative->Core.aVars[idxVarDst].enmKind == kIemNativeVarKind_Invalid,
+    PIEMNATIVEVAR const pVarDst = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarDst)];
+    AssertStmt(pVarDst->enmKind == kIemNativeVarKind_Invalid, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
+    Assert(   pVarDst->cbVar == sizeof(uint16_t)
+           || pVarDst->cbVar == sizeof(uint32_t));
+
+    IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarSrc);
+    PIEMNATIVEVAR const pVarSrc = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarSrc)];
+    AssertStmt(   pVarSrc->enmKind == kIemNativeVarKind_Stack
+               || pVarSrc->enmKind == kIemNativeVarKind_Immediate,
                IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
-    Assert(   pReNative->Core.aVars[idxVarDst].cbVar == sizeof(uint16_t)
-           || pReNative->Core.aVars[idxVarDst].cbVar == sizeof(uint32_t));
-
-    IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarSrc);
-    AssertStmt(   pReNative->Core.aVars[idxVarSrc].enmKind == kIemNativeVarKind_Stack
-               || pReNative->Core.aVars[idxVarSrc].enmKind == kIemNativeVarKind_Immediate,
-               IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
-
-    Assert(pReNative->Core.aVars[idxVarDst].cbVar < pReNative->Core.aVars[idxVarSrc].cbVar);
+
+    Assert(pVarDst->cbVar < pVarSrc->cbVar);
 
     /*
      * Special case for immediates.
      */
-    if (pReNative->Core.aVars[idxVarSrc].enmKind == kIemNativeVarKind_Immediate)
-    {
-        switch (pReNative->Core.aVars[idxVarDst].cbVar)
+    if (pVarSrc->enmKind == kIemNativeVarKind_Immediate)
+    {
+        switch (pVarDst->cbVar)
         {
             case sizeof(uint16_t):
-                iemNativeVarSetKindToConst(pReNative, idxVarDst, (uint16_t)pReNative->Core.aVars[idxVarSrc].u.uValue);
+                iemNativeVarSetKindToConst(pReNative, idxVarDst, (uint16_t)pVarSrc->u.uValue);
                 break;
             case sizeof(uint32_t):
-                iemNativeVarSetKindToConst(pReNative, idxVarDst, (uint32_t)pReNative->Core.aVars[idxVarSrc].u.uValue);
+                iemNativeVarSetKindToConst(pReNative, idxVarDst, (uint32_t)pVarSrc->u.uValue);
                 break;
             default: AssertFailed(); break;
@@ -8557 +8576 @@
         uint8_t const idxRegDst = iemNativeVarRegisterAcquire(pReNative, idxVarDst, &off);
         uint8_t const idxRegSrc = iemNativeVarRegisterAcquire(pReNative, idxVarSrc, &off);
-        switch (pReNative->Core.aVars[idxVarDst].cbVar)
+        switch (pVarDst->cbVar)
         {
             case sizeof(uint16_t):
@@ -8708 +8727 @@
                 if (idxRegOld < RT_ELEMENTS(pReNative->Core.aHstRegs))
                 {
-                    uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, idxVar);
-                    Log12(("iemNativeEmitCallCommon: spilling idxVar=%d/idxReg=%d (referred to by %d) onto the stack (slot %#x bp+%d, off=%#x)\n",
-                           idxVar, idxRegOld, pReNative->Core.aVars[idxVar].idxReferrerVar,
+                    uint8_t const idxStackSlot = iemNativeVarGetStackSlot(pReNative, IEMNATIVE_VAR_IDX_PACK(idxVar));
+                    Log12(("iemNativeEmitCallCommon: spilling idxVar=%d/%#x/idxReg=%d (referred to by %d) onto the stack (slot %#x bp+%d, off=%#x)\n",
+                           idxVar, IEMNATIVE_VAR_IDX_PACK(idxVar), idxRegOld, pReNative->Core.aVars[idxVar].idxReferrerVar,
                            idxStackSlot, iemNativeStackCalcBpDisp(idxStackSlot), off));
                     off = iemNativeEmitStoreGprByBp(pReNative, off, iemNativeStackCalcBpDisp(idxStackSlot), idxRegOld);
@@ -8742 +8761 @@
                 if (pReNative->Core.aHstRegs[idxArgReg].enmWhat == kIemNativeWhat_Var)
                 {
-                    uint8_t const idxVar = pReNative->Core.aHstRegs[idxArgReg].idxVar;
-                    Assert(idxVar < RT_ELEMENTS(pReNative->Core.aVars));
-                    Assert(pReNative->Core.aVars[idxVar].idxReg == idxArgReg);
-                    uint8_t const uArgNo = pReNative->Core.aVars[idxVar].uArgNo;
+                    uint8_t const       idxVar = pReNative->Core.aHstRegs[idxArgReg].idxVar;
+                    IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
+                    PIEMNATIVEVAR const pVar   = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)];
+                    Assert(pVar->idxReg == idxArgReg);
+                    uint8_t const       uArgNo = pVar->uArgNo;
                     if (uArgNo == i)
                     { /* prefect */ }
@@ -8768 +8788 @@
                         AssertStmt(uArgNo == UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_REG_IPE_10));
 
-                        if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack)
+                        if (pVar->enmKind == kIemNativeVarKind_Stack)
                             off = iemNativeRegMoveOrSpillStackVar(pReNative, off, idxVar);
                         else
                         {
                             /* just free it, can be reloaded if used again */
-                            pReNative->Core.aVars[idxVar].idxReg = UINT8_MAX;
+                            pVar->idxReg               = UINT8_MAX;
                             pReNative->Core.bmHstRegs &= ~RT_BIT_32(idxArgReg);
                             iemNativeRegClearGstRegShadowing(pReNative, idxArgReg, off);
@@ -8802 +8822 @@
         for (unsigned i = IEMNATIVE_CALL_ARG_GREG_COUNT; i < cArgs; i++)
         {
-            uint8_t const idxVar    = pReNative->Core.aidxArgVars[i];
-            int32_t const offBpDisp = g_aoffIemNativeCallStackArgBpDisp[i - IEMNATIVE_CALL_ARG_GREG_COUNT];
-            if (pReNative->Core.aVars[idxVar].idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs))
+            PIEMNATIVEVAR const pVar      = &pReNative->Core.aVars[pReNative->Core.aidxArgVars[i]]; /* unpacked */
+            int32_t const       offBpDisp = g_aoffIemNativeCallStackArgBpDisp[i - IEMNATIVE_CALL_ARG_GREG_COUNT];
+            if (pVar->idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs))
             {
-                Assert(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack); /* Imm as well? */
-                off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, pReNative->Core.aVars[idxVar].idxReg);
-                pReNative->Core.bmHstRegs &= ~RT_BIT_32(pReNative->Core.aVars[idxVar].idxReg);
-                pReNative->Core.aVars[idxVar].idxReg = UINT8_MAX;
+                Assert(pVar->enmKind == kIemNativeVarKind_Stack); /* Imm as well? */
+                off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, pVar->idxReg);
+                pReNative->Core.bmHstRegs &= ~RT_BIT_32(pVar->idxReg);
+                pVar->idxReg = UINT8_MAX;
             }
             else
             {
                 /* Use ARG0 as temp for stuff we need registers for. */
-                switch (pReNative->Core.aVars[idxVar].enmKind)
+                switch (pVar->enmKind)
                 {
                     case kIemNativeVarKind_Stack:
                     {
-                        uint8_t const idxStackSlot = pReNative->Core.aVars[idxVar].idxStackSlot;
+                        uint8_t const idxStackSlot = pVar->idxStackSlot;
                         AssertStmt(idxStackSlot != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_NOT_INITIALIZED));
                         off = iemNativeEmitLoadGprByBp(pReNative, off, IEMNATIVE_CALL_ARG0_GREG /* is free */,
@@ -8827 +8847 @@
 
                     case kIemNativeVarKind_Immediate:
-                        off = iemNativeEmitStoreImm64ByBp(pReNative, off, offBpDisp, pReNative->Core.aVars[idxVar].u.uValue);
+                        off = iemNativeEmitStoreImm64ByBp(pReNative, off, offBpDisp, pVar->u.uValue);
                         continue;
 
                     case kIemNativeVarKind_VarRef:
                     {
-                        uint8_t const idxOtherVar    = pReNative->Core.aVars[idxVar].u.idxRefVar;
+                        uint8_t const idxOtherVar    = pVar->u.idxRefVar; /* unpacked */
                         Assert(idxOtherVar < RT_ELEMENTS(pReNative->Core.aVars));
-                        uint8_t const idxStackSlot   = iemNativeVarGetStackSlot(pReNative, idxOtherVar);
+                        uint8_t const idxStackSlot   = iemNativeVarGetStackSlot(pReNative, IEMNATIVE_VAR_IDX_PACK(idxOtherVar));
                         int32_t const offBpDispOther = iemNativeStackCalcBpDisp(idxStackSlot);
                         uint8_t const idxRegOther    = pReNative->Core.aVars[idxOtherVar].idxReg;
@@ -8852 +8872 @@
                     case kIemNativeVarKind_GstRegRef:
                         off = iemNativeEmitLeaGprByGstRegRef(pReNative, off, IEMNATIVE_CALL_ARG0_GREG,
-                                                             pReNative->Core.aVars[idxVar].u.GstRegRef.enmClass,
-                                                             pReNative->Core.aVars[idxVar].u.GstRegRef.idx);
+                                                             pVar->u.GstRegRef.enmClass, pVar->u.GstRegRef.idx);
                         off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, IEMNATIVE_CALL_ARG0_GREG);
                         continue;
@@ -8885 +8904 @@
             uint8_t const idxArgReg = g_aidxIemNativeCallRegs[i];
             if (pReNative->Core.bmHstRegs & RT_BIT_32(idxArgReg))
-                Assert(   pReNative->Core.aHstRegs[idxArgReg].idxVar == pReNative->Core.aidxArgVars[i]
+                Assert(   pReNative->Core.aHstRegs[idxArgReg].idxVar == IEMNATIVE_VAR_IDX_PACK(pReNative->Core.aidxArgVars[i])
                        && pReNative->Core.aVars[pReNative->Core.aidxArgVars[i]].uArgNo == i
                        && pReNative->Core.aVars[pReNative->Core.aidxArgVars[i]].idxReg == idxArgReg);
             else
             {
-                uint8_t const idxVar = pReNative->Core.aidxArgVars[i];
-                if (pReNative->Core.aVars[idxVar].idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs))
+                PIEMNATIVEVAR const pVar = &pReNative->Core.aVars[pReNative->Core.aidxArgVars[i]]; /* unpacked */
+                if (pVar->idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs))
                 {
-                    Assert(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack);
-                    off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxArgReg, pReNative->Core.aVars[idxVar].idxReg);
-                    pReNative->Core.bmHstRegs = (pReNative->Core.bmHstRegs & ~RT_BIT_32(pReNative->Core.aVars[idxVar].idxReg))
+                    Assert(pVar->enmKind == kIemNativeVarKind_Stack);
+                    off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxArgReg, pVar->idxReg);
+                    pReNative->Core.bmHstRegs = (pReNative->Core.bmHstRegs & ~RT_BIT_32(pVar->idxReg))
                                               | RT_BIT_32(idxArgReg);
-                    pReNative->Core.aVars[idxVar].idxReg = idxArgReg;
+                    pVar->idxReg = idxArgReg;
                 }
                 else
                 {
                     /* Use ARG0 as temp for stuff we need registers for. */
-                    switch (pReNative->Core.aVars[idxVar].enmKind)
+                    switch (pVar->enmKind)
                     {
                         case kIemNativeVarKind_Stack:
                         {
-                            uint8_t const idxStackSlot = pReNative->Core.aVars[idxVar].idxStackSlot;
+                            uint8_t const idxStackSlot = pVar->idxStackSlot;
                             AssertStmt(idxStackSlot != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_NOT_INITIALIZED));
                             off = iemNativeEmitLoadGprByBp(pReNative, off, idxArgReg, iemNativeStackCalcBpDisp(idxStackSlot));
@@ -8913 +8932 @@
 
                         case kIemNativeVarKind_Immediate:
-                            off = iemNativeEmitLoadGprImm64(pReNative, off, idxArgReg, pReNative->Core.aVars[idxVar].u.uValue);
+                            off = iemNativeEmitLoadGprImm64(pReNative, off, idxArgReg, pVar->u.uValue);
                             continue;
 
                         case kIemNativeVarKind_VarRef:
                         {
-                            uint8_t const idxOtherVar    = pReNative->Core.aVars[idxVar].u.idxRefVar;
+                            uint8_t const idxOtherVar    = pVar->u.idxRefVar; /* unpacked */
                             Assert(idxOtherVar < RT_ELEMENTS(pReNative->Core.aVars));
-                            uint8_t const idxStackSlot   = iemNativeVarGetStackSlot(pReNative, idxOtherVar);
+                            uint8_t const idxStackSlot   = iemNativeVarGetStackSlot(pReNative,
+                                                                                    IEMNATIVE_VAR_IDX_PACK(idxOtherVar));
                             int32_t const offBpDispOther = iemNativeStackCalcBpDisp(idxStackSlot);
                             uint8_t const idxRegOther    = pReNative->Core.aVars[idxOtherVar].idxReg;
@@ -8937 +8957 @@
                         case kIemNativeVarKind_GstRegRef:
                             off = iemNativeEmitLeaGprByGstRegRef(pReNative, off, idxArgReg,
-                                                                 pReNative->Core.aVars[idxVar].u.GstRegRef.enmClass,
-                                                                 pReNative->Core.aVars[idxVar].u.GstRegRef.idx);
+                                                                 pVar->u.GstRegRef.enmClass, pVar->u.GstRegRef.idx);
                             continue;
 
@@ -8973 +8992 @@
     for (uint32_t i = cHiddenArgs; i < cArgs; i++)
     {
-        uint8_t const idxVar = pReNative->Core.aidxArgVars[i];
+        uint8_t const idxVar = pReNative->Core.aidxArgVars[i]; /* unpacked */
         Assert(idxVar < RT_ELEMENTS(pReNative->Core.aVars));
 
@@ -9140 +9159 @@
     {
         IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarRc);
-        AssertStmt(pReNative->Core.aVars[idxVarRc].uArgNo == UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_8));
-        AssertStmt(pReNative->Core.aVars[idxVarRc].cbVar <= sizeof(uint64_t), IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_9));
+        PIEMNATIVEVAR const pVarRc = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarRc)];
+        AssertStmt(pVarRc->uArgNo == UINT8_MAX,       IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_8));
+        AssertStmt(pVarRc->cbVar <= sizeof(uint64_t), IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_IPE_9));
     }
 
@@ -9153 +9173 @@
      */
     off = iemNativeEmitCallImm(pReNative, off, pfnAImpl);
-    if (idxVarRc < RT_ELEMENTS(pReNative->Core.aVars))
-    {
-pReNative->pInstrBuf[off++] = 0xcc; /** @todo test IEM_MC_CALL_AIMPL_3 and IEM_MC_CALL_AIMPL_4 return codes. */
+    if (idxVarRc != UINT8_MAX)
+    {
+off = iemNativeEmitBrk(pReNative, off, 0x4222); /** @todo test IEM_MC_CALL_AIMPL_3 and IEM_MC_CALL_AIMPL_4 return codes. */
         iemNativeVarRegisterSet(pReNative, idxVarRc, IEMNATIVE_CALL_RET_GREG, off);
     }
@@ -9271 +9291 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
-    Assert(pReNative->Core.aVars[idxDstVar].cbVar == cbZeroExtended); RT_NOREF(cbZeroExtended);
+    IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, cbZeroExtended); RT_NOREF(cbZeroExtended);
     Assert(iGRegEx < 20);
 
@@ -9307 +9327 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
-    Assert(pReNative->Core.aVars[idxDstVar].cbVar == cbSignExtended);
+    IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, cbSignExtended);
     Assert(iGRegEx < 20);
 
@@ -9372 +9392 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
-    Assert(pReNative->Core.aVars[idxDstVar].cbVar == cbZeroExtended); RT_NOREF(cbZeroExtended);
+    IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, cbZeroExtended); RT_NOREF(cbZeroExtended);
     Assert(iGReg < 16);
 
@@ -9406 +9426 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
-    Assert(pReNative->Core.aVars[idxDstVar].cbVar == cbSignExtended);
+    IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, cbSignExtended);
     Assert(iGReg < 16);
 
@@ -9446 +9466 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
-    Assert(pReNative->Core.aVars[idxDstVar].cbVar == cbZeroExtended); RT_NOREF_PV(cbZeroExtended);
+    IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, cbZeroExtended); RT_NOREF(cbZeroExtended);
     Assert(iGReg < 16);
 
@@ -9477 +9497 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
-    Assert(pReNative->Core.aVars[idxDstVar].cbVar == sizeof(uint64_t));
+    IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, sizeof(uint64_t));
     Assert(iGReg < 16);
 
@@ -9512 +9532 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
-    Assert(pReNative->Core.aVars[idxDstVar].cbVar == sizeof(uint64_t));
+    IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, sizeof(uint64_t));
     Assert(iGReg < 16);
 
@@ -9624 +9644 @@
      * IEM_MC_STORE_GREG_U8_CONST statement.
      */
-    if (pReNative->Core.aVars[idxValueVar].enmKind == kIemNativeVarKind_Stack)
+    PIEMNATIVEVAR const pValueVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxValueVar)];
+    if (pValueVar->enmKind == kIemNativeVarKind_Stack)
     { /* likely */ }
     else
     {
-        AssertStmt(pReNative->Core.aVars[idxValueVar].enmKind == kIemNativeVarKind_Immediate,
+        AssertStmt(pValueVar->enmKind == kIemNativeVarKind_Immediate,
                    IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
-        return iemNativeEmitStoreGregU8Const(pReNative, off, iGRegEx, (uint8_t)pReNative->Core.aVars[idxValueVar].u.uValue);
+        return iemNativeEmitStoreGregU8Const(pReNative, off, iGRegEx, (uint8_t)pValueVar->u.uValue);
     }
 
@@ -9756 +9777 @@
      * IEM_MC_STORE_GREG_U16_CONST statement.
      */
-    if (pReNative->Core.aVars[idxValueVar].enmKind == kIemNativeVarKind_Stack)
+    PIEMNATIVEVAR const pValueVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxValueVar)];
+    if (pValueVar->enmKind == kIemNativeVarKind_Stack)
     { /* likely */ }
     else
     {
-        AssertStmt(pReNative->Core.aVars[idxValueVar].enmKind == kIemNativeVarKind_Immediate,
+        AssertStmt(pValueVar->enmKind == kIemNativeVarKind_Immediate,
                    IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
-        return iemNativeEmitStoreGregU16Const(pReNative, off, iGReg, (uint16_t)pReNative->Core.aVars[idxValueVar].u.uValue);
+        return iemNativeEmitStoreGregU16Const(pReNative, off, iGReg, (uint16_t)pValueVar->u.uValue);
     }
 
@@ -9772 +9794 @@
     uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 12);
     pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
-    if (pReNative->Core.aVars[idxValueVar].idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs))
-    {
-        if (idxGstTmpReg >= 8 || pReNative->Core.aVars[idxValueVar].idxReg >= 8)
-            pbCodeBuf[off++] = (idxGstTmpReg >= 8                              ? X86_OP_REX_R : 0)
-                             | (pReNative->Core.aVars[idxValueVar].idxReg >= 8 ? X86_OP_REX_B : 0);
+    if (pValueVar->idxReg < RT_ELEMENTS(pReNative->Core.aHstRegs))
+    {
+        if (idxGstTmpReg >= 8 || pValueVar->idxReg >= 8)
+            pbCodeBuf[off++] = (idxGstTmpReg      >= 8 ? X86_OP_REX_R : 0)
+                             | (pValueVar->idxReg >= 8 ? X86_OP_REX_B : 0);
         pbCodeBuf[off++] = 0x8b;
-        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxGstTmpReg & 7, pReNative->Core.aVars[idxValueVar].idxReg & 7);
+        pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxGstTmpReg & 7, pValueVar->idxReg & 7);
     }
     else
     {
-        uint8_t const idxStackSlot = pReNative->Core.aVars[idxValueVar].idxStackSlot;
+        uint8_t const idxStackSlot = pValueVar->idxStackSlot;
         AssertStmt(idxStackSlot != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_NOT_INITIALIZED));
         if (idxGstTmpReg >= 8)
@@ -9840 +9862 @@
      * IEM_MC_STORE_GREG_U32_CONST statement.
      */
-    if (pReNative->Core.aVars[idxValueVar].enmKind == kIemNativeVarKind_Stack)
+    PIEMNATIVEVAR const pValueVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxValueVar)];
+    if (pValueVar->enmKind == kIemNativeVarKind_Stack)
     { /* likely */ }
     else
     {
-        AssertStmt(pReNative->Core.aVars[idxValueVar].enmKind == kIemNativeVarKind_Immediate,
+        AssertStmt(pValueVar->enmKind == kIemNativeVarKind_Immediate,
                    IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
-        return iemNativeEmitStoreGregU32Const(pReNative, off, iGReg, (uint32_t)pReNative->Core.aVars[idxValueVar].u.uValue);
+        return iemNativeEmitStoreGregU32Const(pReNative, off, iGReg, (uint32_t)pValueVar->u.uValue);
     }
 
@@ -9894 +9917 @@
      * IEM_MC_STORE_GREG_U64_CONST statement.
      */
-    if (pReNative->Core.aVars[idxValueVar].enmKind == kIemNativeVarKind_Stack)
+    PIEMNATIVEVAR const pValueVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxValueVar)];
+    if (pValueVar->enmKind == kIemNativeVarKind_Stack)
     { /* likely */ }
     else
     {
-        AssertStmt(pReNative->Core.aVars[idxValueVar].enmKind == kIemNativeVarKind_Immediate,
+        AssertStmt(pValueVar->enmKind == kIemNativeVarKind_Immediate,
                    IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
-        return iemNativeEmitStoreGregU64Const(pReNative, off, iGReg, pReNative->Core.aVars[idxValueVar].u.uValue);
+        return iemNativeEmitStoreGregU64Const(pReNative, off, iGReg, pValueVar->u.uValue);
     }
 
@@ -10168 +10192 @@
 iemNativeEmitAndLocal(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar, uint64_t uMask, uint8_t cbMask)
 {
-    Assert(pReNative->Core.aVars[idxVar].cbVar == cbMask);
 #ifdef VBOX_STRICT
     switch (cbMask)
@@ -10181 +10204 @@
 
     uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxVar, &off, true /*fInitialized*/);
+    IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVar, cbMask);
+
     if (cbMask <= sizeof(uint32_t))
         off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxVarReg, uMask);
     else
         off = iemNativeEmitAndGprByImm(pReNative, off, idxVarReg, uMask);
+
     iemNativeVarRegisterRelease(pReNative, idxVar);
     return off;
@@ -10206 +10232 @@
 iemNativeEmitOrLocal(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar, uint64_t uMask, uint8_t cbMask)
 {
-    Assert(pReNative->Core.aVars[idxVar].cbVar == cbMask);
 #ifdef VBOX_STRICT
     switch (cbMask)
@@ -10219 +10244 @@
 
     uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxVar, &off, true /*fInitialized*/);
+    IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVar, cbMask);
+
     if (cbMask <= sizeof(uint32_t))
         off = iemNativeEmitOrGpr32ByImm(pReNative, off, idxVarReg, uMask);
     else
         off = iemNativeEmitOrGprByImm(pReNative, off, idxVarReg, uMask);
+
     iemNativeVarRegisterRelease(pReNative, idxVar);
     return off;
@@ -10241 +10269 @@
 iemNativeEmitBswapLocal(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVar, uint8_t cbLocal)
 {
-    Assert(pReNative->Core.aVars[idxVar].cbVar == cbLocal);
-
     uint8_t const idxVarReg = iemNativeVarRegisterAcquire(pReNative, idxVar, &off, true /*fInitialized*/);
+    IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVar, cbLocal);
 
     switch (cbLocal)
@@ -10336 +10363 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarEFlags);
-    Assert(pReNative->Core.aVars[idxVarEFlags].cbVar == sizeof(uint32_t));
+    IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVarEFlags, sizeof(uint32_t));
     RT_NOREF(fEflInput, fEflOutput);
 
@@ -10389 +10416 @@
 iemNativeEmitCommitEFlags(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarEFlags, uint32_t fEflOutput)
 {
-    IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarEFlags);
-    Assert(pReNative->Core.aVars[idxVarEFlags].cbVar == sizeof(uint32_t));
     RT_NOREF(fEflOutput);
-
     uint8_t const idxReg = iemNativeVarRegisterAcquire(pReNative, idxVarEFlags, &off, true /*fInitialized*/);
+    IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVarEFlags, sizeof(uint32_t));
 
 #ifdef VBOX_STRICT
@@ -10439 +10464 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
-    Assert(pReNative->Core.aVars[idxDstVar].cbVar == cbVar); RT_NOREF(cbVar);
+    IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, cbVar); RT_NOREF(cbVar);
     Assert(iSReg < X86_SREG_COUNT);
 
@@ -10479 +10504 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarRef);
-    Assert(pReNative->Core.aVars[idxVarRef].cbVar == sizeof(void *));
+    IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVarRef, sizeof(void *));
     Assert(iGRegEx < 20);
 
@@ -10531 +10556 @@
 iemNativeEmitRefGregUxx(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarRef, uint8_t iGReg, bool fConst)
 {
-    IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarRef);
-    Assert(pReNative->Core.aVars[idxVarRef].cbVar == sizeof(void *));
     Assert(iGReg < 16);
-
     iemNativeVarSetKindToGstRegRef(pReNative, idxVarRef, kIemNativeGstRegRef_Gpr, iGReg);
+    IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVarRef, sizeof(void *));
 
     /* If we've delayed writing back the register value, flush it now. */
@@ -10557 +10580 @@
 iemNativeEmitRefEFlags(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxVarRef)
 {
-    IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarRef);
-    Assert(pReNative->Core.aVars[idxVarRef].cbVar == sizeof(void *));
-
     iemNativeVarSetKindToGstRegRef(pReNative, idxVarRef, kIemNativeGstRegRef_EFlags, 0);
+    IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxVarRef, sizeof(void *));
 
     /* If we've delayed writing back the register value, flush it now. */
@@ -11445 +11466 @@
      */
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarValue);
+    PIEMNATIVEVAR const pVarValue = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarValue)];
     Assert(   enmOp != kIemNativeEmitMemOp_Store
-           || pReNative->Core.aVars[idxVarValue].enmKind == kIemNativeVarKind_Immediate
-           || pReNative->Core.aVars[idxVarValue].enmKind == kIemNativeVarKind_Stack);
+           || pVarValue->enmKind == kIemNativeVarKind_Immediate
+           || pVarValue->enmKind == kIemNativeVarKind_Stack);
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarGCPtrMem);
-    AssertStmt(   pReNative->Core.aVars[idxVarGCPtrMem].enmKind == kIemNativeVarKind_Immediate
-               || pReNative->Core.aVars[idxVarGCPtrMem].enmKind == kIemNativeVarKind_Stack,
+    PIEMNATIVEVAR const pVarGCPtrMem = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarGCPtrMem)];
+    AssertStmt(   pVarGCPtrMem->enmKind == kIemNativeVarKind_Immediate
+               || pVarGCPtrMem->enmKind == kIemNativeVarKind_Stack,
                IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
     Assert(iSegReg < 6 || iSegReg == UINT8_MAX);
@@ -11583 +11606 @@
     uint8_t  const idxRegValueStore  =    !TlbState.fSkip
                                        && enmOp == kIemNativeEmitMemOp_Store
-                                       && pReNative->Core.aVars[idxVarValue].enmKind != kIemNativeVarKind_Immediate
+                                       && pVarValue->enmKind != kIemNativeVarKind_Immediate
                                      ? iemNativeVarRegisterAcquire(pReNative, idxVarValue, &off)
                                      : UINT8_MAX;
@@ -11659 +11682 @@
     if (enmOp != kIemNativeEmitMemOp_Store)
     {
-        Assert(idxRegValueFetch == pReNative->Core.aVars[idxVarValue].idxReg);
+        Assert(idxRegValueFetch == pVarValue->idxReg);
         if (idxRegValueFetch != IEMNATIVE_CALL_RET_GREG)
             off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegValueFetch, IEMNATIVE_CALL_RET_GREG);
@@ -11697 +11720 @@
         {
             case kIemNativeEmitMemOp_Store:
-                if (pReNative->Core.aVars[idxVarValue].enmKind != kIemNativeVarKind_Immediate)
+                if (pVarValue->enmKind != kIemNativeVarKind_Immediate)
                 {
                     switch (cbMem)
@@ -11722 +11745 @@
                     {
                         case 1:
-                            off = iemNativeEmitStoreImm8ByGprEx(pCodeBuf, off,
-                                                                (uint8_t)pReNative->Core.aVars[idxVarValue].u.uValue,
+                            off = iemNativeEmitStoreImm8ByGprEx(pCodeBuf, off, (uint8_t)pVarValue->u.uValue,
                                                                 idxRegMemResult, TlbState.idxReg1);
                             break;
                         case 2:
-                            off = iemNativeEmitStoreImm16ByGprEx(pCodeBuf, off,
-                                                                 (uint16_t)pReNative->Core.aVars[idxVarValue].u.uValue,
+                            off = iemNativeEmitStoreImm16ByGprEx(pCodeBuf, off, (uint16_t)pVarValue->u.uValue,
                                                                  idxRegMemResult, TlbState.idxReg1);
                             break;
                         case 4:
-                            off = iemNativeEmitStoreImm32ByGprEx(pCodeBuf, off,
-                                                                 (uint32_t)pReNative->Core.aVars[idxVarValue].u.uValue,
+                            off = iemNativeEmitStoreImm32ByGprEx(pCodeBuf, off, (uint32_t)pVarValue->u.uValue,
                                                                  idxRegMemResult, TlbState.idxReg1);
                             break;
                         case 8:
-                            off = iemNativeEmitStoreImm64ByGprEx(pCodeBuf, off, pReNative->Core.aVars[idxVarValue].u.uValue,
+                            off = iemNativeEmitStoreImm64ByGprEx(pCodeBuf, off, pVarValue->u.uValue,
                                                                  idxRegMemResult, TlbState.idxReg1);
                             break;
@@ -12200 +12220 @@
      */
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarValue);
+    PIEMNATIVEVAR const pVarValue = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarValue)];
 #ifdef VBOX_STRICT
     if (RT_BYTE2(cBitsVarAndFlat) != 0)
@@ -12304 +12325 @@
                                      : UINT32_MAX;
     uint8_t const  idxRegValue       =    !TlbState.fSkip
-                                       && pReNative->Core.aVars[idxVarValue].enmKind != kIemNativeVarKind_Immediate
+                                       && pVarValue->enmKind != kIemNativeVarKind_Immediate
                                      ? iemNativeVarRegisterAcquire(pReNative, idxVarValue, &off, true /*fInitialized*/,
                                                                    IEMNATIVE_CALL_ARG2_GREG /*idxRegPref*/)
@@ -12468 +12489 @@
             {
                 case 2:
-                    off = iemNativeEmitStoreImm16ByGprEx(pCodeBuf, off,
-                                                         (uint16_t)pReNative->Core.aVars[idxVarValue].u.uValue,
+                    off = iemNativeEmitStoreImm16ByGprEx(pCodeBuf, off, (uint16_t)pVarValue->u.uValue,
                                                          idxRegMemResult, TlbState.idxReg1);
                     break;
                 case 4:
                     Assert(!fIsSegReg);
-                    off = iemNativeEmitStoreImm32ByGprEx(pCodeBuf, off,
-                                                         (uint32_t)pReNative->Core.aVars[idxVarValue].u.uValue,
+                    off = iemNativeEmitStoreImm32ByGprEx(pCodeBuf, off, (uint32_t)pVarValue->u.uValue,
                                                          idxRegMemResult, TlbState.idxReg1);
                     break;
                 case 8:
-                    off = iemNativeEmitStoreImm64ByGprEx(pCodeBuf, off, pReNative->Core.aVars[idxVarValue].u.uValue,
-                                                         idxRegMemResult, TlbState.idxReg1);
+                    off = iemNativeEmitStoreImm64ByGprEx(pCodeBuf, off, pVarValue->u.uValue, idxRegMemResult, TlbState.idxReg1);
                     break;
                 default:
@@ -13128 +13146 @@
      */
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarMem);
-    AssertStmt(   pReNative->Core.aVars[idxVarMem].enmKind == kIemNativeVarKind_Invalid
-               && pReNative->Core.aVars[idxVarMem].cbVar   == sizeof(void *),
+    PIEMNATIVEVAR const pVarMem = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarMem)];
+    AssertStmt(   pVarMem->enmKind == kIemNativeVarKind_Invalid
+               && pVarMem->cbVar   == sizeof(void *),
                IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
 
+    PIEMNATIVEVAR const pVarUnmapInfo = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarUnmapInfo)];
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarUnmapInfo);
-    AssertStmt(   pReNative->Core.aVars[idxVarUnmapInfo].enmKind == kIemNativeVarKind_Invalid
-               && pReNative->Core.aVars[idxVarUnmapInfo].cbVar   == sizeof(uint8_t),
+    AssertStmt(   pVarUnmapInfo->enmKind == kIemNativeVarKind_Invalid
+               && pVarUnmapInfo->cbVar   == sizeof(uint8_t),
                IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
 
+    PIEMNATIVEVAR const pVarGCPtrMem = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarGCPtrMem)];
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarGCPtrMem);
-    AssertStmt(   pReNative->Core.aVars[idxVarGCPtrMem].enmKind == kIemNativeVarKind_Immediate
-               || pReNative->Core.aVars[idxVarGCPtrMem].enmKind == kIemNativeVarKind_Stack,
+    AssertStmt(   pVarGCPtrMem->enmKind == kIemNativeVarKind_Immediate
+               || pVarGCPtrMem->enmKind == kIemNativeVarKind_Stack,
                IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
 
@@ -13305 +13326 @@
      * Put the output in the right registers.
      */
-    Assert(idxRegMemResult == pReNative->Core.aVars[idxVarMem].idxReg);
+    Assert(idxRegMemResult == pVarMem->idxReg);
     if (idxRegMemResult != IEMNATIVE_CALL_RET_GREG)
         off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegMemResult, IEMNATIVE_CALL_RET_GREG);
@@ -13315 +13336 @@
 #endif
 
-    Assert(pReNative->Core.aVars[idxVarUnmapInfo].idxReg == idxRegUnmapInfo);
+    Assert(pVarUnmapInfo->idxReg == idxRegUnmapInfo);
     off = iemNativeEmitLoadGprByBpU8(pReNative, off, idxRegUnmapInfo, offBpDispVarUnmapInfo);
 
@@ -13385 +13406 @@
      */
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVarUnmapInfo);
-    Assert(pReNative->Core.aVars[idxVarUnmapInfo].enmKind == kIemNativeVarKind_Stack);
-    Assert(   pReNative->Core.aVars[idxVarUnmapInfo].idxReg       < RT_ELEMENTS(pReNative->Core.aHstRegs)
-           || pReNative->Core.aVars[idxVarUnmapInfo].idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS); /* must be initialized */
+    PIEMNATIVEVAR const pVarUnmapInfo = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVarUnmapInfo)];
+    Assert(pVarUnmapInfo->enmKind == kIemNativeVarKind_Stack);
+    Assert(   pVarUnmapInfo->idxReg       < RT_ELEMENTS(pReNative->Core.aHstRegs)
+           || pVarUnmapInfo->idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS); /* must be initialized */
 #ifdef VBOX_STRICT
     switch (fAccess & (IEM_ACCESS_TYPE_MASK | IEM_ACCESS_ATOMIC))
@@ -13437 +13459 @@
      *        are enabled. */
 #ifdef RT_ARCH_AMD64
-    if (pReNative->Core.aVars[idxVarUnmapInfo].idxReg == UINT8_MAX)
+    if (pVarUnmapInfo->idxReg == UINT8_MAX)
     {
         /* test byte [rbp - xxx], 0ffh  */
         uint8_t * const pbCodeBuf    = iemNativeInstrBufEnsure(pReNative, off, 7);
         pbCodeBuf[off++] = 0xf6;
-        uint8_t const   idxStackSlot = pReNative->Core.aVars[idxVarUnmapInfo].idxStackSlot;
+        uint8_t const   idxStackSlot = pVarUnmapInfo->idxStackSlot;
         off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, 0, iemNativeStackCalcBpDisp(idxStackSlot), pReNative);
         pbCodeBuf[off++] = 0xff;
@@ -13527 +13549 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
-    Assert(pReNative->Core.aVars[idxDstVar].cbVar == sizeof(uint16_t));
+    IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, sizeof(uint16_t));
 
     /* Allocate a temporary FCW register. */
@@ -13550 +13572 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxDstVar);
-    Assert(pReNative->Core.aVars[idxDstVar].cbVar == sizeof(uint16_t));
+    IEMNATIVE_ASSERT_VAR_SIZE(pReNative, idxDstVar, sizeof(uint16_t));
 
     /* Allocate a temporary FSW register. */

trunk/src/VBox/VMM/include/IEMN8veRecompiler.h

@@ -794 +794 @@
     /** The argument number if argument, UINT8_MAX if regular variable. */
     uint8_t             uArgNo;
-    /** If referenced, the index of the variable referencing this one, otherwise
-     *  UINT8_MAX.  A referenced variable must only be placed on the stack and
-     *  must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
+    /** If referenced, the index (unpacked) of the variable referencing this one,
+     * otherwise UINT8_MAX.  A referenced variable must only be placed on the stack
+     * and must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
     uint8_t             idxReferrerVar;
     /** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not.
@@ -809 +809 @@
         /** kIemNativeVarKind_Immediate: The immediate value. */
         uint64_t            uValue;
-        /** kIemNativeVarKind_VarRef: The index of the variable being referenced. */
+        /** kIemNativeVarKind_VarRef: The index (unpacked) of the variable being referenced. */
         uint8_t             idxRefVar;
         /** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
@@ -821 +821 @@
     } u;
 } IEMNATIVEVAR;
+/** Pointer to a variable or argument. */
+typedef IEMNATIVEVAR *PIEMNATIVEVAR;
+/** Pointer to a const variable or argument. */
+typedef IEMNATIVEVAR const *PCIEMNATIVEVAR;
 
 /** What is being kept in a host register. */
@@ -878 +882 @@
     /** What is being kept in this register. */
     IEMNATIVEWHAT   enmWhat;
-    /** Variable index if holding a variable, otherwise UINT8_MAX. */
+    /** Variable index (packed) if holding a variable, otherwise UINT8_MAX. */
     uint8_t         idxVar;
     /** Stack slot assigned by iemNativeVarSaveVolatileRegsPreHlpCall and freed
@@ -906 +910 @@
     union
     {
-        /** Index of variable arguments, UINT8_MAX if not valid. */
+        /** Index of variable (unpacked) arguments, UINT8_MAX if not valid. */
         uint8_t                 aidxArgVars[8];
         /** For more efficient resetting. */
@@ -933 +937 @@
 /** Pointer to const core state. */
 typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;
+
+/** @def IEMNATIVE_VAR_IDX_UNPACK
+ * @returns Index into IEMNATIVECORESTATE::aVars.
+ * @param   a_idxVar    Variable index w/ magic (in strict builds).
+ */
+/** @def IEMNATIVE_VAR_IDX_PACK
+ * @returns Variable index w/ magic (in strict builds).
+ * @param   a_idxVar    Index into IEMNATIVECORESTATE::aVars.
+ */
+#ifdef VBOX_STRICT
+# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) ((a_idxVar) & IEMNATIVE_VAR_IDX_MASK)
+# define IEMNATIVE_VAR_IDX_PACK(a_idxVar)   ((a_idxVar) | IEMNATIVE_VAR_IDX_MAGIC)
+# define IEMNATIVE_VAR_IDX_MAGIC            UINT8_C(0xd0)
+# define IEMNATIVE_VAR_IDX_MAGIC_MASK       UINT8_C(0xf0)
+# define IEMNATIVE_VAR_IDX_MASK             UINT8_C(0x0f)
+#else
+# define IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) (a_idxVar)
+# define IEMNATIVE_VAR_IDX_PACK(a_idxVar)   (a_idxVar)
+#endif
 
 
@@ -1181 +1204 @@
                                                                             IEMNATIVEGSTREG enmGstReg);
 
-DECL_HIDDEN_THROW(uint8_t)  iemNativeRegAllocVar(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t idxVar);
 DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);
 DECL_HIDDEN_THROW(uint8_t)  iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);
@@ -1260 +1282 @@
  * Checks that a variable index is valid.
  */
-#define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
+#ifdef IEMNATIVE_VAR_IDX_MAGIC
+# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
+    AssertMsg(   ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
+              && (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
+              && ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))), \
+              ("%s=%#x\n", #a_idxVar, a_idxVar))
+#else
+# define IEMNATIVE_ASSERT_VAR_IDX(a_pReNative, a_idxVar) \
     AssertMsg(   (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
               && ((a_pReNative)->Core.bmVars & RT_BIT_32(a_idxVar)), ("%s=%d\n", #a_idxVar, a_idxVar))
+#endif
 
 /**
@@ -1269 +1299 @@
  * This also adds a RT_NOREF of a_idxVar.
  */
-#define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
+#ifdef IEMNATIVE_VAR_IDX_MAGIC
+# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
+        RT_NOREF_PV(a_idxVar); \
+        AssertMsg(   ((a_idxVar) & IEMNATIVE_VAR_IDX_MAGIC_MASK) == IEMNATIVE_VAR_IDX_MAGIC \
+                  && (unsigned)IEMNATIVE_VAR_IDX_UNPACK(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
+                  && ((a_pReNative)->Core.bmVars & RT_BIT_32(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar))) \
+                  && (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].uArgNo == (a_uArgNo), \
+                  ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
+                   (a_pReNative)->Core.aVars[RT_MIN(IEMNATIVE_VAR_IDX_UNPACK(a_idxVar), \
+                                                    RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, \
+                   a_uArgNo)); \
+    } while (0)
+#else
+# define IEMNATIVE_ASSERT_ARG_VAR_IDX(a_pReNative, a_idxVar, a_uArgNo) do { \
         RT_NOREF_PV(a_idxVar); \
         AssertMsg(   (unsigned)(a_idxVar) < RT_ELEMENTS((a_pReNative)->Core.aVars) \
@@ -1275 +1318 @@
                   && (a_pReNative)->Core.aVars[a_idxVar].uArgNo == (a_uArgNo) \
                   , ("%s=%d; uArgNo=%d, expected %u\n", #a_idxVar, a_idxVar, \
-                     (a_pReNative)->Core.aVars[RT_MAX(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
+                     (a_pReNative)->Core.aVars[RT_MIN(a_idxVar, RT_ELEMENTS((a_pReNative)->Core.aVars)) - 1].uArgNo, a_uArgNo)); \
     } while (0)
+#endif
+
+
+/**
+ * Checks that a variable has the expected size.
+ */
+#define IEMNATIVE_ASSERT_VAR_SIZE(a_pReNative, a_idxVar, a_cbVar) \
+    AssertMsg((a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar), \
+              ("%s=%#x: cbVar=%#x, expected %#x!\n", #a_idxVar, a_idxVar, \
+              (a_pReNative)->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVar)].cbVar == (a_cbVar)))
+
 
 /**
@@ -1299 +1353 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
-    Assert(pReNative->Core.aVars[idxVar].fRegAcquired);
-    pReNative->Core.aVars[idxVar].fRegAcquired = false;
+    Assert(pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired);
+    pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)].fRegAcquired = false;
 }
 

trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h

@@ -6720 +6720 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
-    AssertStmt(pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
-               IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
-
-    uint8_t const idxRegVar = pReNative->Core.aVars[idxVar].idxReg;
+    PIEMNATIVEVAR const pVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)];
+    AssertStmt(pVar->enmKind == kIemNativeVarKind_Stack, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
+
+    uint8_t const idxRegVar = pVar->idxReg;
     if (   idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs)
         && (   (RT_BIT_32(idxRegVar) & (~IEMNATIVE_CALL_VOLATILE_GREG_MASK | fHstVolatileRegsAllowed))
@@ -6741 +6741 @@
     else
     {
-        uint8_t const idxStackSlot = pReNative->Core.aVars[idxVar].idxStackSlot;
+        uint8_t const idxStackSlot = pVar->idxStackSlot;
         AssertStmt(idxStackSlot != UINT8_MAX, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_NOT_INITIALIZED));
         off = iemNativeEmitLoadGprByBp(pReNative, off, idxRegArg, iemNativeStackCalcBpDisp(idxStackSlot));
@@ -6762 +6762 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
-    if (pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Immediate)
-        off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegArg, pReNative->Core.aVars[idxVar].u.uValue + offAddend);
+    PIEMNATIVEVAR const pVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)];
+    if (pVar->enmKind == kIemNativeVarKind_Immediate)
+        off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegArg, pVar->u.uValue + offAddend);
     else
         off = iemNativeEmitLoadArgGregFromStackVar(pReNative, off, idxRegArg, idxVar, offAddend,
@@ -6781 +6782 @@
 {
     IEMNATIVE_ASSERT_VAR_IDX(pReNative, idxVar);
-    AssertStmt(   pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Invalid
-               || pReNative->Core.aVars[idxVar].enmKind == kIemNativeVarKind_Stack,
+    PIEMNATIVEVAR const pVar = &pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(idxVar)];
+    AssertStmt(   pVar->enmKind == kIemNativeVarKind_Invalid
+               || pVar->enmKind == kIemNativeVarKind_Stack,
                IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_VAR_UNEXPECTED_KIND));
 
@@ -6788 +6790 @@
     int32_t const offBpDisp      = iemNativeStackCalcBpDisp(idxStackSlot);
 
-    uint8_t const idxRegVar      = pReNative->Core.aVars[idxVar].idxReg;
+    uint8_t const idxRegVar      = pVar->idxReg;
     if (idxRegVar < RT_ELEMENTS(pReNative->Core.aHstRegs))
     {
         off = iemNativeEmitStoreGprByBp(pReNative, off, offBpDisp, idxRegVar);
         iemNativeRegFreeVar(pReNative, idxRegVar, fFlushShadows);
-        Assert(pReNative->Core.aVars[idxVar].idxReg == UINT8_MAX);
-    }
-    Assert(   pReNative->Core.aVars[idxVar].idxStackSlot != UINT8_MAX
-           && pReNative->Core.aVars[idxVar].idxReg       == UINT8_MAX);
+        Assert(pVar->idxReg == UINT8_MAX);
+    }
+    Assert(   pVar->idxStackSlot != UINT8_MAX
+           && pVar->idxReg       == UINT8_MAX);
 
     return iemNativeEmitLeaGprByBp(pReNative, off, idxRegArg, offBpDisp);

trunk/src/VBox/VMM/include/IEMN8veRecompilerTlbLookup.h

@@ -87 +87 @@
 #ifdef IEMNATIVE_WITH_TLB_LOOKUP
         /* 32-bit and 64-bit wraparound will require special handling, so skip these for absolute addresses. */
-        :           fSkip(   a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind == kIemNativeVarKind_Immediate
+        :           fSkip(      a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].enmKind
+                             == kIemNativeVarKind_Immediate
                           &&   (  (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT
                                 ? (uint64_t)(UINT32_MAX - a_cbMem - a_offDisp)
                                 : (uint64_t)(UINT64_MAX - a_cbMem - a_offDisp))
-                             < a_pReNative->Core.aVars[a_idxVarGCPtrMem].u.uValue)
+                             < a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].u.uValue)
 #else
         :           fSkip(true)
@@ -98 +99 @@
         ,    idxRegPtrHlp(UINT8_MAX)
 #else
-        ,    idxRegPtrHlp(   a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate
+        ,    idxRegPtrHlp(      a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].enmKind
+                             != kIemNativeVarKind_Immediate
                           || fSkip
                           ? UINT8_MAX
-                          : iemNativeRegAllocTmpImm(a_pReNative, a_poff, a_pReNative->Core.aVars[a_idxVarGCPtrMem].u.uValue) )
-#endif
-        ,       idxRegPtr(a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate && !fSkip
+                          : iemNativeRegAllocTmpImm(a_pReNative, a_poff,
+                                                    a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].u.uValue))
+#endif
+        ,       idxRegPtr(      a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].enmKind
+                             != kIemNativeVarKind_Immediate
+                          && !fSkip
                           ? iemNativeVarRegisterAcquire(a_pReNative, a_idxVarGCPtrMem, a_poff,
                                                         true /*fInitialized*/, IEMNATIVE_CALL_ARG2_GREG)
@@ -121 +126 @@
         ,         idxReg3(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
 #endif
-        ,         uAbsPtr(  a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate || fSkip
+        ,         uAbsPtr(      a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].enmKind
+                             != kIemNativeVarKind_Immediate
+                          || fSkip
                           ? UINT64_MAX
-                          : a_pReNative->Core.aVars[a_idxVarGCPtrMem].u.uValue)
+                          : a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].u.uValue)
 
     {