Changeset 102977 in vbox

Timestamp:

Jan 19, 2024 11:11:30 PM (12 months ago)

Author:

vboxsync

Message:

VMM/IEM: Implemented generic fallback for misaligned x86 locking that is not compatible with the host. Using the existing split-lock solution with VINF_EM_EMULATE_SPLIT_LOCK from bugref:10052. We keep ignoring the 'lock' prefix in the recompiler for single CPU VMs (now also on amd64 hosts). bugref:10547

Location:

trunk

Files:

• include/VBox/err.h (modified) (1 diff)
• src/VBox/VMM/VMMAll/IEMAll.cpp (modified) (7 diffs)
• src/VBox/VMM/VMMAll/IEMAllAImpl-arm64.S (modified) (1 diff)
• src/VBox/VMM/VMMAll/IEMAllInstOneByte.cpp.h (modified) (41 diffs)
• src/VBox/VMM/VMMAll/IEMAllInstPython.py (modified) (31 diffs)
• src/VBox/VMM/VMMAll/IEMAllInstTwoByte0f.cpp.h (modified) (49 diffs)
• src/VBox/VMM/VMMAll/IEMAllMemRWTmpl.cpp.h (modified) (1 diff)
• src/VBox/VMM/VMMAll/IEMAllMemRWTmplInline.cpp.h (modified) (3 diffs)
• src/VBox/VMM/VMMAll/IEMAllN8vePython.py (modified) (2 diffs)
• src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp (modified) (20 diffs)
• src/VBox/VMM/VMMAll/IEMAllThrdPython.py (modified) (1 diff)
• src/VBox/VMM/VMMAll/IEMAllThrdRecompiler.cpp (modified) (1 diff)
• src/VBox/VMM/VMMR3/EM.cpp (modified) (5 diffs)
• src/VBox/VMM/VMMR3/IEMR3.cpp (modified) (1 diff)
• src/VBox/VMM/include/IEMInline.h (modified) (8 diffs)
• src/VBox/VMM/include/IEMInternal.h (modified) (6 diffs)
• src/VBox/VMM/include/IEMMc.h (modified) (11 diffs)
• src/VBox/VMM/include/IEMN8veRecompiler.h (modified) (1 diff)
• src/VBox/VMM/testcase/tstIEMCheckMc.cpp (modified) (1 diff)

trunk/include/VBox/err.h

@@ -273 +273 @@
 /** Trick for resuming EMHistoryExec after a VMCPU_FF_IOM is handled. */
 #define VINF_EM_RESUME_R3_HISTORY_EXEC       1161
-/** Emulate split-lock access on SMP. */
+/** Emulate split-lock access on SMP.
+ * This is also used for dealing with locked alignment conflicts with the host
+ * in general from IEM. */
 #define VINF_EM_EMULATE_SPLIT_LOCK           1162
 /** @} */

trunk/src/VBox/VMM/VMMAll/IEMAll.cpp

@@ -6219 +6219 @@
  * @param   GCPtrMem    The address of the guest memory.
  * @param   fAccess     How the memory is being accessed.  The
- *                      IEM_ACCESS_TYPE_XXX bit is used to figure out how to map
- *                      the memory, while the IEM_ACCESS_WHAT_XXX bit is used
- *                      when raising exceptions.
+ *                      IEM_ACCESS_TYPE_XXX part is used to figure out how to
+ *                      map the memory, while the IEM_ACCESS_WHAT_XXX part is
+ *                      used when raising exceptions.  The IEM_ACCESS_ATOMIC and
+ *                      IEM_ACCESS_PARTIAL_WRITE bits are also allowed to be
+ *                      set.
  * @param   uAlignCtl   Alignment control:
  *                          - Bits 15:0 is the alignment mask.
@@ -6238 +6240 @@
     Assert(   cbMem <= 64 || cbMem == 512 || cbMem == 256 || cbMem == 108 || cbMem == 104 || cbMem == 102 || cbMem == 94
            || (iSegReg == UINT8_MAX && uAlignCtl == 0 && fAccess == IEM_ACCESS_DATA_R /* for the CPUID logging interface */) );
-    Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK | IEM_ACCESS_WHAT_MASK)));
+    Assert(!(fAccess & ~(IEM_ACCESS_TYPE_MASK | IEM_ACCESS_WHAT_MASK | IEM_ACCESS_ATOMIC | IEM_ACCESS_PARTIAL_WRITE)));
     Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
 
@@ -6297 +6299 @@
                 return iemRaiseGeneralProtectionFault0(pVCpu);
         }
+
+#if (defined(RT_ARCH_AMD64) && defined(RT_OS_LINUX)) || defined(RT_ARCH_ARM64)
+        /* If the access is atomic there are host platform alignmnet restrictions
+           we need to conform with. */
+        if (   !(fAccess & IEM_ACCESS_ATOMIC)
+# if defined(RT_ARCH_AMD64)
+            || (64U - (GCPtrMem & 63U) >= cbMem) /* split-lock detection. ASSUMES 64 byte cache line. */
+# elif defined(RT_ARCH_ARM64)
+            || (16U - (GCPtrMem & 15U) >= cbMem) /* LSE2 allows atomics anywhere within a 16 byte sized & aligned block. */
+# else
+#  error port me
+# endif
+           )
+        { /* okay */ }
+        else
+        {
+            LogEx(LOG_GROUP_IEM, ("iemMemMap: GCPtrMem=%RGv LB %u - misaligned atomic fallback.\n", GCPtrMem, cbMem));
+            pVCpu->iem.s.cMisalignedAtomics += 1;
+            return VINF_EM_EMULATE_SPLIT_LOCK;
+        }
+#endif
     }
 
@@ -6586 +6609 @@
  *                      is required (for IDT, GDT and LDT accesses).
  * @param   GCPtrMem    The address of the guest memory.
- * @param   fAccess     How the memory is being accessed.  The
- *                      IEM_ACCESS_TYPE_XXX bit is used to figure out
- *                      how to map the memory, while the
- *                      IEM_ACCESS_WHAT_XXX bit is used when raising
- *                      exceptions.
+ * @param   fAccess     How the memory is being accessed. The
+ *                      IEM_ACCESS_TYPE_XXX part is used to figure out how to
+ *                      map the memory, while the IEM_ACCESS_WHAT_XXX part is
+ *                      used when raising exceptions. The IEM_ACCESS_ATOMIC and
+ *                      IEM_ACCESS_PARTIAL_WRITE bits are also allowed to be
+ *                      set.
  * @param   uAlignCtl   Alignment control:
  *                          - Bits 15:0 is the alignment mask.
@@ -6606 +6630 @@
      */
     Assert(cbMem <= 64 || cbMem == 512 || cbMem == 108 || cbMem == 104 || cbMem == 94); /* 512 is the max! */
-    Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK | IEM_ACCESS_WHAT_MASK)));
+    Assert(!(fAccess & ~(IEM_ACCESS_TYPE_MASK | IEM_ACCESS_WHAT_MASK | IEM_ACCESS_ATOMIC | IEM_ACCESS_PARTIAL_WRITE)));
     Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
 
@@ -6642 +6666 @@
                 iemRaiseGeneralProtectionFault0Jmp(pVCpu);
         }
+
+#if (defined(RT_ARCH_AMD64) && defined(RT_OS_LINUX)) || defined(RT_ARCH_ARM64)
+        /* If the access is atomic there are host platform alignmnet restrictions
+           we need to conform with. */
+        if (   !(fAccess & IEM_ACCESS_ATOMIC)
+# if defined(RT_ARCH_AMD64)
+            || (64U - (GCPtrMem & 63U) >= cbMem) /* split-lock detection. ASSUMES 64 byte cache line. */
+# elif defined(RT_ARCH_ARM64)
+            || (16U - (GCPtrMem & 15U) >= cbMem) /* LSE2 allows atomics anywhere within a 16 byte sized & aligned block. */
+# else
+#  error port me
+# endif
+           )
+        { /* okay */ }
+        else
+        {
+            LogEx(LOG_GROUP_IEM, ("iemMemMap: GCPtrMem=%RGv LB %u - misaligned atomic fallback.\n", GCPtrMem, cbMem));
+            pVCpu->iem.s.cMisalignedAtomics += 1;
+            IEM_DO_LONGJMP(pVCpu, VINF_EM_EMULATE_SPLIT_LOCK);
+        }
+#endif
     }
 
@@ -6924 +6969 @@
 /** Fallback for iemMemCommitAndUnmapRwJmp.  */
 void iemMemCommitAndUnmapRwSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    Assert(((bUnmapInfo >> 4) & IEM_ACCESS_TYPE_MASK) == (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE));
+    iemMemCommitAndUnmapJmp(pVCpu, bUnmapInfo);
+}
+
+
+/** Fallback for iemMemCommitAndUnmapAtJmp.  */
+void iemMemCommitAndUnmapAtSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP
 {
     Assert(((bUnmapInfo >> 4) & IEM_ACCESS_TYPE_MASK) == (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE));

trunk/src/VBox/VMM/VMMAll/IEMAllAImpl-arm64.S

@@ -38 +38 @@
         ret
 
+/* Some sketches.
+
+// IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u8_locked, (uint8_t  *pu8Mem,  uint8_t  *pu8Reg));
+        .p2align        2
+        .private_extern NAME(iemAImpl_xchg_u8_locked)
+        .globl          NAME(iemAImpl_xchg_u8_locked)
+NAME(iemAImpl_xchg_u8_locked):
+        ldrb    w2, [x1]
+        swpalb  w2, w2, [x0]
+        strb    w2, [x1]
+        ret
+
+// IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u16_locked,(uint16_t *pu16Mem, uint16_t *pu16Reg));
+        .p2align        2
+        .private_extern NAME(iemAImpl_xchg_u16_locked)
+        .globl          NAME(iemAImpl_xchg_u16_locked)
+NAME(iemAImpl_xchg_u16_locked):
+        ldrh    w2, [x1]
+        swpalh  w2, w2, [x0]
+        strh    w2, [x1]
+        ret
+
+// IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u32_locked,(uint32_t *pu32Mem, uint32_t *pu32Reg));
+// IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u64_locked,(uint64_t *pu64Mem, uint64_t *pu64Reg));
+
+*/

trunk/src/VBox/VMM/VMMAll/IEMAllInstOneByte.cpp.h

@@ -60 +60 @@
  * Body for instructions like ADD, AND, OR, TEST, CMP, ++ with a byte
  * memory/register as the destination.
- *
- * Used with IEMOP_BODY_BINARY_rm_r8_NO_LOCK or IEMOP_BODY_BINARY_rm_r8_LOCKED.
- */
-#define IEMOP_BODY_BINARY_rm_r8_RW(a_fnNormalU8) \
+ */
+#define IEMOP_BODY_BINARY_rm_r8_RW(a_fnNormalU8, a_fnLockedU8) \
     uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); \
     \
@@ -115 +113 @@
         else \
         { \
-            (void)0
+            IEM_MC_BEGIN(3, 3, 0, 0); \
+            IEM_MC_ARG(uint8_t *,  pu8Dst,           0); \
+            IEM_MC_ARG(uint8_t,    u8Src,            1); \
+            IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); \
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); \
+            IEM_MC_LOCAL(uint8_t, bMapInfoDst); \
+            \
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); \
+            IEMOP_HLP_DONE_DECODING(); \
+            IEM_MC_MEM_MAP_U8_ATOMIC(pu8Dst, bMapInfoDst, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+            IEM_MC_FETCH_GREG_U8(u8Src, IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_FETCH_EFLAGS(EFlags); \
+            IEM_MC_CALL_VOID_AIMPL_3(a_fnLockedU8, pu8Dst, u8Src, pEFlags); \
+            \
+            IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bMapInfoDst); \
+            IEM_MC_COMMIT_EFLAGS(EFlags); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+    } \
+    (void)0
 
 /**
  * Body for instructions like TEST & CMP, ++ with a byte memory/registers as
  * operands.
- *
- * Used with IEMOP_BODY_BINARY_rm_r8_NO_LOCK or IEMOP_BODY_BINARY_rm_r8_LOCKED.
  */
 #define IEMOP_BODY_BINARY_rm_r8_RO(a_fnNormalU8) \
@@ -152 +168 @@
          *       after the memory. \
          */ \
-        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK) || (pVCpu->iem.s.fExec & IEM_F_X86_DISREGARD_LOCK)) \
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) \
         { \
             IEM_MC_BEGIN(3, 3, 0, 0); \
@@ -175 +191 @@
         else \
         { \
-            (void)0
-
-#define IEMOP_BODY_BINARY_rm_r8_NO_LOCK() \
             IEMOP_HLP_DONE_DECODING(); \
             IEMOP_RAISE_INVALID_LOCK_PREFIX_RET(); \
-        } \
-    } \
-    (void)0
-
-#define IEMOP_BODY_BINARY_rm_r8_LOCKED(a_fnLockedU8) \
-            IEM_MC_BEGIN(3, 3, 0, 0); \
-            IEM_MC_ARG(uint8_t *,  pu8Dst,           0); \
-            IEM_MC_ARG(uint8_t,    u8Src,            1); \
-            IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); \
-            IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); \
-            IEM_MC_LOCAL(uint8_t, bMapInfoDst); \
-            \
-            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); \
-            IEMOP_HLP_DONE_DECODING(); \
-            IEM_MC_MEM_MAP_U8_RW(pu8Dst, bMapInfoDst, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
-            IEM_MC_FETCH_GREG_U8(u8Src, IEM_GET_MODRM_REG(pVCpu, bRm)); \
-            IEM_MC_FETCH_EFLAGS(EFlags); \
-            IEM_MC_CALL_VOID_AIMPL_3(a_fnLockedU8, pu8Dst, u8Src, pEFlags); \
-            \
-            IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bMapInfoDst); \
-            IEM_MC_COMMIT_EFLAGS(EFlags); \
-            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
-            IEM_MC_END(); \
         } \
     } \
@@ -417 +407 @@
                     IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); \
                     IEMOP_HLP_DONE_DECODING(); \
-                    IEM_MC_MEM_MAP_U16_RW(pu16Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U16_ATOMIC(pu16Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm)); \
                     IEM_MC_FETCH_EFLAGS(EFlags); \
                     IEM_MC_CALL_VOID_AIMPL_3(a_fnLockedU16, pu16Dst, u16Src, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_ADVANCE_RIP_AND_FINISH(); \
@@ -438 +428 @@
                     IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); \
                     IEMOP_HLP_DONE_DECODING(); \
-                    IEM_MC_MEM_MAP_U32_RW(pu32Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U32_ATOMIC(pu32Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm)); \
                     IEM_MC_FETCH_EFLAGS(EFlags); \
                     IEM_MC_CALL_VOID_AIMPL_3(a_fnLockedU32, pu32Dst, u32Src, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo /* CMP,TEST */); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo /* CMP,TEST */); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_ADVANCE_RIP_AND_FINISH(); \
@@ -459 +449 @@
                     IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); \
                     IEMOP_HLP_DONE_DECODING(); \
-                    IEM_MC_MEM_MAP_U64_RW(pu64Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U64_ATOMIC(pu64Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm)); \
                     IEM_MC_FETCH_EFLAGS(EFlags); \
                     IEM_MC_CALL_VOID_AIMPL_3(a_fnLockedU64, pu64Dst, u64Src, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_ADVANCE_RIP_AND_FINISH(); \
@@ -737 +727 @@
 {
     IEMOP_MNEMONIC2(MR, ADD, add, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
-    IEMOP_BODY_BINARY_rm_r8_RW(    iemAImpl_add_u8);
-    IEMOP_BODY_BINARY_rm_r8_LOCKED(iemAImpl_add_u8_locked);
+    IEMOP_BODY_BINARY_rm_r8_RW(iemAImpl_add_u8, iemAImpl_add_u8_locked);
 }
 
@@ -860 +849 @@
     IEMOP_MNEMONIC2(MR, OR, or, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
     IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
-    IEMOP_BODY_BINARY_rm_r8_RW(    iemAImpl_or_u8);
-    IEMOP_BODY_BINARY_rm_r8_LOCKED(iemAImpl_or_u8_locked);
+    IEMOP_BODY_BINARY_rm_r8_RW(iemAImpl_or_u8, iemAImpl_or_u8_locked);
 }
 
@@ -1029 +1017 @@
 {
     IEMOP_MNEMONIC2(MR, ADC, adc, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
-    IEMOP_BODY_BINARY_rm_r8_RW(    iemAImpl_adc_u8);
-    IEMOP_BODY_BINARY_rm_r8_LOCKED(iemAImpl_adc_u8_locked);
+    IEMOP_BODY_BINARY_rm_r8_RW(iemAImpl_adc_u8, iemAImpl_adc_u8_locked);
 }
 
@@ -1150 +1137 @@
 {
     IEMOP_MNEMONIC2(MR, SBB, sbb, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
-    IEMOP_BODY_BINARY_rm_r8_RW(    iemAImpl_sbb_u8);
-    IEMOP_BODY_BINARY_rm_r8_LOCKED(iemAImpl_sbb_u8_locked);
+    IEMOP_BODY_BINARY_rm_r8_RW(iemAImpl_sbb_u8, iemAImpl_sbb_u8_locked);
 }
 
@@ -1263 +1249 @@
     IEMOP_MNEMONIC2(MR, AND, and, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
     IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
-    IEMOP_BODY_BINARY_rm_r8_RW(    iemAImpl_and_u8);
-    IEMOP_BODY_BINARY_rm_r8_LOCKED(iemAImpl_and_u8_locked);
+    IEMOP_BODY_BINARY_rm_r8_RW(iemAImpl_and_u8, iemAImpl_and_u8_locked);
 }
 
@@ -1388 +1373 @@
 {
     IEMOP_MNEMONIC2(MR, SUB, sub, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
-    IEMOP_BODY_BINARY_rm_r8_RW(    iemAImpl_sub_u8);
-    IEMOP_BODY_BINARY_rm_r8_LOCKED(iemAImpl_sub_u8_locked);
+    IEMOP_BODY_BINARY_rm_r8_RW(iemAImpl_sub_u8, iemAImpl_sub_u8_locked);
 }
 
@@ -1501 +1485 @@
     IEMOP_MNEMONIC2(MR, XOR, xor, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
     IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
-    IEMOP_BODY_BINARY_rm_r8_RW(    iemAImpl_xor_u8);
-    IEMOP_BODY_BINARY_rm_r8_LOCKED(iemAImpl_xor_u8_locked);
+    IEMOP_BODY_BINARY_rm_r8_RW(iemAImpl_xor_u8, iemAImpl_xor_u8_locked);
 }
 
@@ -1658 +1641 @@
     IEMOP_MNEMONIC(cmp_Eb_Gb, "cmp Eb,Gb");
     IEMOP_BODY_BINARY_rm_r8_RO(iemAImpl_cmp_u8);
-    IEMOP_BODY_BINARY_rm_r8_NO_LOCK();
 }
 
@@ -3957 +3939 @@
             IEMOP_HLP_DONE_DECODING(); \
             \
-            IEM_MC_MEM_MAP_U8_RW(pu8Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+            IEM_MC_MEM_MAP_U8_ATOMIC(pu8Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
             IEM_MC_FETCH_EFLAGS(EFlags); \
             IEM_MC_CALL_VOID_AIMPL_3(a_fnLockedU8, pu8Dst, u8Src, pEFlags); \
             \
-            IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+            IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo); \
             IEM_MC_COMMIT_EFLAGS(EFlags); \
             IEM_MC_ADVANCE_RIP_AND_FINISH(); \
@@ -4313 +4295 @@
                     IEM_MC_LOCAL(uint8_t,       bUnmapInfo); \
                     IEM_MC_ARG(uint16_t *,      pu16Dst,                0); \
-                    IEM_MC_MEM_MAP_U16_RW(pu16Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U16_ATOMIC(pu16Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     \
                     IEM_MC_ARG_CONST(uint16_t,  u16Src,  u16Imm,        1); \
@@ -4320 +4302 @@
                     IEM_MC_CALL_VOID_AIMPL_3(a_fnLockedU16, pu16Dst, u16Src, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_ADVANCE_RIP_AND_FINISH(); \
@@ -4338 +4320 @@
                     IEM_MC_LOCAL(uint8_t,       bUnmapInfo); \
                     IEM_MC_ARG(uint32_t *,      pu32Dst,                0); \
-                    IEM_MC_MEM_MAP_U32_RW(pu32Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U32_ATOMIC(pu32Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     \
                     IEM_MC_ARG_CONST(uint32_t,  u32Src,  u32Imm,        1); \
@@ -4345 +4327 @@
                     IEM_MC_CALL_VOID_AIMPL_3(a_fnLockedU32, pu32Dst, u32Src, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_ADVANCE_RIP_AND_FINISH(); \
@@ -4363 +4345 @@
                     IEM_MC_LOCAL(uint8_t,       bUnmapInfo); \
                     IEM_MC_ARG(uint64_t *,      pu64Dst,                0); \
-                    IEM_MC_MEM_MAP_U64_RW(pu64Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U64_ATOMIC(pu64Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     \
                     IEM_MC_ARG_CONST(uint64_t,  u64Src,  u64Imm,        1); \
@@ -4370 +4352 @@
                     IEM_MC_CALL_VOID_AIMPL_3(a_fnLockedU64, pu64Dst, u64Src, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_ADVANCE_RIP_AND_FINISH(); \
@@ -4829 +4811 @@
                     IEM_MC_LOCAL(uint8_t,       bUnmapInfo); \
                     IEM_MC_ARG(uint16_t *,      pu16Dst,                            0); \
-                    IEM_MC_MEM_MAP_U16_RW(pu16Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U16_ATOMIC(pu16Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     \
                     IEM_MC_ARG_CONST(uint16_t,  u16Src,  (int16_t)(int8_t)u8Imm,    1); \
@@ -4836 +4818 @@
                     IEM_MC_CALL_VOID_AIMPL_3(a_fnLockedU16, pu16Dst, u16Src, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_ADVANCE_RIP_AND_FINISH(); \
@@ -4852 +4834 @@
                     IEM_MC_LOCAL(uint8_t,       bUnmapInfo); \
                     IEM_MC_ARG(uint32_t *,      pu32Dst,                    0); \
-                    IEM_MC_MEM_MAP_U32_RW(pu32Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U32_ATOMIC(pu32Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     \
                     IEM_MC_ARG_CONST(uint32_t,  u32Src,  (int32_t)(int8_t)u8Imm,    1); \
@@ -4859 +4841 @@
                     IEM_MC_CALL_VOID_AIMPL_3(a_fnLockedU32, pu32Dst, u32Src, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_ADVANCE_RIP_AND_FINISH(); \
@@ -4875 +4857 @@
                     IEM_MC_LOCAL(uint8_t,       bUnmapInfo); \
                     IEM_MC_ARG(uint64_t *,      pu64Dst,                            0); \
-                    IEM_MC_MEM_MAP_U64_RW(pu64Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U64_ATOMIC(pu64Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     \
                     IEM_MC_ARG_CONST(uint64_t,  u64Src, (int64_t)(int8_t)u8Imm,     1); \
@@ -4882 +4864 @@
                     IEM_MC_CALL_VOID_AIMPL_3(a_fnLockedU64, pu64Dst, u64Src, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_ADVANCE_RIP_AND_FINISH(); \
@@ -5168 +5150 @@
     IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
     IEMOP_BODY_BINARY_rm_r8_RO(iemAImpl_test_u8);
-    IEMOP_BODY_BINARY_rm_r8_NO_LOCK();
 }
 
@@ -5214 +5195 @@
          * We're accessing memory.
          */
-#define IEMOP_XCHG_BYTE(a_fnWorker) \
+#define IEMOP_XCHG_BYTE(a_fnWorker, a_Style) \
             IEM_MC_BEGIN(2, 4, 0, 0); \
             IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); \
@@ -5224 +5205 @@
             IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); \
             IEMOP_HLP_DONE_DECODING(); /** @todo testcase: lock xchg */ \
-            IEM_MC_MEM_MAP_U8_RW(pu8Mem, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+            IEM_MC_MEM_MAP_U8_##a_Style(pu8Mem, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
             IEM_MC_FETCH_GREG_U8(uTmpReg, IEM_GET_MODRM_REG(pVCpu, bRm)); \
             IEM_MC_CALL_VOID_AIMPL_2(a_fnWorker, pu8Mem, pu8Reg); \
-            IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+            IEM_MC_MEM_COMMIT_AND_UNMAP_##a_Style(bUnmapInfo); \
             IEM_MC_STORE_GREG_U8(IEM_GET_MODRM_REG(pVCpu, bRm), uTmpReg); \
             \
@@ -5235 +5216 @@
         if (!(pVCpu->iem.s.fExec & IEM_F_X86_DISREGARD_LOCK))
         {
-            IEMOP_XCHG_BYTE(iemAImpl_xchg_u8_locked);
+            IEMOP_XCHG_BYTE(iemAImpl_xchg_u8_locked,ATOMIC);
         }
         else
         {
-            IEMOP_XCHG_BYTE(iemAImpl_xchg_u8_unlocked);
+            IEMOP_XCHG_BYTE(iemAImpl_xchg_u8_unlocked,RW);
         }
     }
@@ -5313 +5294 @@
          * We're accessing memory.
          */
-#define IEMOP_XCHG_EV_GV(a_fnWorker16, a_fnWorker32, a_fnWorker64) \
+#define IEMOP_XCHG_EV_GV(a_fnWorker16, a_fnWorker32, a_fnWorker64, a_Type) \
             do { \
                 switch (pVCpu->iem.s.enmEffOpSize) \
@@ -5327 +5308 @@
                         IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); \
                         IEMOP_HLP_DONE_DECODING(); /** @todo testcase: lock xchg */ \
-                        IEM_MC_MEM_MAP_U16_RW(pu16Mem, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                        IEM_MC_MEM_MAP_U16_##a_Type(pu16Mem, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                         IEM_MC_FETCH_GREG_U16(uTmpReg, IEM_GET_MODRM_REG(pVCpu, bRm)); \
                         IEM_MC_CALL_VOID_AIMPL_2(a_fnWorker16, pu16Mem, pu16Reg); \
-                        IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                        IEM_MC_MEM_COMMIT_AND_UNMAP_##a_Type(bUnmapInfo); \
                         IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), uTmpReg); \
                         \
@@ -5347 +5328 @@
                         IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); \
                         IEMOP_HLP_DONE_DECODING(); \
-                        IEM_MC_MEM_MAP_U32_RW(pu32Mem, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                        IEM_MC_MEM_MAP_U32_##a_Type(pu32Mem, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                         IEM_MC_FETCH_GREG_U32(uTmpReg, IEM_GET_MODRM_REG(pVCpu, bRm)); \
                         IEM_MC_CALL_VOID_AIMPL_2(a_fnWorker32, pu32Mem, pu32Reg); \
-                        IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                        IEM_MC_MEM_COMMIT_AND_UNMAP_##a_Type(bUnmapInfo); \
                         IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), uTmpReg); \
                         \
@@ -5367 +5348 @@
                         IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); \
                         IEMOP_HLP_DONE_DECODING(); \
-                        IEM_MC_MEM_MAP_U64_RW(pu64Mem, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                        IEM_MC_MEM_MAP_U64_##a_Type(pu64Mem, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                         IEM_MC_FETCH_GREG_U64(uTmpReg, IEM_GET_MODRM_REG(pVCpu, bRm)); \
                         IEM_MC_CALL_VOID_AIMPL_2(a_fnWorker64, pu64Mem, pu64Reg); \
-                        IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                        IEM_MC_MEM_COMMIT_AND_UNMAP_##a_Type(bUnmapInfo); \
                         IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), uTmpReg); \
                         \
@@ -5382 +5363 @@
         if (!(pVCpu->iem.s.fExec & IEM_F_X86_DISREGARD_LOCK))
         {
-            IEMOP_XCHG_EV_GV(iemAImpl_xchg_u16_locked, iemAImpl_xchg_u32_locked, iemAImpl_xchg_u64_locked);
+            IEMOP_XCHG_EV_GV(iemAImpl_xchg_u16_locked, iemAImpl_xchg_u32_locked, iemAImpl_xchg_u64_locked,ATOMIC);
         }
         else
         {
-            IEMOP_XCHG_EV_GV(iemAImpl_xchg_u16_unlocked, iemAImpl_xchg_u32_unlocked, iemAImpl_xchg_u64_unlocked);
+            IEMOP_XCHG_EV_GV(iemAImpl_xchg_u16_unlocked, iemAImpl_xchg_u32_unlocked, iemAImpl_xchg_u64_unlocked,RW);
         }
     }
@@ -13021 +13002 @@
             IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, a_bRm, 0); \
             IEMOP_HLP_DONE_DECODING(); \
-            IEM_MC_MEM_MAP_U8_RW(pu8Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+            IEM_MC_MEM_MAP_U8_ATOMIC(pu8Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
             IEM_MC_FETCH_EFLAGS(EFlags); \
             IEM_MC_CALL_VOID_AIMPL_2(a_fnLockedU8, pu8Dst, pEFlags); \
             \
-            IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+            IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo); \
             IEM_MC_COMMIT_EFLAGS(EFlags); \
             IEM_MC_ADVANCE_RIP_AND_FINISH(); \
@@ -13170 +13151 @@
                     IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); \
                     IEMOP_HLP_DONE_DECODING(); \
-                    IEM_MC_MEM_MAP_U16_RW(pu16Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U16_ATOMIC(pu16Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     IEM_MC_FETCH_EFLAGS(EFlags); \
                     IEM_MC_CALL_VOID_AIMPL_2(a_fnLockedU16, pu16Dst, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_ADVANCE_RIP_AND_FINISH(); \
@@ -13189 +13170 @@
                     IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); \
                     IEMOP_HLP_DONE_DECODING(); \
-                    IEM_MC_MEM_MAP_U32_RW(pu32Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U32_ATOMIC(pu32Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     IEM_MC_FETCH_EFLAGS(EFlags); \
                     IEM_MC_CALL_VOID_AIMPL_2(a_fnLockedU32, pu32Dst, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_ADVANCE_RIP_AND_FINISH(); \
@@ -13208 +13189 @@
                     IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); \
                     IEMOP_HLP_DONE_DECODING(); \
-                    IEM_MC_MEM_MAP_U64_RW(pu64Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U64_ATOMIC(pu64Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     IEM_MC_FETCH_EFLAGS(EFlags); \
                     IEM_MC_CALL_VOID_AIMPL_2(a_fnLockedU64, pu64Dst, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_ADVANCE_RIP_AND_FINISH(); \

trunk/src/VBox/VMM/VMMAll/IEMAllInstPython.py

@@ -3026 +3026 @@
     'IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT':                       (McBlock.parseMcGeneric,           True,  True,  False, ),
     'IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE':              (McBlock.parseMcGeneric,           True,  True,  False, ),
+    'IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC':                        (McBlock.parseMcGeneric,           True,  True,  True,  ),
     'IEM_MC_MEM_COMMIT_AND_UNMAP_RW':                            (McBlock.parseMcGeneric,           True,  True,  True,  ),
     'IEM_MC_MEM_COMMIT_AND_UNMAP_RO':                            (McBlock.parseMcGeneric,           True,  True,  True,  ),
@@ -3037 +3038 @@
     'IEM_MC_MEM_MAP_R64_WO':                                     (McBlock.parseMcGeneric,           True,  True,  True,  ),
     'IEM_MC_MEM_MAP_R80_WO':                                     (McBlock.parseMcGeneric,           True,  True,  True,  ),
+    'IEM_MC_MEM_MAP_U8_ATOMIC':                                  (McBlock.parseMcGeneric,           True,  True,  True,  ),
     'IEM_MC_MEM_MAP_U8_RW':                                      (McBlock.parseMcGeneric,           True,  True,  True,  ),
     'IEM_MC_MEM_MAP_U8_RO':                                      (McBlock.parseMcGeneric,           True,  True,  True,  ),
     'IEM_MC_MEM_MAP_U8_WO':                                      (McBlock.parseMcGeneric,           True,  True,  True,  ),
+    'IEM_MC_MEM_MAP_U16_ATOMIC':                                 (McBlock.parseMcGeneric,           True,  True,  True,  ),
     'IEM_MC_MEM_MAP_U16_RW':                                     (McBlock.parseMcGeneric,           True,  True,  True,  ),
     'IEM_MC_MEM_MAP_U16_RO':                                     (McBlock.parseMcGeneric,           True,  True,  True,  ),
     'IEM_MC_MEM_MAP_U16_WO':                                     (McBlock.parseMcGeneric,           True,  True,  True,  ),
+    'IEM_MC_MEM_MAP_U32_ATOMIC':                                 (McBlock.parseMcGeneric,           True,  True,  True,  ),
     'IEM_MC_MEM_MAP_U32_RW':                                     (McBlock.parseMcGeneric,           True,  True,  True,  ),
     'IEM_MC_MEM_MAP_U32_RO':                                     (McBlock.parseMcGeneric,           True,  True,  True,  ),
     'IEM_MC_MEM_MAP_U32_WO':                                     (McBlock.parseMcGeneric,           True,  True,  True,  ),
+    'IEM_MC_MEM_MAP_U64_ATOMIC':                                 (McBlock.parseMcGeneric,           True,  True,  True,  ),
     'IEM_MC_MEM_MAP_U64_RW':                                     (McBlock.parseMcGeneric,           True,  True,  True,  ),
     'IEM_MC_MEM_MAP_U64_RO':                                     (McBlock.parseMcGeneric,           True,  True,  True,  ),
     'IEM_MC_MEM_MAP_U64_WO':                                     (McBlock.parseMcGeneric,           True,  True,  True,  ),
+    'IEM_MC_MEM_MAP_U128_ATOMIC':                                (McBlock.parseMcGeneric,           True,  True,  True,  ),
     'IEM_MC_MEM_MAP_U128_RW':                                    (McBlock.parseMcGeneric,           True,  True,  True,  ),
     'IEM_MC_MEM_MAP_U128_RO':                                    (McBlock.parseMcGeneric,           True,  True,  True,  ),
@@ -3488 +3494 @@
         self.cTotalMcBlocks = 0;
 
-        self.oReMacroName   = re.compile('^[A-Za-z_][A-Za-z0-9_]*$');
-        self.oReMnemonic    = re.compile('^[A-Za-z_][A-Za-z0-9_]*$');
-        self.oReStatsName   = re.compile('^[A-Za-z_][A-Za-z0-9_]*$');
-        self.oReFunctionName= re.compile('^iemOp_[A-Za-z_][A-Za-z0-9_]*$');
-        self.oReGroupName   = re.compile('^og_[a-z0-9]+(|_[a-z0-9]+|_[a-z0-9]+_[a-z0-9]+)$');
-        self.oReDisEnum     = re.compile('^OP_[A-Z0-9_]+$');
-        self.oReFunTable    = re.compile('^(IEM_STATIC|static) +const +PFNIEMOP +g_apfn[A-Za-z0-9_]+ *\[ *\d* *\] *= *$');
-        self.oReComment     = re.compile('//.*?$|/\*.*?\*/'); ## Full comments.
-        self.oReHashDefine2 = re.compile('(?s)\A\s*([A-Za-z_][A-Za-z0-9_]*)\(([^)]*)\)\s*(.*)\Z'); ##< With arguments.
-        self.oReHashDefine3 = re.compile('(?s)\A\s*([A-Za-z_][A-Za-z0-9_]*)[^(]\s*(.*)\Z');        ##< Simple, no arguments.
+        self.oReMacroName   = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$');
+        self.oReMnemonic    = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$');
+        self.oReStatsName   = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$');
+        self.oReFunctionName= re.compile(r'^iemOp_[A-Za-z_][A-Za-z0-9_]*$');
+        self.oReGroupName   = re.compile(r'^og_[a-z0-9]+(|_[a-z0-9]+|_[a-z0-9]+_[a-z0-9]+)$');
+        self.oReDisEnum     = re.compile(r'^OP_[A-Z0-9_]+$');
+        self.oReFunTable    = re.compile(r'^(IEM_STATIC|static) +const +PFNIEMOP +g_apfn[A-Za-z0-9_]+ *\[ *\d* *\] *= *$');
+        self.oReComment     = re.compile(r'//.*?$|/\*.*?\*/'); ## Full comments.
+        self.oReHashDefine2 = re.compile(r'(?s)\A\s*([A-Za-z_][A-Za-z0-9_]*)\(([^)]*)\)\s*(.*)\Z'); ##< With arguments.
+        self.oReHashDefine3 = re.compile(r'(?s)\A\s*([A-Za-z_][A-Za-z0-9_]*)[^(]\s*(.*)\Z');        ##< Simple, no arguments.
         self.oReMcBeginEnd  = re.compile(r'\bIEM_MC_(BEGIN|END|DEFER_TO_CIMPL_[1-5]_RET)\s*\('); ##> Not DEFER_TO_CIMPL_0_RET!
         self.fDebug         = True;
@@ -3619 +3625 @@
         # Extract the table name.
         #
-        sName = re.search(' *([a-zA-Z_0-9]+) *\[', sLine).group(1);
+        sName = re.search(r' *([a-zA-Z_0-9]+) *\[', sLine).group(1);
         oMap  = g_dInstructionMapsByIemName.get(sName);
         if not oMap:
@@ -3635 +3641 @@
         asPrefixes        = ('none', '0x66', '0xf3', '0xf2');
 
-        oEntriesMatch = re.search('\[ *(256|32) *\]', sLine);
+        oEntriesMatch = re.search(r'\[ *(256|32) *\]', sLine);
         if oEntriesMatch:
             cEntriesPerByte   = 1;
@@ -3954 +3960 @@
     def parseTagOpBrief(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:    \@opbrief
+        Tag:    @opbrief
         Value:  Text description, multiple sections, appended.
 
@@ -3984 +3990 @@
     def parseTagOpDesc(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:    \@opdesc
+        Tag:    @opdesc
         Value:  Text description, multiple sections, appended.
 
@@ -4022 +4028 @@
     def parseTagOpOperandN(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tags:  \@op1, \@op2, \@op3, \@op4
+        Tags:  @op1, @op2, @op3, @op4
         Value: [where:]type
 
@@ -4072 +4078 @@
     def parseTagOpMaps(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:    \@opmaps
+        Tag:    @opmaps
         Value:  map[,map2]
 
@@ -4107 +4113 @@
     def parseTagOpPfx(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:        \@oppfx
+        Tag:        @oppfx
         Value:      n/a|none|0x66|0xf3|0xf2
 
@@ -4145 +4151 @@
     def parseTagOpcode(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:        \@opcode
+        Tag:        @opcode
         Value:      0x?? | /reg (TODO: | mr/reg | 11 /reg | !11 /reg | 11 mr/reg | !11 mr/reg)
 
@@ -4175 +4181 @@
     def parseTagOpcodeSub(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:        \@opcodesub
+        Tag:        @opcodesub
         Value:      none | 11 mr/reg | !11 mr/reg | rex.w=0 | rex.w=1 | vex.l=0 | vex.l=1
                     | 11 mr/reg vex.l=0 | 11 mr/reg vex.l=1 | !11 mr/reg vex.l=0 | !11 mr/reg vex.l=1
@@ -4202 +4208 @@
     def parseTagOpEnc(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:        \@openc
+        Tag:        @openc
         Value:      ModR/M|fixed|prefix|<map name>
 
@@ -4238 +4244 @@
     def parseTagOpEFlags(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tags:   \@opfltest, \@opflmodify, \@opflundef, \@opflset, \@opflclear
+        Tags:   @opfltest, @opflmodify, @opflundef, @opflset, @opflclear
         Value:  <eflags specifier>
 
@@ -4270 +4276 @@
     def parseTagOpHints(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:        \@ophints
+        Tag:        @ophints
         Value:      Comma or space separated list of flags and hints.
 
@@ -4304 +4310 @@
     def parseTagOpDisEnum(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:        \@opdisenum
+        Tag:        @opdisenum
         Value:      OP_XXXX
 
@@ -4333 +4339 @@
     def parseTagOpMinCpu(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:        \@opmincpu
+        Tag:        @opmincpu
         Value:      <simple CPU name>
 
@@ -4363 +4369 @@
     def parseTagOpCpuId(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:        \@opcpuid
+        Tag:        @opcpuid
         Value:      none | <CPUID flag specifier>
 
@@ -4397 +4403 @@
     def parseTagOpGroup(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:        \@opgroup
+        Tag:        @opgroup
         Value:      op_grp1[_subgrp2[_subsubgrp3]]
 
@@ -4423 +4429 @@
     def parseTagOpUnusedInvalid(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:    \@opunused, \@opinvalid, \@opinvlstyle
+        Tag:    @opunused, @opinvalid, @opinvlstyle
         Value:  <invalid opcode behaviour style>
 
-        The \@opunused indicates the specification is for a currently unused
+        The @opunused indicates the specification is for a currently unused
         instruction encoding.
 
-        The \@opinvalid indicates the specification is for an invalid currently
+        The @opinvalid indicates the specification is for an invalid currently
         instruction encoding (like UD2).
 
-        The \@opinvlstyle just indicates how CPUs decode the instruction when
-        not supported (\@opcpuid, \@opmincpu) or disabled.
+        The @opinvlstyle just indicates how CPUs decode the instruction when
+        not supported (@opcpuid, @opmincpu) or disabled.
         """
         oInstr = self.ensureInstructionForOpTag(iTagLine);
@@ -4461 +4467 @@
     def parseTagOpTest(self, sTag, aasSections, iTagLine, iEndLine): # pylint: disable=too-many-locals
         """
-        Tag:        \@optest
+        Tag:        @optest
         Value:      [<selectors>[ ]?] <inputs> -> <outputs>
         Example:    mode==64bit / in1=0xfffffffe:dw in2=1:dw -> out1=0xffffffff:dw outfl=a?,p?
@@ -4606 +4612 @@
     def parseTagOpTestNum(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Numbered \@optest tag.  Either \@optest42 or \@optest[42].
+        Numbered @optest tag.  Either @optest42 or @optest[42].
         """
         oInstr = self.ensureInstructionForOpTag(iTagLine);
@@ -4622 +4628 @@
     def parseTagOpTestIgnore(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:        \@optestign | \@optestignore
+        Tag:        @optestign | @optestignore
         Value:      <value is ignored>
 
         This is a simple trick to ignore a test while debugging another.
 
-        See also \@oponlytest.
+        See also @oponlytest.
         """
         _ = sTag; _ = aasSections; _ = iTagLine; _ = iEndLine;
@@ -4634 +4640 @@
     def parseTagOpCopyTests(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:        \@opcopytests
+        Tag:        @opcopytests
         Value:      <opstat | function> [..]
-        Example:    \@opcopytests add_Eb_Gb
+        Example:    @opcopytests add_Eb_Gb
 
         Trick to avoid duplicating tests for different encodings of the same
@@ -4663 +4669 @@
     def parseTagOpOnlyTest(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:        \@oponlytest | \@oponly
+        Tag:        @oponlytest | @oponly
         Value:      none
 
@@ -4669 +4675 @@
         for singling out one or two new instructions or tests.
 
-        See also \@optestignore.
+        See also @optestignore.
         """
         oInstr = self.ensureInstructionForOpTag(iTagLine);
@@ -4686 +4692 @@
     def parseTagOpXcptType(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:        \@opxcpttype
+        Tag:        @opxcpttype
         Value:      [none|1|2|3|4|4UA|5|6|7|8|11|12|E1|E1NF|E2|E3|E3NF|E4|E4NF|E5|E5NF|E6|E6NF|E7NF|E9|E9NF|E10|E11|E12|E12NF]
 
@@ -4713 +4719 @@
     def parseTagOpFunction(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:        \@opfunction
+        Tag:        @opfunction
         Value:      <VMM function name>
 
@@ -4741 +4747 @@
     def parseTagOpStats(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:        \@opstats
+        Tag:        @opstats
         Value:      <VMM statistics base name>
 
@@ -4769 +4775 @@
     def parseTagOpDone(self, sTag, aasSections, iTagLine, iEndLine):
         """
-        Tag:    \@opdone
+        Tag:    @opdone
         Value:  none
 
@@ -5487 +5493 @@
             for sName, oMacro in self.dMacros.items():
                 if sRegex:
-                    sRegex += '|' + sName;
+                    sRegex += r'|' + sName;
                 else:
-                    sRegex  = '\\b(' + sName;
+                    sRegex  = r'\b(' + sName;
                 if oMacro.asArgs is not None:
-                    sRegex += '\s*\(';
+                    sRegex += r'\s*\(';
                 else:
-                    sRegex += '\\b';
+                    sRegex += r'\b';
             sRegex += ')';
             self.oReMacros = re.compile(sRegex);

trunk/src/VBox/VMM/VMMAll/IEMAllInstTwoByte0f.cpp.h

@@ -9157 +9157 @@
                     IEM_MC_LOCAL(uint8_t,               bUnmapInfo); \
                     IEM_MC_ARG(uint16_t *,              pu16Dst,                0); \
-                    IEM_MC_MEM_MAP_U16_RW(pu16Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U16_ATOMIC(pu16Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     \
                     IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        2); \
@@ -9163 +9163 @@
                     IEM_MC_CALL_VOID_AIMPL_3(a_fnLockedU16, pu16Dst, u16Src, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_ADVANCE_RIP_AND_FINISH(); \
@@ -9185 +9185 @@
                     IEM_MC_LOCAL(uint8_t,               bUnmapInfo); \
                     IEM_MC_ARG(uint32_t *,              pu32Dst,                0); \
-                    IEM_MC_MEM_MAP_U32_RW(pu32Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U32_ATOMIC(pu32Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     \
                     IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        2); \
@@ -9191 +9191 @@
                     IEM_MC_CALL_VOID_AIMPL_3(a_fnLockedU32, pu32Dst, u32Src, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_ADVANCE_RIP_AND_FINISH(); \
@@ -9213 +9213 @@
                     IEM_MC_LOCAL(uint8_t,               bUnmapInfo); \
                     IEM_MC_ARG(uint64_t *,              pu64Dst,                0); \
-                    IEM_MC_MEM_MAP_U64_RW(pu64Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U64_ATOMIC(pu64Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     \
                     IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        2); \
@@ -9219 +9219 @@
                     IEM_MC_CALL_VOID_AIMPL_3(a_fnLockedU64, pu64Dst, u64Src, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_ADVANCE_RIP_AND_FINISH(); \
@@ -9299 +9299 @@
         /* memory destination. */ \
         /** @todo test negative bit offsets! */ \
-        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK) || (pVCpu->iem.s.fExec & IEM_F_X86_DISREGARD_LOCK)) \
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) \
         { \
             switch (pVCpu->iem.s.enmEffOpSize) \
@@ -10274 +10274 @@
     else
     {
-#define IEMOP_BODY_CMPXCHG_BYTE(a_fnWorker) \
+#define IEMOP_BODY_CMPXCHG_BYTE(a_fnWorker, a_Type) \
             IEM_MC_BEGIN(4, 4, IEM_MC_F_MIN_486, 0); \
             IEM_MC_LOCAL(RTGCPTR,       GCPtrEffDst); \
@@ -10282 +10282 @@
             IEM_MC_LOCAL(uint8_t,           bUnmapInfo); \
             IEM_MC_ARG(uint8_t *,           pu8Dst,                 0); \
-            IEM_MC_MEM_MAP_U8_RW(pu8Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+            IEM_MC_MEM_MAP_U8_##a_Type(pu8Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
             \
             IEM_MC_ARG(uint8_t,             u8Src,                  2); \
@@ -10295 +10295 @@
             IEM_MC_CALL_VOID_AIMPL_4(a_fnWorker, pu8Dst, pu8Al, u8Src, pEFlags); \
             \
-            IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+            IEM_MC_MEM_COMMIT_AND_UNMAP_##a_Type(bUnmapInfo); \
             IEM_MC_COMMIT_EFLAGS(EFlags); \
             IEM_MC_STORE_GREG_U8(X86_GREG_xAX, u8Al); \
@@ -10303 +10303 @@
         if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK) || (pVCpu->iem.s.fExec & IEM_F_X86_DISREGARD_LOCK))
         {
-            IEMOP_BODY_CMPXCHG_BYTE(iemAImpl_cmpxchg_u8);
+            IEMOP_BODY_CMPXCHG_BYTE(iemAImpl_cmpxchg_u8,RW);
         }
         else
         {
-            IEMOP_BODY_CMPXCHG_BYTE(iemAImpl_cmpxchg_u8_locked);
+            IEMOP_BODY_CMPXCHG_BYTE(iemAImpl_cmpxchg_u8_locked,ATOMIC);
         }
     }
@@ -10388 +10388 @@
     else
     {
-#define IEMOP_BODY_CMPXCHG_EV_GV(a_fnWorker16, a_fnWorker32, a_fnWorker64) \
+#define IEMOP_BODY_CMPXCHG_EV_GV(a_fnWorker16, a_fnWorker32, a_fnWorker64,a_Type) \
         do { \
             switch (pVCpu->iem.s.enmEffOpSize) \
@@ -10401 +10401 @@
                     \
                     IEM_MC_ARG(uint16_t *,              pu16Dst,                0); \
-                    IEM_MC_MEM_MAP_U16_RW(pu16Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U16_##a_Type(pu16Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     \
                     IEM_MC_ARG(uint16_t,                u16Src,                 2); \
@@ -10414 +10414 @@
                     IEM_MC_CALL_VOID_AIMPL_4(a_fnWorker16, pu16Dst, pu16Ax, u16Src, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_##a_Type(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_STORE_GREG_U16(X86_GREG_xAX, u16Ax); \
@@ -10429 +10429 @@
                     IEM_MC_LOCAL(uint8_t,               bUnmapInfo); \
                     IEM_MC_ARG(uint32_t *,              pu32Dst,                0); \
-                    IEM_MC_MEM_MAP_U32_RW(pu32Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U32_##a_Type(pu32Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     \
                     IEM_MC_ARG(uint32_t,                u32Src,                 2); \
@@ -10442 +10442 @@
                     IEM_MC_CALL_VOID_AIMPL_4(a_fnWorker32, pu32Dst, pu32Eax, u32Src, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_##a_Type(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     \
@@ -10461 +10461 @@
                     IEM_MC_LOCAL(uint8_t,               bUnmapInfo); \
                     IEM_MC_ARG(uint64_t *,              pu64Dst,                0); \
-                    IEM_MC_MEM_MAP_U64_RW(pu64Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U64_##a_Type(pu64Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     \
                     IEM_MC_ARG(uint64_t,                u64Src,                 2); \
@@ -10475 +10475 @@
                     IEM_MC_CALL_VOID_AIMPL_4(a_fnWorker64, pu64Dst, pu64Rax, u64Src, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_##a_Type(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_STORE_GREG_U64(X86_GREG_xAX, u64Rax); \
@@ -10488 +10488 @@
         if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK) || (pVCpu->iem.s.fExec & IEM_F_X86_DISREGARD_LOCK))
         {
-            IEMOP_BODY_CMPXCHG_EV_GV(iemAImpl_cmpxchg_u16, iemAImpl_cmpxchg_u32, iemAImpl_cmpxchg_u64);
+            IEMOP_BODY_CMPXCHG_EV_GV(iemAImpl_cmpxchg_u16, iemAImpl_cmpxchg_u32, iemAImpl_cmpxchg_u64,RW);
         }
         else
         {
-            IEMOP_BODY_CMPXCHG_EV_GV(iemAImpl_cmpxchg_u16_locked, iemAImpl_cmpxchg_u32_locked, iemAImpl_cmpxchg_u64_locked);
+            IEMOP_BODY_CMPXCHG_EV_GV(iemAImpl_cmpxchg_u16_locked, iemAImpl_cmpxchg_u32_locked, iemAImpl_cmpxchg_u64_locked,ATOMIC);
         }
     }
@@ -10914 +10914 @@
                     IEM_MC_ARG(uint16_t *,              pu16Dst,                    0); \
                     IEM_MC_LOCAL(uint8_t,               bUnmapInfo); \
-                    IEM_MC_MEM_MAP_U16_RW(pu16Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U16_ATOMIC(pu16Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     \
                     IEM_MC_ARG_CONST(uint16_t,          u16Src, /*=*/ bImm & 0x0f,  1); \
@@ -10921 +10921 @@
                     IEM_MC_CALL_VOID_AIMPL_3(a_fnLockedU16, pu16Dst, u16Src, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_ADVANCE_RIP_AND_FINISH(); \
@@ -10937 +10937 @@
                     IEM_MC_LOCAL(uint8_t,               bUnmapInfo); \
                     IEM_MC_ARG(uint32_t *,              pu32Dst,                    0); \
-                    IEM_MC_MEM_MAP_U32_RW(pu32Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U32_ATOMIC(pu32Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     \
                     IEM_MC_ARG_CONST(uint32_t,          u32Src, /*=*/ bImm & 0x1f,  1); \
@@ -10944 +10944 @@
                     IEM_MC_CALL_VOID_AIMPL_3(a_fnLockedU32, pu32Dst, u32Src, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_ADVANCE_RIP_AND_FINISH(); \
@@ -10960 +10960 @@
                     IEM_MC_LOCAL(uint8_t,               bUnmapInfo); \
                     IEM_MC_ARG(uint64_t *,              pu64Dst,                    0); \
-                    IEM_MC_MEM_MAP_U64_RW(pu64Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U64_ATOMIC(pu64Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     \
                     IEM_MC_ARG_CONST(uint64_t,          u64Src, /*=*/ bImm & 0x3f,  1); \
@@ -10967 +10967 @@
                     IEM_MC_CALL_VOID_AIMPL_3(a_fnLockedU64, pu64Dst, u64Src, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_ADVANCE_RIP_AND_FINISH(); \
@@ -11042 +11042 @@
         /* memory destination. */ \
         /** @todo test negative bit offsets! */ \
-        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK) || (pVCpu->iem.s.fExec & IEM_F_X86_DISREGARD_LOCK)) \
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) \
         { \
             switch (pVCpu->iem.s.enmEffOpSize) \
@@ -11609 +11609 @@
          * We're accessing memory.
          */
-#define IEMOP_BODY_XADD_BYTE(a_fnWorker) \
+#define IEMOP_BODY_XADD_BYTE(a_fnWorker, a_Type) \
             IEM_MC_BEGIN(3, 4, IEM_MC_F_MIN_486, 0); \
             IEM_MC_LOCAL(RTGCPTR,           GCPtrEffDst); \
@@ -11617 +11617 @@
             IEM_MC_LOCAL(uint8_t,           bUnmapInfo); \
             IEM_MC_ARG(uint8_t *,           pu8Dst,                 0); \
-            IEM_MC_MEM_MAP_U8_RW(pu8Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+            IEM_MC_MEM_MAP_U8_##a_Type(pu8Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
             \
             IEM_MC_LOCAL(uint8_t,           u8RegCopy); \
@@ -11627 +11627 @@
             IEM_MC_CALL_VOID_AIMPL_3(a_fnWorker, pu8Dst, pu8Reg, pEFlags); \
             \
-            IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+            IEM_MC_MEM_COMMIT_AND_UNMAP_##a_Type(bUnmapInfo); \
             IEM_MC_COMMIT_EFLAGS(EFlags); \
             IEM_MC_STORE_GREG_U8(IEM_GET_MODRM_REG(pVCpu, bRm), u8RegCopy); \
@@ -11634 +11634 @@
         if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK) || (pVCpu->iem.s.fExec & IEM_F_X86_DISREGARD_LOCK))
         {
-            IEMOP_BODY_XADD_BYTE(iemAImpl_xadd_u8);
+            IEMOP_BODY_XADD_BYTE(iemAImpl_xadd_u8,RW);
         }
         else
         {
-            IEMOP_BODY_XADD_BYTE(iemAImpl_xadd_u8_locked);
+            IEMOP_BODY_XADD_BYTE(iemAImpl_xadd_u8_locked,ATOMIC);
         }
     }
@@ -11716 +11716 @@
          * We're accessing memory.
          */
-#define IEMOP_BODY_XADD_EV_GV(a_fnWorker16, a_fnWorker32, a_fnWorker64) \
+#define IEMOP_BODY_XADD_EV_GV(a_fnWorker16, a_fnWorker32, a_fnWorker64, a_Type) \
         do { \
             switch (pVCpu->iem.s.enmEffOpSize) \
@@ -11728 +11728 @@
                     IEM_MC_LOCAL(uint8_t,               bUnmapInfo); \
                     IEM_MC_ARG(uint16_t *,              pu16Dst,                0); \
-                    IEM_MC_MEM_MAP_U16_RW(pu16Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U16_##a_Type(pu16Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     \
                     IEM_MC_LOCAL(uint16_t,              u16RegCopy); \
@@ -11738 +11738 @@
                     IEM_MC_CALL_VOID_AIMPL_3(a_fnWorker16, pu16Dst, pu16Reg, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_##a_Type(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16RegCopy); \
@@ -11753 +11753 @@
                     IEM_MC_LOCAL(uint8_t,               bUnmapInfo); \
                     IEM_MC_ARG(uint32_t *,              pu32Dst,                0); \
-                    IEM_MC_MEM_MAP_U32_RW(pu32Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U32_##a_Type(pu32Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     \
                     IEM_MC_LOCAL(uint32_t,              u32RegCopy); \
@@ -11763 +11763 @@
                     IEM_MC_CALL_VOID_AIMPL_3(a_fnWorker32, pu32Dst, pu32Reg, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_##a_Type(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32RegCopy); \
@@ -11778 +11778 @@
                     IEM_MC_LOCAL(uint8_t,               bUnmapInfo); \
                     IEM_MC_ARG(uint64_t *,              pu64Dst,                0); \
-                    IEM_MC_MEM_MAP_U64_RW(pu64Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+                    IEM_MC_MEM_MAP_U64_##a_Type(pu64Dst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
                     \
                     IEM_MC_LOCAL(uint64_t,              u64RegCopy); \
@@ -11788 +11788 @@
                     IEM_MC_CALL_VOID_AIMPL_3(a_fnWorker64, pu64Dst, pu64Reg, pEFlags); \
                     \
-                    IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+                    IEM_MC_MEM_COMMIT_AND_UNMAP_##a_Type(bUnmapInfo); \
                     IEM_MC_COMMIT_EFLAGS(EFlags); \
                     IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64RegCopy); \
@@ -11801 +11801 @@
         if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK) || (pVCpu->iem.s.fExec & IEM_F_X86_DISREGARD_LOCK))
         {
-            IEMOP_BODY_XADD_EV_GV(iemAImpl_xadd_u16, iemAImpl_xadd_u32, iemAImpl_xadd_u64);
+            IEMOP_BODY_XADD_EV_GV(iemAImpl_xadd_u16, iemAImpl_xadd_u32, iemAImpl_xadd_u64,RW);
         }
         else
         {
-            IEMOP_BODY_XADD_EV_GV(iemAImpl_xadd_u16_locked, iemAImpl_xadd_u32_locked, iemAImpl_xadd_u64_locked);
+            IEMOP_BODY_XADD_EV_GV(iemAImpl_xadd_u16_locked, iemAImpl_xadd_u32_locked, iemAImpl_xadd_u64_locked,ATOMIC);
         }
     }
@@ -12433 +12433 @@
 {
     IEMOP_MNEMONIC(cmpxchg8b, "cmpxchg8b Mq");
-#define IEMOP_BODY_CMPXCHG8B(a_fnWorker) \
+#define IEMOP_BODY_CMPXCHG8B(a_fnWorker, a_Type) \
         IEM_MC_BEGIN(4, 5, IEM_MC_F_NOT_286_OR_OLDER, 0); \
         IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst); \
@@ -12441 +12441 @@
         IEM_MC_LOCAL(uint8_t,               bUnmapInfo); \
         IEM_MC_ARG(uint64_t *,              pu64MemDst,             0); \
-        IEM_MC_MEM_MAP_U64_RW(pu64MemDst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+        IEM_MC_MEM_MAP_U64_##a_Type(pu64MemDst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
         \
         IEM_MC_LOCAL(RTUINT64U,             u64EaxEdx); \
@@ -12455 +12455 @@
         IEM_MC_CALL_VOID_AIMPL_4(a_fnWorker, pu64MemDst, pu64EaxEdx, pu64EbxEcx, pEFlags); \
         \
-        IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+        IEM_MC_MEM_COMMIT_AND_UNMAP_##a_Type(bUnmapInfo); \
         IEM_MC_COMMIT_EFLAGS(EFlags); \
         IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF) { \
@@ -12465 +12465 @@
     if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK) || (pVCpu->iem.s.fExec & IEM_F_X86_DISREGARD_LOCK))
     {
-        IEMOP_BODY_CMPXCHG8B(iemAImpl_cmpxchg8b);
-    }
-    else
-    {
-        IEMOP_BODY_CMPXCHG8B(iemAImpl_cmpxchg8b_locked);
+        IEMOP_BODY_CMPXCHG8B(iemAImpl_cmpxchg8b,RW);
+    }
+    else
+    {
+        IEMOP_BODY_CMPXCHG8B(iemAImpl_cmpxchg8b_locked,ATOMIC);
     }
 }
@@ -12485 +12485 @@
          * the patterns IEMAllThrdPython.py requires for the code morphing.
          */
-#define BODY_CMPXCHG16B_HEAD(bUnmapInfoStmt) \
+#define BODY_CMPXCHG16B_HEAD(bUnmapInfoStmt, a_Type) \
             IEM_MC_BEGIN(5, 4, IEM_MC_F_64BIT, 0); \
             IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst); \
@@ -12494 +12494 @@
             bUnmapInfoStmt; \
             IEM_MC_ARG(PRTUINT128U,             pu128MemDst,                0); \
-            IEM_MC_MEM_MAP_U128_RW(pu128MemDst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
+            IEM_MC_MEM_MAP_U128_##a_Type(pu128MemDst, bUnmapInfo, pVCpu->iem.s.iEffSeg, GCPtrEffDst); \
             \
             IEM_MC_LOCAL(RTUINT128U, u128RaxRdx); \
@@ -12507 +12507 @@
             IEM_MC_FETCH_EFLAGS(EFlags)
 
-#define BODY_CMPXCHG16B_TAIL \
-            IEM_MC_MEM_COMMIT_AND_UNMAP_RW(bUnmapInfo); \
+#define BODY_CMPXCHG16B_TAIL(a_Type) \
+            IEM_MC_MEM_COMMIT_AND_UNMAP_##a_Type(bUnmapInfo); \
             IEM_MC_COMMIT_EFLAGS(EFlags); \
             IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF) { \
@@ -12521 +12521 @@
             if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK) || (pVCpu->iem.s.fExec & IEM_F_X86_DISREGARD_LOCK))
             {
-                BODY_CMPXCHG16B_HEAD(IEM_MC_LOCAL(uint8_t, bUnmapInfo));
-                IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
-                BODY_CMPXCHG16B_TAIL;
+                BODY_CMPXCHG16B_HEAD(IEM_MC_LOCAL(uint8_t, bUnmapInfo),RW);
+                IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags,RW);
+                BODY_CMPXCHG16B_TAIL(RW);
             }
             else
             {
-                BODY_CMPXCHG16B_HEAD(IEM_MC_LOCAL(uint8_t, bUnmapInfo));
-                IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b_locked, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
-                BODY_CMPXCHG16B_TAIL;
+                BODY_CMPXCHG16B_HEAD(IEM_MC_LOCAL(uint8_t, bUnmapInfo),ATOMIC);
+                IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b_locked, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags,ATOMIC);
+                BODY_CMPXCHG16B_TAIL(ATOMIC);
             }
         }
@@ -12536 +12536 @@
             if (pVCpu->CTX_SUFF(pVM)->cCpus == 1)
             {
-                BODY_CMPXCHG16B_HEAD(IEM_MC_LOCAL(uint8_t, bUnmapInfo));
-                IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b_fallback, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
-                BODY_CMPXCHG16B_TAIL;
+                BODY_CMPXCHG16B_HEAD(IEM_MC_LOCAL(uint8_t, bUnmapInfo),RW);
+                IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b_fallback, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags,RW);
+                BODY_CMPXCHG16B_TAIL(RW);
             }
             else
             {
-                BODY_CMPXCHG16B_HEAD(IEM_MC_ARG(uint8_t, bUnmapInfo, 4));
+                BODY_CMPXCHG16B_HEAD(IEM_MC_ARG(uint8_t, bUnmapInfo, 4),RW);
                 IEM_MC_CALL_CIMPL_5(IEM_CIMPL_F_STATUS_FLAGS,
                                       RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xAX)
@@ -12556 +12556 @@
         if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK) || (pVCpu->iem.s.fExec & IEM_F_X86_DISREGARD_LOCK))
         {
-            BODY_CMPXCHG16B_HEAD(IEM_MC_LOCAL(uint8_t, bUnmapInfo));
+            BODY_CMPXCHG16B_HEAD(IEM_MC_LOCAL(uint8_t, bUnmapInfo),RW);
             IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
-            BODY_CMPXCHG16B_TAIL;
+            BODY_CMPXCHG16B_TAIL(RW);
         }
         else
         {
-            BODY_CMPXCHG16B_HEAD(IEM_MC_LOCAL(uint8_t, bUnmapInfo));
+            BODY_CMPXCHG16B_HEAD(IEM_MC_LOCAL(uint8_t, bUnmapInfo),ATOMIC);
             IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b_locked, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
-            BODY_CMPXCHG16B_TAIL;
+            BODY_CMPXCHG16B_TAIL(ATOMIC);
         }
 
@@ -12574 +12574 @@
         if (pVCpu->CTX_SUFF(pVM)->cCpus == 1)
         {
-            BODY_CMPXCHG16B_HEAD(IEM_MC_LOCAL(uint8_t, bUnmapInfo));
+            BODY_CMPXCHG16B_HEAD(IEM_MC_LOCAL(uint8_t, bUnmapInfo),RW);
             IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b_fallback, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
-            BODY_CMPXCHG16B_TAIL;
+            BODY_CMPXCHG16B_TAIL(RW);
         }
         else
         {
-            BODY_CMPXCHG16B_HEAD(IEM_MC_ARG(uint8_t, bUnmapInfo, 4));
+            BODY_CMPXCHG16B_HEAD(IEM_MC_ARG(uint8_t, bUnmapInfo, 4),RW);
             IEM_MC_CALL_CIMPL_4(IEM_CIMPL_F_STATUS_FLAGS,
                                   RT_BIT_64(kIemNativeGstReg_GprFirst + X86_GREG_xAX)

trunk/src/VBox/VMM/VMMAll/IEMAllMemRWTmpl.cpp.h

@@ -182 +182 @@
 
 #ifdef IEM_WITH_SETJMP
+
+/**
+ * Maps a data buffer for atomic read+write direct access (or via a bounce
+ * buffer), longjmp on error.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pbUnmapInfo         Pointer to unmap info variable.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+TMPL_MEM_TYPE *
+RT_CONCAT3(iemMemMapData,TMPL_MEM_FN_SUFF,AtSafeJmp)(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo,
+                                                     uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
+{
+# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
+    pVCpu->iem.s.DataTlb.cTlbSafeWritePath++;
+# endif
+    Log8(("IEM AT/map " TMPL_MEM_FMT_DESC " %d|%RGv\n", iSegReg, GCPtrMem));
+    *pbUnmapInfo = 1 | ((IEM_ACCESS_TYPE_READ  | IEM_ACCESS_TYPE_WRITE) << 4); /* zero is for the TLB hit */
+    return (TMPL_MEM_TYPE *)iemMemMapJmp(pVCpu, pbUnmapInfo, sizeof(TMPL_MEM_TYPE), iSegReg, GCPtrMem,
+                                         IEM_ACCESS_DATA_ATOMIC, TMPL_MEM_TYPE_ALIGN);
+}
+
 
 /**

trunk/src/VBox/VMM/VMMAll/IEMAllMemRWTmplInline.cpp.h

@@ -382 +382 @@
 /**
  * Inlined read-write memory mapping function that longjumps on error.
+ *
+ * Almost identical to RT_CONCAT3(iemMemMapData,TMPL_MEM_FN_SUFF,AtJmp).
  */
 DECL_INLINE_THROW(TMPL_MEM_TYPE *)
@@ -438 +440 @@
 /**
  * Inlined flat read-write memory mapping function that longjumps on error.
+ *
+ * Almost identical to RT_CONCAT3(iemMemFlatMapData,TMPL_MEM_FN_SUFF,AtJmp).
  */
 DECL_INLINE_THROW(TMPL_MEM_TYPE *)
@@ -490 +494 @@
 }
 
+#  ifdef TMPL_MEM_WITH_ATOMIC_MAPPING
+
+/**
+ * Inlined atomic read-write memory mapping function that longjumps on error.
+ *
+ * Almost identical to RT_CONCAT3(iemMemMapData,TMPL_MEM_FN_SUFF,RwJmp).
+ */
+DECL_INLINE_THROW(TMPL_MEM_TYPE *)
+RT_CONCAT3(iemMemMapData,TMPL_MEM_FN_SUFF,AtJmp)(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo,
+                                                 uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
+{
+#  if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3) && !defined(TMPL_MEM_NO_INLINE)
+    /*
+     * Convert from segmented to flat address and check that it doesn't cross a page boundrary.
+     */
+    RTGCPTR const GCPtrEff = iemMemApplySegmentToWriteJmp(pVCpu, iSegReg, sizeof(TMPL_MEM_TYPE), GCPtrMem);
+#  if TMPL_MEM_TYPE_SIZE > 1
+    if (RT_LIKELY(!(GCPtrEff & TMPL_MEM_TYPE_ALIGN))) /* strictly aligned, otherwise do fall back which knows th details. */
+#  endif
+    {
+        /*
+         * TLB lookup.
+         */
+        uint64_t const uTag  = IEMTLB_CALC_TAG(    &pVCpu->iem.s.DataTlb, GCPtrEff);
+        PIEMTLBENTRY   pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
+        if (RT_LIKELY(pTlbe->uTag == uTag))
+        {
+            /*
+             * Check TLB page table level access flags.
+             */
+            AssertCompile(IEMTLBE_F_PT_NO_USER == 4);
+            uint64_t const fNoUser = (IEM_GET_CPL(pVCpu) + 1) & IEMTLBE_F_PT_NO_USER;
+            if (RT_LIKELY(   (pTlbe->fFlagsAndPhysRev & (  IEMTLBE_F_PHYS_REV       | IEMTLBE_F_NO_MAPPINGR3
+                                                         | IEMTLBE_F_PG_UNASSIGNED  | IEMTLBE_F_PG_NO_WRITE   | IEMTLBE_F_PG_NO_READ
+                                                         | IEMTLBE_F_PT_NO_ACCESSED | IEMTLBE_F_PT_NO_DIRTY   | IEMTLBE_F_PT_NO_WRITE
+                                                         | fNoUser))
+                          == pVCpu->iem.s.DataTlb.uTlbPhysRev))
+            {
+                /*
+                 * Return the address.
+                 */
+                STAM_STATS({pVCpu->iem.s.DataTlb.cTlbHits++;});
+                Assert(pTlbe->pbMappingR3); /* (Only ever cleared by the owning EMT.) */
+                Assert(!((uintptr_t)pTlbe->pbMappingR3 & GUEST_PAGE_OFFSET_MASK));
+                *pbUnmapInfo = 0;
+                Log7Ex(LOG_GROUP_IEM_MEM,("IEM AT/map " TMPL_MEM_FMT_DESC " %d|%RGv=%RGv: %p\n",
+                                          iSegReg, GCPtrMem, GCPtrEff, &pTlbe->pbMappingR3[GCPtrEff & GUEST_PAGE_OFFSET_MASK]));
+                return (TMPL_MEM_TYPE *)&pTlbe->pbMappingR3[GCPtrEff & GUEST_PAGE_OFFSET_MASK];
+            }
+        }
+    }
+
+    /* Fall back on the slow careful approach in case of TLB miss, MMIO, exception
+       outdated page pointer, or other troubles.  (This will do a TLB load.) */
+    Log8Ex(LOG_GROUP_IEM_MEM,(LOG_FN_FMT ": %u:%RGv falling back\n", LOG_FN_NAME, iSegReg, GCPtrMem));
+#  endif
+    return RT_CONCAT3(iemMemMapData,TMPL_MEM_FN_SUFF,AtSafeJmp)(pVCpu, pbUnmapInfo, iSegReg, GCPtrMem);
+}
+
+
+/**
+ * Inlined flat read-write memory mapping function that longjumps on error.
+ *
+ * Almost identical to RT_CONCAT3(iemMemFlatMapData,TMPL_MEM_FN_SUFF,RwJmp).
+ */
+DECL_INLINE_THROW(TMPL_MEM_TYPE *)
+RT_CONCAT3(iemMemFlatMapData,TMPL_MEM_FN_SUFF,AtJmp)(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo,
+                                                     RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
+{
+#  if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3) && !defined(TMPL_MEM_NO_INLINE)
+    /*
+     * Check that the address doesn't cross a page boundrary.
+     */
+#  if TMPL_MEM_TYPE_SIZE > 1
+    if (RT_LIKELY(!(GCPtrMem & TMPL_MEM_TYPE_ALIGN))) /* strictly aligned, otherwise do fall back which knows th details. */
+#  endif
+    {
+        /*
+         * TLB lookup.
+         */
+        uint64_t const uTag  = IEMTLB_CALC_TAG(    &pVCpu->iem.s.DataTlb, GCPtrMem);
+        PIEMTLBENTRY   pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
+        if (RT_LIKELY(pTlbe->uTag == uTag))
+        {
+            /*
+             * Check TLB page table level access flags.
+             */
+            AssertCompile(IEMTLBE_F_PT_NO_USER == 4);
+            uint64_t const fNoUser = (IEM_GET_CPL(pVCpu) + 1) & IEMTLBE_F_PT_NO_USER;
+            if (RT_LIKELY(   (pTlbe->fFlagsAndPhysRev & (  IEMTLBE_F_PHYS_REV       | IEMTLBE_F_NO_MAPPINGR3
+                                                         | IEMTLBE_F_PG_UNASSIGNED  | IEMTLBE_F_PG_NO_WRITE   | IEMTLBE_F_PG_NO_READ
+                                                         | IEMTLBE_F_PT_NO_ACCESSED | IEMTLBE_F_PT_NO_DIRTY   | IEMTLBE_F_PT_NO_WRITE
+                                                         | fNoUser))
+                          == pVCpu->iem.s.DataTlb.uTlbPhysRev))
+            {
+                /*
+                 * Return the address.
+                 */
+                STAM_STATS({pVCpu->iem.s.DataTlb.cTlbHits++;});
+                Assert(pTlbe->pbMappingR3); /* (Only ever cleared by the owning EMT.) */
+                Assert(!((uintptr_t)pTlbe->pbMappingR3 & GUEST_PAGE_OFFSET_MASK));
+                *pbUnmapInfo = 0;
+                Log7Ex(LOG_GROUP_IEM_MEM,("IEM AT/map " TMPL_MEM_FMT_DESC " %RGv: %p\n",
+                                          GCPtrMem, &pTlbe->pbMappingR3[GCPtrMem & GUEST_PAGE_OFFSET_MASK]));
+                return (TMPL_MEM_TYPE *)&pTlbe->pbMappingR3[GCPtrMem & GUEST_PAGE_OFFSET_MASK];
+            }
+        }
+    }
+
+    /* Fall back on the slow careful approach in case of TLB miss, MMIO, exception
+       outdated page pointer, or other troubles.  (This will do a TLB load.) */
+    Log8Ex(LOG_GROUP_IEM_MEM,(LOG_FN_FMT ": %RGv falling back\n", LOG_FN_NAME, GCPtrMem));
+#  endif
+    return RT_CONCAT3(iemMemMapData,TMPL_MEM_FN_SUFF,AtSafeJmp)(pVCpu, pbUnmapInfo, UINT8_MAX, GCPtrMem);
+}
+
+#  endif /* TMPL_MEM_WITH_ATOMIC_MAPPING */
 
 /**

trunk/src/VBox/VMM/VMMAll/IEMAllN8vePython.py

@@ -169 +169 @@
     'IEM_MC_MEM_FLAT_MAP_R64_WO':                                        (None, True,  True,  True,  ),
     'IEM_MC_MEM_FLAT_MAP_R80_WO':                                        (None, True,  True,  True,  ),
+    'IEM_MC_MEM_FLAT_MAP_U8_ATOMIC':                                     (None, True,  True,  True,  ),
     'IEM_MC_MEM_FLAT_MAP_U8_RO':                                         (None, True,  True,  True,  ),
     'IEM_MC_MEM_FLAT_MAP_U8_RW':                                         (None, True,  True,  True,  ),
+    'IEM_MC_MEM_FLAT_MAP_U16_ATOMIC':                                    (None, True,  True,  True,  ),
     'IEM_MC_MEM_FLAT_MAP_U16_RO':                                        (None, True,  True,  True,  ),
     'IEM_MC_MEM_FLAT_MAP_U16_RW':                                        (None, True,  True,  True,  ),
+    'IEM_MC_MEM_FLAT_MAP_U32_ATOMIC':                                    (None, True,  True,  True,  ),
     'IEM_MC_MEM_FLAT_MAP_U32_RO':                                        (None, True,  True,  True,  ),
     'IEM_MC_MEM_FLAT_MAP_U32_RW':                                        (None, True,  True,  True,  ),
+    'IEM_MC_MEM_FLAT_MAP_U64_ATOMIC':                                    (None, True,  True,  True,  ),
     'IEM_MC_MEM_FLAT_MAP_U64_RO':                                        (None, True,  True,  True,  ),
     'IEM_MC_MEM_FLAT_MAP_U64_RW':                                        (None, True,  True,  True,  ),
+    'IEM_MC_MEM_FLAT_MAP_U128_ATOMIC':                                   (None, True,  True,  True,  ),
     'IEM_MC_MEM_FLAT_MAP_U128_RW':                                       (None, True,  True,  True,  ),
     'IEM_MC_STORE_MEM_FLAT_U128_ALIGN_SSE':                              (None, True,  True,  False, ),
@@ -398 +403 @@
                 elif oStmt.sName in ('IEM_MC_MEM_COMMIT_AND_UNMAP_RW', 'IEM_MC_MEM_COMMIT_AND_UNMAP_RO',
                                      'IEM_MC_MEM_COMMIT_AND_UNMAP_WO', 'IEM_MC_MEM_ROLLBACK_AND_UNMAP_WO',
+                                     'IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC',
                                      'IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE_WO'):
                     #

trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompiler.cpp

@@ -2228 +2228 @@
 
 /**
+ * Used by TB code to map unsigned 8-bit data for atomic read-write w/
+ * segmentation.
+ */
+IEM_DECL_NATIVE_HLP_DEF(uint8_t *, iemNativeHlpMemMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo,
+                                                                   RTGCPTR GCPtrMem, uint8_t iSegReg))
+{
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_MAPPED
+    return iemMemMapDataU8AtSafeJmp(pVCpu, pbUnmapInfo, iSegReg, GCPtrMem);
+#else
+    return iemMemMapDataU8AtJmp(pVCpu, pbUnmapInfo, iSegReg, GCPtrMem);
+#endif
+}
+
+
+/**
  * Used by TB code to map unsigned 8-bit data read-write w/ segmentation.
  */
@@ -2270 +2285 @@
 
 /**
+ * Used by TB code to map unsigned 16-bit data for atomic read-write w/
+ * segmentation.
+ */
+IEM_DECL_NATIVE_HLP_DEF(uint16_t *, iemNativeHlpMemMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo,
+                                                                     RTGCPTR GCPtrMem, uint8_t iSegReg))
+{
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_MAPPED
+    return iemMemMapDataU16AtSafeJmp(pVCpu, pbUnmapInfo, iSegReg, GCPtrMem);
+#else
+    return iemMemMapDataU16AtJmp(pVCpu, pbUnmapInfo, iSegReg, GCPtrMem);
+#endif
+}
+
+
+/**
  * Used by TB code to map unsigned 16-bit data read-write w/ segmentation.
  */
@@ -2312 +2342 @@
 
 /**
+ * Used by TB code to map unsigned 32-bit data for atomic read-write w/
+ * segmentation.
+ */
+IEM_DECL_NATIVE_HLP_DEF(uint32_t *, iemNativeHlpMemMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo,
+                                                                     RTGCPTR GCPtrMem, uint8_t iSegReg))
+{
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_MAPPED
+    return iemMemMapDataU32AtSafeJmp(pVCpu, pbUnmapInfo, iSegReg, GCPtrMem);
+#else
+    return iemMemMapDataU32AtJmp(pVCpu, pbUnmapInfo, iSegReg, GCPtrMem);
+#endif
+}
+
+
+/**
  * Used by TB code to map unsigned 32-bit data read-write w/ segmentation.
  */
@@ -2354 +2399 @@
 
 /**
+ * Used by TB code to map unsigned 64-bit data for atomic read-write w/
+ * segmentation.
+ */
+IEM_DECL_NATIVE_HLP_DEF(uint64_t *, iemNativeHlpMemMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo,
+                                                                     RTGCPTR GCPtrMem, uint8_t iSegReg))
+{
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_MAPPED
+    return iemMemMapDataU64AtSafeJmp(pVCpu, pbUnmapInfo, iSegReg, GCPtrMem);
+#else
+    return iemMemMapDataU64AtJmp(pVCpu, pbUnmapInfo, iSegReg, GCPtrMem);
+#endif
+}
+
+
+/**
  * Used by TB code to map unsigned 64-bit data read-write w/ segmentation.
  */
@@ -2419 +2479 @@
 #else
     return iemMemMapDataD80WoJmp(pVCpu, pbUnmapInfo, iSegReg, GCPtrMem);
+#endif
+}
+
+
+/**
+ * Used by TB code to map unsigned 128-bit data for atomic read-write w/
+ * segmentation.
+ */
+IEM_DECL_NATIVE_HLP_DEF(RTUINT128U *, iemNativeHlpMemMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo,
+                                                                        RTGCPTR GCPtrMem, uint8_t iSegReg))
+{
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_MAPPED
+    return iemMemMapDataU128AtSafeJmp(pVCpu, pbUnmapInfo, iSegReg, GCPtrMem);
+#else
+    return iemMemMapDataU128AtJmp(pVCpu, pbUnmapInfo, iSegReg, GCPtrMem);
 #endif
 }
@@ -2470 +2545 @@
 
 /**
+ * Used by TB code to map unsigned 8-bit data for atomic read-write w/ flat
+ * address.
+ */
+IEM_DECL_NATIVE_HLP_DEF(uint8_t *, iemNativeHlpMemFlatMapDataU8Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem))
+{
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_MAPPED
+    return iemMemMapDataU8AtSafeJmp(pVCpu, pbUnmapInfo, UINT8_MAX, GCPtrMem);
+#else
+    return iemMemFlatMapDataU8AtJmp(pVCpu, pbUnmapInfo, GCPtrMem);
+#endif
+}
+
+
+/**
  * Used by TB code to map unsigned 8-bit data read-write w/ flat address.
  */
@@ -2509 +2598 @@
 
 /**
+ * Used by TB code to map unsigned 16-bit data for atomic read-write w/ flat
+ * address.
+ */
+IEM_DECL_NATIVE_HLP_DEF(uint16_t *, iemNativeHlpMemFlatMapDataU16Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem))
+{
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_MAPPED
+    return iemMemMapDataU16AtSafeJmp(pVCpu, pbUnmapInfo, UINT8_MAX, GCPtrMem);
+#else
+    return iemMemFlatMapDataU16AtJmp(pVCpu, pbUnmapInfo, GCPtrMem);
+#endif
+}
+
+
+/**
  * Used by TB code to map unsigned 16-bit data read-write w/ flat address.
  */
@@ -2548 +2651 @@
 
 /**
+ * Used by TB code to map unsigned 32-bit data for atomic read-write w/ flat
+ * address.
+ */
+IEM_DECL_NATIVE_HLP_DEF(uint32_t *, iemNativeHlpMemFlatMapDataU32Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem))
+{
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_MAPPED
+    return iemMemMapDataU32AtSafeJmp(pVCpu, pbUnmapInfo, UINT8_MAX, GCPtrMem);
+#else
+    return iemMemFlatMapDataU32AtJmp(pVCpu, pbUnmapInfo, GCPtrMem);
+#endif
+}
+
+
+/**
  * Used by TB code to map unsigned 32-bit data read-write w/ flat address.
  */
@@ -2587 +2704 @@
 
 /**
+ * Used by TB code to map unsigned 64-bit data for atomic read-write w/ flat
+ * address.
+ */
+IEM_DECL_NATIVE_HLP_DEF(uint64_t *, iemNativeHlpMemFlatMapDataU64Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem))
+{
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_MAPPED
+    return iemMemMapDataU64AtSafeJmp(pVCpu, pbUnmapInfo, UINT8_MAX, GCPtrMem);
+#else
+    return iemMemFlatMapDataU64AtJmp(pVCpu, pbUnmapInfo, GCPtrMem);
+#endif
+}
+
+
+/**
  * Used by TB code to map unsigned 64-bit data read-write w/ flat address.
  */
@@ -2647 +2778 @@
 #else
     return iemMemFlatMapDataD80WoJmp(pVCpu, pbUnmapInfo, GCPtrMem);
+#endif
+}
+
+
+/**
+ * Used by TB code to map unsigned 128-bit data for atomic read-write w/ flat
+ * address.
+ */
+IEM_DECL_NATIVE_HLP_DEF(RTUINT128U *, iemNativeHlpMemFlatMapDataU128Atomic,(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, RTGCPTR GCPtrMem))
+{
+#ifdef IEMNATIVE_WITH_TLB_LOOKUP_MAPPED
+    return iemMemMapDataU128AtSafeJmp(pVCpu, pbUnmapInfo, UINT8_MAX, GCPtrMem);
+#else
+    return iemMemFlatMapDataU128AtJmp(pVCpu, pbUnmapInfo, GCPtrMem);
 #endif
 }
@@ -2693 +2838 @@
 *   Helpers: Commit, rollback & unmap                                                                                            *
 *********************************************************************************************************************************/
+
+/**
+ * Used by TB code to commit and unmap a read-write memory mapping.
+ */
+IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemCommitAndUnmapAtomic,(PVMCPUCC pVCpu, uint8_t bUnmapInfo))
+{
+    return iemMemCommitAndUnmapAtSafeJmp(pVCpu, bUnmapInfo);
+}
+
 
 /**
@@ -11903 +12057 @@
 *********************************************************************************************************************************/
 
+#define IEM_MC_MEM_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
+    off = iemNativeEmitMemMapCommon(pReNative, off, a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint8_t), \
+                                    IEM_ACCESS_DATA_ATOMIC,  0 /*fAlignMask*/, \
+                                    (uintptr_t)iemNativeHlpMemMapDataU8Atomic, pCallEntry->idxInstr)
+
 #define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint8_t), \
-                                    IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE,  0 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_RW,  0 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemMapDataU8Rw, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint8_t), \
-                                    IEM_ACCESS_TYPE_WRITE,  0 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_W,  0 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemMapDataU8Wo, pCallEntry->idxInstr) \
 
 #define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint8_t), \
-                                    IEM_ACCESS_TYPE_READ,  0 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_R,  0 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemMapDataU8Ro, pCallEntry->idxInstr)
 
+
+#define IEM_MC_MEM_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
+    off = iemNativeEmitMemMapCommon(pReNative, off, a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint16_t), \
+                                    IEM_ACCESS_DATA_ATOMIC,  sizeof(uint16_t) - 1 /*fAlignMask*/, \
+                                    (uintptr_t)iemNativeHlpMemMapDataU16Atomic, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint16_t), \
-                                    IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE,  sizeof(uint16_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_RW,  sizeof(uint16_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemMapDataU16Rw, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint16_t), \
-                                    IEM_ACCESS_TYPE_WRITE, sizeof(uint16_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemMapDataU16Wo, pCallEntry->idxInstr) \
 
 #define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint16_t), \
-                                    IEM_ACCESS_TYPE_READ,  sizeof(uint16_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_R,  sizeof(uint16_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemMapDataU16Ro, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(int16_t), \
-                                    IEM_ACCESS_TYPE_WRITE, sizeof(uint16_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemMapDataU16Wo, pCallEntry->idxInstr) \
 
+
+#define IEM_MC_MEM_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
+    off = iemNativeEmitMemMapCommon(pReNative, off, a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint32_t), \
+                                    IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1 /*fAlignMask*/, \
+                                    (uintptr_t)iemNativeHlpMemMapDataU32Atomic, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint32_t), \
-                                    IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE, sizeof(uint32_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemMapDataU32Rw, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint32_t), \
-                                    IEM_ACCESS_TYPE_WRITE, sizeof(uint32_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemMapDataU32Wo, pCallEntry->idxInstr) \
 
 #define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint32_t), \
-                                    IEM_ACCESS_TYPE_READ,  sizeof(uint32_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_R,  sizeof(uint32_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemMapDataU32Ro, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(int32_t), \
-                                    IEM_ACCESS_TYPE_WRITE, sizeof(uint32_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemMapDataU32Wo, pCallEntry->idxInstr) \
 
+
+#define IEM_MC_MEM_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
+    off = iemNativeEmitMemMapCommon(pReNative, off, a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint64_t), \
+                                    IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1 /*fAlignMask*/, \
+                                    (uintptr_t)iemNativeHlpMemMapDataU64Atomic, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint64_t), \
-                                    IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE, sizeof(uint64_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemMapDataU64Rw, pCallEntry->idxInstr)
-
 #define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint64_t), \
-                                    IEM_ACCESS_TYPE_WRITE, sizeof(uint64_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemMapDataU64Wo, pCallEntry->idxInstr) \
 
 #define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(uint64_t), \
-                                    IEM_ACCESS_TYPE_READ,  sizeof(uint64_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_R,  sizeof(uint64_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemMapDataU64Ro, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(int64_t), \
-                                    IEM_ACCESS_TYPE_WRITE, sizeof(uint64_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemMapDataU64Wo, pCallEntry->idxInstr) \
 
@@ -11984 +12157 @@
 #define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(RTFLOAT80U), \
-                                    IEM_ACCESS_TYPE_WRITE, sizeof(uint64_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemMapDataR80Wo, pCallEntry->idxInstr) \
 
 #define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(RTFLOAT80U), \
-                                    IEM_ACCESS_TYPE_WRITE, sizeof(uint64_t) - 1 /*fAlignMask*/, /** @todo check BCD align */ \
+                                    IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1 /*fAlignMask*/, /** @todo check BCD align */ \
                                     (uintptr_t)iemNativeHlpMemMapDataD80Wo, pCallEntry->idxInstr) \
 
+
+#define IEM_MC_MEM_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
+    off = iemNativeEmitMemMapCommon(pReNative, off, a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(RTUINT128U), \
+                                    IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128U) - 1 /*fAlignMask*/, \
+                                    (uintptr_t)iemNativeHlpMemMapDataU128Atomic, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(RTUINT128U), \
-                                    IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE, sizeof(RTUINT128U) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_RW, sizeof(RTUINT128U) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemMapDataU128Rw, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(RTUINT128U), \
-                                    IEM_ACCESS_TYPE_WRITE, sizeof(RTUINT128U) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_W, sizeof(RTUINT128U) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemMapDataU128Wo, pCallEntry->idxInstr) \
 
 #define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem, sizeof(RTUINT128U), \
-                                    IEM_ACCESS_TYPE_READ,  sizeof(RTUINT128U) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_R,  sizeof(RTUINT128U) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemMapDataU128Ro, pCallEntry->idxInstr)
 
 
+
+#define IEM_MC_MEM_FLAT_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
+    off = iemNativeEmitMemMapCommon(pReNative, off, a_pu8Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint8_t), \
+                                    IEM_ACCESS_DATA_ATOMIC,  0 /*fAlignMask*/, \
+                                    (uintptr_t)iemNativeHlpMemFlatMapDataU8Atomic, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu8Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint8_t), \
-                                    IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE,  0 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_RW,  0 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataU8Rw, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu8Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint8_t), \
-                                    IEM_ACCESS_TYPE_WRITE,  0 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_W,  0 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataU8Wo, pCallEntry->idxInstr) \
 
 #define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu8Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint8_t), \
-                                    IEM_ACCESS_TYPE_READ,  0 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_R,  0 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataU8Ro, pCallEntry->idxInstr)
 
+
+#define IEM_MC_MEM_FLAT_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
+    off = iemNativeEmitMemMapCommon(pReNative, off, a_pu16Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint16_t), \
+                                    IEM_ACCESS_DATA_ATOMIC,  sizeof(uint16_t) - 1 /*fAlignMask*/, \
+                                    (uintptr_t)iemNativeHlpMemFlatMapDataU16Atomic, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu16Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint16_t), \
-                                    IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE,  sizeof(uint16_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_RW,  sizeof(uint16_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataU16Rw, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu16Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint16_t), \
-                                    IEM_ACCESS_TYPE_WRITE, sizeof(uint16_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataU16Wo, pCallEntry->idxInstr) \
 
 #define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu16Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint16_t), \
-                                    IEM_ACCESS_TYPE_READ,  sizeof(uint16_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_R,  sizeof(uint16_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataU16Ro, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_FLAT_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pi16Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(int16_t), \
-                                    IEM_ACCESS_TYPE_WRITE, sizeof(uint16_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataU16Wo, pCallEntry->idxInstr) \
 
+
+#define IEM_MC_MEM_FLAT_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
+    off = iemNativeEmitMemMapCommon(pReNative, off, a_pu32Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint32_t), \
+                                    IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1 /*fAlignMask*/, \
+                                    (uintptr_t)iemNativeHlpMemFlatMapDataU32Atomic, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu32Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint32_t), \
-                                    IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE, sizeof(uint32_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataU32Rw, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu32Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint32_t), \
-                                    IEM_ACCESS_TYPE_WRITE, sizeof(uint32_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataU32Wo, pCallEntry->idxInstr) \
 
 #define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu32Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint32_t), \
-                                    IEM_ACCESS_TYPE_READ,  sizeof(uint32_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_R,  sizeof(uint32_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataU32Ro, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_FLAT_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pi32Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(int32_t), \
-                                    IEM_ACCESS_TYPE_WRITE, sizeof(uint32_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataU32Wo, pCallEntry->idxInstr) \
 
+
+#define IEM_MC_MEM_FLAT_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
+    off = iemNativeEmitMemMapCommon(pReNative, off, a_pu64Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint64_t), \
+                                    IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1 /*fAlignMask*/, \
+                                    (uintptr_t)iemNativeHlpMemFlatMapDataU64Atomic, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu64Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint64_t), \
-                                    IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE, sizeof(uint64_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataU64Rw, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu64Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint64_t), \
-                                    IEM_ACCESS_TYPE_WRITE, sizeof(uint64_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataU64Wo, pCallEntry->idxInstr) \
 
 #define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu64Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(uint64_t), \
-                                    IEM_ACCESS_TYPE_READ,  sizeof(uint64_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_R,  sizeof(uint64_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataU64Ro, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_FLAT_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pi64Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(int64_t), \
-                                    IEM_ACCESS_TYPE_WRITE, sizeof(uint64_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataU64Wo, pCallEntry->idxInstr) \
 
@@ -12091 +12289 @@
 #define IEM_MC_MEM_FLAT_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pr80Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(RTFLOAT80U), \
-                                    IEM_ACCESS_TYPE_WRITE, sizeof(uint64_t) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataR80Wo, pCallEntry->idxInstr) \
 
 #define IEM_MC_MEM_FLAT_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pd80Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(RTFLOAT80U), \
-                                    IEM_ACCESS_TYPE_WRITE, sizeof(uint64_t) - 1 /*fAlignMask*/, /** @todo check BCD align */ \
+                                    IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1 /*fAlignMask*/, /** @todo check BCD align */ \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataD80Wo, pCallEntry->idxInstr) \
 
+
+#define IEM_MC_MEM_FLAT_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
+    off = iemNativeEmitMemMapCommon(pReNative, off, a_pu128Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(RTUINT128U), \
+                                    IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128U) - 1 /*fAlignMask*/, \
+                                    (uintptr_t)iemNativeHlpMemFlatMapDataU128Atomic, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_FLAT_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu128Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(RTUINT128U), \
-                                    IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE, sizeof(RTUINT128U) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_RW, sizeof(RTUINT128U) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataU128Rw, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_FLAT_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu128Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(RTUINT128U), \
-                                    IEM_ACCESS_TYPE_WRITE, sizeof(RTUINT128U) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_W, sizeof(RTUINT128U) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataU128Wo, pCallEntry->idxInstr) \
 
 #define IEM_MC_MEM_FLAT_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
     off = iemNativeEmitMemMapCommon(pReNative, off, a_pu128Mem, a_bUnmapInfo, UINT8_MAX, a_GCPtrMem, sizeof(RTUINT128U), \
-                                    IEM_ACCESS_TYPE_READ,  sizeof(RTUINT128U) - 1 /*fAlignMask*/, \
+                                    IEM_ACCESS_DATA_R,  sizeof(RTUINT128U) - 1 /*fAlignMask*/, \
                                     (uintptr_t)iemNativeHlpMemFlatMapDataU128Ro, pCallEntry->idxInstr)
 
@@ -12144 +12347 @@
 
 #ifdef VBOX_STRICT
+# define IEM_MAP_HLP_FN_NO_AT(a_fAccess, a_fnBase) \
+        (  ((a_fAccess) & (IEM_ACCESS_TYPE_MASK | IEM_ACCESS_ATOMIC)) == (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_TYPE_READ) \
+         ? (uintptr_t)RT_CONCAT(a_fnBase,Rw) \
+         : ((a_fAccess) & (IEM_ACCESS_TYPE_MASK | IEM_ACCESS_ATOMIC)) == IEM_ACCESS_TYPE_READ \
+         ? (uintptr_t)RT_CONCAT(a_fnBase,Ro) : (uintptr_t)RT_CONCAT(a_fnBase,Wo) )
 # define IEM_MAP_HLP_FN(a_fAccess, a_fnBase) \
-        (  ((a_fAccess) & IEM_ACCESS_TYPE_MASK) == (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_TYPE_READ) \
-         ? (uintptr_t)RT_CONCAT(a_fnBase,Rw) \
-         : ((a_fAccess) & IEM_ACCESS_TYPE_MASK) == IEM_ACCESS_TYPE_READ \
-         ? (uintptr_t)RT_CONCAT(a_fnBase,Ro) : (uintptr_t)RT_CONCAT(a_fnBase,Wo) )
+        (  ((a_fAccess) & (IEM_ACCESS_TYPE_MASK | IEM_ACCESS_ATOMIC)) == (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_TYPE_READ | IEM_ACCESS_ATOMIC) \
+         ? (uintptr_t)RT_CONCAT(a_fnBase,Atomic) \
+         : IEM_MAP_HLP_FN_NO_AT(a_fAccess, a_fnBase) )
 
     if (iSegReg == UINT8_MAX)
@@ -12168 +12375 @@
             case 16: Assert(pfnFunction == IEM_MAP_HLP_FN(fAccess, iemNativeHlpMemFlatMapDataU128)); break;
 # if 0
-            case 32: Assert(pfnFunction == IEM_MAP_HLP_FN(fAccess, iemNativeHlpMemFlatMapDataU256)); break;
-            case 64: Assert(pfnFunction == IEM_MAP_HLP_FN(fAccess, iemNativeHlpMemFlatMapDataU512)); break;
+            case 32: Assert(pfnFunction == IEM_MAP_HLP_FN_NO_AT(fAccess, iemNativeHlpMemFlatMapDataU256)); break;
+            case 64: Assert(pfnFunction == IEM_MAP_HLP_FN_NO_AT(fAccess, iemNativeHlpMemFlatMapDataU512)); break;
 # endif
             default: AssertFailed(); break;
@@ -12190 +12397 @@
             case 16: Assert(pfnFunction == IEM_MAP_HLP_FN(fAccess, iemNativeHlpMemMapDataU128)); break;
 # if 0
-            case 32: Assert(pfnFunction == IEM_MAP_HLP_FN(fAccess, iemNativeHlpMemMapDataU256)); break;
-            case 64: Assert(pfnFunction == IEM_MAP_HLP_FN(fAccess, iemNativeHlpMemMapDataU512)); break;
+            case 32: Assert(pfnFunction == IEM_MAP_HLP_FN_NO_AT(fAccess, iemNativeHlpMemMapDataU256)); break;
+            case 64: Assert(pfnFunction == IEM_MAP_HLP_FN_NO_AT(fAccess, iemNativeHlpMemMapDataU512)); break;
 # endif
             default: AssertFailed(); break;
@@ -12197 +12404 @@
     }
 # undef IEM_MAP_HLP_FN
+# undef IEM_MAP_HLP_FN_NO_AT
 #endif
 
@@ -12353 +12561 @@
 
 
+#define IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(a_bMapInfo) \
+    off = iemNativeEmitMemCommitAndUnmap(pReNative, off, (a_bMapInfo), IEM_ACCESS_DATA_ATOMIC, \
+                                         (uintptr_t)iemNativeHlpMemCommitAndUnmapAtomic, pCallEntry->idxInstr)
+
 #define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo) \
-    off = iemNativeEmitMemCommitAndUnmap(pReNative, off, (a_bMapInfo), IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE, \
+    off = iemNativeEmitMemCommitAndUnmap(pReNative, off, (a_bMapInfo), IEM_ACCESS_DATA_RW, \
                                          (uintptr_t)iemNativeHlpMemCommitAndUnmapRw, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_bMapInfo) \
-    off = iemNativeEmitMemCommitAndUnmap(pReNative, off, (a_bMapInfo), IEM_ACCESS_TYPE_WRITE, \
+    off = iemNativeEmitMemCommitAndUnmap(pReNative, off, (a_bMapInfo), IEM_ACCESS_DATA_W, \
                                          (uintptr_t)iemNativeHlpMemCommitAndUnmapWo, pCallEntry->idxInstr)
 
 #define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_bMapInfo) \
-    off = iemNativeEmitMemCommitAndUnmap(pReNative, off, (a_bMapInfo), IEM_ACCESS_TYPE_READ, \
+    off = iemNativeEmitMemCommitAndUnmap(pReNative, off, (a_bMapInfo), IEM_ACCESS_DATA_R, \
                                          (uintptr_t)iemNativeHlpMemCommitAndUnmapRo, pCallEntry->idxInstr)
 
@@ -12377 +12589 @@
            || pReNative->Core.aVars[idxVarUnmapInfo].idxStackSlot < IEMNATIVE_FRAME_VAR_SLOTS); /* must be initialized */
 #ifdef VBOX_STRICT
-    switch (fAccess & IEM_ACCESS_TYPE_MASK)
-    {
-        case IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE: Assert(pfnFunction == (uintptr_t)iemNativeHlpMemCommitAndUnmapRw); break;
-        case IEM_ACCESS_TYPE_WRITE:                        Assert(pfnFunction == (uintptr_t)iemNativeHlpMemCommitAndUnmapWo); break;
-        case IEM_ACCESS_TYPE_READ:                         Assert(pfnFunction == (uintptr_t)iemNativeHlpMemCommitAndUnmapRo); break;
+    switch (fAccess & (IEM_ACCESS_TYPE_MASK | IEM_ACCESS_ATOMIC))
+    {
+        case IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_ATOMIC:
+            Assert(pfnFunction == (uintptr_t)iemNativeHlpMemCommitAndUnmapAtomic); break;
+        case IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE:
+            Assert(pfnFunction == (uintptr_t)iemNativeHlpMemCommitAndUnmapRw); break;
+        case IEM_ACCESS_TYPE_WRITE:
+            Assert(pfnFunction == (uintptr_t)iemNativeHlpMemCommitAndUnmapWo); break;
+        case IEM_ACCESS_TYPE_READ:
+            Assert(pfnFunction == (uintptr_t)iemNativeHlpMemCommitAndUnmapRo); break;
         default: AssertFailed();
     }

trunk/src/VBox/VMM/VMMAll/IEMAllThrdPython.py

@@ -882 +882 @@
         'IEM_MC_MEM_MAP_R64_WO':                  (  2, 'IEM_MC_MEM_FLAT_MAP_R64_WO' ),
         'IEM_MC_MEM_MAP_R80_WO':                  (  2, 'IEM_MC_MEM_FLAT_MAP_R80_WO' ),
+        'IEM_MC_MEM_MAP_U8_ATOMIC':               (  2, 'IEM_MC_MEM_FLAT_MAP_U8_ATOMIC' ),
         'IEM_MC_MEM_MAP_U8_RW':                   (  2, 'IEM_MC_MEM_FLAT_MAP_U8_RW' ),
         'IEM_MC_MEM_MAP_U8_RO':                   (  2, 'IEM_MC_MEM_FLAT_MAP_U8_RO' ),
         'IEM_MC_MEM_MAP_U8_WO':                   (  2, 'IEM_MC_MEM_FLAT_MAP_U8_WO' ),
+        'IEM_MC_MEM_MAP_U16_ATOMIC':              (  2, 'IEM_MC_MEM_FLAT_MAP_U16_ATOMIC' ),
         'IEM_MC_MEM_MAP_U16_RW':                  (  2, 'IEM_MC_MEM_FLAT_MAP_U16_RW' ),
         'IEM_MC_MEM_MAP_U16_RO':                  (  2, 'IEM_MC_MEM_FLAT_MAP_U16_RO' ),
         'IEM_MC_MEM_MAP_U16_WO':                  (  2, 'IEM_MC_MEM_FLAT_MAP_U16_WO' ),
+        'IEM_MC_MEM_MAP_U32_ATOMIC':              (  2, 'IEM_MC_MEM_FLAT_MAP_U32_ATOMIC' ),
         'IEM_MC_MEM_MAP_U32_RW':                  (  2, 'IEM_MC_MEM_FLAT_MAP_U32_RW' ),
         'IEM_MC_MEM_MAP_U32_RO':                  (  2, 'IEM_MC_MEM_FLAT_MAP_U32_RO' ),
         'IEM_MC_MEM_MAP_U32_WO':                  (  2, 'IEM_MC_MEM_FLAT_MAP_U32_WO' ),
+        'IEM_MC_MEM_MAP_U64_ATOMIC':              (  2, 'IEM_MC_MEM_FLAT_MAP_U64_ATOMIC' ),
         'IEM_MC_MEM_MAP_U64_RW':                  (  2, 'IEM_MC_MEM_FLAT_MAP_U64_RW' ),
         'IEM_MC_MEM_MAP_U64_RO':                  (  2, 'IEM_MC_MEM_FLAT_MAP_U64_RO' ),
         'IEM_MC_MEM_MAP_U64_WO':                  (  2, 'IEM_MC_MEM_FLAT_MAP_U64_WO' ),
+        'IEM_MC_MEM_MAP_U128_ATOMIC':             (  2, 'IEM_MC_MEM_FLAT_MAP_U128_ATOMIC' ),
         'IEM_MC_MEM_MAP_U128_RW':                 (  2, 'IEM_MC_MEM_FLAT_MAP_U128_RW' ),
         'IEM_MC_MEM_MAP_U128_RO':                 (  2, 'IEM_MC_MEM_FLAT_MAP_U128_RO' ),

trunk/src/VBox/VMM/VMMAll/IEMAllThrdRecompiler.cpp

@@ -2665 +2665 @@
      * Init the execution environment.
      */
-#ifdef RT_ARCH_ARM64 /** @todo ARM64: fix unaligned locked instructions properly. @bugref{10547} */
+#if 1 /** @todo this seems like a good idea, however if we ever share memory
+       * directly with other threads on the host, it isn't necessarily... */
     if (pVM->cCpus == 1)
         iemInitExec(pVCpu, IEM_F_X86_DISREGARD_LOCK /*fExecOpts*/);

trunk/src/VBox/VMM/VMMR3/EM.cpp

@@ -706 +706 @@
 #endif /* LOG_ENABLED || VBOX_STRICT */
 
-
 #if !defined(VBOX_VMM_TARGET_ARMV8)
+
 /**
  * Handle pending ring-3 I/O port write.
@@ -851 +851 @@
     return VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE, emR3ExecuteSplitLockInstructionRendezvous, pVCpu);
 }
+
 #endif /* VBOX_VMM_TARGET_ARMV8 */
-
 
 /**
@@ -893 +893 @@
                         rc = VINF_EM_DBG_STEPPED;
                 }
+#ifndef VBOX_VMM_TARGET_ARMV8
+                if (rc != VINF_EM_EMULATE_SPLIT_LOCK)
+                { /* likely */ }
+                else
+                {
+                    rc = emR3ExecuteSplitLockInstruction(pVM, pVCpu);
+                    if (rc == VINF_SUCCESS || rc == VINF_EM_RESCHEDULE)
+                        rc = VINF_EM_DBG_STEPPED;
+                }
+#endif
                 break;
 
@@ -1116 +1126 @@
         if (rcStrict != VINF_SUCCESS)
         {
+#ifndef VBOX_VMM_TARGET_ARMV8
+            if (rcStrict == VINF_EM_EMULATE_SPLIT_LOCK)
+                rcStrict = emR3ExecuteSplitLockInstruction(pVM, pVCpu);
+#endif
+            if (rcStrict != VINF_SUCCESS)
+            {
 #if 0
-            if (RT_LIKELY(rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST))
+                if (RT_LIKELY(rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST))
+                    break;
+                /* Fatal error: */
+#endif
                 break;
-            /* Fatal error: */
-#endif
-            break;
+            }
         }
 
@@ -2556 +2573 @@
                     else if (rc == VINF_SUCCESS)
                         rc = VINF_EM_RESCHEDULE; /* Need to check whether we can run in HM or NEM again. */
+#ifndef VBOX_VMM_TARGET_ARMV8
+                    if (rc != VINF_EM_EMULATE_SPLIT_LOCK)
+                    { /* likely */ }
+                    else
+                        rc = VBOXSTRICTRC_TODO(emR3ExecuteSplitLockInstruction(pVM, pVCpu));
+#endif
                     fFFDone = false;
                     break;

trunk/src/VBox/VMM/VMMR3/IEMR3.cpp

@@ -291 +291 @@
         STAMR3RegisterF(pVM, &pVCpu->iem.s.cPendingCommit,              STAMTYPE_U32,       STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
                         "Times RC/R0 had to postpone instruction committing to ring-3", "/IEM/CPU%u/cPendingCommit", idCpu);
+        STAMR3RegisterF(pVM, &pVCpu->iem.s.cMisalignedAtomics,          STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,
+                        "Number of misaligned (for the host) atomic instructions", "/IEM/CPU%u/cMisalignedAtomics", idCpu);
 
         STAMR3RegisterF(pVM, &pVCpu->iem.s.CodeTlb.cTlbMisses,          STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,

trunk/src/VBox/VMM/include/IEMInline.h

@@ -56 +56 @@
             )
         {
-            if (pVCpu->iem.s.rcPassUp == VINF_SUCCESS)
-                rcStrict = VINF_SUCCESS;
+            rcStrict = pVCpu->iem.s.rcPassUp;
+            if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+            { /* likely */ }
             else
-            {
                 pVCpu->iem.s.cRetPassUpStatus++;
-                rcStrict = pVCpu->iem.s.rcPassUp;
-            }
         }
         else if (RT_SUCCESS(rcStrict))
@@ -79 +77 @@
                       || rcStrict == VINF_EM_RAW_TO_R3
                       || rcStrict == VINF_EM_TRIPLE_FAULT
+                      || rcStrict == VINF_EM_EMULATE_SPLIT_LOCK
                       || rcStrict == VINF_GIM_R3_HYPERCALL
                       /* raw-mode / virt handlers only: */
@@ -117 +116 @@
             pVCpu->iem.s.cRetErrStatuses++;
     }
-    else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
-    {
-        pVCpu->iem.s.cRetPassUpStatus++;
+    else
+    {
         rcStrict = pVCpu->iem.s.rcPassUp;
+        if (rcStrict != VINF_SUCCESS)
+            pVCpu->iem.s.cRetPassUpStatus++;
     }
 
@@ -3936 +3936 @@
 
 
+DECL_INLINE_THROW(void) iemMemCommitAndUnmapAtJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP
+{
+# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
+    if (RT_LIKELY(bMapInfo == 0))
+        return;
+# endif
+    iemMemCommitAndUnmapAtSafeJmp(pVCpu, bMapInfo);
+}
+
+
 DECL_INLINE_THROW(void) iemMemCommitAndUnmapWoJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP
 {
@@ -3991 +4001 @@
 #endif
 
+#define TMPL_MEM_WITH_ATOMIC_MAPPING
+
 #define TMPL_MEM_TYPE       uint8_t
 #define TMPL_MEM_TYPE_ALIGN 0
@@ -4028 +4040 @@
 
 #undef TMPL_MEM_WITH_STACK
+#undef TMPL_MEM_WITH_ATOMIC_MAPPING
 
 #define TMPL_MEM_NO_STORE
@@ -4058 +4071 @@
 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
 
+#define TMPL_MEM_WITH_ATOMIC_MAPPING
 #define TMPL_MEM_TYPE       RTUINT128U
 #define TMPL_MEM_TYPE_ALIGN 15
@@ -4065 +4079 @@
 #define TMPL_MEM_FMT_DESC   "dqword"
 #include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
+#undef  TMPL_MEM_WITH_ATOMIC_MAPPING
 
 #undef TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK

trunk/src/VBox/VMM/include/IEMInternal.h

@@ -1604 +1604 @@
     /** Number of times RZ left with instruction commit pending for ring-3. */
     uint32_t                cPendingCommit;
+    /** Number of misaligned (host sense) atomic instruction accesses. */
+    uint32_t                cMisalignedAtomics;
     /** Number of long jumps. */
     uint32_t                cLongJumps;
@@ -1641 +1643 @@
     uint8_t                 cLogRelWrMsr;
     /** Alignment padding. */
-    uint8_t                 abAlignment9[46];
+    uint8_t                 abAlignment9[42];
 
     /** @name Recompilation
@@ -1989 +1991 @@
 /** Not locked, accessed via the TLB. */
 #define IEM_ACCESS_NOT_LOCKED           UINT32_C(0x00001000)
+/** Atomic access.
+ * This enables special alignment checks and the VINF_EM_EMULATE_SPLIT_LOCK
+ * fallback for misaligned stuff. See @bugref{10547}. */
+#define IEM_ACCESS_ATOMIC               UINT32_C(0x00002000)
 /** Valid bit mask. */
-#define IEM_ACCESS_VALID_MASK           UINT32_C(0x00001fff)
+#define IEM_ACCESS_VALID_MASK           UINT32_C(0x00003fff)
 /** Shift count for the TLB flags (upper word). */
 #define IEM_ACCESS_SHIFT_TLB_FLAGS      16
 
+/** Atomic read+write data alias. */
+#define IEM_ACCESS_DATA_ATOMIC          (IEM_ACCESS_TYPE_READ  | IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_DATA | IEM_ACCESS_ATOMIC)
 /** Read+write data alias. */
 #define IEM_ACCESS_DATA_RW              (IEM_ACCESS_TYPE_READ  | IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_WHAT_DATA)
@@ -5201 +5209 @@
 #ifdef IEM_WITH_SETJMP
 uint8_t        *iemMemMapDataU8RwSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
+uint8_t        *iemMemMapDataU8AtSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
 uint8_t        *iemMemMapDataU8WoSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
 uint8_t const  *iemMemMapDataU8RoSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
 uint16_t       *iemMemMapDataU16RwSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
+uint16_t       *iemMemMapDataU16AtSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
 uint16_t       *iemMemMapDataU16WoSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
 uint16_t const *iemMemMapDataU16RoSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
 uint32_t       *iemMemMapDataU32RwSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
+uint32_t       *iemMemMapDataU32AtSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
 uint32_t       *iemMemMapDataU32WoSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
 uint32_t const *iemMemMapDataU32RoSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
 uint64_t       *iemMemMapDataU64RwSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
+uint64_t       *iemMemMapDataU64AtSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
 uint64_t       *iemMemMapDataU64WoSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
 uint64_t const *iemMemMapDataU64RoSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
@@ -5219 +5231 @@
 PCRTPBCD80U     iemMemMapDataD80RoSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
 PRTUINT128U     iemMemMapDataU128RwSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
+PRTUINT128U     iemMemMapDataU128AtSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
 PRTUINT128U     iemMemMapDataU128WoSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
 PCRTUINT128U    iemMemMapDataU128RoSafeJmp(PVMCPUCC pVCpu, uint8_t *pbUnmapInfo, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP;
@@ -5224 +5237 @@
 void            iemMemCommitAndUnmapJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
 void            iemMemCommitAndUnmapRwSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
+void            iemMemCommitAndUnmapAtSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
 void            iemMemCommitAndUnmapWoSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;
 void            iemMemCommitAndUnmapRoSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP;

trunk/src/VBox/VMM/include/IEMMc.h

@@ -1508 +1508 @@
 
 /**
+ * Maps guest memory for byte atomic read+write direct (or bounce) buffer
+ * acccess, for atomic operations.
+ *
+ * @param[out] a_pu8Mem     Where to return the pointer to the mapping.
+ * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
+ * @param[in]  a_iSeg       The segment register to access via. No UINT8_MAX!
+ * @param[in]  a_GCPtrMem   The memory address.
+ * @remarks Will return/long jump on errors.
+ * @see     IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_MEM_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
+                                       (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, 0))
+#else
+# define IEM_MC_MEM_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
+    (a_pu8Mem) = iemMemMapDataU8AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
+#endif
+
+/**
  * Maps guest memory for byte read+write direct (or bounce) buffer acccess.
  *
@@ -1565 +1585 @@
 
 /**
+ * Maps guest memory for byte atomic read+write direct (or bounce) buffer
+ * acccess, flat address variant.
+ *
+ * @param[out] a_pu8Mem     Where to return the pointer to the mapping.
+ * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
+ * @param[in]  a_GCPtrMem   The memory address.
+ * @remarks Will return/long jump on errors.
+ * @see     IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_MEM_FLAT_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
+                                       (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, 0))
+#else
+# define IEM_MC_MEM_FLAT_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
+    (a_pu8Mem) = iemMemFlatMapDataU8AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
+#endif
+
+/**
  * Maps guest memory for byte read+write direct (or bounce) buffer acccess, flat
  * address variant.
@@ -1625 +1664 @@
 
 /**
+ * Maps guest memory for word atomic read+write direct (or bounce) buffer acccess.
+ *
+ * @param[out] a_pu16Mem    Where to return the pointer to the mapping.
+ * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
+ * @param[in]  a_iSeg       The segment register to access via. No UINT8_MAX!
+ * @param[in]  a_GCPtrMem   The memory address.
+ * @remarks Will return/long jump on errors.
+ * @see     IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_MEM_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
+                                       (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint16_t) - 1))
+#else
+# define IEM_MC_MEM_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
+    (a_pu16Mem) = iemMemMapDataU16AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
+#endif
+
+/**
  * Maps guest memory for word read+write direct (or bounce) buffer acccess.
  *
@@ -1682 +1740 @@
 
 /**
+ * Maps guest memory for word atomic read+write direct (or bounce) buffer
+ * acccess, flat address variant.
+ *
+ * @param[out] a_pu16Mem    Where to return the pointer to the mapping.
+ * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
+ * @param[in]  a_GCPtrMem   The memory address.
+ * @remarks Will return/long jump on errors.
+ * @see     IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_MEM_FLAT_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
+                                       (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint16_t) - 1))
+#else
+# define IEM_MC_MEM_FLAT_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
+    (a_pu16Mem) = iemMemFlatMapDataU16AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
+#endif
+
+/**
  * Maps guest memory for word read+write direct (or bounce) buffer acccess, flat
  * address variant.
@@ -1760 +1837 @@
 
 /**
+ * Maps guest memory for dword atomic read+write direct (or bounce) buffer acccess.
+ *
+ * @param[out] a_pu32Mem    Where to return the pointer to the mapping.
+ * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
+ * @param[in]  a_iSeg       The segment register to access via. No UINT8_MAX!
+ * @param[in]  a_GCPtrMem   The memory address.
+ * @remarks Will return/long jump on errors.
+ * @see     IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_MEM_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
+                                       (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1))
+#else
+# define IEM_MC_MEM_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
+    (a_pu32Mem) = iemMemMapDataU32AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
+#endif
+
+/**
  * Maps guest memory for dword read+write direct (or bounce) buffer acccess.
  *
@@ -1817 +1913 @@
 
 /**
+ * Maps guest memory for dword atomic read+write direct (or bounce) buffer
+ * acccess, flat address variant.
+ *
+ * @param[out] a_pu32Mem    Where to return the pointer to the mapping.
+ * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
+ * @param[in]  a_GCPtrMem   The memory address.
+ * @remarks Will return/long jump on errors.
+ * @see     IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_MEM_FLAT_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
+                                       (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1))
+#else
+# define IEM_MC_MEM_FLAT_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
+    (a_pu32Mem) = iemMemFlatMapDataU32AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
+#endif
+
+/**
  * Maps guest memory for dword read+write direct (or bounce) buffer acccess,
  * flat address variant.
@@ -1913 +2028 @@
 
 /**
+ * Maps guest memory for qword atomic read+write direct (or bounce) buffer acccess.
+ *
+ * @param[out] a_pu64Mem    Where to return the pointer to the mapping.
+ * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
+ * @param[in]  a_iSeg       The segment register to access via. No UINT8_MAX!
+ * @param[in]  a_GCPtrMem   The memory address.
+ * @remarks Will return/long jump on errors.
+ * @see     IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_MEM_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
+                                       (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1))
+#else
+# define IEM_MC_MEM_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
+    (a_pu64Mem) = iemMemMapDataU64AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
+#endif
+
+/**
  * Maps guest memory for qword read+write direct (or bounce) buffer acccess.
  *
@@ -1970 +2104 @@
 
 /**
+ * Maps guest memory for qword atomic read+write direct (or bounce) buffer
+ * acccess, flat address variant.
+ *
+ * @param[out] a_pu64Mem    Where to return the pointer to the mapping.
+ * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
+ * @param[in]  a_GCPtrMem   The memory address.
+ * @remarks Will return/long jump on errors.
+ * @see     IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_MEM_FLAT_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
+                                       (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1))
+#else
+# define IEM_MC_MEM_FLAT_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
+    (a_pu64Mem) = iemMemFlatMapDataU64AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
+#endif
+
+/**
  * Maps guest memory for qword read+write direct (or bounce) buffer acccess,
  * flat address variant.
@@ -2066 +2219 @@
 
 /**
+ * Maps guest memory for dqword atomic read+write direct (or bounce) buffer acccess.
+ *
+ * @param[out] a_pu128Mem   Where to return the pointer to the mapping.
+ * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
+ * @param[in]  a_iSeg       The segment register to access via. No UINT8_MAX!
+ * @param[in]  a_GCPtrMem   The memory address.
+ * @remarks Will return/long jump on errors.
+ * @see     IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_MEM_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128U), (a_iSeg), \
+                                       (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128U) - 1))
+#else
+# define IEM_MC_MEM_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
+    (a_pu128Mem) = iemMemMapDataU128AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
+#endif
+
+/**
  * Maps guest memory for dqword read+write direct (or bounce) buffer acccess.
  *
@@ -2123 +2295 @@
 
 /**
+ * Maps guest memory for dqword atomic read+write direct (or bounce) buffer
+ * access, flat address variant.
+ *
+ * @param[out] a_pu128Mem   Where to return the pointer to the mapping.
+ * @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
+ * @param[in]  a_GCPtrMem   The memory address.
+ * @remarks Will return/long jump on errors.
+ * @see     IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_MEM_FLAT_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
+    IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
+                                       (a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128) - 1))
+#else
+# define IEM_MC_MEM_FLAT_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
+    (a_pu128Mem) = iemMemFlatMapDataU128AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
+#endif
+
+/**
  * Maps guest memory for dqword read+write direct (or bounce) buffer acccess,
  * flat address variant.
@@ -2269 +2460 @@
 #else
 # define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo)         iemMemCommitAndUnmapRwJmp(pVCpu, (a_bMapInfo))
+#endif
+
+/** Commits the memory and unmaps guest memory previously mapped ATOMIC.
+ * @remarks     May return.
+ * @note        Implictly frees the a_bMapInfo variable.
+ */
+#ifndef IEM_WITH_SETJMP
+# define IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(a_bMapInfo)     IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
+#else
+# define IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(a_bMapInfo)     iemMemCommitAndUnmapRwJmp(pVCpu, (a_bMapInfo))
 #endif
 

trunk/src/VBox/VMM/include/IEMN8veRecompiler.h

@@ -44 +44 @@
 #endif
 
+#ifdef VBOX_WITH_STATISTICS
 /** Always count instructions for now. */
-#define IEMNATIVE_WITH_INSTRUCTION_COUNTING
+# define IEMNATIVE_WITH_INSTRUCTION_COUNTING
+#endif
 
 

trunk/src/VBox/VMM/testcase/tstIEMCheckMc.cpp

@@ -931 +931 @@
 #define IEM_MC_POP_GREG_U64(a_iGReg)                                    do { CHK_GREG_IDX(a_iGReg); (void)fMcBegin; } while (0)
 
-#define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { CHK_VAR(a_pd80Mem); (a_pd80Mem) = NULL; CHK_PTYPE(RTPBCD80U *,      a_pd80Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { CHK_VAR(a_pi16Mem); (a_pi16Mem) = NULL; CHK_PTYPE(int16_t *,        a_pi16Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { CHK_VAR(a_pi32Mem); (a_pi32Mem) = NULL; CHK_PTYPE(int32_t *,        a_pi32Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { CHK_VAR(a_pi64Mem); (a_pi64Mem) = NULL; CHK_PTYPE(int64_t *,        a_pi64Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { CHK_VAR(a_pr32Mem); (a_pr32Mem) = NULL; CHK_PTYPE(RTFLOAT32U *,     a_pr32Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { CHK_VAR(a_pr64Mem); (a_pr64Mem) = NULL; CHK_PTYPE(RTFLOAT64U *,     a_pr64Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { CHK_VAR(a_pr80Mem); (a_pr80Mem) = NULL; CHK_PTYPE(RTFLOAT80U *,     a_pr80Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)   do { CHK_VAR(a_pu8Mem);  (a_pu8Mem)  = NULL; CHK_PTYPE(uint8_t *,        a_pu8Mem);  CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)   do { CHK_VAR(a_pu8Mem);  (a_pu8Mem)  = NULL; CHK_PTYPE(uint8_t const *,  a_pu8Mem);  CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)   do { CHK_VAR(a_pu8Mem);  (a_pu8Mem)  = NULL; CHK_PTYPE(uint8_t *,        a_pu8Mem);  CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { CHK_VAR(a_pu16Mem); (a_pu16Mem) = NULL; CHK_PTYPE(uint16_t *,       a_pu16Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { CHK_VAR(a_pu16Mem); (a_pu16Mem) = NULL; CHK_PTYPE(uint16_t const *, a_pu16Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { CHK_VAR(a_pu16Mem); (a_pu16Mem) = NULL; CHK_PTYPE(uint16_t *,       a_pu16Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { CHK_VAR(a_pu32Mem); (a_pu32Mem) = NULL; CHK_PTYPE(uint32_t *,       a_pu32Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { CHK_VAR(a_pu32Mem); (a_pu32Mem) = NULL; CHK_PTYPE(uint32_t const *, a_pu32Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { CHK_VAR(a_pu32Mem); (a_pu32Mem) = NULL; CHK_PTYPE(uint32_t *,       a_pu32Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { CHK_VAR(a_pu64Mem); (a_pu64Mem) = NULL; CHK_PTYPE(uint64_t *,       a_pu64Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { CHK_VAR(a_pu64Mem); (a_pu64Mem) = NULL; CHK_PTYPE(uint64_t const *, a_pu64Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { CHK_VAR(a_pu64Mem); (a_pu64Mem) = NULL; CHK_PTYPE(uint64_t *,       a_pu64Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { CHK_VAR(a_pu128Mem); (a_pu128Mem) = NULL; CHK_PTYPE(RTUINT128U *,       a_pu128Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { CHK_VAR(a_pu128Mem); (a_pu128Mem) = NULL; CHK_PTYPE(RTUINT128U const *, a_pu128Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-#define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { CHK_VAR(a_pu128Mem); (a_pu128Mem) = NULL; CHK_PTYPE(RTUINT128U *,       a_pu128Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
-
+#define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)      do { CHK_VAR(a_pd80Mem); (a_pd80Mem) = NULL; CHK_PTYPE(RTPBCD80U *,      a_pd80Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)      do { CHK_VAR(a_pi16Mem); (a_pi16Mem) = NULL; CHK_PTYPE(int16_t *,        a_pi16Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)      do { CHK_VAR(a_pi32Mem); (a_pi32Mem) = NULL; CHK_PTYPE(int32_t *,        a_pi32Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)      do { CHK_VAR(a_pi64Mem); (a_pi64Mem) = NULL; CHK_PTYPE(int64_t *,        a_pi64Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)      do { CHK_VAR(a_pr32Mem); (a_pr32Mem) = NULL; CHK_PTYPE(RTFLOAT32U *,     a_pr32Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)      do { CHK_VAR(a_pr64Mem); (a_pr64Mem) = NULL; CHK_PTYPE(RTFLOAT64U *,     a_pr64Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)      do { CHK_VAR(a_pr80Mem); (a_pr80Mem) = NULL; CHK_PTYPE(RTFLOAT80U *,     a_pr80Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)    do { CHK_VAR(a_pu8Mem);  (a_pu8Mem)  = NULL; CHK_PTYPE(uint8_t *,        a_pu8Mem);  CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)        do { CHK_VAR(a_pu8Mem);  (a_pu8Mem)  = NULL; CHK_PTYPE(uint8_t *,        a_pu8Mem);  CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)        do { CHK_VAR(a_pu8Mem);  (a_pu8Mem)  = NULL; CHK_PTYPE(uint8_t const *,  a_pu8Mem);  CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)        do { CHK_VAR(a_pu8Mem);  (a_pu8Mem)  = NULL; CHK_PTYPE(uint8_t *,        a_pu8Mem);  CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)  do { CHK_VAR(a_pu16Mem); (a_pu16Mem) = NULL; CHK_PTYPE(uint16_t *,       a_pu16Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)      do { CHK_VAR(a_pu16Mem); (a_pu16Mem) = NULL; CHK_PTYPE(uint16_t *,       a_pu16Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)      do { CHK_VAR(a_pu16Mem); (a_pu16Mem) = NULL; CHK_PTYPE(uint16_t const *, a_pu16Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)      do { CHK_VAR(a_pu16Mem); (a_pu16Mem) = NULL; CHK_PTYPE(uint16_t *,       a_pu16Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)  do { CHK_VAR(a_pu32Mem); (a_pu32Mem) = NULL; CHK_PTYPE(uint32_t *,       a_pu32Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)      do { CHK_VAR(a_pu32Mem); (a_pu32Mem) = NULL; CHK_PTYPE(uint32_t *,       a_pu32Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)      do { CHK_VAR(a_pu32Mem); (a_pu32Mem) = NULL; CHK_PTYPE(uint32_t const *, a_pu32Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)      do { CHK_VAR(a_pu32Mem); (a_pu32Mem) = NULL; CHK_PTYPE(uint32_t *,       a_pu32Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)  do { CHK_VAR(a_pu64Mem); (a_pu64Mem) = NULL; CHK_PTYPE(uint64_t *,       a_pu64Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)      do { CHK_VAR(a_pu64Mem); (a_pu64Mem) = NULL; CHK_PTYPE(uint64_t *,       a_pu64Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)      do { CHK_VAR(a_pu64Mem); (a_pu64Mem) = NULL; CHK_PTYPE(uint64_t const *, a_pu64Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)      do { CHK_VAR(a_pu64Mem); (a_pu64Mem) = NULL; CHK_PTYPE(uint64_t *,       a_pu64Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do{ CHK_VAR(a_pu128Mem); (a_pu128Mem) = NULL; CHK_PTYPE(RTUINT128U *,       a_pu128Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)    do { CHK_VAR(a_pu128Mem); (a_pu128Mem) = NULL; CHK_PTYPE(RTUINT128U *,       a_pu128Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)    do { CHK_VAR(a_pu128Mem); (a_pu128Mem) = NULL; CHK_PTYPE(RTUINT128U const *, a_pu128Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+#define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)    do { CHK_VAR(a_pu128Mem); (a_pu128Mem) = NULL; CHK_PTYPE(RTUINT128U *,       a_pu128Mem); CHK_VAR(a_bUnmapInfo); CHK_TYPE(uint8_t, a_bUnmapInfo); a_bUnmapInfo = 1; CHK_GCPTR(a_GCPtrMem); CHK_VAR(a_GCPtrMem); CHK_SEG_IDX(a_iSeg); (void)fMcBegin; } while (0)
+
+#define IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(a_bMapInfo)                      do { CHK_VAR(a_bMapInfo); CHK_TYPE(uint8_t, a_bMapInfo); (void)fMcBegin; } while (0)
 #define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo)                          do { CHK_VAR(a_bMapInfo); CHK_TYPE(uint8_t, a_bMapInfo); (void)fMcBegin; } while (0)
 #define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_bMapInfo)                          do { CHK_VAR(a_bMapInfo); CHK_TYPE(uint8_t, a_bMapInfo); (void)fMcBegin; } while (0)