Changeset 93837 in vbox
- Timestamp:
- Feb 18, 2022 11:14:59 AM (3 years ago)
- svn:sync-xref-src-repo-rev:
- 150043
- Location:
- trunk
- Files:
-
- 2 edited
-
include/iprt/asm.h (modified) (5 diffs)
-
src/VBox/Runtime/testcase/tstRTInlineAsm.cpp (modified) (9 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/include/iprt/asm.h
r93759 r93837 1430 1430 } 1431 1431 1432 #if defined(RT_ARCH_AMD64) || defined( DOXYGEN_RUNNING)1432 #if defined(RT_ARCH_AMD64) || defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING) 1433 1433 1434 1434 /** @def RTASM_HAVE_CMP_WRITE_U128 1435 * Indicates that we've got ASMAtomicCmpWriteU128() available. */ 1435 * Indicates that we've got ASMAtomicCmpWriteU128(), ASMAtomicCmpWriteU128v2() 1436 * and ASMAtomicCmpWriteExU128() available. */ 1436 1437 # define RTASM_HAVE_CMP_WRITE_U128 1 1437 1438 … … 1463 1464 ai64Cmp[1] = u64OldHi; 1464 1465 return _InterlockedCompareExchange128((__int64 volatile *)pu128, u64NewHi, u64NewLo, ai64Cmp) != 0; 1466 1467 # elif (defined(__clang_major__) || defined(__GNUC__)) && defined(RT_ARCH_ARM64) 1468 return __sync_bool_compare_and_swap(pu128, ((uint128_t)u64OldHi << 64) | u64OldLo, ((uint128_t)u64NewHi << 64) | u64NewLo); 1465 1469 1466 1470 # elif defined(RT_ARCH_AMD64) … … 1506 1510 { 1507 1511 # ifdef RT_COMPILER_WITH_128BIT_INT_TYPES 1512 # if (defined(__clang_major__) || defined(__GNUC__)) && defined(RT_ARCH_ARM64) 1513 return __sync_bool_compare_and_swap(pu128, u128Old, u128New); 1514 # else 1508 1515 return ASMAtomicCmpWriteU128v2(pu128, (uint64_t)(u128New >> 64), (uint64_t)u128New, 1509 1516 (uint64_t)(u128Old >> 64), (uint64_t)u128Old); 1517 # endif 1510 1518 # else 1511 1519 return ASMAtomicCmpWriteU128v2(pu128, u128New.Hi, u128New.Lo, u128Old.Hi, u128Old.Lo); … … 1520 1528 const RTUINT128U u128Old) RT_NOTHROW_DEF 1521 1529 { 1530 # if (defined(__clang_major__) || defined(__GNUC__)) && defined(RT_ARCH_ARM64) 1531 return ASMAtomicCmpWriteU128(&pu128->u, u128New.u, u128Old.u); 1532 # else 1522 1533 return ASMAtomicCmpWriteU128v2(&pu128->u, u128New.s.Hi, u128New.s.Lo, u128Old.s.Hi, u128Old.s.Lo); 1523 } 1524 1525 #endif /* RT_ARCH_AMD64 */ 1526 1534 # endif 1535 } 1536 1537 #endif /* RT_ARCH_AMD64 || RT_ARCH_ARM64 */ 1527 1538 1528 1539 /** … … 2180 2191 return ASMAtomicCmpXchgExU64((volatile uint64_t RT_FAR *)pi64, (uint64_t)i64, (uint64_t)i64Old, (uint64_t RT_FAR *)pi64Old); 2181 2192 } 2193 2194 #if defined(RT_ARCH_AMD64) || defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING) 2195 2196 /** @def RTASM_HAVE_CMP_XCHG_U128 2197 * Indicates that we've got ASMAtomicCmpSwapU128(), ASMAtomicCmpSwapU128v2() 2198 * and ASMAtomicCmpSwapExU128() available. */ 2199 # define RTASM_HAVE_CMP_XCHG_U128 1 2200 2201 2202 /** 2203 * Atomically compare and exchange an unsigned 128-bit value, ordered. 2204 * 2205 * @returns true if exchange was done. 2206 * @returns false if exchange wasn't done. 2207 * 2208 * @param pu128 Pointer to the 128-bit variable to update. 2209 * @param u64NewHi The high 64 bits of the value to assign to *pu128. 2210 * @param u64NewLo The low 64 bits of the value to assign to *pu128. 2211 * @param u64OldHi The high 64-bit of the value to compare with. 2212 * @param u64OldLo The low 64-bit of the value to compare with. 2213 * @param pu128Old Where to return the old value. 2214 * 2215 * @remarks AMD64: Not present in the earliest CPUs, so check CPUID. 2216 */ 2217 # if (RT_INLINE_ASM_EXTERNAL_TMP_ARM && !RT_INLINE_ASM_USES_INTRIN) 2218 DECLASM(bool) ASMAtomicCmpXchgU128v2(volatile uint128_t *pu128, const uint64_t u64NewHi, const uint64_t u64NewLo, 2219 const uint64_t u64OldHi, const uint64_t u64OldLo, uint128_t *pu128Old) RT_NOTHROW_PROTO; 2220 # else 2221 DECLINLINE(bool) ASMAtomicCmpXchgU128v2(volatile uint128_t *pu128, const uint64_t u64NewHi, const uint64_t u64NewLo, 2222 const uint64_t u64OldHi, const uint64_t u64OldLo, uint128_t *pu128Old) RT_NOTHROW_DEF 2223 { 2224 # if RT_INLINE_ASM_USES_INTRIN 2225 pu128Old->Hi = u64OldHi; 2226 pu128Old->Lo = u64OldLo; 2227 AssertCompileMemberOffset(uint128_t, Lo, 0); 2228 return _InterlockedCompareExchange128((__int64 volatile *)pu128, u64NewHi, u64NewLo, (__int64 *)&pu128Old->Lo) != 0; 2229 2230 # elif (defined(__clang_major__) || defined(__GNUC__)) && defined(RT_ARCH_ARM64) 2231 uint128_t const uCmp = ((uint128_t)u64OldHi << 64) | u64OldLo; 2232 uint128_t const uOld = __sync_val_compare_and_swap(pu128, uCmp, ((uint128_t)u64NewHi << 64) | u64NewLo); 2233 *pu128Old = uOld; 2234 return uCmp == uOld; 2235 2236 # elif defined(RT_ARCH_AMD64) 2237 # if RT_INLINE_ASM_GNU_STYLE 2238 uint8_t bRet; 2239 uint64_t u64RetHi, u64RetLo; 2240 __asm__ __volatile__("lock; cmpxchg16b %3\n\t" 2241 "setz %b0\n\t" 2242 : "=r" (bRet) 2243 , "=a" (u64RetLo) 2244 , "=d" (u64RetHi) 2245 , "+m" (*pu128) 2246 : "a" (u64OldLo) 2247 , "d" (u64OldHi) 2248 , "b" (u64NewLo) 2249 , "c" (u64NewHi) 2250 : "cc"); 2251 *pu128Old = ((uint128_t)u64RetHi << 64) | u64RetLo; 2252 return (bool)bRet; 2253 # else 2254 # error "Port me" 2255 # endif 2256 # else 2257 # error "Port me" 2258 # endif 2259 } 2260 # endif 2261 2262 2263 /** 2264 * Atomically compare and exchange an unsigned 128-bit value, ordered. 2265 * 2266 * @returns true if exchange was done. 2267 * @returns false if exchange wasn't done. 2268 * 2269 * @param pu128 Pointer to the 128-bit variable to update. 2270 * @param u128New The 128-bit value to assign to *pu128. 2271 * @param u128Old The value to compare with. 2272 * @param pu128Old Where to return the old value. 2273 * 2274 * @remarks AMD64: Not present in the earliest CPUs, so check CPUID. 2275 */ 2276 DECLINLINE(bool) ASMAtomicCmpXchgU128(volatile uint128_t *pu128, const uint128_t u128New, 2277 const uint128_t u128Old, uint128_t *pu128Old) RT_NOTHROW_DEF 2278 { 2279 # ifdef RT_COMPILER_WITH_128BIT_INT_TYPES 2280 # if (defined(__clang_major__) || defined(__GNUC__)) && defined(RT_ARCH_ARM64) 2281 uint128_t const uSwapped = __sync_val_compare_and_swap(pu128, u128Old, u128New); 2282 *pu128Old = uSwapped; 2283 return uSwapped == u128Old; 2284 # else 2285 return ASMAtomicCmpXchgU128v2(pu128, (uint64_t)(u128New >> 64), (uint64_t)u128New, 2286 (uint64_t)(u128Old >> 64), (uint64_t)u128Old, pu128Old); 2287 # endif 2288 # else 2289 return ASMAtomicCmpXchgU128v2(pu128, u128New.Hi, u128New.Lo, u128Old.Hi, u128Old.Lo, pu128Old); 2290 # endif 2291 } 2292 2293 2294 /** 2295 * RTUINT128U wrapper for ASMAtomicCmpXchgU128. 2296 */ 2297 DECLINLINE(bool) ASMAtomicCmpXchgU128U(volatile RTUINT128U *pu128, const RTUINT128U u128New, 2298 const RTUINT128U u128Old, PRTUINT128U pu128Old) RT_NOTHROW_DEF 2299 { 2300 # if (defined(__clang_major__) || defined(__GNUC__)) && defined(RT_ARCH_ARM64) 2301 return ASMAtomicCmpXchgU128(&pu128->u, u128New.u, u128Old.u, &pu128Old->u); 2302 # else 2303 return ASMAtomicCmpXchgU128v2(&pu128->u, u128New.s.Hi, u128New.s.Lo, u128Old.s.Hi, u128Old.s.Lo, &pu128Old->u); 2304 # endif 2305 } 2306 2307 #endif /* RT_ARCH_AMD64 || RT_ARCH_ARM64 */ 2308 2309 2182 2310 2183 2311 /** @def ASMAtomicCmpXchgExHandle -
trunk/src/VBox/Runtime/testcase/tstRTInlineAsm.cpp
r93755 r93837 1549 1549 u128B = RTUINT128_INIT_C(0x80040008008efd, 0x40080004004def)), 1550 1550 true, 0xfff40ff8f08ef3, 0x4ee8ee04cc4de4); 1551 1552 /* Make sure the v2 version works too (arm) */ 1553 CHECK_OP_AND_VAL_128_C(bool, "%d", pu128, ASMAtomicCmpWriteU128v2(&pu128->u, 1554 UINT64_C(0x95487930069587), UINT64_C(0x89958490385964), 1555 UINT64_C(0xfff40ff8f08ef3), UINT64_C(0x4ee8ee04cc4de4)), 1556 true, 0x95487930069587, 0x89958490385964); 1557 CHECK_OP_AND_VAL_128_C(bool, "%d", pu128, ASMAtomicCmpWriteU128v2(&pu128->u, 1558 UINT64_C(0x99969404869434), UINT64_C(0x11049309994567), 1559 UINT64_C(0x33f40ff8f08eff), UINT64_C(0x99e8ee04cc4dee)), 1560 false, 0x95487930069587, 0x89958490385964); 1551 1561 } 1552 1562 #endif /* RTASM_HAVE_CMP_WRITE_U128 */ … … 1698 1708 } 1699 1709 1710 DECLINLINE(void) tstASMAtomicCmpXchgU128Worker(RTUINT128U volatile *pu128) 1711 { 1712 pu128->s.Lo = UINT64_C(0xffffffffffffff); 1713 pu128->s.Hi = UINT64_C(0xffffffffffffff); 1714 1715 RTUINT128U u128A, u128B; 1716 RTUINT128U const u128OldInit = RTUINT128_INIT_C(0x4242424242424242, 0x2222222222222222); 1717 RTUINT128U u128Old = u128OldInit; 1718 CHECK_OP_AND_VAL_128_C(bool, "%d", pu128, ASMAtomicCmpXchgU128U(pu128, 1719 u128A = RTUINT128_INIT_C(0, 0), 1720 u128B = RTUINT128_INIT_C(0, 0), 1721 &u128Old), 1722 false, 0xffffffffffffff, 0xffffffffffffff); 1723 CHECKVAL128_C(&u128Old, 0xffffffffffffff, 0xffffffffffffff); 1724 1725 u128Old = u128OldInit; 1726 CHECK_OP_AND_VAL_128_C(bool, "%d", pu128, ASMAtomicCmpXchgU128U(pu128, 1727 u128A = RTUINT128_INIT_C(0, 0), 1728 u128B = RTUINT128_INIT_C(0xffffffffffffff, 0xffffffffffffff), 1729 &u128Old), 1730 true, 0, 0); 1731 CHECKVAL128_C(&u128Old, 0xffffffffffffff, 0xffffffffffffff); 1732 1733 u128Old = u128OldInit; 1734 CHECK_OP_AND_VAL_128_C(bool, "%d", pu128, ASMAtomicCmpXchgU128U(pu128, 1735 u128A = RTUINT128_INIT_C(0x80040008008efd, 0x40080004004def), 1736 u128B = RTUINT128_INIT_C(0, 1), 1737 &u128Old), 1738 false, 0, 0); 1739 CHECKVAL128_C(&u128Old, 0, 0); 1740 1741 u128Old = u128OldInit; 1742 CHECK_OP_AND_VAL_128_C(bool, "%d", pu128, ASMAtomicCmpXchgU128U(pu128, 1743 u128A = RTUINT128_INIT_C(0x80040008008efd, 0x40080004004def), 1744 u128B = RTUINT128_INIT_C(1, 0), 1745 &u128Old), 1746 false, 0, 0); 1747 CHECKVAL128_C(&u128Old, 0, 0); 1748 1749 u128Old = u128OldInit; 1750 CHECK_OP_AND_VAL_128_C(bool, "%d", pu128, ASMAtomicCmpXchgU128U(pu128, 1751 u128A = RTUINT128_INIT_C(0x80040008008efd, 0x40080004004def), 1752 u128B = RTUINT128_INIT_C(0, 0), 1753 &u128Old), 1754 true, 0x80040008008efd, 0x40080004004def); 1755 CHECKVAL128_C(&u128Old, 0, 0); 1756 1757 u128Old = u128OldInit; 1758 CHECK_OP_AND_VAL_128_C(bool, "%d", pu128, ASMAtomicCmpXchgU128U(pu128, 1759 u128A = RTUINT128_INIT_C(0xfff40ff8f08ef3, 0x4ee8ee04cc4de4), 1760 u128B = RTUINT128_INIT_C(0x80040008008efd, 0), 1761 &u128Old), 1762 false, 0x80040008008efd, 0x40080004004def); 1763 CHECKVAL128_C(&u128Old, 0x80040008008efd, 0x40080004004def); 1764 1765 u128Old = u128OldInit; 1766 CHECK_OP_AND_VAL_128_C(bool, "%d", pu128, ASMAtomicCmpXchgU128U(pu128, 1767 u128A = RTUINT128_INIT_C(0xfff40ff8f08ef3, 0x4ee8ee04cc4de4), 1768 u128B = RTUINT128_INIT_C(0, 0x40080004004def), 1769 &u128Old), 1770 false, 0x80040008008efd, 0x40080004004def); 1771 CHECKVAL128_C(&u128Old, 0x80040008008efd, 0x40080004004def); 1772 1773 u128Old = u128OldInit; 1774 CHECK_OP_AND_VAL_128_C(bool, "%d", pu128, ASMAtomicCmpXchgU128U(pu128, 1775 u128A = RTUINT128_INIT_C(0xfff40ff8f08ef3, 0x4ee8ee04cc4de4), 1776 u128B = RTUINT128_INIT_C(0x80040008008efd, 0x40080004004def), 1777 &u128Old), 1778 true, 0xfff40ff8f08ef3, 0x4ee8ee04cc4de4); 1779 CHECKVAL128_C(&u128Old, 0x80040008008efd, 0x40080004004def); 1780 1781 /* Make sure the v2 version works too (arm) */ 1782 u128Old = u128OldInit; 1783 CHECK_OP_AND_VAL_128_C(bool, "%d", pu128, ASMAtomicCmpXchgU128v2(&pu128->u, 1784 UINT64_C(0x78039485960543), UINT64_C(0x97058437294586), 1785 UINT64_C(0xfff40ff8f08ef3), UINT64_C(0x4ee8ee04cc4de4), 1786 &u128Old.u), 1787 true, 0x78039485960543, 0x97058437294586); 1788 CHECKVAL128_C(&u128Old, 0xfff40ff8f08ef3, 0x4ee8ee04cc4de4); 1789 1790 u128Old = u128OldInit; 1791 CHECK_OP_AND_VAL_128_C(bool, "%d", pu128, ASMAtomicCmpXchgU128v2(&pu128->u, 1792 UINT64_C(0x13495874560495), UINT64_C(0x12304896098597), 1793 UINT64_C(0xfff40ff8f08ef3), UINT64_C(0x4ee8ee04cc4de4), 1794 &u128Old.u), 1795 false, 0x78039485960543, 0x97058437294586); 1796 CHECKVAL128_C(&u128Old, 0x78039485960543, 0x97058437294586); 1797 } 1798 1700 1799 1701 1800 static void tstASMAtomicCmpXchgEx(void) … … 1705 1804 DO_SIMPLE_TEST(ASMAtomicCmpXchgExU32, uint32_t); 1706 1805 DO_SIMPLE_TEST(ASMAtomicCmpXchgExU64, uint64_t); 1806 #ifdef RTASM_HAVE_CMP_XCHG_U128 1807 # ifdef RT_ARCH_AMD64 1808 if (ASMCpuId_ECX(1) & X86_CPUID_FEATURE_ECX_CX16) 1809 # endif 1810 { 1811 RTTestISub("ASMAtomicCmpXchgU128"); 1812 DO_SIMPLE_TEST_NO_SUB_NO_STACK(tstASMAtomicCmpXchgU128Worker, RTUINT128U); 1813 } 1814 #endif 1707 1815 } 1708 1816 … … 2801 2909 static uint64_t volatile s_u64; 2802 2910 static int64_t volatile s_i64; 2911 #if defined(RTASM_HAVE_CMP_WRITE_U128) || defined(RTASM_HAVE_CMP_XCHG_U128) 2912 static RTUINT128U volatile s_u128; 2913 #endif 2803 2914 static uint8_t s_u8Old; 2804 2915 static int8_t s_i8Old; … … 2809 2920 static uint64_t s_u64Old; 2810 2921 static int64_t s_i64Old; 2922 #if defined(RTASM_HAVE_CMP_WRITE_U128) || defined(RTASM_HAVE_CMP_XCHG_U128) 2923 static RTUINT128U s_u128Old; 2924 RTUINT128U u128Tmp1, u128Tmp2; 2925 # ifdef RT_ARCH_AMD64 2926 bool const fHaveCmpXchg128 = RT_BOOL(ASMCpuId_ECX(1) & X86_CPUID_FEATURE_ECX_CX16); 2927 # else 2928 bool const fHaveCmpXchg128 = true; 2929 # endif 2930 #endif 2811 2931 unsigned i; 2812 2932 const unsigned cRounds = _16M; /* Must be multiple of 8 */ … … 2917 3037 BENCH(ASMAtomicCmpXchgU64(&s_u64, 0, 0), "ASMAtomicCmpXchgU64"); 2918 3038 BENCH(ASMAtomicCmpXchgS64(&s_i64, 0, 0), "ASMAtomicCmpXchgS64"); 3039 #ifdef RTASM_HAVE_CMP_WRITE_U128 3040 if (fHaveCmpXchg128) 3041 BENCH(ASMAtomicCmpWriteU128U(&s_u128, u128Tmp1 = RTUINT128_INIT_C(0, 0), u128Tmp2 = RTUINT128_INIT_C(0, 0)), 3042 "ASMAtomicCmpWriteU128U"); 3043 #endif 2919 3044 BENCH(ASMAtomicCmpXchgU8(&s_u8, 0, 1), "ASMAtomicCmpXchgU8/neg"); 2920 3045 BENCH(ASMAtomicCmpXchgS8(&s_i8, 0, 1), "ASMAtomicCmpXchgS8/neg"); … … 2925 3050 BENCH(ASMAtomicCmpXchgU64(&s_u64, 0, 1), "ASMAtomicCmpXchgU64/neg"); 2926 3051 BENCH(ASMAtomicCmpXchgS64(&s_i64, 0, 1), "ASMAtomicCmpXchgS64/neg"); 3052 #ifdef RTASM_HAVE_CMP_WRITE_U128 3053 if (fHaveCmpXchg128) 3054 BENCH(ASMAtomicCmpWriteU128U(&s_u128, u128Tmp1 = RTUINT128_INIT_C(0, 0), u128Tmp2 = RTUINT128_INIT_C(0, 1)), 3055 "ASMAtomicCmpWriteU128U/neg"); 3056 #endif 2927 3057 BENCH(ASMAtomicCmpXchgExU8(&s_u8, 0, 0, &s_u8Old), "ASMAtomicCmpXchgExU8"); 2928 3058 BENCH(ASMAtomicCmpXchgExS8(&s_i8, 0, 0, &s_i8Old), "ASMAtomicCmpXchgExS8"); … … 2933 3063 BENCH(ASMAtomicCmpXchgExU64(&s_u64, 0, 0, &s_u64Old), "ASMAtomicCmpXchgExU64"); 2934 3064 BENCH(ASMAtomicCmpXchgExS64(&s_i64, 0, 0, &s_i64Old), "ASMAtomicCmpXchgExS64"); 3065 #ifdef RTASM_HAVE_CMP_XCHG_U128 3066 if (fHaveCmpXchg128) 3067 BENCH(ASMAtomicCmpXchgU128U(&s_u128, u128Tmp1 = RTUINT128_INIT_C(0, 0), u128Tmp2 = RTUINT128_INIT_C(0, 0), &s_u128Old), 3068 "ASMAtomicCmpXchgU128U"); 3069 #endif 2935 3070 BENCH(ASMAtomicCmpXchgExU8(&s_u8, 0, 1, &s_u8Old), "ASMAtomicCmpXchgExU8/neg"); 2936 3071 BENCH(ASMAtomicCmpXchgExS8(&s_i8, 0, 1, &s_i8Old), "ASMAtomicCmpXchgExS8/neg"); … … 2941 3076 BENCH(ASMAtomicCmpXchgExU64(&s_u64, 0, 1, &s_u64Old), "ASMAtomicCmpXchgExU64/neg"); 2942 3077 BENCH(ASMAtomicCmpXchgExS64(&s_i64, 0, 1, &s_i64Old), "ASMAtomicCmpXchgExS64/neg"); 3078 #ifdef RTASM_HAVE_CMP_XCHG_U128 3079 if (fHaveCmpXchg128) 3080 BENCH(ASMAtomicCmpXchgU128U(&s_u128, u128Tmp1 = RTUINT128_INIT_C(0, 0), u128Tmp2 = RTUINT128_INIT_C(0, 1), &s_u128Old), 3081 "ASMAtomicCmpXchgU128U/neg"); 3082 #endif 2943 3083 BENCH(ASMAtomicIncU32(&s_u32), "ASMAtomicIncU32"); 2944 3084 BENCH(ASMAtomicIncS32(&s_i32), "ASMAtomicIncS32");
Note:
See TracChangeset
for help on using the changeset viewer.
