VirtualBox

source: vbox/trunk/include/iprt/asm.h@ 30254

Last change on this file since 30254 was 30112, checked in by vboxsync, 14 years ago

iprt/asm.h,*: Added ASMAtomicWriteNullPtr and ASMAtomicUoWriteNullPtr to better deal with NULL being 0 in C++.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 121.7 KB
Line 
1/** @file
2 * IPRT - Assembly Functions.
3 */
4
5/*
6 * Copyright (C) 2006-2010 Oracle Corporation
7 *
8 * This file is part of VirtualBox Open Source Edition (OSE), as
9 * available from http://www.virtualbox.org. This file is free software;
10 * you can redistribute it and/or modify it under the terms of the GNU
11 * General Public License (GPL) as published by the Free Software
12 * Foundation, in version 2 as it comes in the "COPYING" file of the
13 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
14 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
15 *
16 * The contents of this file may alternatively be used under the terms
17 * of the Common Development and Distribution License Version 1.0
18 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
19 * VirtualBox OSE distribution, in which case the provisions of the
20 * CDDL are applicable instead of those of the GPL.
21 *
22 * You may elect to license modified versions of this file under the
23 * terms and conditions of either the GPL or the CDDL or both.
24 */
25
26#ifndef ___iprt_asm_h
27#define ___iprt_asm_h
28
29#include <iprt/cdefs.h>
30#include <iprt/types.h>
31#include <iprt/assert.h>
32/** @def RT_INLINE_ASM_USES_INTRIN
33 * Defined as 1 if we're using a _MSC_VER 1400.
34 * Otherwise defined as 0.
35 */
36
37/* Solaris 10 header ugliness */
38#ifdef u
39# undef u
40#endif
41
42#if defined(_MSC_VER) && RT_INLINE_ASM_USES_INTRIN
43# include <intrin.h>
44 /* Emit the intrinsics at all optimization levels. */
45# pragma intrinsic(_ReadWriteBarrier)
46# pragma intrinsic(__cpuid)
47# pragma intrinsic(__stosd)
48# pragma intrinsic(__stosw)
49# pragma intrinsic(__stosb)
50# pragma intrinsic(_BitScanForward)
51# pragma intrinsic(_BitScanReverse)
52# pragma intrinsic(_bittest)
53# pragma intrinsic(_bittestandset)
54# pragma intrinsic(_bittestandreset)
55# pragma intrinsic(_bittestandcomplement)
56# pragma intrinsic(_byteswap_ushort)
57# pragma intrinsic(_byteswap_ulong)
58# pragma intrinsic(_interlockedbittestandset)
59# pragma intrinsic(_interlockedbittestandreset)
60# pragma intrinsic(_InterlockedAnd)
61# pragma intrinsic(_InterlockedOr)
62# pragma intrinsic(_InterlockedIncrement)
63# pragma intrinsic(_InterlockedDecrement)
64# pragma intrinsic(_InterlockedExchange)
65# pragma intrinsic(_InterlockedExchangeAdd)
66# pragma intrinsic(_InterlockedCompareExchange)
67# pragma intrinsic(_InterlockedCompareExchange64)
68# ifdef RT_ARCH_AMD64
69# pragma intrinsic(__stosq)
70# pragma intrinsic(_byteswap_uint64)
71# pragma intrinsic(_InterlockedExchange64)
72# endif
73#endif
74
75
76/** @defgroup grp_rt_asm ASM - Assembly Routines
77 * @ingroup grp_rt
78 *
79 * @remarks The difference between ordered and unordered atomic operations are that
80 * the former will complete outstanding reads and writes before continuing
81 * while the latter doesn't make any promisses about the order. Ordered
82 * operations doesn't, it seems, make any 100% promise wrt to whether
83 * the operation will complete before any subsequent memory access.
84 * (please, correct if wrong.)
85 *
86 * ASMAtomicSomething operations are all ordered, while ASMAtomicUoSomething
87 * are unordered (note the Uo).
88 *
89 * @remarks Some remarks about __volatile__: Without this keyword gcc is allowed to reorder
90 * or even optimize assembler instructions away. For instance, in the following code
91 * the second rdmsr instruction is optimized away because gcc treats that instruction
92 * as deterministic:
93 *
94 * @code
95 * static inline uint64_t rdmsr_low(int idx)
96 * {
97 * uint32_t low;
98 * __asm__ ("rdmsr" : "=a"(low) : "c"(idx) : "edx");
99 * }
100 * ...
101 * uint32_t msr1 = rdmsr_low(1);
102 * foo(msr1);
103 * msr1 = rdmsr_low(1);
104 * bar(msr1);
105 * @endcode
106 *
107 * The input parameter of rdmsr_low is the same for both calls and therefore gcc will
108 * use the result of the first call as input parameter for bar() as well. For rdmsr this
109 * is not acceptable as this instruction is _not_ deterministic. This applies to reading
110 * machine status information in general.
111 *
112 * @{
113 */
114
115
116/** @def RT_INLINE_ASM_GCC_4_3_X_X86
117 * Used to work around some 4.3.x register allocation issues in this version of
118 * the compiler. So far this workaround is still required for 4.4 and 4.5. */
119#ifdef __GNUC__
120# define RT_INLINE_ASM_GCC_4_3_X_X86 (__GNUC__ == 4 && __GNUC_MINOR__ >= 3 && defined(__i386__))
121#endif
122#ifndef RT_INLINE_ASM_GCC_4_3_X_X86
123# define RT_INLINE_ASM_GCC_4_3_X_X86 0
124#endif
125
126/** @def RT_INLINE_DONT_USE_CMPXCHG8B
127 * i686-apple-darwin9-gcc-4.0.1 (GCC) 4.0.1 (Apple Inc. build 5493) screws up
128 * RTSemRWRequestWrite semsemrw-lockless-generic.cpp in release builds. PIC
129 * mode, x86.
130 *
131 * Some gcc 4.3.x versions may have register allocation issues with cmpxchg8b
132 * when in PIC mode on x86.
133 */
134#ifndef RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
135# define RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC \
136 ( (defined(PIC) || defined(__PIC__)) \
137 && defined(RT_ARCH_X86) \
138 && ( RT_INLINE_ASM_GCC_4_3_X_X86 \
139 || defined(RT_OS_DARWIN)) )
140#endif
141
142
143/** @def ASMReturnAddress
144 * Gets the return address of the current (or calling if you like) function or method.
145 */
146#ifdef _MSC_VER
147# ifdef __cplusplus
148extern "C"
149# endif
150void * _ReturnAddress(void);
151# pragma intrinsic(_ReturnAddress)
152# define ASMReturnAddress() _ReturnAddress()
153#elif defined(__GNUC__) || defined(DOXYGEN_RUNNING)
154# define ASMReturnAddress() __builtin_return_address(0)
155#else
156# error "Unsupported compiler."
157#endif
158
159
160/**
161 * Compiler memory barrier.
162 *
163 * Ensure that the compiler does not use any cached (register/tmp stack) memory
164 * values or any outstanding writes when returning from this function.
165 *
166 * This function must be used if non-volatile data is modified by a
167 * device or the VMM. Typical cases are port access, MMIO access,
168 * trapping instruction, etc.
169 */
170#if RT_INLINE_ASM_GNU_STYLE
171# define ASMCompilerBarrier() do { __asm__ __volatile__("" : : : "memory"); } while (0)
172#elif RT_INLINE_ASM_USES_INTRIN
173# define ASMCompilerBarrier() do { _ReadWriteBarrier(); } while (0)
174#else /* 2003 should have _ReadWriteBarrier() but I guess we're at 2002 level then... */
175DECLINLINE(void) ASMCompilerBarrier(void)
176{
177 __asm
178 {
179 }
180}
181#endif
182
183
184
185/**
186 * Atomically Exchange an unsigned 8-bit value, ordered.
187 *
188 * @returns Current *pu8 value
189 * @param pu8 Pointer to the 8-bit variable to update.
190 * @param u8 The 8-bit value to assign to *pu8.
191 */
192#if RT_INLINE_ASM_EXTERNAL
193DECLASM(uint8_t) ASMAtomicXchgU8(volatile uint8_t *pu8, uint8_t u8);
194#else
195DECLINLINE(uint8_t) ASMAtomicXchgU8(volatile uint8_t *pu8, uint8_t u8)
196{
197# if RT_INLINE_ASM_GNU_STYLE
198 __asm__ __volatile__("xchgb %0, %1\n\t"
199 : "=m" (*pu8),
200 "=q" (u8) /* =r - busted on g++ (GCC) 3.4.4 20050721 (Red Hat 3.4.4-2) */
201 : "1" (u8),
202 "m" (*pu8));
203# else
204 __asm
205 {
206# ifdef RT_ARCH_AMD64
207 mov rdx, [pu8]
208 mov al, [u8]
209 xchg [rdx], al
210 mov [u8], al
211# else
212 mov edx, [pu8]
213 mov al, [u8]
214 xchg [edx], al
215 mov [u8], al
216# endif
217 }
218# endif
219 return u8;
220}
221#endif
222
223
224/**
225 * Atomically Exchange a signed 8-bit value, ordered.
226 *
227 * @returns Current *pu8 value
228 * @param pi8 Pointer to the 8-bit variable to update.
229 * @param i8 The 8-bit value to assign to *pi8.
230 */
231DECLINLINE(int8_t) ASMAtomicXchgS8(volatile int8_t *pi8, int8_t i8)
232{
233 return (int8_t)ASMAtomicXchgU8((volatile uint8_t *)pi8, (uint8_t)i8);
234}
235
236
237/**
238 * Atomically Exchange a bool value, ordered.
239 *
240 * @returns Current *pf value
241 * @param pf Pointer to the 8-bit variable to update.
242 * @param f The 8-bit value to assign to *pi8.
243 */
244DECLINLINE(bool) ASMAtomicXchgBool(volatile bool *pf, bool f)
245{
246#ifdef _MSC_VER
247 return !!ASMAtomicXchgU8((volatile uint8_t *)pf, (uint8_t)f);
248#else
249 return (bool)ASMAtomicXchgU8((volatile uint8_t *)pf, (uint8_t)f);
250#endif
251}
252
253
254/**
255 * Atomically Exchange an unsigned 16-bit value, ordered.
256 *
257 * @returns Current *pu16 value
258 * @param pu16 Pointer to the 16-bit variable to update.
259 * @param u16 The 16-bit value to assign to *pu16.
260 */
261#if RT_INLINE_ASM_EXTERNAL
262DECLASM(uint16_t) ASMAtomicXchgU16(volatile uint16_t *pu16, uint16_t u16);
263#else
264DECLINLINE(uint16_t) ASMAtomicXchgU16(volatile uint16_t *pu16, uint16_t u16)
265{
266# if RT_INLINE_ASM_GNU_STYLE
267 __asm__ __volatile__("xchgw %0, %1\n\t"
268 : "=m" (*pu16),
269 "=r" (u16)
270 : "1" (u16),
271 "m" (*pu16));
272# else
273 __asm
274 {
275# ifdef RT_ARCH_AMD64
276 mov rdx, [pu16]
277 mov ax, [u16]
278 xchg [rdx], ax
279 mov [u16], ax
280# else
281 mov edx, [pu16]
282 mov ax, [u16]
283 xchg [edx], ax
284 mov [u16], ax
285# endif
286 }
287# endif
288 return u16;
289}
290#endif
291
292
293/**
294 * Atomically Exchange a signed 16-bit value, ordered.
295 *
296 * @returns Current *pu16 value
297 * @param pi16 Pointer to the 16-bit variable to update.
298 * @param i16 The 16-bit value to assign to *pi16.
299 */
300DECLINLINE(int16_t) ASMAtomicXchgS16(volatile int16_t *pi16, int16_t i16)
301{
302 return (int16_t)ASMAtomicXchgU16((volatile uint16_t *)pi16, (uint16_t)i16);
303}
304
305
306/**
307 * Atomically Exchange an unsigned 32-bit value, ordered.
308 *
309 * @returns Current *pu32 value
310 * @param pu32 Pointer to the 32-bit variable to update.
311 * @param u32 The 32-bit value to assign to *pu32.
312 */
313#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
314DECLASM(uint32_t) ASMAtomicXchgU32(volatile uint32_t *pu32, uint32_t u32);
315#else
316DECLINLINE(uint32_t) ASMAtomicXchgU32(volatile uint32_t *pu32, uint32_t u32)
317{
318# if RT_INLINE_ASM_GNU_STYLE
319 __asm__ __volatile__("xchgl %0, %1\n\t"
320 : "=m" (*pu32),
321 "=r" (u32)
322 : "1" (u32),
323 "m" (*pu32));
324
325# elif RT_INLINE_ASM_USES_INTRIN
326 u32 = _InterlockedExchange((long *)pu32, u32);
327
328# else
329 __asm
330 {
331# ifdef RT_ARCH_AMD64
332 mov rdx, [pu32]
333 mov eax, u32
334 xchg [rdx], eax
335 mov [u32], eax
336# else
337 mov edx, [pu32]
338 mov eax, u32
339 xchg [edx], eax
340 mov [u32], eax
341# endif
342 }
343# endif
344 return u32;
345}
346#endif
347
348
349/**
350 * Atomically Exchange a signed 32-bit value, ordered.
351 *
352 * @returns Current *pu32 value
353 * @param pi32 Pointer to the 32-bit variable to update.
354 * @param i32 The 32-bit value to assign to *pi32.
355 */
356DECLINLINE(int32_t) ASMAtomicXchgS32(volatile int32_t *pi32, int32_t i32)
357{
358 return (int32_t)ASMAtomicXchgU32((volatile uint32_t *)pi32, (uint32_t)i32);
359}
360
361
362/**
363 * Atomically Exchange an unsigned 64-bit value, ordered.
364 *
365 * @returns Current *pu64 value
366 * @param pu64 Pointer to the 64-bit variable to update.
367 * @param u64 The 64-bit value to assign to *pu64.
368 */
369#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
370 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
371DECLASM(uint64_t) ASMAtomicXchgU64(volatile uint64_t *pu64, uint64_t u64);
372#else
373DECLINLINE(uint64_t) ASMAtomicXchgU64(volatile uint64_t *pu64, uint64_t u64)
374{
375# if defined(RT_ARCH_AMD64)
376# if RT_INLINE_ASM_USES_INTRIN
377 u64 = _InterlockedExchange64((__int64 *)pu64, u64);
378
379# elif RT_INLINE_ASM_GNU_STYLE
380 __asm__ __volatile__("xchgq %0, %1\n\t"
381 : "=m" (*pu64),
382 "=r" (u64)
383 : "1" (u64),
384 "m" (*pu64));
385# else
386 __asm
387 {
388 mov rdx, [pu64]
389 mov rax, [u64]
390 xchg [rdx], rax
391 mov [u64], rax
392 }
393# endif
394# else /* !RT_ARCH_AMD64 */
395# if RT_INLINE_ASM_GNU_STYLE
396# if defined(PIC) || defined(__PIC__)
397 uint32_t u32EBX = (uint32_t)u64;
398 __asm__ __volatile__(/*"xchgl %%esi, %5\n\t"*/
399 "xchgl %%ebx, %3\n\t"
400 "1:\n\t"
401 "lock; cmpxchg8b (%5)\n\t"
402 "jnz 1b\n\t"
403 "movl %3, %%ebx\n\t"
404 /*"xchgl %%esi, %5\n\t"*/
405 : "=A" (u64),
406 "=m" (*pu64)
407 : "0" (*pu64),
408 "m" ( u32EBX ),
409 "c" ( (uint32_t)(u64 >> 32) ),
410 "S" (pu64));
411# else /* !PIC */
412 __asm__ __volatile__("1:\n\t"
413 "lock; cmpxchg8b %1\n\t"
414 "jnz 1b\n\t"
415 : "=A" (u64),
416 "=m" (*pu64)
417 : "0" (*pu64),
418 "b" ( (uint32_t)u64 ),
419 "c" ( (uint32_t)(u64 >> 32) ));
420# endif
421# else
422 __asm
423 {
424 mov ebx, dword ptr [u64]
425 mov ecx, dword ptr [u64 + 4]
426 mov edi, pu64
427 mov eax, dword ptr [edi]
428 mov edx, dword ptr [edi + 4]
429 retry:
430 lock cmpxchg8b [edi]
431 jnz retry
432 mov dword ptr [u64], eax
433 mov dword ptr [u64 + 4], edx
434 }
435# endif
436# endif /* !RT_ARCH_AMD64 */
437 return u64;
438}
439#endif
440
441
442/**
443 * Atomically Exchange an signed 64-bit value, ordered.
444 *
445 * @returns Current *pi64 value
446 * @param pi64 Pointer to the 64-bit variable to update.
447 * @param i64 The 64-bit value to assign to *pi64.
448 */
449DECLINLINE(int64_t) ASMAtomicXchgS64(volatile int64_t *pi64, int64_t i64)
450{
451 return (int64_t)ASMAtomicXchgU64((volatile uint64_t *)pi64, (uint64_t)i64);
452}
453
454
455/**
456 * Atomically Exchange a pointer value, ordered.
457 *
458 * @returns Current *ppv value
459 * @param ppv Pointer to the pointer variable to update.
460 * @param pv The pointer value to assign to *ppv.
461 */
462DECLINLINE(void *) ASMAtomicXchgPtr(void * volatile *ppv, const void *pv)
463{
464#if ARCH_BITS == 32
465 return (void *)ASMAtomicXchgU32((volatile uint32_t *)(void *)ppv, (uint32_t)pv);
466#elif ARCH_BITS == 64
467 return (void *)ASMAtomicXchgU64((volatile uint64_t *)(void *)ppv, (uint64_t)pv);
468#else
469# error "ARCH_BITS is bogus"
470#endif
471}
472
473
474/**
475 * Convenience macro for avoiding the annoying casting with ASMAtomicXchgPtr.
476 *
477 * @returns Current *pv value
478 * @param ppv Pointer to the pointer variable to update.
479 * @param pv The pointer value to assign to *ppv.
480 * @param Type The type of *ppv, sans volatile.
481 */
482#ifdef __GNUC__
483# define ASMAtomicXchgPtrT(ppv, pv, Type) \
484 __extension__ \
485 ({\
486 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
487 Type const pvTypeChecked = (pv); \
488 Type pvTypeCheckedRet = (__typeof__(*(ppv))) ASMAtomicXchgPtr((void * volatile *)ppvTypeChecked, (void *)pvTypeChecked); \
489 pvTypeCheckedRet; \
490 })
491#else
492# define ASMAtomicXchgPtrT(ppv, pv, Type) \
493 (Type)ASMAtomicXchgPtr((void * volatile *)(ppv), (void *)(pv))
494#endif
495
496
497/**
498 * Atomically Exchange a raw-mode context pointer value, ordered.
499 *
500 * @returns Current *ppv value
501 * @param ppvRC Pointer to the pointer variable to update.
502 * @param pvRC The pointer value to assign to *ppv.
503 */
504DECLINLINE(RTRCPTR) ASMAtomicXchgRCPtr(RTRCPTR volatile *ppvRC, RTRCPTR pvRC)
505{
506 return (RTRCPTR)ASMAtomicXchgU32((uint32_t volatile *)(void *)ppvRC, (uint32_t)pvRC);
507}
508
509
510/**
511 * Atomically Exchange a ring-0 pointer value, ordered.
512 *
513 * @returns Current *ppv value
514 * @param ppvR0 Pointer to the pointer variable to update.
515 * @param pvR0 The pointer value to assign to *ppv.
516 */
517DECLINLINE(RTR0PTR) ASMAtomicXchgR0Ptr(RTR0PTR volatile *ppvR0, RTR0PTR pvR0)
518{
519#if R0_ARCH_BITS == 32
520 return (RTR0PTR)ASMAtomicXchgU32((volatile uint32_t *)(void *)ppvR0, (uint32_t)pvR0);
521#elif R0_ARCH_BITS == 64
522 return (RTR0PTR)ASMAtomicXchgU64((volatile uint64_t *)(void *)ppvR0, (uint64_t)pvR0);
523#else
524# error "R0_ARCH_BITS is bogus"
525#endif
526}
527
528
529/**
530 * Atomically Exchange a ring-3 pointer value, ordered.
531 *
532 * @returns Current *ppv value
533 * @param ppvR3 Pointer to the pointer variable to update.
534 * @param pvR3 The pointer value to assign to *ppv.
535 */
536DECLINLINE(RTR3PTR) ASMAtomicXchgR3Ptr(RTR3PTR volatile *ppvR3, RTR3PTR pvR3)
537{
538#if R3_ARCH_BITS == 32
539 return (RTR3PTR)ASMAtomicXchgU32((volatile uint32_t *)(void *)ppvR3, (uint32_t)pvR3);
540#elif R3_ARCH_BITS == 64
541 return (RTR3PTR)ASMAtomicXchgU64((volatile uint64_t *)(void *)ppvR3, (uint64_t)pvR3);
542#else
543# error "R3_ARCH_BITS is bogus"
544#endif
545}
546
547
548/** @def ASMAtomicXchgHandle
549 * Atomically Exchange a typical IPRT handle value, ordered.
550 *
551 * @param ph Pointer to the value to update.
552 * @param hNew The new value to assigned to *pu.
553 * @param phRes Where to store the current *ph value.
554 *
555 * @remarks This doesn't currently work for all handles (like RTFILE).
556 */
557#if HC_ARCH_BITS == 32
558# define ASMAtomicXchgHandle(ph, hNew, phRes) \
559 do { \
560 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
561 AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
562 *(uint32_t *)(phRes) = ASMAtomicXchgU32((uint32_t volatile *)(ph), (const uint32_t)(hNew)); \
563 } while (0)
564#elif HC_ARCH_BITS == 64
565# define ASMAtomicXchgHandle(ph, hNew, phRes) \
566 do { \
567 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
568 AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
569 *(uint64_t *)(phRes) = ASMAtomicXchgU64((uint64_t volatile *)(ph), (const uint64_t)(hNew)); \
570 } while (0)
571#else
572# error HC_ARCH_BITS
573#endif
574
575
576/**
577 * Atomically Exchange a value which size might differ
578 * between platforms or compilers, ordered.
579 *
580 * @param pu Pointer to the variable to update.
581 * @param uNew The value to assign to *pu.
582 * @todo This is busted as its missing the result argument.
583 */
584#define ASMAtomicXchgSize(pu, uNew) \
585 do { \
586 switch (sizeof(*(pu))) { \
587 case 1: ASMAtomicXchgU8((volatile uint8_t *)(void *)(pu), (uint8_t)(uNew)); break; \
588 case 2: ASMAtomicXchgU16((volatile uint16_t *)(void *)(pu), (uint16_t)(uNew)); break; \
589 case 4: ASMAtomicXchgU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew)); break; \
590 case 8: ASMAtomicXchgU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew)); break; \
591 default: AssertMsgFailed(("ASMAtomicXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
592 } \
593 } while (0)
594
595/**
596 * Atomically Exchange a value which size might differ
597 * between platforms or compilers, ordered.
598 *
599 * @param pu Pointer to the variable to update.
600 * @param uNew The value to assign to *pu.
601 * @param puRes Where to store the current *pu value.
602 */
603#define ASMAtomicXchgSizeCorrect(pu, uNew, puRes) \
604 do { \
605 switch (sizeof(*(pu))) { \
606 case 1: *(uint8_t *)(puRes) = ASMAtomicXchgU8((volatile uint8_t *)(void *)(pu), (uint8_t)(uNew)); break; \
607 case 2: *(uint16_t *)(puRes) = ASMAtomicXchgU16((volatile uint16_t *)(void *)(pu), (uint16_t)(uNew)); break; \
608 case 4: *(uint32_t *)(puRes) = ASMAtomicXchgU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew)); break; \
609 case 8: *(uint64_t *)(puRes) = ASMAtomicXchgU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew)); break; \
610 default: AssertMsgFailed(("ASMAtomicXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
611 } \
612 } while (0)
613
614
615
616/**
617 * Atomically Compare and Exchange an unsigned 8-bit value, ordered.
618 *
619 * @returns true if xchg was done.
620 * @returns false if xchg wasn't done.
621 *
622 * @param pu8 Pointer to the value to update.
623 * @param u8New The new value to assigned to *pu8.
624 * @param u8Old The old value to *pu8 compare with.
625 */
626#if RT_INLINE_ASM_EXTERNAL || !RT_INLINE_ASM_GNU_STYLE
627DECLASM(bool) ASMAtomicCmpXchgU8(volatile uint8_t *pu8, const uint8_t u8New, const uint8_t u8Old);
628#else
629DECLINLINE(bool) ASMAtomicCmpXchgU8(volatile uint8_t *pu8, const uint8_t u8New, uint8_t u8Old)
630{
631 uint8_t u8Ret;
632 __asm__ __volatile__("lock; cmpxchgb %3, %0\n\t"
633 "setz %1\n\t"
634 : "=m" (*pu8),
635 "=qm" (u8Ret),
636 "=a" (u8Old)
637 : "q" (u8New),
638 "2" (u8Old),
639 "m" (*pu8));
640 return (bool)u8Ret;
641}
642#endif
643
644
645/**
646 * Atomically Compare and Exchange a signed 8-bit value, ordered.
647 *
648 * @returns true if xchg was done.
649 * @returns false if xchg wasn't done.
650 *
651 * @param pi8 Pointer to the value to update.
652 * @param i8New The new value to assigned to *pi8.
653 * @param i8Old The old value to *pi8 compare with.
654 */
655DECLINLINE(bool) ASMAtomicCmpXchgS8(volatile int8_t *pi8, const int8_t i8New, const int8_t i8Old)
656{
657 return ASMAtomicCmpXchgU8((volatile uint8_t *)pi8, (const uint8_t)i8New, (const uint8_t)i8Old);
658}
659
660
661/**
662 * Atomically Compare and Exchange a bool value, ordered.
663 *
664 * @returns true if xchg was done.
665 * @returns false if xchg wasn't done.
666 *
667 * @param pf Pointer to the value to update.
668 * @param fNew The new value to assigned to *pf.
669 * @param fOld The old value to *pf compare with.
670 */
671DECLINLINE(bool) ASMAtomicCmpXchgBool(volatile bool *pf, const bool fNew, const bool fOld)
672{
673 return ASMAtomicCmpXchgU8((volatile uint8_t *)pf, (const uint8_t)fNew, (const uint8_t)fOld);
674}
675
676
677/**
678 * Atomically Compare and Exchange an unsigned 32-bit value, ordered.
679 *
680 * @returns true if xchg was done.
681 * @returns false if xchg wasn't done.
682 *
683 * @param pu32 Pointer to the value to update.
684 * @param u32New The new value to assigned to *pu32.
685 * @param u32Old The old value to *pu32 compare with.
686 */
687#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
688DECLASM(bool) ASMAtomicCmpXchgU32(volatile uint32_t *pu32, const uint32_t u32New, const uint32_t u32Old);
689#else
690DECLINLINE(bool) ASMAtomicCmpXchgU32(volatile uint32_t *pu32, const uint32_t u32New, uint32_t u32Old)
691{
692# if RT_INLINE_ASM_GNU_STYLE
693 uint8_t u8Ret;
694 __asm__ __volatile__("lock; cmpxchgl %3, %0\n\t"
695 "setz %1\n\t"
696 : "=m" (*pu32),
697 "=qm" (u8Ret),
698 "=a" (u32Old)
699 : "r" (u32New),
700 "2" (u32Old),
701 "m" (*pu32));
702 return (bool)u8Ret;
703
704# elif RT_INLINE_ASM_USES_INTRIN
705 return _InterlockedCompareExchange((long *)pu32, u32New, u32Old) == u32Old;
706
707# else
708 uint32_t u32Ret;
709 __asm
710 {
711# ifdef RT_ARCH_AMD64
712 mov rdx, [pu32]
713# else
714 mov edx, [pu32]
715# endif
716 mov eax, [u32Old]
717 mov ecx, [u32New]
718# ifdef RT_ARCH_AMD64
719 lock cmpxchg [rdx], ecx
720# else
721 lock cmpxchg [edx], ecx
722# endif
723 setz al
724 movzx eax, al
725 mov [u32Ret], eax
726 }
727 return !!u32Ret;
728# endif
729}
730#endif
731
732
733/**
734 * Atomically Compare and Exchange a signed 32-bit value, ordered.
735 *
736 * @returns true if xchg was done.
737 * @returns false if xchg wasn't done.
738 *
739 * @param pi32 Pointer to the value to update.
740 * @param i32New The new value to assigned to *pi32.
741 * @param i32Old The old value to *pi32 compare with.
742 */
743DECLINLINE(bool) ASMAtomicCmpXchgS32(volatile int32_t *pi32, const int32_t i32New, const int32_t i32Old)
744{
745 return ASMAtomicCmpXchgU32((volatile uint32_t *)pi32, (uint32_t)i32New, (uint32_t)i32Old);
746}
747
748
749/**
750 * Atomically Compare and exchange an unsigned 64-bit value, ordered.
751 *
752 * @returns true if xchg was done.
753 * @returns false if xchg wasn't done.
754 *
755 * @param pu64 Pointer to the 64-bit variable to update.
756 * @param u64New The 64-bit value to assign to *pu64.
757 * @param u64Old The value to compare with.
758 */
759#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
760 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
761DECLASM(bool) ASMAtomicCmpXchgU64(volatile uint64_t *pu64, const uint64_t u64New, const uint64_t u64Old);
762#else
763DECLINLINE(bool) ASMAtomicCmpXchgU64(volatile uint64_t *pu64, uint64_t u64New, uint64_t u64Old)
764{
765# if RT_INLINE_ASM_USES_INTRIN
766 return _InterlockedCompareExchange64((__int64 *)pu64, u64New, u64Old) == u64Old;
767
768# elif defined(RT_ARCH_AMD64)
769# if RT_INLINE_ASM_GNU_STYLE
770 uint8_t u8Ret;
771 __asm__ __volatile__("lock; cmpxchgq %3, %0\n\t"
772 "setz %1\n\t"
773 : "=m" (*pu64),
774 "=qm" (u8Ret),
775 "=a" (u64Old)
776 : "r" (u64New),
777 "2" (u64Old),
778 "m" (*pu64));
779 return (bool)u8Ret;
780# else
781 bool fRet;
782 __asm
783 {
784 mov rdx, [pu32]
785 mov rax, [u64Old]
786 mov rcx, [u64New]
787 lock cmpxchg [rdx], rcx
788 setz al
789 mov [fRet], al
790 }
791 return fRet;
792# endif
793# else /* !RT_ARCH_AMD64 */
794 uint32_t u32Ret;
795# if RT_INLINE_ASM_GNU_STYLE
796# if defined(PIC) || defined(__PIC__)
797 uint32_t u32EBX = (uint32_t)u64New;
798 uint32_t u32Spill;
799 __asm__ __volatile__("xchgl %%ebx, %4\n\t"
800 "lock; cmpxchg8b (%6)\n\t"
801 "setz %%al\n\t"
802 "movl %4, %%ebx\n\t"
803 "movzbl %%al, %%eax\n\t"
804 : "=a" (u32Ret),
805 "=d" (u32Spill),
806# if (__GNUC__ * 100 + __GNUC_MINOR__) >= 403
807 "+m" (*pu64)
808# else
809 "=m" (*pu64)
810# endif
811 : "A" (u64Old),
812 "m" ( u32EBX ),
813 "c" ( (uint32_t)(u64New >> 32) ),
814 "S" (pu64));
815# else /* !PIC */
816 uint32_t u32Spill;
817 __asm__ __volatile__("lock; cmpxchg8b %2\n\t"
818 "setz %%al\n\t"
819 "movzbl %%al, %%eax\n\t"
820 : "=a" (u32Ret),
821 "=d" (u32Spill),
822 "+m" (*pu64)
823 : "A" (u64Old),
824 "b" ( (uint32_t)u64New ),
825 "c" ( (uint32_t)(u64New >> 32) ));
826# endif
827 return (bool)u32Ret;
828# else
829 __asm
830 {
831 mov ebx, dword ptr [u64New]
832 mov ecx, dword ptr [u64New + 4]
833 mov edi, [pu64]
834 mov eax, dword ptr [u64Old]
835 mov edx, dword ptr [u64Old + 4]
836 lock cmpxchg8b [edi]
837 setz al
838 movzx eax, al
839 mov dword ptr [u32Ret], eax
840 }
841 return !!u32Ret;
842# endif
843# endif /* !RT_ARCH_AMD64 */
844}
845#endif
846
847
848/**
849 * Atomically Compare and exchange a signed 64-bit value, ordered.
850 *
851 * @returns true if xchg was done.
852 * @returns false if xchg wasn't done.
853 *
854 * @param pi64 Pointer to the 64-bit variable to update.
855 * @param i64 The 64-bit value to assign to *pu64.
856 * @param i64Old The value to compare with.
857 */
858DECLINLINE(bool) ASMAtomicCmpXchgS64(volatile int64_t *pi64, const int64_t i64, const int64_t i64Old)
859{
860 return ASMAtomicCmpXchgU64((volatile uint64_t *)pi64, (uint64_t)i64, (uint64_t)i64Old);
861}
862
863
864/**
865 * Atomically Compare and Exchange a pointer value, ordered.
866 *
867 * @returns true if xchg was done.
868 * @returns false if xchg wasn't done.
869 *
870 * @param ppv Pointer to the value to update.
871 * @param pvNew The new value to assigned to *ppv.
872 * @param pvOld The old value to *ppv compare with.
873 */
874DECLINLINE(bool) ASMAtomicCmpXchgPtrVoid(void * volatile *ppv, const void *pvNew, const void *pvOld)
875{
876#if ARCH_BITS == 32
877 return ASMAtomicCmpXchgU32((volatile uint32_t *)(void *)ppv, (uint32_t)pvNew, (uint32_t)pvOld);
878#elif ARCH_BITS == 64
879 return ASMAtomicCmpXchgU64((volatile uint64_t *)(void *)ppv, (uint64_t)pvNew, (uint64_t)pvOld);
880#else
881# error "ARCH_BITS is bogus"
882#endif
883}
884
885
886/**
887 * Atomically Compare and Exchange a pointer value, ordered.
888 *
889 * @returns true if xchg was done.
890 * @returns false if xchg wasn't done.
891 *
892 * @param ppv Pointer to the value to update.
893 * @param pvNew The new value to assigned to *ppv.
894 * @param pvOld The old value to *ppv compare with.
895 *
896 * @remarks This is relatively type safe on GCC platforms.
897 */
898#ifdef __GNUC__
899# define ASMAtomicCmpXchgPtr(ppv, pvNew, pvOld) \
900 __extension__ \
901 ({\
902 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
903 __typeof__(*(ppv)) const pvNewTypeChecked = (pvNew); \
904 __typeof__(*(ppv)) const pvOldTypeChecked = (pvOld); \
905 bool fMacroRet = ASMAtomicCmpXchgPtrVoid((void * volatile *)ppvTypeChecked, \
906 (void *)pvNewTypeChecked, (void *)pvOldTypeChecked); \
907 fMacroRet; \
908 })
909#else
910# define ASMAtomicCmpXchgPtr(ppv, pvNew, pvOld) \
911 ASMAtomicCmpXchgPtrVoid((void * volatile *)(ppv), (void *)(pvNew), (void *)(pvOld))
912#endif
913
914
915/** @def ASMAtomicCmpXchgHandle
916 * Atomically Compare and Exchange a typical IPRT handle value, ordered.
917 *
918 * @param ph Pointer to the value to update.
919 * @param hNew The new value to assigned to *pu.
920 * @param hOld The old value to *pu compare with.
921 * @param fRc Where to store the result.
922 *
923 * @remarks This doesn't currently work for all handles (like RTFILE).
924 */
925#if HC_ARCH_BITS == 32
926# define ASMAtomicCmpXchgHandle(ph, hNew, hOld, fRc) \
927 do { \
928 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
929 (fRc) = ASMAtomicCmpXchgU32((uint32_t volatile *)(ph), (const uint32_t)(hNew), (const uint32_t)(hOld)); \
930 } while (0)
931#elif HC_ARCH_BITS == 64
932# define ASMAtomicCmpXchgHandle(ph, hNew, hOld, fRc) \
933 do { \
934 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
935 (fRc) = ASMAtomicCmpXchgU64((uint64_t volatile *)(ph), (const uint64_t)(hNew), (const uint64_t)(hOld)); \
936 } while (0)
937#else
938# error HC_ARCH_BITS
939#endif
940
941
942/** @def ASMAtomicCmpXchgSize
943 * Atomically Compare and Exchange a value which size might differ
944 * between platforms or compilers, ordered.
945 *
946 * @param pu Pointer to the value to update.
947 * @param uNew The new value to assigned to *pu.
948 * @param uOld The old value to *pu compare with.
949 * @param fRc Where to store the result.
950 */
951#define ASMAtomicCmpXchgSize(pu, uNew, uOld, fRc) \
952 do { \
953 switch (sizeof(*(pu))) { \
954 case 4: (fRc) = ASMAtomicCmpXchgU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew), (uint32_t)(uOld)); \
955 break; \
956 case 8: (fRc) = ASMAtomicCmpXchgU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew), (uint64_t)(uOld)); \
957 break; \
958 default: AssertMsgFailed(("ASMAtomicCmpXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
959 (fRc) = false; \
960 break; \
961 } \
962 } while (0)
963
964
965/**
966 * Atomically Compare and Exchange an unsigned 32-bit value, additionally
967 * passes back old value, ordered.
968 *
969 * @returns true if xchg was done.
970 * @returns false if xchg wasn't done.
971 *
972 * @param pu32 Pointer to the value to update.
973 * @param u32New The new value to assigned to *pu32.
974 * @param u32Old The old value to *pu32 compare with.
975 * @param pu32Old Pointer store the old value at.
976 */
977#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
978DECLASM(bool) ASMAtomicCmpXchgExU32(volatile uint32_t *pu32, const uint32_t u32New, const uint32_t u32Old, uint32_t *pu32Old);
979#else
980DECLINLINE(bool) ASMAtomicCmpXchgExU32(volatile uint32_t *pu32, const uint32_t u32New, const uint32_t u32Old, uint32_t *pu32Old)
981{
982# if RT_INLINE_ASM_GNU_STYLE
983 uint8_t u8Ret;
984 __asm__ __volatile__("lock; cmpxchgl %3, %0\n\t"
985 "setz %1\n\t"
986 : "=m" (*pu32),
987 "=qm" (u8Ret),
988 "=a" (*pu32Old)
989 : "r" (u32New),
990 "a" (u32Old),
991 "m" (*pu32));
992 return (bool)u8Ret;
993
994# elif RT_INLINE_ASM_USES_INTRIN
995 return (*pu32Old =_InterlockedCompareExchange((long *)pu32, u32New, u32Old)) == u32Old;
996
997# else
998 uint32_t u32Ret;
999 __asm
1000 {
1001# ifdef RT_ARCH_AMD64
1002 mov rdx, [pu32]
1003# else
1004 mov edx, [pu32]
1005# endif
1006 mov eax, [u32Old]
1007 mov ecx, [u32New]
1008# ifdef RT_ARCH_AMD64
1009 lock cmpxchg [rdx], ecx
1010 mov rdx, [pu32Old]
1011 mov [rdx], eax
1012# else
1013 lock cmpxchg [edx], ecx
1014 mov edx, [pu32Old]
1015 mov [edx], eax
1016# endif
1017 setz al
1018 movzx eax, al
1019 mov [u32Ret], eax
1020 }
1021 return !!u32Ret;
1022# endif
1023}
1024#endif
1025
1026
1027/**
1028 * Atomically Compare and Exchange a signed 32-bit value, additionally
1029 * passes back old value, ordered.
1030 *
1031 * @returns true if xchg was done.
1032 * @returns false if xchg wasn't done.
1033 *
1034 * @param pi32 Pointer to the value to update.
1035 * @param i32New The new value to assigned to *pi32.
1036 * @param i32Old The old value to *pi32 compare with.
1037 * @param pi32Old Pointer store the old value at.
1038 */
1039DECLINLINE(bool) ASMAtomicCmpXchgExS32(volatile int32_t *pi32, const int32_t i32New, const int32_t i32Old, int32_t *pi32Old)
1040{
1041 return ASMAtomicCmpXchgExU32((volatile uint32_t *)pi32, (uint32_t)i32New, (uint32_t)i32Old, (uint32_t *)pi32Old);
1042}
1043
1044
1045/**
1046 * Atomically Compare and exchange an unsigned 64-bit value, additionally
1047 * passing back old value, ordered.
1048 *
1049 * @returns true if xchg was done.
1050 * @returns false if xchg wasn't done.
1051 *
1052 * @param pu64 Pointer to the 64-bit variable to update.
1053 * @param u64New The 64-bit value to assign to *pu64.
1054 * @param u64Old The value to compare with.
1055 * @param pu64Old Pointer store the old value at.
1056 */
1057#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
1058 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
1059DECLASM(bool) ASMAtomicCmpXchgExU64(volatile uint64_t *pu64, const uint64_t u64New, const uint64_t u64Old, uint64_t *pu64Old);
1060#else
1061DECLINLINE(bool) ASMAtomicCmpXchgExU64(volatile uint64_t *pu64, const uint64_t u64New, const uint64_t u64Old, uint64_t *pu64Old)
1062{
1063# if RT_INLINE_ASM_USES_INTRIN
1064 return (*pu64Old =_InterlockedCompareExchange64((__int64 *)pu64, u64New, u64Old)) == u64Old;
1065
1066# elif defined(RT_ARCH_AMD64)
1067# if RT_INLINE_ASM_GNU_STYLE
1068 uint8_t u8Ret;
1069 __asm__ __volatile__("lock; cmpxchgq %3, %0\n\t"
1070 "setz %1\n\t"
1071 : "=m" (*pu64),
1072 "=qm" (u8Ret),
1073 "=a" (*pu64Old)
1074 : "r" (u64New),
1075 "a" (u64Old),
1076 "m" (*pu64));
1077 return (bool)u8Ret;
1078# else
1079 bool fRet;
1080 __asm
1081 {
1082 mov rdx, [pu32]
1083 mov rax, [u64Old]
1084 mov rcx, [u64New]
1085 lock cmpxchg [rdx], rcx
1086 mov rdx, [pu64Old]
1087 mov [rdx], rax
1088 setz al
1089 mov [fRet], al
1090 }
1091 return fRet;
1092# endif
1093# else /* !RT_ARCH_AMD64 */
1094# if RT_INLINE_ASM_GNU_STYLE
1095 uint64_t u64Ret;
1096# if defined(PIC) || defined(__PIC__)
1097 /* NB: this code uses a memory clobber description, because the clean
1098 * solution with an output value for *pu64 makes gcc run out of registers.
1099 * This will cause suboptimal code, and anyone with a better solution is
1100 * welcome to improve this. */
1101 __asm__ __volatile__("xchgl %%ebx, %1\n\t"
1102 "lock; cmpxchg8b %3\n\t"
1103 "xchgl %%ebx, %1\n\t"
1104 : "=A" (u64Ret)
1105 : "DS" ((uint32_t)u64New),
1106 "c" ((uint32_t)(u64New >> 32)),
1107 "m" (*pu64),
1108 "0" (u64Old)
1109 : "memory" );
1110# else /* !PIC */
1111 __asm__ __volatile__("lock; cmpxchg8b %4\n\t"
1112 : "=A" (u64Ret),
1113 "=m" (*pu64)
1114 : "b" ((uint32_t)u64New),
1115 "c" ((uint32_t)(u64New >> 32)),
1116 "m" (*pu64),
1117 "0" (u64Old));
1118# endif
1119 *pu64Old = u64Ret;
1120 return u64Ret == u64Old;
1121# else
1122 uint32_t u32Ret;
1123 __asm
1124 {
1125 mov ebx, dword ptr [u64New]
1126 mov ecx, dword ptr [u64New + 4]
1127 mov edi, [pu64]
1128 mov eax, dword ptr [u64Old]
1129 mov edx, dword ptr [u64Old + 4]
1130 lock cmpxchg8b [edi]
1131 mov ebx, [pu64Old]
1132 mov [ebx], eax
1133 setz al
1134 movzx eax, al
1135 add ebx, 4
1136 mov [ebx], edx
1137 mov dword ptr [u32Ret], eax
1138 }
1139 return !!u32Ret;
1140# endif
1141# endif /* !RT_ARCH_AMD64 */
1142}
1143#endif
1144
1145
1146/**
1147 * Atomically Compare and exchange a signed 64-bit value, additionally
1148 * passing back old value, ordered.
1149 *
1150 * @returns true if xchg was done.
1151 * @returns false if xchg wasn't done.
1152 *
1153 * @param pi64 Pointer to the 64-bit variable to update.
1154 * @param i64 The 64-bit value to assign to *pu64.
1155 * @param i64Old The value to compare with.
1156 * @param pi64Old Pointer store the old value at.
1157 */
1158DECLINLINE(bool) ASMAtomicCmpXchgExS64(volatile int64_t *pi64, const int64_t i64, const int64_t i64Old, int64_t *pi64Old)
1159{
1160 return ASMAtomicCmpXchgExU64((volatile uint64_t *)pi64, (uint64_t)i64, (uint64_t)i64Old, (uint64_t *)pi64Old);
1161}
1162
1163/** @def ASMAtomicCmpXchgExHandle
1164 * Atomically Compare and Exchange a typical IPRT handle value, ordered.
1165 *
1166 * @param ph Pointer to the value to update.
1167 * @param hNew The new value to assigned to *pu.
1168 * @param hOld The old value to *pu compare with.
1169 * @param fRc Where to store the result.
1170 * @param phOldVal Pointer to where to store the old value.
1171 *
1172 * @remarks This doesn't currently work for all handles (like RTFILE).
1173 */
1174#if HC_ARCH_BITS == 32
1175# define ASMAtomicCmpXchgExHandle(ph, hNew, hOld, fRc, phOldVal) \
1176 do { \
1177 AssertCompile(sizeof(*ph) == sizeof(uint32_t)); \
1178 AssertCompile(sizeof(*phOldVal) == sizeof(uint32_t)); \
1179 (fRc) = ASMAtomicCmpXchgExU32((volatile uint32_t *)(pu), (uint32_t)(uNew), (uint32_t)(uOld), (uint32_t *)(puOldVal)); \
1180 } while (0)
1181#elif HC_ARCH_BITS == 64
1182# define ASMAtomicCmpXchgExHandle(ph, hNew, hOld, fRc, phOldVal) \
1183 do { \
1184 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
1185 AssertCompile(sizeof(*(phOldVal)) == sizeof(uint64_t)); \
1186 (fRc) = ASMAtomicCmpXchgExU64((volatile uint64_t *)(pu), (uint64_t)(uNew), (uint64_t)(uOld), (uint64_t *)(puOldVal)); \
1187 } while (0)
1188#else
1189# error HC_ARCH_BITS
1190#endif
1191
1192
1193/** @def ASMAtomicCmpXchgExSize
1194 * Atomically Compare and Exchange a value which size might differ
1195 * between platforms or compilers. Additionally passes back old value.
1196 *
1197 * @param pu Pointer to the value to update.
1198 * @param uNew The new value to assigned to *pu.
1199 * @param uOld The old value to *pu compare with.
1200 * @param fRc Where to store the result.
1201 * @param puOldVal Pointer to where to store the old value.
1202 */
1203#define ASMAtomicCmpXchgExSize(pu, uNew, uOld, fRc, puOldVal) \
1204 do { \
1205 switch (sizeof(*(pu))) { \
1206 case 4: (fRc) = ASMAtomicCmpXchgExU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew), (uint32_t)(uOld), (uint32_t *)(uOldVal)); \
1207 break; \
1208 case 8: (fRc) = ASMAtomicCmpXchgExU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew), (uint64_t)(uOld), (uint64_t *)(uOldVal)); \
1209 break; \
1210 default: AssertMsgFailed(("ASMAtomicCmpXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
1211 (fRc) = false; \
1212 (uOldVal) = 0; \
1213 break; \
1214 } \
1215 } while (0)
1216
1217
1218/**
1219 * Atomically Compare and Exchange a pointer value, additionally
1220 * passing back old value, ordered.
1221 *
1222 * @returns true if xchg was done.
1223 * @returns false if xchg wasn't done.
1224 *
1225 * @param ppv Pointer to the value to update.
1226 * @param pvNew The new value to assigned to *ppv.
1227 * @param pvOld The old value to *ppv compare with.
1228 * @param ppvOld Pointer store the old value at.
1229 */
1230DECLINLINE(bool) ASMAtomicCmpXchgExPtrVoid(void * volatile *ppv, const void *pvNew, const void *pvOld, void **ppvOld)
1231{
1232#if ARCH_BITS == 32
1233 return ASMAtomicCmpXchgExU32((volatile uint32_t *)(void *)ppv, (uint32_t)pvNew, (uint32_t)pvOld, (uint32_t *)ppvOld);
1234#elif ARCH_BITS == 64
1235 return ASMAtomicCmpXchgExU64((volatile uint64_t *)(void *)ppv, (uint64_t)pvNew, (uint64_t)pvOld, (uint64_t *)ppvOld);
1236#else
1237# error "ARCH_BITS is bogus"
1238#endif
1239}
1240
1241
1242/**
1243 * Atomically Compare and Exchange a pointer value, additionally
1244 * passing back old value, ordered.
1245 *
1246 * @returns true if xchg was done.
1247 * @returns false if xchg wasn't done.
1248 *
1249 * @param ppv Pointer to the value to update.
1250 * @param pvNew The new value to assigned to *ppv.
1251 * @param pvOld The old value to *ppv compare with.
1252 * @param ppvOld Pointer store the old value at.
1253 *
1254 * @remarks This is relatively type safe on GCC platforms.
1255 */
1256#ifdef __GNUC__
1257# define ASMAtomicCmpXchgExPtr(ppv, pvNew, pvOld, ppvOld) \
1258 __extension__ \
1259 ({\
1260 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
1261 __typeof__(*(ppv)) const pvNewTypeChecked = (pvNew); \
1262 __typeof__(*(ppv)) const pvOldTypeChecked = (pvOld); \
1263 __typeof__(*(ppv)) * const ppvOldTypeChecked = (ppvOld); \
1264 bool fMacroRet = ASMAtomicCmpXchgExPtrVoid((void * volatile *)ppvTypeChecked, \
1265 (void *)pvNewTypeChecked, (void *)pvOldTypeChecked, \
1266 (void **)ppvOld); \
1267 fMacroRet; \
1268 })
1269#else
1270# define ASMAtomicCmpXchgExPtr(ppv, pvNew, pvOld, ppvOld) \
1271 ASMAtomicCmpXchgExPtrVoid((void * volatile *)(ppv), (void *)(pvNew), (void *)pvOld, (void **)ppvOld)
1272#endif
1273
1274
1275/**
1276 * Atomically exchanges and adds to a 32-bit value, ordered.
1277 *
1278 * @returns The old value.
1279 * @param pu32 Pointer to the value.
1280 * @param u32 Number to add.
1281 */
1282#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1283DECLASM(uint32_t) ASMAtomicAddU32(uint32_t volatile *pu32, uint32_t u32);
1284#else
1285DECLINLINE(uint32_t) ASMAtomicAddU32(uint32_t volatile *pu32, uint32_t u32)
1286{
1287# if RT_INLINE_ASM_USES_INTRIN
1288 u32 = _InterlockedExchangeAdd((long *)pu32, u32);
1289 return u32;
1290
1291# elif RT_INLINE_ASM_GNU_STYLE
1292 __asm__ __volatile__("lock; xaddl %0, %1\n\t"
1293 : "=r" (u32),
1294 "=m" (*pu32)
1295 : "0" (u32),
1296 "m" (*pu32)
1297 : "memory");
1298 return u32;
1299# else
1300 __asm
1301 {
1302 mov eax, [u32]
1303# ifdef RT_ARCH_AMD64
1304 mov rdx, [pu32]
1305 lock xadd [rdx], eax
1306# else
1307 mov edx, [pu32]
1308 lock xadd [edx], eax
1309# endif
1310 mov [u32], eax
1311 }
1312 return u32;
1313# endif
1314}
1315#endif
1316
1317
1318/**
1319 * Atomically exchanges and adds to a signed 32-bit value, ordered.
1320 *
1321 * @returns The old value.
1322 * @param pi32 Pointer to the value.
1323 * @param i32 Number to add.
1324 */
1325DECLINLINE(int32_t) ASMAtomicAddS32(int32_t volatile *pi32, int32_t i32)
1326{
1327 return (int32_t)ASMAtomicAddU32((uint32_t volatile *)pi32, (uint32_t)i32);
1328}
1329
1330
1331/**
1332 * Atomically exchanges and subtracts to an unsigned 32-bit value, ordered.
1333 *
1334 * @returns The old value.
1335 * @param pu32 Pointer to the value.
1336 * @param u32 Number to subtract.
1337 */
1338DECLINLINE(uint32_t) ASMAtomicSubU32(uint32_t volatile *pu32, uint32_t u32)
1339{
1340 return ASMAtomicAddU32(pu32, (uint32_t)-(int32_t)u32);
1341}
1342
1343
1344/**
1345 * Atomically exchanges and subtracts to a signed 32-bit value, ordered.
1346 *
1347 * @returns The old value.
1348 * @param pi32 Pointer to the value.
1349 * @param i32 Number to subtract.
1350 */
1351DECLINLINE(int32_t) ASMAtomicSubS32(int32_t volatile *pi32, int32_t i32)
1352{
1353 return (int32_t)ASMAtomicAddU32((uint32_t volatile *)pi32, (uint32_t)-i32);
1354}
1355
1356
1357/**
1358 * Atomically increment a 32-bit value, ordered.
1359 *
1360 * @returns The new value.
1361 * @param pu32 Pointer to the value to increment.
1362 */
1363#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1364DECLASM(uint32_t) ASMAtomicIncU32(uint32_t volatile *pu32);
1365#else
1366DECLINLINE(uint32_t) ASMAtomicIncU32(uint32_t volatile *pu32)
1367{
1368 uint32_t u32;
1369# if RT_INLINE_ASM_USES_INTRIN
1370 u32 = _InterlockedIncrement((long *)pu32);
1371 return u32;
1372
1373# elif RT_INLINE_ASM_GNU_STYLE
1374 __asm__ __volatile__("lock; xaddl %0, %1\n\t"
1375 : "=r" (u32),
1376 "=m" (*pu32)
1377 : "0" (1),
1378 "m" (*pu32)
1379 : "memory");
1380 return u32+1;
1381# else
1382 __asm
1383 {
1384 mov eax, 1
1385# ifdef RT_ARCH_AMD64
1386 mov rdx, [pu32]
1387 lock xadd [rdx], eax
1388# else
1389 mov edx, [pu32]
1390 lock xadd [edx], eax
1391# endif
1392 mov u32, eax
1393 }
1394 return u32+1;
1395# endif
1396}
1397#endif
1398
1399
1400/**
1401 * Atomically increment a signed 32-bit value, ordered.
1402 *
1403 * @returns The new value.
1404 * @param pi32 Pointer to the value to increment.
1405 */
1406DECLINLINE(int32_t) ASMAtomicIncS32(int32_t volatile *pi32)
1407{
1408 return (int32_t)ASMAtomicIncU32((uint32_t volatile *)pi32);
1409}
1410
1411
1412/**
1413 * Atomically decrement an unsigned 32-bit value, ordered.
1414 *
1415 * @returns The new value.
1416 * @param pu32 Pointer to the value to decrement.
1417 */
1418#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1419DECLASM(uint32_t) ASMAtomicDecU32(uint32_t volatile *pu32);
1420#else
1421DECLINLINE(uint32_t) ASMAtomicDecU32(uint32_t volatile *pu32)
1422{
1423 uint32_t u32;
1424# if RT_INLINE_ASM_USES_INTRIN
1425 u32 = _InterlockedDecrement((long *)pu32);
1426 return u32;
1427
1428# elif RT_INLINE_ASM_GNU_STYLE
1429 __asm__ __volatile__("lock; xaddl %0, %1\n\t"
1430 : "=r" (u32),
1431 "=m" (*pu32)
1432 : "0" (-1),
1433 "m" (*pu32)
1434 : "memory");
1435 return u32-1;
1436# else
1437 __asm
1438 {
1439 mov eax, -1
1440# ifdef RT_ARCH_AMD64
1441 mov rdx, [pu32]
1442 lock xadd [rdx], eax
1443# else
1444 mov edx, [pu32]
1445 lock xadd [edx], eax
1446# endif
1447 mov u32, eax
1448 }
1449 return u32-1;
1450# endif
1451}
1452#endif
1453
1454
1455/**
1456 * Atomically decrement a signed 32-bit value, ordered.
1457 *
1458 * @returns The new value.
1459 * @param pi32 Pointer to the value to decrement.
1460 */
1461DECLINLINE(int32_t) ASMAtomicDecS32(int32_t volatile *pi32)
1462{
1463 return (int32_t)ASMAtomicDecU32((uint32_t volatile *)pi32);
1464}
1465
1466
1467/**
1468 * Atomically Or an unsigned 32-bit value, ordered.
1469 *
1470 * @param pu32 Pointer to the pointer variable to OR u32 with.
1471 * @param u32 The value to OR *pu32 with.
1472 */
1473#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1474DECLASM(void) ASMAtomicOrU32(uint32_t volatile *pu32, uint32_t u32);
1475#else
1476DECLINLINE(void) ASMAtomicOrU32(uint32_t volatile *pu32, uint32_t u32)
1477{
1478# if RT_INLINE_ASM_USES_INTRIN
1479 _InterlockedOr((long volatile *)pu32, (long)u32);
1480
1481# elif RT_INLINE_ASM_GNU_STYLE
1482 __asm__ __volatile__("lock; orl %1, %0\n\t"
1483 : "=m" (*pu32)
1484 : "ir" (u32),
1485 "m" (*pu32));
1486# else
1487 __asm
1488 {
1489 mov eax, [u32]
1490# ifdef RT_ARCH_AMD64
1491 mov rdx, [pu32]
1492 lock or [rdx], eax
1493# else
1494 mov edx, [pu32]
1495 lock or [edx], eax
1496# endif
1497 }
1498# endif
1499}
1500#endif
1501
1502
1503/**
1504 * Atomically Or a signed 32-bit value, ordered.
1505 *
1506 * @param pi32 Pointer to the pointer variable to OR u32 with.
1507 * @param i32 The value to OR *pu32 with.
1508 */
1509DECLINLINE(void) ASMAtomicOrS32(int32_t volatile *pi32, int32_t i32)
1510{
1511 ASMAtomicOrU32((uint32_t volatile *)pi32, i32);
1512}
1513
1514
1515/**
1516 * Atomically And an unsigned 32-bit value, ordered.
1517 *
1518 * @param pu32 Pointer to the pointer variable to AND u32 with.
1519 * @param u32 The value to AND *pu32 with.
1520 */
1521#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1522DECLASM(void) ASMAtomicAndU32(uint32_t volatile *pu32, uint32_t u32);
1523#else
1524DECLINLINE(void) ASMAtomicAndU32(uint32_t volatile *pu32, uint32_t u32)
1525{
1526# if RT_INLINE_ASM_USES_INTRIN
1527 _InterlockedAnd((long volatile *)pu32, u32);
1528
1529# elif RT_INLINE_ASM_GNU_STYLE
1530 __asm__ __volatile__("lock; andl %1, %0\n\t"
1531 : "=m" (*pu32)
1532 : "ir" (u32),
1533 "m" (*pu32));
1534# else
1535 __asm
1536 {
1537 mov eax, [u32]
1538# ifdef RT_ARCH_AMD64
1539 mov rdx, [pu32]
1540 lock and [rdx], eax
1541# else
1542 mov edx, [pu32]
1543 lock and [edx], eax
1544# endif
1545 }
1546# endif
1547}
1548#endif
1549
1550
1551/**
1552 * Atomically And a signed 32-bit value, ordered.
1553 *
1554 * @param pi32 Pointer to the pointer variable to AND i32 with.
1555 * @param i32 The value to AND *pi32 with.
1556 */
1557DECLINLINE(void) ASMAtomicAndS32(int32_t volatile *pi32, int32_t i32)
1558{
1559 ASMAtomicAndU32((uint32_t volatile *)pi32, (uint32_t)i32);
1560}
1561
1562
1563/**
1564 * Serialize Instruction.
1565 */
1566#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1567DECLASM(void) ASMSerializeInstruction(void);
1568#else
1569DECLINLINE(void) ASMSerializeInstruction(void)
1570{
1571# if RT_INLINE_ASM_GNU_STYLE
1572 RTCCUINTREG xAX = 0;
1573# ifdef RT_ARCH_AMD64
1574 __asm__ ("cpuid"
1575 : "=a" (xAX)
1576 : "0" (xAX)
1577 : "rbx", "rcx", "rdx");
1578# elif (defined(PIC) || defined(__PIC__)) && defined(__i386__)
1579 __asm__ ("push %%ebx\n\t"
1580 "cpuid\n\t"
1581 "pop %%ebx\n\t"
1582 : "=a" (xAX)
1583 : "0" (xAX)
1584 : "ecx", "edx");
1585# else
1586 __asm__ ("cpuid"
1587 : "=a" (xAX)
1588 : "0" (xAX)
1589 : "ebx", "ecx", "edx");
1590# endif
1591
1592# elif RT_INLINE_ASM_USES_INTRIN
1593 int aInfo[4];
1594 __cpuid(aInfo, 0);
1595
1596# else
1597 __asm
1598 {
1599 push ebx
1600 xor eax, eax
1601 cpuid
1602 pop ebx
1603 }
1604# endif
1605}
1606#endif
1607
1608
1609/**
1610 * Memory fence, waits for any pending writes and reads to complete.
1611 */
1612DECLINLINE(void) ASMMemoryFence(void)
1613{
1614 /** @todo use mfence? check if all cpus we care for support it. */
1615 uint32_t volatile u32;
1616 ASMAtomicXchgU32(&u32, 0);
1617}
1618
1619
1620/**
1621 * Write fence, waits for any pending writes to complete.
1622 */
1623DECLINLINE(void) ASMWriteFence(void)
1624{
1625 /** @todo use sfence? check if all cpus we care for support it. */
1626 ASMMemoryFence();
1627}
1628
1629
1630/**
1631 * Read fence, waits for any pending reads to complete.
1632 */
1633DECLINLINE(void) ASMReadFence(void)
1634{
1635 /** @todo use lfence? check if all cpus we care for support it. */
1636 ASMMemoryFence();
1637}
1638
1639
1640/**
1641 * Atomically reads an unsigned 8-bit value, ordered.
1642 *
1643 * @returns Current *pu8 value
1644 * @param pu8 Pointer to the 8-bit variable to read.
1645 */
1646DECLINLINE(uint8_t) ASMAtomicReadU8(volatile uint8_t *pu8)
1647{
1648 ASMMemoryFence();
1649 return *pu8; /* byte reads are atomic on x86 */
1650}
1651
1652
1653/**
1654 * Atomically reads an unsigned 8-bit value, unordered.
1655 *
1656 * @returns Current *pu8 value
1657 * @param pu8 Pointer to the 8-bit variable to read.
1658 */
1659DECLINLINE(uint8_t) ASMAtomicUoReadU8(volatile uint8_t *pu8)
1660{
1661 return *pu8; /* byte reads are atomic on x86 */
1662}
1663
1664
1665/**
1666 * Atomically reads a signed 8-bit value, ordered.
1667 *
1668 * @returns Current *pi8 value
1669 * @param pi8 Pointer to the 8-bit variable to read.
1670 */
1671DECLINLINE(int8_t) ASMAtomicReadS8(volatile int8_t *pi8)
1672{
1673 ASMMemoryFence();
1674 return *pi8; /* byte reads are atomic on x86 */
1675}
1676
1677
1678/**
1679 * Atomically reads a signed 8-bit value, unordered.
1680 *
1681 * @returns Current *pi8 value
1682 * @param pi8 Pointer to the 8-bit variable to read.
1683 */
1684DECLINLINE(int8_t) ASMAtomicUoReadS8(volatile int8_t *pi8)
1685{
1686 return *pi8; /* byte reads are atomic on x86 */
1687}
1688
1689
1690/**
1691 * Atomically reads an unsigned 16-bit value, ordered.
1692 *
1693 * @returns Current *pu16 value
1694 * @param pu16 Pointer to the 16-bit variable to read.
1695 */
1696DECLINLINE(uint16_t) ASMAtomicReadU16(volatile uint16_t *pu16)
1697{
1698 ASMMemoryFence();
1699 Assert(!((uintptr_t)pu16 & 1));
1700 return *pu16;
1701}
1702
1703
1704/**
1705 * Atomically reads an unsigned 16-bit value, unordered.
1706 *
1707 * @returns Current *pu16 value
1708 * @param pu16 Pointer to the 16-bit variable to read.
1709 */
1710DECLINLINE(uint16_t) ASMAtomicUoReadU16(volatile uint16_t *pu16)
1711{
1712 Assert(!((uintptr_t)pu16 & 1));
1713 return *pu16;
1714}
1715
1716
1717/**
1718 * Atomically reads a signed 16-bit value, ordered.
1719 *
1720 * @returns Current *pi16 value
1721 * @param pi16 Pointer to the 16-bit variable to read.
1722 */
1723DECLINLINE(int16_t) ASMAtomicReadS16(volatile int16_t *pi16)
1724{
1725 ASMMemoryFence();
1726 Assert(!((uintptr_t)pi16 & 1));
1727 return *pi16;
1728}
1729
1730
1731/**
1732 * Atomically reads a signed 16-bit value, unordered.
1733 *
1734 * @returns Current *pi16 value
1735 * @param pi16 Pointer to the 16-bit variable to read.
1736 */
1737DECLINLINE(int16_t) ASMAtomicUoReadS16(volatile int16_t *pi16)
1738{
1739 Assert(!((uintptr_t)pi16 & 1));
1740 return *pi16;
1741}
1742
1743
1744/**
1745 * Atomically reads an unsigned 32-bit value, ordered.
1746 *
1747 * @returns Current *pu32 value
1748 * @param pu32 Pointer to the 32-bit variable to read.
1749 */
1750DECLINLINE(uint32_t) ASMAtomicReadU32(volatile uint32_t *pu32)
1751{
1752 ASMMemoryFence();
1753 Assert(!((uintptr_t)pu32 & 3));
1754 return *pu32;
1755}
1756
1757
1758/**
1759 * Atomically reads an unsigned 32-bit value, unordered.
1760 *
1761 * @returns Current *pu32 value
1762 * @param pu32 Pointer to the 32-bit variable to read.
1763 */
1764DECLINLINE(uint32_t) ASMAtomicUoReadU32(volatile uint32_t *pu32)
1765{
1766 Assert(!((uintptr_t)pu32 & 3));
1767 return *pu32;
1768}
1769
1770
1771/**
1772 * Atomically reads a signed 32-bit value, ordered.
1773 *
1774 * @returns Current *pi32 value
1775 * @param pi32 Pointer to the 32-bit variable to read.
1776 */
1777DECLINLINE(int32_t) ASMAtomicReadS32(volatile int32_t *pi32)
1778{
1779 ASMMemoryFence();
1780 Assert(!((uintptr_t)pi32 & 3));
1781 return *pi32;
1782}
1783
1784
1785/**
1786 * Atomically reads a signed 32-bit value, unordered.
1787 *
1788 * @returns Current *pi32 value
1789 * @param pi32 Pointer to the 32-bit variable to read.
1790 */
1791DECLINLINE(int32_t) ASMAtomicUoReadS32(volatile int32_t *pi32)
1792{
1793 Assert(!((uintptr_t)pi32 & 3));
1794 return *pi32;
1795}
1796
1797
1798/**
1799 * Atomically reads an unsigned 64-bit value, ordered.
1800 *
1801 * @returns Current *pu64 value
1802 * @param pu64 Pointer to the 64-bit variable to read.
1803 * The memory pointed to must be writable.
1804 * @remark This will fault if the memory is read-only!
1805 */
1806#if (RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)) \
1807 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
1808DECLASM(uint64_t) ASMAtomicReadU64(volatile uint64_t *pu64);
1809#else
1810DECLINLINE(uint64_t) ASMAtomicReadU64(volatile uint64_t *pu64)
1811{
1812 uint64_t u64;
1813# ifdef RT_ARCH_AMD64
1814 Assert(!((uintptr_t)pu64 & 7));
1815/*# if RT_INLINE_ASM_GNU_STYLE
1816 __asm__ __volatile__( "mfence\n\t"
1817 "movq %1, %0\n\t"
1818 : "=r" (u64)
1819 : "m" (*pu64));
1820# else
1821 __asm
1822 {
1823 mfence
1824 mov rdx, [pu64]
1825 mov rax, [rdx]
1826 mov [u64], rax
1827 }
1828# endif*/
1829 ASMMemoryFence();
1830 u64 = *pu64;
1831# else /* !RT_ARCH_AMD64 */
1832# if RT_INLINE_ASM_GNU_STYLE
1833# if defined(PIC) || defined(__PIC__)
1834 uint32_t u32EBX = 0;
1835 Assert(!((uintptr_t)pu64 & 7));
1836 __asm__ __volatile__("xchgl %%ebx, %3\n\t"
1837 "lock; cmpxchg8b (%5)\n\t"
1838 "movl %3, %%ebx\n\t"
1839 : "=A" (u64),
1840# if (__GNUC__ * 100 + __GNUC_MINOR__) >= 403
1841 "+m" (*pu64)
1842# else
1843 "=m" (*pu64)
1844# endif
1845 : "0" (0),
1846 "m" (u32EBX),
1847 "c" (0),
1848 "S" (pu64));
1849# else /* !PIC */
1850 __asm__ __volatile__("lock; cmpxchg8b %1\n\t"
1851 : "=A" (u64),
1852 "+m" (*pu64)
1853 : "0" (0),
1854 "b" (0),
1855 "c" (0));
1856# endif
1857# else
1858 Assert(!((uintptr_t)pu64 & 7));
1859 __asm
1860 {
1861 xor eax, eax
1862 xor edx, edx
1863 mov edi, pu64
1864 xor ecx, ecx
1865 xor ebx, ebx
1866 lock cmpxchg8b [edi]
1867 mov dword ptr [u64], eax
1868 mov dword ptr [u64 + 4], edx
1869 }
1870# endif
1871# endif /* !RT_ARCH_AMD64 */
1872 return u64;
1873}
1874#endif
1875
1876
1877/**
1878 * Atomically reads an unsigned 64-bit value, unordered.
1879 *
1880 * @returns Current *pu64 value
1881 * @param pu64 Pointer to the 64-bit variable to read.
1882 * The memory pointed to must be writable.
1883 * @remark This will fault if the memory is read-only!
1884 */
1885#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
1886 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
1887DECLASM(uint64_t) ASMAtomicUoReadU64(volatile uint64_t *pu64);
1888#else
1889DECLINLINE(uint64_t) ASMAtomicUoReadU64(volatile uint64_t *pu64)
1890{
1891 uint64_t u64;
1892# ifdef RT_ARCH_AMD64
1893 Assert(!((uintptr_t)pu64 & 7));
1894/*# if RT_INLINE_ASM_GNU_STYLE
1895 Assert(!((uintptr_t)pu64 & 7));
1896 __asm__ __volatile__("movq %1, %0\n\t"
1897 : "=r" (u64)
1898 : "m" (*pu64));
1899# else
1900 __asm
1901 {
1902 mov rdx, [pu64]
1903 mov rax, [rdx]
1904 mov [u64], rax
1905 }
1906# endif */
1907 u64 = *pu64;
1908# else /* !RT_ARCH_AMD64 */
1909# if RT_INLINE_ASM_GNU_STYLE
1910# if defined(PIC) || defined(__PIC__)
1911 uint32_t u32EBX = 0;
1912 uint32_t u32Spill;
1913 Assert(!((uintptr_t)pu64 & 7));
1914 __asm__ __volatile__("xor %%eax,%%eax\n\t"
1915 "xor %%ecx,%%ecx\n\t"
1916 "xor %%edx,%%edx\n\t"
1917 "xchgl %%ebx, %3\n\t"
1918 "lock; cmpxchg8b (%4)\n\t"
1919 "movl %3, %%ebx\n\t"
1920 : "=A" (u64),
1921# if (__GNUC__ * 100 + __GNUC_MINOR__) >= 403
1922 "+m" (*pu64),
1923# else
1924 "=m" (*pu64),
1925# endif
1926 "=c" (u32Spill)
1927 : "m" (u32EBX),
1928 "S" (pu64));
1929# else /* !PIC */
1930 __asm__ __volatile__("lock; cmpxchg8b %1\n\t"
1931 : "=A" (u64),
1932 "+m" (*pu64)
1933 : "0" (0),
1934 "b" (0),
1935 "c" (0));
1936# endif
1937# else
1938 Assert(!((uintptr_t)pu64 & 7));
1939 __asm
1940 {
1941 xor eax, eax
1942 xor edx, edx
1943 mov edi, pu64
1944 xor ecx, ecx
1945 xor ebx, ebx
1946 lock cmpxchg8b [edi]
1947 mov dword ptr [u64], eax
1948 mov dword ptr [u64 + 4], edx
1949 }
1950# endif
1951# endif /* !RT_ARCH_AMD64 */
1952 return u64;
1953}
1954#endif
1955
1956
1957/**
1958 * Atomically reads a signed 64-bit value, ordered.
1959 *
1960 * @returns Current *pi64 value
1961 * @param pi64 Pointer to the 64-bit variable to read.
1962 * The memory pointed to must be writable.
1963 * @remark This will fault if the memory is read-only!
1964 */
1965DECLINLINE(int64_t) ASMAtomicReadS64(volatile int64_t *pi64)
1966{
1967 return (int64_t)ASMAtomicReadU64((volatile uint64_t *)pi64);
1968}
1969
1970
1971/**
1972 * Atomically reads a signed 64-bit value, unordered.
1973 *
1974 * @returns Current *pi64 value
1975 * @param pi64 Pointer to the 64-bit variable to read.
1976 * The memory pointed to must be writable.
1977 * @remark This will fault if the memory is read-only!
1978 */
1979DECLINLINE(int64_t) ASMAtomicUoReadS64(volatile int64_t *pi64)
1980{
1981 return (int64_t)ASMAtomicUoReadU64((volatile uint64_t *)pi64);
1982}
1983
1984
1985/**
1986 * Atomically reads a pointer value, ordered.
1987 *
1988 * @returns Current *pv value
1989 * @param ppv Pointer to the pointer variable to read.
1990 *
1991 * @remarks Please use ASMAtomicReadPtrT, it provides better type safety and
1992 * requires less typing (no casts).
1993 */
1994DECLINLINE(void *) ASMAtomicReadPtr(void * volatile *ppv)
1995{
1996#if ARCH_BITS == 32
1997 return (void *)ASMAtomicReadU32((volatile uint32_t *)(void *)ppv);
1998#elif ARCH_BITS == 64
1999 return (void *)ASMAtomicReadU64((volatile uint64_t *)(void *)ppv);
2000#else
2001# error "ARCH_BITS is bogus"
2002#endif
2003}
2004
2005/**
2006 * Convenience macro for avoiding the annoying casting with ASMAtomicReadPtr.
2007 *
2008 * @returns Current *pv value
2009 * @param ppv Pointer to the pointer variable to read.
2010 * @param Type The type of *ppv, sans volatile.
2011 */
2012#ifdef __GNUC__
2013# define ASMAtomicReadPtrT(ppv, Type) \
2014 __extension__ \
2015 ({\
2016 __typeof__(*(ppv)) volatile *ppvTypeChecked = (ppv); \
2017 Type pvTypeChecked = (__typeof__(*(ppv))) ASMAtomicReadPtr((void * volatile *)ppvTypeChecked); \
2018 pvTypeChecked; \
2019 })
2020#else
2021# define ASMAtomicReadPtrT(ppv, Type) \
2022 (Type)ASMAtomicReadPtr((void * volatile *)(ppv))
2023#endif
2024
2025
2026/**
2027 * Atomically reads a pointer value, unordered.
2028 *
2029 * @returns Current *pv value
2030 * @param ppv Pointer to the pointer variable to read.
2031 *
2032 * @remarks Please use ASMAtomicUoReadPtrT, it provides better type safety and
2033 * requires less typing (no casts).
2034 */
2035DECLINLINE(void *) ASMAtomicUoReadPtr(void * volatile *ppv)
2036{
2037#if ARCH_BITS == 32
2038 return (void *)ASMAtomicUoReadU32((volatile uint32_t *)(void *)ppv);
2039#elif ARCH_BITS == 64
2040 return (void *)ASMAtomicUoReadU64((volatile uint64_t *)(void *)ppv);
2041#else
2042# error "ARCH_BITS is bogus"
2043#endif
2044}
2045
2046
2047/**
2048 * Convenience macro for avoiding the annoying casting with ASMAtomicUoReadPtr.
2049 *
2050 * @returns Current *pv value
2051 * @param ppv Pointer to the pointer variable to read.
2052 * @param Type The type of *ppv, sans volatile.
2053 */
2054#ifdef __GNUC__
2055# define ASMAtomicUoReadPtrT(ppv, Type) \
2056 __extension__ \
2057 ({\
2058 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
2059 Type pvTypeChecked = (__typeof__(*(ppv))) ASMAtomicUoReadPtr((void * volatile *)ppvTypeChecked); \
2060 pvTypeChecked; \
2061 })
2062#else
2063# define ASMAtomicUoReadPtrT(ppv, Type) \
2064 (Type)ASMAtomicUoReadPtr((void * volatile *)(ppv))
2065#endif
2066
2067
2068/**
2069 * Atomically reads a boolean value, ordered.
2070 *
2071 * @returns Current *pf value
2072 * @param pf Pointer to the boolean variable to read.
2073 */
2074DECLINLINE(bool) ASMAtomicReadBool(volatile bool *pf)
2075{
2076 ASMMemoryFence();
2077 return *pf; /* byte reads are atomic on x86 */
2078}
2079
2080
2081/**
2082 * Atomically reads a boolean value, unordered.
2083 *
2084 * @returns Current *pf value
2085 * @param pf Pointer to the boolean variable to read.
2086 */
2087DECLINLINE(bool) ASMAtomicUoReadBool(volatile bool *pf)
2088{
2089 return *pf; /* byte reads are atomic on x86 */
2090}
2091
2092
2093/**
2094 * Atomically read a typical IPRT handle value, ordered.
2095 *
2096 * @param ph Pointer to the handle variable to read.
2097 * @param phRes Where to store the result.
2098 *
2099 * @remarks This doesn't currently work for all handles (like RTFILE).
2100 */
2101#if HC_ARCH_BITS == 32
2102# define ASMAtomicReadHandle(ph, phRes) \
2103 do { \
2104 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
2105 AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
2106 *(uint32_t *)(phRes) = ASMAtomicReadU32((uint32_t volatile *)(ph)); \
2107 } while (0)
2108#elif HC_ARCH_BITS == 64
2109# define ASMAtomicReadHandle(ph, phRes) \
2110 do { \
2111 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
2112 AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
2113 *(uint64_t *)(phRes) = ASMAtomicReadU64((uint64_t volatile *)(ph)); \
2114 } while (0)
2115#else
2116# error HC_ARCH_BITS
2117#endif
2118
2119
2120/**
2121 * Atomically read a typical IPRT handle value, unordered.
2122 *
2123 * @param ph Pointer to the handle variable to read.
2124 * @param phRes Where to store the result.
2125 *
2126 * @remarks This doesn't currently work for all handles (like RTFILE).
2127 */
2128#if HC_ARCH_BITS == 32
2129# define ASMAtomicUoReadHandle(ph, phRes) \
2130 do { \
2131 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
2132 AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
2133 *(uint32_t *)(phRes) = ASMAtomicUoReadU32((uint32_t volatile *)(ph)); \
2134 } while (0)
2135#elif HC_ARCH_BITS == 64
2136# define ASMAtomicUoReadHandle(ph, phRes) \
2137 do { \
2138 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
2139 AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
2140 *(uint64_t *)(phRes) = ASMAtomicUoReadU64((uint64_t volatile *)(ph)); \
2141 } while (0)
2142#else
2143# error HC_ARCH_BITS
2144#endif
2145
2146
2147/**
2148 * Atomically read a value which size might differ
2149 * between platforms or compilers, ordered.
2150 *
2151 * @param pu Pointer to the variable to update.
2152 * @param puRes Where to store the result.
2153 */
2154#define ASMAtomicReadSize(pu, puRes) \
2155 do { \
2156 switch (sizeof(*(pu))) { \
2157 case 1: *(uint8_t *)(puRes) = ASMAtomicReadU8( (volatile uint8_t *)(void *)(pu)); break; \
2158 case 2: *(uint16_t *)(puRes) = ASMAtomicReadU16((volatile uint16_t *)(void *)(pu)); break; \
2159 case 4: *(uint32_t *)(puRes) = ASMAtomicReadU32((volatile uint32_t *)(void *)(pu)); break; \
2160 case 8: *(uint64_t *)(puRes) = ASMAtomicReadU64((volatile uint64_t *)(void *)(pu)); break; \
2161 default: AssertMsgFailed(("ASMAtomicReadSize: size %d is not supported\n", sizeof(*(pu)))); \
2162 } \
2163 } while (0)
2164
2165
2166/**
2167 * Atomically read a value which size might differ
2168 * between platforms or compilers, unordered.
2169 *
2170 * @param pu Pointer to the variable to read.
2171 * @param puRes Where to store the result.
2172 */
2173#define ASMAtomicUoReadSize(pu, puRes) \
2174 do { \
2175 switch (sizeof(*(pu))) { \
2176 case 1: *(uint8_t *)(puRes) = ASMAtomicUoReadU8( (volatile uint8_t *)(void *)(pu)); break; \
2177 case 2: *(uint16_t *)(puRes) = ASMAtomicUoReadU16((volatile uint16_t *)(void *)(pu)); break; \
2178 case 4: *(uint32_t *)(puRes) = ASMAtomicUoReadU32((volatile uint32_t *)(void *)(pu)); break; \
2179 case 8: *(uint64_t *)(puRes) = ASMAtomicUoReadU64((volatile uint64_t *)(void *)(pu)); break; \
2180 default: AssertMsgFailed(("ASMAtomicReadSize: size %d is not supported\n", sizeof(*(pu)))); \
2181 } \
2182 } while (0)
2183
2184
2185/**
2186 * Atomically writes an unsigned 8-bit value, ordered.
2187 *
2188 * @param pu8 Pointer to the 8-bit variable.
2189 * @param u8 The 8-bit value to assign to *pu8.
2190 */
2191DECLINLINE(void) ASMAtomicWriteU8(volatile uint8_t *pu8, uint8_t u8)
2192{
2193 ASMAtomicXchgU8(pu8, u8);
2194}
2195
2196
2197/**
2198 * Atomically writes an unsigned 8-bit value, unordered.
2199 *
2200 * @param pu8 Pointer to the 8-bit variable.
2201 * @param u8 The 8-bit value to assign to *pu8.
2202 */
2203DECLINLINE(void) ASMAtomicUoWriteU8(volatile uint8_t *pu8, uint8_t u8)
2204{
2205 *pu8 = u8; /* byte writes are atomic on x86 */
2206}
2207
2208
2209/**
2210 * Atomically writes a signed 8-bit value, ordered.
2211 *
2212 * @param pi8 Pointer to the 8-bit variable to read.
2213 * @param i8 The 8-bit value to assign to *pi8.
2214 */
2215DECLINLINE(void) ASMAtomicWriteS8(volatile int8_t *pi8, int8_t i8)
2216{
2217 ASMAtomicXchgS8(pi8, i8);
2218}
2219
2220
2221/**
2222 * Atomically writes a signed 8-bit value, unordered.
2223 *
2224 * @param pi8 Pointer to the 8-bit variable to read.
2225 * @param i8 The 8-bit value to assign to *pi8.
2226 */
2227DECLINLINE(void) ASMAtomicUoWriteS8(volatile int8_t *pi8, int8_t i8)
2228{
2229 *pi8 = i8; /* byte writes are atomic on x86 */
2230}
2231
2232
2233/**
2234 * Atomically writes an unsigned 16-bit value, ordered.
2235 *
2236 * @param pu16 Pointer to the 16-bit variable.
2237 * @param u16 The 16-bit value to assign to *pu16.
2238 */
2239DECLINLINE(void) ASMAtomicWriteU16(volatile uint16_t *pu16, uint16_t u16)
2240{
2241 ASMAtomicXchgU16(pu16, u16);
2242}
2243
2244
2245/**
2246 * Atomically writes an unsigned 16-bit value, unordered.
2247 *
2248 * @param pu16 Pointer to the 16-bit variable.
2249 * @param u16 The 16-bit value to assign to *pu16.
2250 */
2251DECLINLINE(void) ASMAtomicUoWriteU16(volatile uint16_t *pu16, uint16_t u16)
2252{
2253 Assert(!((uintptr_t)pu16 & 1));
2254 *pu16 = u16;
2255}
2256
2257
2258/**
2259 * Atomically writes a signed 16-bit value, ordered.
2260 *
2261 * @param pi16 Pointer to the 16-bit variable to read.
2262 * @param i16 The 16-bit value to assign to *pi16.
2263 */
2264DECLINLINE(void) ASMAtomicWriteS16(volatile int16_t *pi16, int16_t i16)
2265{
2266 ASMAtomicXchgS16(pi16, i16);
2267}
2268
2269
2270/**
2271 * Atomically writes a signed 16-bit value, unordered.
2272 *
2273 * @param pi16 Pointer to the 16-bit variable to read.
2274 * @param i16 The 16-bit value to assign to *pi16.
2275 */
2276DECLINLINE(void) ASMAtomicUoWriteS16(volatile int16_t *pi16, int16_t i16)
2277{
2278 Assert(!((uintptr_t)pi16 & 1));
2279 *pi16 = i16;
2280}
2281
2282
2283/**
2284 * Atomically writes an unsigned 32-bit value, ordered.
2285 *
2286 * @param pu32 Pointer to the 32-bit variable.
2287 * @param u32 The 32-bit value to assign to *pu32.
2288 */
2289DECLINLINE(void) ASMAtomicWriteU32(volatile uint32_t *pu32, uint32_t u32)
2290{
2291 ASMAtomicXchgU32(pu32, u32);
2292}
2293
2294
2295/**
2296 * Atomically writes an unsigned 32-bit value, unordered.
2297 *
2298 * @param pu32 Pointer to the 32-bit variable.
2299 * @param u32 The 32-bit value to assign to *pu32.
2300 */
2301DECLINLINE(void) ASMAtomicUoWriteU32(volatile uint32_t *pu32, uint32_t u32)
2302{
2303 Assert(!((uintptr_t)pu32 & 3));
2304 *pu32 = u32;
2305}
2306
2307
2308/**
2309 * Atomically writes a signed 32-bit value, ordered.
2310 *
2311 * @param pi32 Pointer to the 32-bit variable to read.
2312 * @param i32 The 32-bit value to assign to *pi32.
2313 */
2314DECLINLINE(void) ASMAtomicWriteS32(volatile int32_t *pi32, int32_t i32)
2315{
2316 ASMAtomicXchgS32(pi32, i32);
2317}
2318
2319
2320/**
2321 * Atomically writes a signed 32-bit value, unordered.
2322 *
2323 * @param pi32 Pointer to the 32-bit variable to read.
2324 * @param i32 The 32-bit value to assign to *pi32.
2325 */
2326DECLINLINE(void) ASMAtomicUoWriteS32(volatile int32_t *pi32, int32_t i32)
2327{
2328 Assert(!((uintptr_t)pi32 & 3));
2329 *pi32 = i32;
2330}
2331
2332
2333/**
2334 * Atomically writes an unsigned 64-bit value, ordered.
2335 *
2336 * @param pu64 Pointer to the 64-bit variable.
2337 * @param u64 The 64-bit value to assign to *pu64.
2338 */
2339DECLINLINE(void) ASMAtomicWriteU64(volatile uint64_t *pu64, uint64_t u64)
2340{
2341 ASMAtomicXchgU64(pu64, u64);
2342}
2343
2344
2345/**
2346 * Atomically writes an unsigned 64-bit value, unordered.
2347 *
2348 * @param pu64 Pointer to the 64-bit variable.
2349 * @param u64 The 64-bit value to assign to *pu64.
2350 */
2351DECLINLINE(void) ASMAtomicUoWriteU64(volatile uint64_t *pu64, uint64_t u64)
2352{
2353 Assert(!((uintptr_t)pu64 & 7));
2354#if ARCH_BITS == 64
2355 *pu64 = u64;
2356#else
2357 ASMAtomicXchgU64(pu64, u64);
2358#endif
2359}
2360
2361
2362/**
2363 * Atomically writes a signed 64-bit value, ordered.
2364 *
2365 * @param pi64 Pointer to the 64-bit variable.
2366 * @param i64 The 64-bit value to assign to *pi64.
2367 */
2368DECLINLINE(void) ASMAtomicWriteS64(volatile int64_t *pi64, int64_t i64)
2369{
2370 ASMAtomicXchgS64(pi64, i64);
2371}
2372
2373
2374/**
2375 * Atomically writes a signed 64-bit value, unordered.
2376 *
2377 * @param pi64 Pointer to the 64-bit variable.
2378 * @param i64 The 64-bit value to assign to *pi64.
2379 */
2380DECLINLINE(void) ASMAtomicUoWriteS64(volatile int64_t *pi64, int64_t i64)
2381{
2382 Assert(!((uintptr_t)pi64 & 7));
2383#if ARCH_BITS == 64
2384 *pi64 = i64;
2385#else
2386 ASMAtomicXchgS64(pi64, i64);
2387#endif
2388}
2389
2390
2391/**
2392 * Atomically writes a boolean value, unordered.
2393 *
2394 * @param pf Pointer to the boolean variable.
2395 * @param f The boolean value to assign to *pf.
2396 */
2397DECLINLINE(void) ASMAtomicWriteBool(volatile bool *pf, bool f)
2398{
2399 ASMAtomicWriteU8((uint8_t volatile *)pf, f);
2400}
2401
2402
2403/**
2404 * Atomically writes a boolean value, unordered.
2405 *
2406 * @param pf Pointer to the boolean variable.
2407 * @param f The boolean value to assign to *pf.
2408 */
2409DECLINLINE(void) ASMAtomicUoWriteBool(volatile bool *pf, bool f)
2410{
2411 *pf = f; /* byte writes are atomic on x86 */
2412}
2413
2414
2415/**
2416 * Atomically writes a pointer value, ordered.
2417 *
2418 * @param ppv Pointer to the pointer variable.
2419 * @param pv The pointer value to assign to *ppv.
2420 */
2421DECLINLINE(void) ASMAtomicWritePtrVoid(void * volatile *ppv, const void *pv)
2422{
2423#if ARCH_BITS == 32
2424 ASMAtomicWriteU32((volatile uint32_t *)(void *)ppv, (uint32_t)pv);
2425#elif ARCH_BITS == 64
2426 ASMAtomicWriteU64((volatile uint64_t *)(void *)ppv, (uint64_t)pv);
2427#else
2428# error "ARCH_BITS is bogus"
2429#endif
2430}
2431
2432
2433/**
2434 * Atomically writes a pointer value, ordered.
2435 *
2436 * @param ppv Pointer to the pointer variable.
2437 * @param pv The pointer value to assign to *ppv. If NULL use
2438 * ASMAtomicWriteNullPtr or you'll land in trouble.
2439 *
2440 * @remarks This is relatively type safe on GCC platforms when @a pv isn't
2441 * NULL.
2442 */
2443#ifdef __GNUC__
2444# define ASMAtomicWritePtr(ppv, pv) \
2445 do \
2446 { \
2447 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
2448 __typeof__(*(ppv)) const pvTypeChecked = (pv); \
2449 \
2450 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2451 AssertCompile(sizeof(pv) == sizeof(void *)); \
2452 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2453 \
2454 ASMAtomicWritePtrVoid((void * volatile *)(ppvTypeChecked), (void *)(pvTypeChecked)); \
2455 } while (0)
2456#else
2457# define ASMAtomicWritePtr(ppv, pv) \
2458 do \
2459 { \
2460 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2461 AssertCompile(sizeof(pv) == sizeof(void *)); \
2462 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2463 \
2464 ASMAtomicWritePtrVoid((void * volatile *)(ppv), (void *)(pv)); \
2465 } while (0)
2466#endif
2467
2468
2469/**
2470 * Atomically sets a pointer to NULL, ordered.
2471 *
2472 * @param ppv Pointer to the pointer variable that should be set to NULL.
2473 *
2474 * @remarks This is relatively type safe on GCC platforms.
2475 */
2476#ifdef __GNUC__
2477# define ASMAtomicWriteNullPtr(ppv) \
2478 do \
2479 { \
2480 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
2481 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2482 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2483 ASMAtomicWritePtrVoid((void * volatile *)(ppvTypeChecked), NULL); \
2484 } while (0)
2485#else
2486# define ASMAtomicWriteNullPtr(ppv) \
2487 do \
2488 { \
2489 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2490 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2491 ASMAtomicWritePtrVoid((void * volatile *)(ppv), NULL); \
2492 } while (0)
2493#endif
2494
2495
2496/**
2497 * Atomically writes a pointer value, unordered.
2498 *
2499 * @returns Current *pv value
2500 * @param ppv Pointer to the pointer variable.
2501 * @param pv The pointer value to assign to *ppv. If NULL use
2502 * ASMAtomicUoWriteNullPtr or you'll land in trouble.
2503 *
2504 * @remarks This is relatively type safe on GCC platforms when @a pv isn't
2505 * NULL.
2506 */
2507#ifdef __GNUC__
2508# define ASMAtomicUoWritePtr(ppv, pv) \
2509 do \
2510 { \
2511 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
2512 __typeof__(*(ppv)) const pvTypeChecked = (pv); \
2513 \
2514 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2515 AssertCompile(sizeof(pv) == sizeof(void *)); \
2516 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2517 \
2518 *(ppvTypeChecked) = pvTypeChecked; \
2519 } while (0)
2520#else
2521# define ASMAtomicUoWritePtr(ppv, pv) \
2522 do \
2523 { \
2524 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2525 AssertCompile(sizeof(pv) == sizeof(void *)); \
2526 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2527 *(ppv) = pv; \
2528 } while (0)
2529#endif
2530
2531
2532/**
2533 * Atomically sets a pointer to NULL, unordered.
2534 *
2535 * @param ppv Pointer to the pointer variable that should be set to NULL.
2536 *
2537 * @remarks This is relatively type safe on GCC platforms.
2538 */
2539#ifdef __GNUC__
2540# define ASMAtomicUoWriteNullPtr(ppv) \
2541 do \
2542 { \
2543 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
2544 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2545 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2546 *(ppvTypeChecked) = NULL; \
2547 } while (0)
2548#else
2549# define ASMAtomicUoWriteNullPtr(ppv) \
2550 do \
2551 { \
2552 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2553 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2554 *(ppv) = NULL; \
2555 } while (0)
2556#endif
2557
2558
2559/**
2560 * Atomically write a typical IPRT handle value, ordered.
2561 *
2562 * @param ph Pointer to the variable to update.
2563 * @param hNew The value to assign to *ph.
2564 *
2565 * @remarks This doesn't currently work for all handles (like RTFILE).
2566 */
2567#if HC_ARCH_BITS == 32
2568# define ASMAtomicWriteHandle(ph, hNew) \
2569 do { \
2570 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
2571 ASMAtomicWriteU32((uint32_t volatile *)(ph), (const uint32_t)(hNew)); \
2572 } while (0)
2573#elif HC_ARCH_BITS == 64
2574# define ASMAtomicWriteHandle(ph, hNew) \
2575 do { \
2576 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
2577 ASMAtomicWriteU64((uint64_t volatile *)(ph), (const uint64_t)(hNew)); \
2578 } while (0)
2579#else
2580# error HC_ARCH_BITS
2581#endif
2582
2583
2584/**
2585 * Atomically write a typical IPRT handle value, unordered.
2586 *
2587 * @param ph Pointer to the variable to update.
2588 * @param hNew The value to assign to *ph.
2589 *
2590 * @remarks This doesn't currently work for all handles (like RTFILE).
2591 */
2592#if HC_ARCH_BITS == 32
2593# define ASMAtomicUoWriteHandle(ph, hNew) \
2594 do { \
2595 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
2596 ASMAtomicUoWriteU32((uint32_t volatile *)(ph), (const uint32_t)hNew); \
2597 } while (0)
2598#elif HC_ARCH_BITS == 64
2599# define ASMAtomicUoWriteHandle(ph, hNew) \
2600 do { \
2601 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
2602 ASMAtomicUoWriteU64((uint64_t volatile *)(ph), (const uint64_t)hNew); \
2603 } while (0)
2604#else
2605# error HC_ARCH_BITS
2606#endif
2607
2608
2609/**
2610 * Atomically write a value which size might differ
2611 * between platforms or compilers, ordered.
2612 *
2613 * @param pu Pointer to the variable to update.
2614 * @param uNew The value to assign to *pu.
2615 */
2616#define ASMAtomicWriteSize(pu, uNew) \
2617 do { \
2618 switch (sizeof(*(pu))) { \
2619 case 1: ASMAtomicWriteU8( (volatile uint8_t *)(void *)(pu), (uint8_t )(uNew)); break; \
2620 case 2: ASMAtomicWriteU16((volatile uint16_t *)(void *)(pu), (uint16_t)(uNew)); break; \
2621 case 4: ASMAtomicWriteU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew)); break; \
2622 case 8: ASMAtomicWriteU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew)); break; \
2623 default: AssertMsgFailed(("ASMAtomicWriteSize: size %d is not supported\n", sizeof(*(pu)))); \
2624 } \
2625 } while (0)
2626
2627/**
2628 * Atomically write a value which size might differ
2629 * between platforms or compilers, unordered.
2630 *
2631 * @param pu Pointer to the variable to update.
2632 * @param uNew The value to assign to *pu.
2633 */
2634#define ASMAtomicUoWriteSize(pu, uNew) \
2635 do { \
2636 switch (sizeof(*(pu))) { \
2637 case 1: ASMAtomicUoWriteU8( (volatile uint8_t *)(void *)(pu), (uint8_t )(uNew)); break; \
2638 case 2: ASMAtomicUoWriteU16((volatile uint16_t *)(void *)(pu), (uint16_t)(uNew)); break; \
2639 case 4: ASMAtomicUoWriteU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew)); break; \
2640 case 8: ASMAtomicUoWriteU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew)); break; \
2641 default: AssertMsgFailed(("ASMAtomicWriteSize: size %d is not supported\n", sizeof(*(pu)))); \
2642 } \
2643 } while (0)
2644
2645
2646
2647
2648/** @def RT_ASM_PAGE_SIZE
2649 * We try avoid dragging in iprt/param.h here.
2650 * @internal
2651 */
2652#if defined(RT_ARCH_SPARC64)
2653# define RT_ASM_PAGE_SIZE 0x2000
2654# if defined(PAGE_SIZE) && !defined(NT_INCLUDED)
2655# if PAGE_SIZE != 0x2000
2656# error "PAGE_SIZE is not 0x2000!"
2657# endif
2658# endif
2659#else
2660# define RT_ASM_PAGE_SIZE 0x1000
2661# if defined(PAGE_SIZE) && !defined(NT_INCLUDED)
2662# if PAGE_SIZE != 0x1000
2663# error "PAGE_SIZE is not 0x1000!"
2664# endif
2665# endif
2666#endif
2667
2668/**
2669 * Zeros a 4K memory page.
2670 *
2671 * @param pv Pointer to the memory block. This must be page aligned.
2672 */
2673#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2674DECLASM(void) ASMMemZeroPage(volatile void *pv);
2675# else
2676DECLINLINE(void) ASMMemZeroPage(volatile void *pv)
2677{
2678# if RT_INLINE_ASM_USES_INTRIN
2679# ifdef RT_ARCH_AMD64
2680 __stosq((unsigned __int64 *)pv, 0, RT_ASM_PAGE_SIZE / 8);
2681# else
2682 __stosd((unsigned long *)pv, 0, RT_ASM_PAGE_SIZE / 4);
2683# endif
2684
2685# elif RT_INLINE_ASM_GNU_STYLE
2686 RTCCUINTREG uDummy;
2687# ifdef RT_ARCH_AMD64
2688 __asm__ __volatile__("rep stosq"
2689 : "=D" (pv),
2690 "=c" (uDummy)
2691 : "0" (pv),
2692 "c" (RT_ASM_PAGE_SIZE >> 3),
2693 "a" (0)
2694 : "memory");
2695# else
2696 __asm__ __volatile__("rep stosl"
2697 : "=D" (pv),
2698 "=c" (uDummy)
2699 : "0" (pv),
2700 "c" (RT_ASM_PAGE_SIZE >> 2),
2701 "a" (0)
2702 : "memory");
2703# endif
2704# else
2705 __asm
2706 {
2707# ifdef RT_ARCH_AMD64
2708 xor rax, rax
2709 mov ecx, 0200h
2710 mov rdi, [pv]
2711 rep stosq
2712# else
2713 xor eax, eax
2714 mov ecx, 0400h
2715 mov edi, [pv]
2716 rep stosd
2717# endif
2718 }
2719# endif
2720}
2721# endif
2722
2723
2724/**
2725 * Zeros a memory block with a 32-bit aligned size.
2726 *
2727 * @param pv Pointer to the memory block.
2728 * @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
2729 */
2730#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2731DECLASM(void) ASMMemZero32(volatile void *pv, size_t cb);
2732#else
2733DECLINLINE(void) ASMMemZero32(volatile void *pv, size_t cb)
2734{
2735# if RT_INLINE_ASM_USES_INTRIN
2736# ifdef RT_ARCH_AMD64
2737 if (!(cb & 7))
2738 __stosq((unsigned __int64 *)pv, 0, cb / 8);
2739 else
2740# endif
2741 __stosd((unsigned long *)pv, 0, cb / 4);
2742
2743# elif RT_INLINE_ASM_GNU_STYLE
2744 __asm__ __volatile__("rep stosl"
2745 : "=D" (pv),
2746 "=c" (cb)
2747 : "0" (pv),
2748 "1" (cb >> 2),
2749 "a" (0)
2750 : "memory");
2751# else
2752 __asm
2753 {
2754 xor eax, eax
2755# ifdef RT_ARCH_AMD64
2756 mov rcx, [cb]
2757 shr rcx, 2
2758 mov rdi, [pv]
2759# else
2760 mov ecx, [cb]
2761 shr ecx, 2
2762 mov edi, [pv]
2763# endif
2764 rep stosd
2765 }
2766# endif
2767}
2768#endif
2769
2770
2771/**
2772 * Fills a memory block with a 32-bit aligned size.
2773 *
2774 * @param pv Pointer to the memory block.
2775 * @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
2776 * @param u32 The value to fill with.
2777 */
2778#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2779DECLASM(void) ASMMemFill32(volatile void *pv, size_t cb, uint32_t u32);
2780#else
2781DECLINLINE(void) ASMMemFill32(volatile void *pv, size_t cb, uint32_t u32)
2782{
2783# if RT_INLINE_ASM_USES_INTRIN
2784# ifdef RT_ARCH_AMD64
2785 if (!(cb & 7))
2786 __stosq((unsigned __int64 *)pv, RT_MAKE_U64(u32, u32), cb / 8);
2787 else
2788# endif
2789 __stosd((unsigned long *)pv, u32, cb / 4);
2790
2791# elif RT_INLINE_ASM_GNU_STYLE
2792 __asm__ __volatile__("rep stosl"
2793 : "=D" (pv),
2794 "=c" (cb)
2795 : "0" (pv),
2796 "1" (cb >> 2),
2797 "a" (u32)
2798 : "memory");
2799# else
2800 __asm
2801 {
2802# ifdef RT_ARCH_AMD64
2803 mov rcx, [cb]
2804 shr rcx, 2
2805 mov rdi, [pv]
2806# else
2807 mov ecx, [cb]
2808 shr ecx, 2
2809 mov edi, [pv]
2810# endif
2811 mov eax, [u32]
2812 rep stosd
2813 }
2814# endif
2815}
2816#endif
2817
2818
2819/**
2820 * Checks if a memory page is all zeros.
2821 *
2822 * @returns true / false.
2823 *
2824 * @param pvPage Pointer to the page. Must be aligned on 16 byte
2825 * boundrary
2826 */
2827DECLINLINE(bool) ASMMemIsZeroPage(void const *pvPage)
2828{
2829# if 0 /*RT_INLINE_ASM_GNU_STYLE - this is actually slower... */
2830 union { RTCCUINTREG r; bool f; } uAX;
2831 RTCCUINTREG xCX, xDI;
2832 Assert(!((uintptr_t)pvPage & 15));
2833 __asm__ __volatile__("repe; "
2834# ifdef RT_ARCH_AMD64
2835 "scasq\n\t"
2836# else
2837 "scasl\n\t"
2838# endif
2839 "setnc %%al\n\t"
2840 : "=&c" (xCX),
2841 "=&D" (xDI),
2842 "=&a" (uAX.r)
2843 : "mr" (pvPage),
2844# ifdef RT_ARCH_AMD64
2845 "0" (RT_ASM_PAGE_SIZE/8),
2846# else
2847 "0" (RT_ASM_PAGE_SIZE/4),
2848# endif
2849 "1" (pvPage),
2850 "2" (0));
2851 return uAX.f;
2852# else
2853 uintptr_t const *puPtr = (uintptr_t const *)pvPage;
2854 int cLeft = RT_ASM_PAGE_SIZE / sizeof(uintptr_t) / 8;
2855 Assert(!((uintptr_t)pvPage & 15));
2856 for (;;)
2857 {
2858 if (puPtr[0]) return false;
2859 if (puPtr[4]) return false;
2860
2861 if (puPtr[2]) return false;
2862 if (puPtr[6]) return false;
2863
2864 if (puPtr[1]) return false;
2865 if (puPtr[5]) return false;
2866
2867 if (puPtr[3]) return false;
2868 if (puPtr[7]) return false;
2869
2870 if (!--cLeft)
2871 return true;
2872 puPtr += 8;
2873 }
2874 return true;
2875# endif
2876}
2877
2878
2879/**
2880 * Checks if a memory block is filled with the specified byte.
2881 *
2882 * This is a sort of inverted memchr.
2883 *
2884 * @returns Pointer to the byte which doesn't equal u8.
2885 * @returns NULL if all equal to u8.
2886 *
2887 * @param pv Pointer to the memory block.
2888 * @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
2889 * @param u8 The value it's supposed to be filled with.
2890 *
2891 * @todo Fix name, it is a predicate function but it's not returning boolean!
2892 */
2893DECLINLINE(void *) ASMMemIsAll8(void const *pv, size_t cb, uint8_t u8)
2894{
2895/** @todo rewrite this in inline assembly? */
2896 uint8_t const *pb = (uint8_t const *)pv;
2897 for (; cb; cb--, pb++)
2898 if (RT_UNLIKELY(*pb != u8))
2899 return (void *)pb;
2900 return NULL;
2901}
2902
2903
2904/**
2905 * Checks if a memory block is filled with the specified 32-bit value.
2906 *
2907 * This is a sort of inverted memchr.
2908 *
2909 * @returns Pointer to the first value which doesn't equal u32.
2910 * @returns NULL if all equal to u32.
2911 *
2912 * @param pv Pointer to the memory block.
2913 * @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
2914 * @param u32 The value it's supposed to be filled with.
2915 *
2916 * @todo Fix name, it is a predicate function but it's not returning boolean!
2917 */
2918DECLINLINE(uint32_t *) ASMMemIsAllU32(void const *pv, size_t cb, uint32_t u32)
2919{
2920/** @todo rewrite this in inline assembly? */
2921 uint32_t const *pu32 = (uint32_t const *)pv;
2922 for (; cb; cb -= 4, pu32++)
2923 if (RT_UNLIKELY(*pu32 != u32))
2924 return (uint32_t *)pu32;
2925 return NULL;
2926}
2927
2928
2929/**
2930 * Probes a byte pointer for read access.
2931 *
2932 * While the function will not fault if the byte is not read accessible,
2933 * the idea is to do this in a safe place like before acquiring locks
2934 * and such like.
2935 *
2936 * Also, this functions guarantees that an eager compiler is not going
2937 * to optimize the probing away.
2938 *
2939 * @param pvByte Pointer to the byte.
2940 */
2941#if RT_INLINE_ASM_EXTERNAL
2942DECLASM(uint8_t) ASMProbeReadByte(const void *pvByte);
2943#else
2944DECLINLINE(uint8_t) ASMProbeReadByte(const void *pvByte)
2945{
2946 /** @todo verify that the compiler actually doesn't optimize this away. (intel & gcc) */
2947 uint8_t u8;
2948# if RT_INLINE_ASM_GNU_STYLE
2949 __asm__ __volatile__("movb (%1), %0\n\t"
2950 : "=r" (u8)
2951 : "r" (pvByte));
2952# else
2953 __asm
2954 {
2955# ifdef RT_ARCH_AMD64
2956 mov rax, [pvByte]
2957 mov al, [rax]
2958# else
2959 mov eax, [pvByte]
2960 mov al, [eax]
2961# endif
2962 mov [u8], al
2963 }
2964# endif
2965 return u8;
2966}
2967#endif
2968
2969/**
2970 * Probes a buffer for read access page by page.
2971 *
2972 * While the function will fault if the buffer is not fully read
2973 * accessible, the idea is to do this in a safe place like before
2974 * acquiring locks and such like.
2975 *
2976 * Also, this functions guarantees that an eager compiler is not going
2977 * to optimize the probing away.
2978 *
2979 * @param pvBuf Pointer to the buffer.
2980 * @param cbBuf The size of the buffer in bytes. Must be >= 1.
2981 */
2982DECLINLINE(void) ASMProbeReadBuffer(const void *pvBuf, size_t cbBuf)
2983{
2984 /** @todo verify that the compiler actually doesn't optimize this away. (intel & gcc) */
2985 /* the first byte */
2986 const uint8_t *pu8 = (const uint8_t *)pvBuf;
2987 ASMProbeReadByte(pu8);
2988
2989 /* the pages in between pages. */
2990 while (cbBuf > RT_ASM_PAGE_SIZE)
2991 {
2992 ASMProbeReadByte(pu8);
2993 cbBuf -= RT_ASM_PAGE_SIZE;
2994 pu8 += RT_ASM_PAGE_SIZE;
2995 }
2996
2997 /* the last byte */
2998 ASMProbeReadByte(pu8 + cbBuf - 1);
2999}
3000
3001
3002/** @def ASMBreakpoint
3003 * Debugger Breakpoint.
3004 * @remark In the gnu world we add a nop instruction after the int3 to
3005 * force gdb to remain at the int3 source line.
3006 * @remark The L4 kernel will try make sense of the breakpoint, thus the jmp.
3007 * @internal
3008 */
3009#if RT_INLINE_ASM_GNU_STYLE
3010# if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
3011# ifndef __L4ENV__
3012# define ASMBreakpoint() do { __asm__ __volatile__("int3\n\tnop"); } while (0)
3013# else
3014# define ASMBreakpoint() do { __asm__ __volatile__("int3; jmp 1f; 1:"); } while (0)
3015# endif
3016# elif defined(RT_ARCH_SPARC64)
3017# define ASMBreakpoint() do { __asm__ __volatile__("illtrap 0\n\t") } while (0) /** @todo Sparc64: this is just a wild guess. */
3018# elif defined(RT_ARCH_SPARC)
3019# define ASMBreakpoint() do { __asm__ __volatile__("unimp 0\n\t"); } while (0) /** @todo Sparc: this is just a wild guess (same as Sparc64, just different name). */
3020# else
3021# error "PORTME"
3022# endif
3023#else
3024# define ASMBreakpoint() __debugbreak()
3025#endif
3026
3027
3028/**
3029 * Spinloop hint for platforms that have these, empty function on the other
3030 * platforms.
3031 *
3032 * x86 & AMD64: The PAUSE variant of NOP for helping hyperthreaded CPUs detecing
3033 * spin locks.
3034 */
3035#if RT_INLINE_ASM_EXTERNAL && (defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86))
3036DECLASM(void) ASMNopPause(void);
3037#else
3038DECLINLINE(void) ASMNopPause(void)
3039{
3040# if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
3041# if RT_INLINE_ASM_GNU_STYLE
3042 __asm__ __volatile__(".byte 0xf3,0x90\n\t");
3043# else
3044 __asm {
3045 _emit 0f3h
3046 _emit 090h
3047 }
3048# endif
3049# else
3050 /* dummy */
3051# endif
3052}
3053#endif
3054
3055
3056
3057/** @defgroup grp_inline_bits Bit Operations
3058 * @{
3059 */
3060
3061
3062/**
3063 * Sets a bit in a bitmap.
3064 *
3065 * @param pvBitmap Pointer to the bitmap. This should be 32-bit aligned.
3066 * @param iBit The bit to set.
3067 *
3068 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3069 * However, doing so will yield better performance as well as avoiding
3070 * traps accessing the last bits in the bitmap.
3071 */
3072#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3073DECLASM(void) ASMBitSet(volatile void *pvBitmap, int32_t iBit);
3074#else
3075DECLINLINE(void) ASMBitSet(volatile void *pvBitmap, int32_t iBit)
3076{
3077# if RT_INLINE_ASM_USES_INTRIN
3078 _bittestandset((long *)pvBitmap, iBit);
3079
3080# elif RT_INLINE_ASM_GNU_STYLE
3081 __asm__ __volatile__("btsl %1, %0"
3082 : "=m" (*(volatile long *)pvBitmap)
3083 : "Ir" (iBit),
3084 "m" (*(volatile long *)pvBitmap)
3085 : "memory");
3086# else
3087 __asm
3088 {
3089# ifdef RT_ARCH_AMD64
3090 mov rax, [pvBitmap]
3091 mov edx, [iBit]
3092 bts [rax], edx
3093# else
3094 mov eax, [pvBitmap]
3095 mov edx, [iBit]
3096 bts [eax], edx
3097# endif
3098 }
3099# endif
3100}
3101#endif
3102
3103
3104/**
3105 * Atomically sets a bit in a bitmap, ordered.
3106 *
3107 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3108 * the memory access isn't atomic!
3109 * @param iBit The bit to set.
3110 */
3111#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3112DECLASM(void) ASMAtomicBitSet(volatile void *pvBitmap, int32_t iBit);
3113#else
3114DECLINLINE(void) ASMAtomicBitSet(volatile void *pvBitmap, int32_t iBit)
3115{
3116 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3117# if RT_INLINE_ASM_USES_INTRIN
3118 _interlockedbittestandset((long *)pvBitmap, iBit);
3119# elif RT_INLINE_ASM_GNU_STYLE
3120 __asm__ __volatile__("lock; btsl %1, %0"
3121 : "=m" (*(volatile long *)pvBitmap)
3122 : "Ir" (iBit),
3123 "m" (*(volatile long *)pvBitmap)
3124 : "memory");
3125# else
3126 __asm
3127 {
3128# ifdef RT_ARCH_AMD64
3129 mov rax, [pvBitmap]
3130 mov edx, [iBit]
3131 lock bts [rax], edx
3132# else
3133 mov eax, [pvBitmap]
3134 mov edx, [iBit]
3135 lock bts [eax], edx
3136# endif
3137 }
3138# endif
3139}
3140#endif
3141
3142
3143/**
3144 * Clears a bit in a bitmap.
3145 *
3146 * @param pvBitmap Pointer to the bitmap.
3147 * @param iBit The bit to clear.
3148 *
3149 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3150 * However, doing so will yield better performance as well as avoiding
3151 * traps accessing the last bits in the bitmap.
3152 */
3153#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3154DECLASM(void) ASMBitClear(volatile void *pvBitmap, int32_t iBit);
3155#else
3156DECLINLINE(void) ASMBitClear(volatile void *pvBitmap, int32_t iBit)
3157{
3158# if RT_INLINE_ASM_USES_INTRIN
3159 _bittestandreset((long *)pvBitmap, iBit);
3160
3161# elif RT_INLINE_ASM_GNU_STYLE
3162 __asm__ __volatile__("btrl %1, %0"
3163 : "=m" (*(volatile long *)pvBitmap)
3164 : "Ir" (iBit),
3165 "m" (*(volatile long *)pvBitmap)
3166 : "memory");
3167# else
3168 __asm
3169 {
3170# ifdef RT_ARCH_AMD64
3171 mov rax, [pvBitmap]
3172 mov edx, [iBit]
3173 btr [rax], edx
3174# else
3175 mov eax, [pvBitmap]
3176 mov edx, [iBit]
3177 btr [eax], edx
3178# endif
3179 }
3180# endif
3181}
3182#endif
3183
3184
3185/**
3186 * Atomically clears a bit in a bitmap, ordered.
3187 *
3188 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3189 * the memory access isn't atomic!
3190 * @param iBit The bit to toggle set.
3191 * @remarks No memory barrier, take care on smp.
3192 */
3193#if RT_INLINE_ASM_EXTERNAL
3194DECLASM(void) ASMAtomicBitClear(volatile void *pvBitmap, int32_t iBit);
3195#else
3196DECLINLINE(void) ASMAtomicBitClear(volatile void *pvBitmap, int32_t iBit)
3197{
3198 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3199# if RT_INLINE_ASM_GNU_STYLE
3200 __asm__ __volatile__("lock; btrl %1, %0"
3201 : "=m" (*(volatile long *)pvBitmap)
3202 : "Ir" (iBit),
3203 "m" (*(volatile long *)pvBitmap)
3204 : "memory");
3205# else
3206 __asm
3207 {
3208# ifdef RT_ARCH_AMD64
3209 mov rax, [pvBitmap]
3210 mov edx, [iBit]
3211 lock btr [rax], edx
3212# else
3213 mov eax, [pvBitmap]
3214 mov edx, [iBit]
3215 lock btr [eax], edx
3216# endif
3217 }
3218# endif
3219}
3220#endif
3221
3222
3223/**
3224 * Toggles a bit in a bitmap.
3225 *
3226 * @param pvBitmap Pointer to the bitmap.
3227 * @param iBit The bit to toggle.
3228 *
3229 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3230 * However, doing so will yield better performance as well as avoiding
3231 * traps accessing the last bits in the bitmap.
3232 */
3233#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3234DECLASM(void) ASMBitToggle(volatile void *pvBitmap, int32_t iBit);
3235#else
3236DECLINLINE(void) ASMBitToggle(volatile void *pvBitmap, int32_t iBit)
3237{
3238# if RT_INLINE_ASM_USES_INTRIN
3239 _bittestandcomplement((long *)pvBitmap, iBit);
3240# elif RT_INLINE_ASM_GNU_STYLE
3241 __asm__ __volatile__("btcl %1, %0"
3242 : "=m" (*(volatile long *)pvBitmap)
3243 : "Ir" (iBit),
3244 "m" (*(volatile long *)pvBitmap)
3245 : "memory");
3246# else
3247 __asm
3248 {
3249# ifdef RT_ARCH_AMD64
3250 mov rax, [pvBitmap]
3251 mov edx, [iBit]
3252 btc [rax], edx
3253# else
3254 mov eax, [pvBitmap]
3255 mov edx, [iBit]
3256 btc [eax], edx
3257# endif
3258 }
3259# endif
3260}
3261#endif
3262
3263
3264/**
3265 * Atomically toggles a bit in a bitmap, ordered.
3266 *
3267 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3268 * the memory access isn't atomic!
3269 * @param iBit The bit to test and set.
3270 */
3271#if RT_INLINE_ASM_EXTERNAL
3272DECLASM(void) ASMAtomicBitToggle(volatile void *pvBitmap, int32_t iBit);
3273#else
3274DECLINLINE(void) ASMAtomicBitToggle(volatile void *pvBitmap, int32_t iBit)
3275{
3276 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3277# if RT_INLINE_ASM_GNU_STYLE
3278 __asm__ __volatile__("lock; btcl %1, %0"
3279 : "=m" (*(volatile long *)pvBitmap)
3280 : "Ir" (iBit),
3281 "m" (*(volatile long *)pvBitmap)
3282 : "memory");
3283# else
3284 __asm
3285 {
3286# ifdef RT_ARCH_AMD64
3287 mov rax, [pvBitmap]
3288 mov edx, [iBit]
3289 lock btc [rax], edx
3290# else
3291 mov eax, [pvBitmap]
3292 mov edx, [iBit]
3293 lock btc [eax], edx
3294# endif
3295 }
3296# endif
3297}
3298#endif
3299
3300
3301/**
3302 * Tests and sets a bit in a bitmap.
3303 *
3304 * @returns true if the bit was set.
3305 * @returns false if the bit was clear.
3306 *
3307 * @param pvBitmap Pointer to the bitmap.
3308 * @param iBit The bit to test and set.
3309 *
3310 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3311 * However, doing so will yield better performance as well as avoiding
3312 * traps accessing the last bits in the bitmap.
3313 */
3314#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3315DECLASM(bool) ASMBitTestAndSet(volatile void *pvBitmap, int32_t iBit);
3316#else
3317DECLINLINE(bool) ASMBitTestAndSet(volatile void *pvBitmap, int32_t iBit)
3318{
3319 union { bool f; uint32_t u32; uint8_t u8; } rc;
3320# if RT_INLINE_ASM_USES_INTRIN
3321 rc.u8 = _bittestandset((long *)pvBitmap, iBit);
3322
3323# elif RT_INLINE_ASM_GNU_STYLE
3324 __asm__ __volatile__("btsl %2, %1\n\t"
3325 "setc %b0\n\t"
3326 "andl $1, %0\n\t"
3327 : "=q" (rc.u32),
3328 "=m" (*(volatile long *)pvBitmap)
3329 : "Ir" (iBit),
3330 "m" (*(volatile long *)pvBitmap)
3331 : "memory");
3332# else
3333 __asm
3334 {
3335 mov edx, [iBit]
3336# ifdef RT_ARCH_AMD64
3337 mov rax, [pvBitmap]
3338 bts [rax], edx
3339# else
3340 mov eax, [pvBitmap]
3341 bts [eax], edx
3342# endif
3343 setc al
3344 and eax, 1
3345 mov [rc.u32], eax
3346 }
3347# endif
3348 return rc.f;
3349}
3350#endif
3351
3352
3353/**
3354 * Atomically tests and sets a bit in a bitmap, ordered.
3355 *
3356 * @returns true if the bit was set.
3357 * @returns false if the bit was clear.
3358 *
3359 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3360 * the memory access isn't atomic!
3361 * @param iBit The bit to set.
3362 */
3363#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3364DECLASM(bool) ASMAtomicBitTestAndSet(volatile void *pvBitmap, int32_t iBit);
3365#else
3366DECLINLINE(bool) ASMAtomicBitTestAndSet(volatile void *pvBitmap, int32_t iBit)
3367{
3368 union { bool f; uint32_t u32; uint8_t u8; } rc;
3369 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3370# if RT_INLINE_ASM_USES_INTRIN
3371 rc.u8 = _interlockedbittestandset((long *)pvBitmap, iBit);
3372# elif RT_INLINE_ASM_GNU_STYLE
3373 __asm__ __volatile__("lock; btsl %2, %1\n\t"
3374 "setc %b0\n\t"
3375 "andl $1, %0\n\t"
3376 : "=q" (rc.u32),
3377 "=m" (*(volatile long *)pvBitmap)
3378 : "Ir" (iBit),
3379 "m" (*(volatile long *)pvBitmap)
3380 : "memory");
3381# else
3382 __asm
3383 {
3384 mov edx, [iBit]
3385# ifdef RT_ARCH_AMD64
3386 mov rax, [pvBitmap]
3387 lock bts [rax], edx
3388# else
3389 mov eax, [pvBitmap]
3390 lock bts [eax], edx
3391# endif
3392 setc al
3393 and eax, 1
3394 mov [rc.u32], eax
3395 }
3396# endif
3397 return rc.f;
3398}
3399#endif
3400
3401
3402/**
3403 * Tests and clears a bit in a bitmap.
3404 *
3405 * @returns true if the bit was set.
3406 * @returns false if the bit was clear.
3407 *
3408 * @param pvBitmap Pointer to the bitmap.
3409 * @param iBit The bit to test and clear.
3410 *
3411 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3412 * However, doing so will yield better performance as well as avoiding
3413 * traps accessing the last bits in the bitmap.
3414 */
3415#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3416DECLASM(bool) ASMBitTestAndClear(volatile void *pvBitmap, int32_t iBit);
3417#else
3418DECLINLINE(bool) ASMBitTestAndClear(volatile void *pvBitmap, int32_t iBit)
3419{
3420 union { bool f; uint32_t u32; uint8_t u8; } rc;
3421# if RT_INLINE_ASM_USES_INTRIN
3422 rc.u8 = _bittestandreset((long *)pvBitmap, iBit);
3423
3424# elif RT_INLINE_ASM_GNU_STYLE
3425 __asm__ __volatile__("btrl %2, %1\n\t"
3426 "setc %b0\n\t"
3427 "andl $1, %0\n\t"
3428 : "=q" (rc.u32),
3429 "=m" (*(volatile long *)pvBitmap)
3430 : "Ir" (iBit),
3431 "m" (*(volatile long *)pvBitmap)
3432 : "memory");
3433# else
3434 __asm
3435 {
3436 mov edx, [iBit]
3437# ifdef RT_ARCH_AMD64
3438 mov rax, [pvBitmap]
3439 btr [rax], edx
3440# else
3441 mov eax, [pvBitmap]
3442 btr [eax], edx
3443# endif
3444 setc al
3445 and eax, 1
3446 mov [rc.u32], eax
3447 }
3448# endif
3449 return rc.f;
3450}
3451#endif
3452
3453
3454/**
3455 * Atomically tests and clears a bit in a bitmap, ordered.
3456 *
3457 * @returns true if the bit was set.
3458 * @returns false if the bit was clear.
3459 *
3460 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3461 * the memory access isn't atomic!
3462 * @param iBit The bit to test and clear.
3463 *
3464 * @remarks No memory barrier, take care on smp.
3465 */
3466#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3467DECLASM(bool) ASMAtomicBitTestAndClear(volatile void *pvBitmap, int32_t iBit);
3468#else
3469DECLINLINE(bool) ASMAtomicBitTestAndClear(volatile void *pvBitmap, int32_t iBit)
3470{
3471 union { bool f; uint32_t u32; uint8_t u8; } rc;
3472 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3473# if RT_INLINE_ASM_USES_INTRIN
3474 rc.u8 = _interlockedbittestandreset((long *)pvBitmap, iBit);
3475
3476# elif RT_INLINE_ASM_GNU_STYLE
3477 __asm__ __volatile__("lock; btrl %2, %1\n\t"
3478 "setc %b0\n\t"
3479 "andl $1, %0\n\t"
3480 : "=q" (rc.u32),
3481 "=m" (*(volatile long *)pvBitmap)
3482 : "Ir" (iBit),
3483 "m" (*(volatile long *)pvBitmap)
3484 : "memory");
3485# else
3486 __asm
3487 {
3488 mov edx, [iBit]
3489# ifdef RT_ARCH_AMD64
3490 mov rax, [pvBitmap]
3491 lock btr [rax], edx
3492# else
3493 mov eax, [pvBitmap]
3494 lock btr [eax], edx
3495# endif
3496 setc al
3497 and eax, 1
3498 mov [rc.u32], eax
3499 }
3500# endif
3501 return rc.f;
3502}
3503#endif
3504
3505
3506/**
3507 * Tests and toggles a bit in a bitmap.
3508 *
3509 * @returns true if the bit was set.
3510 * @returns false if the bit was clear.
3511 *
3512 * @param pvBitmap Pointer to the bitmap.
3513 * @param iBit The bit to test and toggle.
3514 *
3515 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3516 * However, doing so will yield better performance as well as avoiding
3517 * traps accessing the last bits in the bitmap.
3518 */
3519#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3520DECLASM(bool) ASMBitTestAndToggle(volatile void *pvBitmap, int32_t iBit);
3521#else
3522DECLINLINE(bool) ASMBitTestAndToggle(volatile void *pvBitmap, int32_t iBit)
3523{
3524 union { bool f; uint32_t u32; uint8_t u8; } rc;
3525# if RT_INLINE_ASM_USES_INTRIN
3526 rc.u8 = _bittestandcomplement((long *)pvBitmap, iBit);
3527
3528# elif RT_INLINE_ASM_GNU_STYLE
3529 __asm__ __volatile__("btcl %2, %1\n\t"
3530 "setc %b0\n\t"
3531 "andl $1, %0\n\t"
3532 : "=q" (rc.u32),
3533 "=m" (*(volatile long *)pvBitmap)
3534 : "Ir" (iBit),
3535 "m" (*(volatile long *)pvBitmap)
3536 : "memory");
3537# else
3538 __asm
3539 {
3540 mov edx, [iBit]
3541# ifdef RT_ARCH_AMD64
3542 mov rax, [pvBitmap]
3543 btc [rax], edx
3544# else
3545 mov eax, [pvBitmap]
3546 btc [eax], edx
3547# endif
3548 setc al
3549 and eax, 1
3550 mov [rc.u32], eax
3551 }
3552# endif
3553 return rc.f;
3554}
3555#endif
3556
3557
3558/**
3559 * Atomically tests and toggles a bit in a bitmap, ordered.
3560 *
3561 * @returns true if the bit was set.
3562 * @returns false if the bit was clear.
3563 *
3564 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3565 * the memory access isn't atomic!
3566 * @param iBit The bit to test and toggle.
3567 */
3568#if RT_INLINE_ASM_EXTERNAL
3569DECLASM(bool) ASMAtomicBitTestAndToggle(volatile void *pvBitmap, int32_t iBit);
3570#else
3571DECLINLINE(bool) ASMAtomicBitTestAndToggle(volatile void *pvBitmap, int32_t iBit)
3572{
3573 union { bool f; uint32_t u32; uint8_t u8; } rc;
3574 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3575# if RT_INLINE_ASM_GNU_STYLE
3576 __asm__ __volatile__("lock; btcl %2, %1\n\t"
3577 "setc %b0\n\t"
3578 "andl $1, %0\n\t"
3579 : "=q" (rc.u32),
3580 "=m" (*(volatile long *)pvBitmap)
3581 : "Ir" (iBit),
3582 "m" (*(volatile long *)pvBitmap)
3583 : "memory");
3584# else
3585 __asm
3586 {
3587 mov edx, [iBit]
3588# ifdef RT_ARCH_AMD64
3589 mov rax, [pvBitmap]
3590 lock btc [rax], edx
3591# else
3592 mov eax, [pvBitmap]
3593 lock btc [eax], edx
3594# endif
3595 setc al
3596 and eax, 1
3597 mov [rc.u32], eax
3598 }
3599# endif
3600 return rc.f;
3601}
3602#endif
3603
3604
3605/**
3606 * Tests if a bit in a bitmap is set.
3607 *
3608 * @returns true if the bit is set.
3609 * @returns false if the bit is clear.
3610 *
3611 * @param pvBitmap Pointer to the bitmap.
3612 * @param iBit The bit to test.
3613 *
3614 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3615 * However, doing so will yield better performance as well as avoiding
3616 * traps accessing the last bits in the bitmap.
3617 */
3618#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3619DECLASM(bool) ASMBitTest(const volatile void *pvBitmap, int32_t iBit);
3620#else
3621DECLINLINE(bool) ASMBitTest(const volatile void *pvBitmap, int32_t iBit)
3622{
3623 union { bool f; uint32_t u32; uint8_t u8; } rc;
3624# if RT_INLINE_ASM_USES_INTRIN
3625 rc.u32 = _bittest((long *)pvBitmap, iBit);
3626# elif RT_INLINE_ASM_GNU_STYLE
3627
3628 __asm__ __volatile__("btl %2, %1\n\t"
3629 "setc %b0\n\t"
3630 "andl $1, %0\n\t"
3631 : "=q" (rc.u32)
3632 : "m" (*(const volatile long *)pvBitmap),
3633 "Ir" (iBit)
3634 : "memory");
3635# else
3636 __asm
3637 {
3638 mov edx, [iBit]
3639# ifdef RT_ARCH_AMD64
3640 mov rax, [pvBitmap]
3641 bt [rax], edx
3642# else
3643 mov eax, [pvBitmap]
3644 bt [eax], edx
3645# endif
3646 setc al
3647 and eax, 1
3648 mov [rc.u32], eax
3649 }
3650# endif
3651 return rc.f;
3652}
3653#endif
3654
3655
3656/**
3657 * Clears a bit range within a bitmap.
3658 *
3659 * @param pvBitmap Pointer to the bitmap.
3660 * @param iBitStart The First bit to clear.
3661 * @param iBitEnd The first bit not to clear.
3662 */
3663DECLINLINE(void) ASMBitClearRange(volatile void *pvBitmap, int32_t iBitStart, int32_t iBitEnd)
3664{
3665 if (iBitStart < iBitEnd)
3666 {
3667 volatile uint32_t *pu32 = (volatile uint32_t *)pvBitmap + (iBitStart >> 5);
3668 int iStart = iBitStart & ~31;
3669 int iEnd = iBitEnd & ~31;
3670 if (iStart == iEnd)
3671 *pu32 &= ((1 << (iBitStart & 31)) - 1) | ~((1 << (iBitEnd & 31)) - 1);
3672 else
3673 {
3674 /* bits in first dword. */
3675 if (iBitStart & 31)
3676 {
3677 *pu32 &= (1 << (iBitStart & 31)) - 1;
3678 pu32++;
3679 iBitStart = iStart + 32;
3680 }
3681
3682 /* whole dword. */
3683 if (iBitStart != iEnd)
3684 ASMMemZero32(pu32, (iEnd - iBitStart) >> 3);
3685
3686 /* bits in last dword. */
3687 if (iBitEnd & 31)
3688 {
3689 pu32 = (volatile uint32_t *)pvBitmap + (iBitEnd >> 5);
3690 *pu32 &= ~((1 << (iBitEnd & 31)) - 1);
3691 }
3692 }
3693 }
3694}
3695
3696
3697/**
3698 * Sets a bit range within a bitmap.
3699 *
3700 * @param pvBitmap Pointer to the bitmap.
3701 * @param iBitStart The First bit to set.
3702 * @param iBitEnd The first bit not to set.
3703 */
3704DECLINLINE(void) ASMBitSetRange(volatile void *pvBitmap, int32_t iBitStart, int32_t iBitEnd)
3705{
3706 if (iBitStart < iBitEnd)
3707 {
3708 volatile uint32_t *pu32 = (volatile uint32_t *)pvBitmap + (iBitStart >> 5);
3709 int iStart = iBitStart & ~31;
3710 int iEnd = iBitEnd & ~31;
3711 if (iStart == iEnd)
3712 *pu32 |= ((1 << (iBitEnd - iBitStart)) - 1) << iBitStart;
3713 else
3714 {
3715 /* bits in first dword. */
3716 if (iBitStart & 31)
3717 {
3718 *pu32 |= ~((1 << (iBitStart & 31)) - 1);
3719 pu32++;
3720 iBitStart = iStart + 32;
3721 }
3722
3723 /* whole dword. */
3724 if (iBitStart != iEnd)
3725 ASMMemFill32(pu32, (iEnd - iBitStart) >> 3, ~0);
3726
3727 /* bits in last dword. */
3728 if (iBitEnd & 31)
3729 {
3730 pu32 = (volatile uint32_t *)pvBitmap + (iBitEnd >> 5);
3731 *pu32 |= (1 << (iBitEnd & 31)) - 1;
3732 }
3733 }
3734 }
3735}
3736
3737
3738/**
3739 * Finds the first clear bit in a bitmap.
3740 *
3741 * @returns Index of the first zero bit.
3742 * @returns -1 if no clear bit was found.
3743 * @param pvBitmap Pointer to the bitmap.
3744 * @param cBits The number of bits in the bitmap. Multiple of 32.
3745 */
3746#if RT_INLINE_ASM_EXTERNAL
3747DECLASM(int) ASMBitFirstClear(const volatile void *pvBitmap, uint32_t cBits);
3748#else
3749DECLINLINE(int) ASMBitFirstClear(const volatile void *pvBitmap, uint32_t cBits)
3750{
3751 if (cBits)
3752 {
3753 int32_t iBit;
3754# if RT_INLINE_ASM_GNU_STYLE
3755 RTCCUINTREG uEAX, uECX, uEDI;
3756 cBits = RT_ALIGN_32(cBits, 32);
3757 __asm__ __volatile__("repe; scasl\n\t"
3758 "je 1f\n\t"
3759# ifdef RT_ARCH_AMD64
3760 "lea -4(%%rdi), %%rdi\n\t"
3761 "xorl (%%rdi), %%eax\n\t"
3762 "subq %5, %%rdi\n\t"
3763# else
3764 "lea -4(%%edi), %%edi\n\t"
3765 "xorl (%%edi), %%eax\n\t"
3766 "subl %5, %%edi\n\t"
3767# endif
3768 "shll $3, %%edi\n\t"
3769 "bsfl %%eax, %%edx\n\t"
3770 "addl %%edi, %%edx\n\t"
3771 "1:\t\n"
3772 : "=d" (iBit),
3773 "=&c" (uECX),
3774 "=&D" (uEDI),
3775 "=&a" (uEAX)
3776 : "0" (0xffffffff),
3777 "mr" (pvBitmap),
3778 "1" (cBits >> 5),
3779 "2" (pvBitmap),
3780 "3" (0xffffffff));
3781# else
3782 cBits = RT_ALIGN_32(cBits, 32);
3783 __asm
3784 {
3785# ifdef RT_ARCH_AMD64
3786 mov rdi, [pvBitmap]
3787 mov rbx, rdi
3788# else
3789 mov edi, [pvBitmap]
3790 mov ebx, edi
3791# endif
3792 mov edx, 0ffffffffh
3793 mov eax, edx
3794 mov ecx, [cBits]
3795 shr ecx, 5
3796 repe scasd
3797 je done
3798
3799# ifdef RT_ARCH_AMD64
3800 lea rdi, [rdi - 4]
3801 xor eax, [rdi]
3802 sub rdi, rbx
3803# else
3804 lea edi, [edi - 4]
3805 xor eax, [edi]
3806 sub edi, ebx
3807# endif
3808 shl edi, 3
3809 bsf edx, eax
3810 add edx, edi
3811 done:
3812 mov [iBit], edx
3813 }
3814# endif
3815 return iBit;
3816 }
3817 return -1;
3818}
3819#endif
3820
3821
3822/**
3823 * Finds the next clear bit in a bitmap.
3824 *
3825 * @returns Index of the first zero bit.
3826 * @returns -1 if no clear bit was found.
3827 * @param pvBitmap Pointer to the bitmap.
3828 * @param cBits The number of bits in the bitmap. Multiple of 32.
3829 * @param iBitPrev The bit returned from the last search.
3830 * The search will start at iBitPrev + 1.
3831 */
3832#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3833DECLASM(int) ASMBitNextClear(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev);
3834#else
3835DECLINLINE(int) ASMBitNextClear(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev)
3836{
3837 const volatile uint32_t *pau32Bitmap = (const volatile uint32_t *)pvBitmap;
3838 int iBit = ++iBitPrev & 31;
3839 if (iBit)
3840 {
3841 /*
3842 * Inspect the 32-bit word containing the unaligned bit.
3843 */
3844 uint32_t u32 = ~pau32Bitmap[iBitPrev / 32] >> iBit;
3845
3846# if RT_INLINE_ASM_USES_INTRIN
3847 unsigned long ulBit = 0;
3848 if (_BitScanForward(&ulBit, u32))
3849 return ulBit + iBitPrev;
3850# else
3851# if RT_INLINE_ASM_GNU_STYLE
3852 __asm__ __volatile__("bsf %1, %0\n\t"
3853 "jnz 1f\n\t"
3854 "movl $-1, %0\n\t"
3855 "1:\n\t"
3856 : "=r" (iBit)
3857 : "r" (u32));
3858# else
3859 __asm
3860 {
3861 mov edx, [u32]
3862 bsf eax, edx
3863 jnz done
3864 mov eax, 0ffffffffh
3865 done:
3866 mov [iBit], eax
3867 }
3868# endif
3869 if (iBit >= 0)
3870 return iBit + iBitPrev;
3871# endif
3872
3873 /*
3874 * Skip ahead and see if there is anything left to search.
3875 */
3876 iBitPrev |= 31;
3877 iBitPrev++;
3878 if (cBits <= (uint32_t)iBitPrev)
3879 return -1;
3880 }
3881
3882 /*
3883 * 32-bit aligned search, let ASMBitFirstClear do the dirty work.
3884 */
3885 iBit = ASMBitFirstClear(&pau32Bitmap[iBitPrev / 32], cBits - iBitPrev);
3886 if (iBit >= 0)
3887 iBit += iBitPrev;
3888 return iBit;
3889}
3890#endif
3891
3892
3893/**
3894 * Finds the first set bit in a bitmap.
3895 *
3896 * @returns Index of the first set bit.
3897 * @returns -1 if no clear bit was found.
3898 * @param pvBitmap Pointer to the bitmap.
3899 * @param cBits The number of bits in the bitmap. Multiple of 32.
3900 */
3901#if RT_INLINE_ASM_EXTERNAL
3902DECLASM(int) ASMBitFirstSet(const volatile void *pvBitmap, uint32_t cBits);
3903#else
3904DECLINLINE(int) ASMBitFirstSet(const volatile void *pvBitmap, uint32_t cBits)
3905{
3906 if (cBits)
3907 {
3908 int32_t iBit;
3909# if RT_INLINE_ASM_GNU_STYLE
3910 RTCCUINTREG uEAX, uECX, uEDI;
3911 cBits = RT_ALIGN_32(cBits, 32);
3912 __asm__ __volatile__("repe; scasl\n\t"
3913 "je 1f\n\t"
3914# ifdef RT_ARCH_AMD64
3915 "lea -4(%%rdi), %%rdi\n\t"
3916 "movl (%%rdi), %%eax\n\t"
3917 "subq %5, %%rdi\n\t"
3918# else
3919 "lea -4(%%edi), %%edi\n\t"
3920 "movl (%%edi), %%eax\n\t"
3921 "subl %5, %%edi\n\t"
3922# endif
3923 "shll $3, %%edi\n\t"
3924 "bsfl %%eax, %%edx\n\t"
3925 "addl %%edi, %%edx\n\t"
3926 "1:\t\n"
3927 : "=d" (iBit),
3928 "=&c" (uECX),
3929 "=&D" (uEDI),
3930 "=&a" (uEAX)
3931 : "0" (0xffffffff),
3932 "mr" (pvBitmap),
3933 "1" (cBits >> 5),
3934 "2" (pvBitmap),
3935 "3" (0));
3936# else
3937 cBits = RT_ALIGN_32(cBits, 32);
3938 __asm
3939 {
3940# ifdef RT_ARCH_AMD64
3941 mov rdi, [pvBitmap]
3942 mov rbx, rdi
3943# else
3944 mov edi, [pvBitmap]
3945 mov ebx, edi
3946# endif
3947 mov edx, 0ffffffffh
3948 xor eax, eax
3949 mov ecx, [cBits]
3950 shr ecx, 5
3951 repe scasd
3952 je done
3953# ifdef RT_ARCH_AMD64
3954 lea rdi, [rdi - 4]
3955 mov eax, [rdi]
3956 sub rdi, rbx
3957# else
3958 lea edi, [edi - 4]
3959 mov eax, [edi]
3960 sub edi, ebx
3961# endif
3962 shl edi, 3
3963 bsf edx, eax
3964 add edx, edi
3965 done:
3966 mov [iBit], edx
3967 }
3968# endif
3969 return iBit;
3970 }
3971 return -1;
3972}
3973#endif
3974
3975
3976/**
3977 * Finds the next set bit in a bitmap.
3978 *
3979 * @returns Index of the next set bit.
3980 * @returns -1 if no set bit was found.
3981 * @param pvBitmap Pointer to the bitmap.
3982 * @param cBits The number of bits in the bitmap. Multiple of 32.
3983 * @param iBitPrev The bit returned from the last search.
3984 * The search will start at iBitPrev + 1.
3985 */
3986#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3987DECLASM(int) ASMBitNextSet(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev);
3988#else
3989DECLINLINE(int) ASMBitNextSet(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev)
3990{
3991 const volatile uint32_t *pau32Bitmap = (const volatile uint32_t *)pvBitmap;
3992 int iBit = ++iBitPrev & 31;
3993 if (iBit)
3994 {
3995 /*
3996 * Inspect the 32-bit word containing the unaligned bit.
3997 */
3998 uint32_t u32 = pau32Bitmap[iBitPrev / 32] >> iBit;
3999
4000# if RT_INLINE_ASM_USES_INTRIN
4001 unsigned long ulBit = 0;
4002 if (_BitScanForward(&ulBit, u32))
4003 return ulBit + iBitPrev;
4004# else
4005# if RT_INLINE_ASM_GNU_STYLE
4006 __asm__ __volatile__("bsf %1, %0\n\t"
4007 "jnz 1f\n\t"
4008 "movl $-1, %0\n\t"
4009 "1:\n\t"
4010 : "=r" (iBit)
4011 : "r" (u32));
4012# else
4013 __asm
4014 {
4015 mov edx, [u32]
4016 bsf eax, edx
4017 jnz done
4018 mov eax, 0ffffffffh
4019 done:
4020 mov [iBit], eax
4021 }
4022# endif
4023 if (iBit >= 0)
4024 return iBit + iBitPrev;
4025# endif
4026
4027 /*
4028 * Skip ahead and see if there is anything left to search.
4029 */
4030 iBitPrev |= 31;
4031 iBitPrev++;
4032 if (cBits <= (uint32_t)iBitPrev)
4033 return -1;
4034 }
4035
4036 /*
4037 * 32-bit aligned search, let ASMBitFirstClear do the dirty work.
4038 */
4039 iBit = ASMBitFirstSet(&pau32Bitmap[iBitPrev / 32], cBits - iBitPrev);
4040 if (iBit >= 0)
4041 iBit += iBitPrev;
4042 return iBit;
4043}
4044#endif
4045
4046
4047/**
4048 * Finds the first bit which is set in the given 32-bit integer.
4049 * Bits are numbered from 1 (least significant) to 32.
4050 *
4051 * @returns index [1..32] of the first set bit.
4052 * @returns 0 if all bits are cleared.
4053 * @param u32 Integer to search for set bits.
4054 * @remark Similar to ffs() in BSD.
4055 */
4056#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4057DECLASM(unsigned) ASMBitFirstSetU32(uint32_t u32);
4058#else
4059DECLINLINE(unsigned) ASMBitFirstSetU32(uint32_t u32)
4060{
4061# if RT_INLINE_ASM_USES_INTRIN
4062 unsigned long iBit;
4063 if (_BitScanForward(&iBit, u32))
4064 iBit++;
4065 else
4066 iBit = 0;
4067# elif RT_INLINE_ASM_GNU_STYLE
4068 uint32_t iBit;
4069 __asm__ __volatile__("bsf %1, %0\n\t"
4070 "jnz 1f\n\t"
4071 "xorl %0, %0\n\t"
4072 "jmp 2f\n"
4073 "1:\n\t"
4074 "incl %0\n"
4075 "2:\n\t"
4076 : "=r" (iBit)
4077 : "rm" (u32));
4078# else
4079 uint32_t iBit;
4080 _asm
4081 {
4082 bsf eax, [u32]
4083 jnz found
4084 xor eax, eax
4085 jmp done
4086 found:
4087 inc eax
4088 done:
4089 mov [iBit], eax
4090 }
4091# endif
4092 return iBit;
4093}
4094#endif
4095
4096
4097/**
4098 * Finds the first bit which is set in the given 32-bit integer.
4099 * Bits are numbered from 1 (least significant) to 32.
4100 *
4101 * @returns index [1..32] of the first set bit.
4102 * @returns 0 if all bits are cleared.
4103 * @param i32 Integer to search for set bits.
4104 * @remark Similar to ffs() in BSD.
4105 */
4106DECLINLINE(unsigned) ASMBitFirstSetS32(int32_t i32)
4107{
4108 return ASMBitFirstSetU32((uint32_t)i32);
4109}
4110
4111
4112/**
4113 * Finds the last bit which is set in the given 32-bit integer.
4114 * Bits are numbered from 1 (least significant) to 32.
4115 *
4116 * @returns index [1..32] of the last set bit.
4117 * @returns 0 if all bits are cleared.
4118 * @param u32 Integer to search for set bits.
4119 * @remark Similar to fls() in BSD.
4120 */
4121#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4122DECLASM(unsigned) ASMBitLastSetU32(uint32_t u32);
4123#else
4124DECLINLINE(unsigned) ASMBitLastSetU32(uint32_t u32)
4125{
4126# if RT_INLINE_ASM_USES_INTRIN
4127 unsigned long iBit;
4128 if (_BitScanReverse(&iBit, u32))
4129 iBit++;
4130 else
4131 iBit = 0;
4132# elif RT_INLINE_ASM_GNU_STYLE
4133 uint32_t iBit;
4134 __asm__ __volatile__("bsrl %1, %0\n\t"
4135 "jnz 1f\n\t"
4136 "xorl %0, %0\n\t"
4137 "jmp 2f\n"
4138 "1:\n\t"
4139 "incl %0\n"
4140 "2:\n\t"
4141 : "=r" (iBit)
4142 : "rm" (u32));
4143# else
4144 uint32_t iBit;
4145 _asm
4146 {
4147 bsr eax, [u32]
4148 jnz found
4149 xor eax, eax
4150 jmp done
4151 found:
4152 inc eax
4153 done:
4154 mov [iBit], eax
4155 }
4156# endif
4157 return iBit;
4158}
4159#endif
4160
4161
4162/**
4163 * Finds the last bit which is set in the given 32-bit integer.
4164 * Bits are numbered from 1 (least significant) to 32.
4165 *
4166 * @returns index [1..32] of the last set bit.
4167 * @returns 0 if all bits are cleared.
4168 * @param i32 Integer to search for set bits.
4169 * @remark Similar to fls() in BSD.
4170 */
4171DECLINLINE(unsigned) ASMBitLastSetS32(int32_t i32)
4172{
4173 return ASMBitLastSetU32((uint32_t)i32);
4174}
4175
4176/**
4177 * Reverse the byte order of the given 16-bit integer.
4178 *
4179 * @returns Revert
4180 * @param u16 16-bit integer value.
4181 */
4182#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4183DECLASM(uint16_t) ASMByteSwapU16(uint16_t u16);
4184#else
4185DECLINLINE(uint16_t) ASMByteSwapU16(uint16_t u16)
4186{
4187# if RT_INLINE_ASM_USES_INTRIN
4188 u16 = _byteswap_ushort(u16);
4189# elif RT_INLINE_ASM_GNU_STYLE
4190 __asm__ ("rorw $8, %0" : "=r" (u16) : "0" (u16));
4191# else
4192 _asm
4193 {
4194 mov ax, [u16]
4195 ror ax, 8
4196 mov [u16], ax
4197 }
4198# endif
4199 return u16;
4200}
4201#endif
4202
4203
4204/**
4205 * Reverse the byte order of the given 32-bit integer.
4206 *
4207 * @returns Revert
4208 * @param u32 32-bit integer value.
4209 */
4210#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4211DECLASM(uint32_t) ASMByteSwapU32(uint32_t u32);
4212#else
4213DECLINLINE(uint32_t) ASMByteSwapU32(uint32_t u32)
4214{
4215# if RT_INLINE_ASM_USES_INTRIN
4216 u32 = _byteswap_ulong(u32);
4217# elif RT_INLINE_ASM_GNU_STYLE
4218 __asm__ ("bswapl %0" : "=r" (u32) : "0" (u32));
4219# else
4220 _asm
4221 {
4222 mov eax, [u32]
4223 bswap eax
4224 mov [u32], eax
4225 }
4226# endif
4227 return u32;
4228}
4229#endif
4230
4231
4232/**
4233 * Reverse the byte order of the given 64-bit integer.
4234 *
4235 * @returns Revert
4236 * @param u64 64-bit integer value.
4237 */
4238DECLINLINE(uint64_t) ASMByteSwapU64(uint64_t u64)
4239{
4240#if defined(RT_ARCH_AMD64) && RT_INLINE_ASM_USES_INTRIN
4241 u64 = _byteswap_uint64(u64);
4242#else
4243 u64 = (uint64_t)ASMByteSwapU32((uint32_t)u64) << 32
4244 | (uint64_t)ASMByteSwapU32((uint32_t)(u64 >> 32));
4245#endif
4246 return u64;
4247}
4248
4249
4250/** @} */
4251
4252
4253/** @} */
4254
4255/* KLUDGE: Play safe for now as I cannot test all solaris and linux usages. */
4256#if !defined(__cplusplus) && !defined(DEBUG)
4257# if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
4258# include <iprt/asm-amd64-x86.h>
4259# endif
4260# include <iprt/asm-math.h>
4261#endif
4262
4263#endif
4264
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette