VirtualBox

source: vbox/trunk/include/iprt/asm.h@ 33199

Last change on this file since 33199 was 33199, checked in by vboxsync, 14 years ago

iprt/asm: C90 warning
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 128.8 KB
Line 
1/** @file
2 * IPRT - Assembly Functions.
3 */
4
5/*
6 * Copyright (C) 2006-2010 Oracle Corporation
7 *
8 * This file is part of VirtualBox Open Source Edition (OSE), as
9 * available from http://www.virtualbox.org. This file is free software;
10 * you can redistribute it and/or modify it under the terms of the GNU
11 * General Public License (GPL) as published by the Free Software
12 * Foundation, in version 2 as it comes in the "COPYING" file of the
13 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
14 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
15 *
16 * The contents of this file may alternatively be used under the terms
17 * of the Common Development and Distribution License Version 1.0
18 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
19 * VirtualBox OSE distribution, in which case the provisions of the
20 * CDDL are applicable instead of those of the GPL.
21 *
22 * You may elect to license modified versions of this file under the
23 * terms and conditions of either the GPL or the CDDL or both.
24 */
25
26#ifndef ___iprt_asm_h
27#define ___iprt_asm_h
28
29#include <iprt/cdefs.h>
30#include <iprt/types.h>
31#include <iprt/assert.h>
32/** @def RT_INLINE_ASM_USES_INTRIN
33 * Defined as 1 if we're using a _MSC_VER 1400.
34 * Otherwise defined as 0.
35 */
36
37/* Solaris 10 header ugliness */
38#ifdef u
39# undef u
40#endif
41
42#if defined(_MSC_VER) && RT_INLINE_ASM_USES_INTRIN
43# include <intrin.h>
44 /* Emit the intrinsics at all optimization levels. */
45# pragma intrinsic(_ReadWriteBarrier)
46# pragma intrinsic(__cpuid)
47# pragma intrinsic(__stosd)
48# pragma intrinsic(__stosw)
49# pragma intrinsic(__stosb)
50# pragma intrinsic(_BitScanForward)
51# pragma intrinsic(_BitScanReverse)
52# pragma intrinsic(_bittest)
53# pragma intrinsic(_bittestandset)
54# pragma intrinsic(_bittestandreset)
55# pragma intrinsic(_bittestandcomplement)
56# pragma intrinsic(_byteswap_ushort)
57# pragma intrinsic(_byteswap_ulong)
58# pragma intrinsic(_interlockedbittestandset)
59# pragma intrinsic(_interlockedbittestandreset)
60# pragma intrinsic(_InterlockedAnd)
61# pragma intrinsic(_InterlockedOr)
62# pragma intrinsic(_InterlockedIncrement)
63# pragma intrinsic(_InterlockedDecrement)
64# pragma intrinsic(_InterlockedExchange)
65# pragma intrinsic(_InterlockedExchangeAdd)
66# pragma intrinsic(_InterlockedCompareExchange)
67# pragma intrinsic(_InterlockedCompareExchange64)
68# ifdef RT_ARCH_AMD64
69# pragma intrinsic(__stosq)
70# pragma intrinsic(_byteswap_uint64)
71# pragma intrinsic(_InterlockedExchange64)
72# pragma intrinsic(_InterlockedExchangeAdd64)
73# pragma intrinsic(_InterlockedAnd64)
74# pragma intrinsic(_InterlockedOr64)
75# pragma intrinsic(_InterlockedIncrement64)
76# pragma intrinsic(_InterlockedDecrement64)
77# endif
78#endif
79
80
81/** @defgroup grp_rt_asm ASM - Assembly Routines
82 * @ingroup grp_rt
83 *
84 * @remarks The difference between ordered and unordered atomic operations are that
85 * the former will complete outstanding reads and writes before continuing
86 * while the latter doesn't make any promisses about the order. Ordered
87 * operations doesn't, it seems, make any 100% promise wrt to whether
88 * the operation will complete before any subsequent memory access.
89 * (please, correct if wrong.)
90 *
91 * ASMAtomicSomething operations are all ordered, while ASMAtomicUoSomething
92 * are unordered (note the Uo).
93 *
94 * @remarks Some remarks about __volatile__: Without this keyword gcc is allowed to reorder
95 * or even optimize assembler instructions away. For instance, in the following code
96 * the second rdmsr instruction is optimized away because gcc treats that instruction
97 * as deterministic:
98 *
99 * @code
100 * static inline uint64_t rdmsr_low(int idx)
101 * {
102 * uint32_t low;
103 * __asm__ ("rdmsr" : "=a"(low) : "c"(idx) : "edx");
104 * }
105 * ...
106 * uint32_t msr1 = rdmsr_low(1);
107 * foo(msr1);
108 * msr1 = rdmsr_low(1);
109 * bar(msr1);
110 * @endcode
111 *
112 * The input parameter of rdmsr_low is the same for both calls and therefore gcc will
113 * use the result of the first call as input parameter for bar() as well. For rdmsr this
114 * is not acceptable as this instruction is _not_ deterministic. This applies to reading
115 * machine status information in general.
116 *
117 * @{
118 */
119
120
121/** @def RT_INLINE_ASM_GCC_4_3_X_X86
122 * Used to work around some 4.3.x register allocation issues in this version of
123 * the compiler. So far this workaround is still required for 4.4 and 4.5. */
124#ifdef __GNUC__
125# define RT_INLINE_ASM_GCC_4_3_X_X86 (__GNUC__ == 4 && __GNUC_MINOR__ >= 3 && defined(__i386__))
126#endif
127#ifndef RT_INLINE_ASM_GCC_4_3_X_X86
128# define RT_INLINE_ASM_GCC_4_3_X_X86 0
129#endif
130
131/** @def RT_INLINE_DONT_USE_CMPXCHG8B
132 * i686-apple-darwin9-gcc-4.0.1 (GCC) 4.0.1 (Apple Inc. build 5493) screws up
133 * RTSemRWRequestWrite semsemrw-lockless-generic.cpp in release builds. PIC
134 * mode, x86.
135 *
136 * Some gcc 4.3.x versions may have register allocation issues with cmpxchg8b
137 * when in PIC mode on x86.
138 */
139#ifndef RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
140# define RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC \
141 ( (defined(PIC) || defined(__PIC__)) \
142 && defined(RT_ARCH_X86) \
143 && ( RT_INLINE_ASM_GCC_4_3_X_X86 \
144 || defined(RT_OS_DARWIN)) )
145#endif
146
147
148/** @def ASMReturnAddress
149 * Gets the return address of the current (or calling if you like) function or method.
150 */
151#ifdef _MSC_VER
152# ifdef __cplusplus
153extern "C"
154# endif
155void * _ReturnAddress(void);
156# pragma intrinsic(_ReturnAddress)
157# define ASMReturnAddress() _ReturnAddress()
158#elif defined(__GNUC__) || defined(DOXYGEN_RUNNING)
159# define ASMReturnAddress() __builtin_return_address(0)
160#else
161# error "Unsupported compiler."
162#endif
163
164
165/**
166 * Compiler memory barrier.
167 *
168 * Ensure that the compiler does not use any cached (register/tmp stack) memory
169 * values or any outstanding writes when returning from this function.
170 *
171 * This function must be used if non-volatile data is modified by a
172 * device or the VMM. Typical cases are port access, MMIO access,
173 * trapping instruction, etc.
174 */
175#if RT_INLINE_ASM_GNU_STYLE
176# define ASMCompilerBarrier() do { __asm__ __volatile__("" : : : "memory"); } while (0)
177#elif RT_INLINE_ASM_USES_INTRIN
178# define ASMCompilerBarrier() do { _ReadWriteBarrier(); } while (0)
179#else /* 2003 should have _ReadWriteBarrier() but I guess we're at 2002 level then... */
180DECLINLINE(void) ASMCompilerBarrier(void)
181{
182 __asm
183 {
184 }
185}
186#endif
187
188
189/** @def ASMBreakpoint
190 * Debugger Breakpoint.
191 * @remark In the gnu world we add a nop instruction after the int3 to
192 * force gdb to remain at the int3 source line.
193 * @remark The L4 kernel will try make sense of the breakpoint, thus the jmp.
194 * @internal
195 */
196#if RT_INLINE_ASM_GNU_STYLE
197# if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
198# ifndef __L4ENV__
199# define ASMBreakpoint() do { __asm__ __volatile__("int3\n\tnop"); } while (0)
200# else
201# define ASMBreakpoint() do { __asm__ __volatile__("int3; jmp 1f; 1:"); } while (0)
202# endif
203# elif defined(RT_ARCH_SPARC64)
204# define ASMBreakpoint() do { __asm__ __volatile__("illtrap 0\n\t") } while (0) /** @todo Sparc64: this is just a wild guess. */
205# elif defined(RT_ARCH_SPARC)
206# define ASMBreakpoint() do { __asm__ __volatile__("unimp 0\n\t"); } while (0) /** @todo Sparc: this is just a wild guess (same as Sparc64, just different name). */
207# else
208# error "PORTME"
209# endif
210#else
211# define ASMBreakpoint() __debugbreak()
212#endif
213
214
215/**
216 * Spinloop hint for platforms that have these, empty function on the other
217 * platforms.
218 *
219 * x86 & AMD64: The PAUSE variant of NOP for helping hyperthreaded CPUs detecing
220 * spin locks.
221 */
222#if RT_INLINE_ASM_EXTERNAL && (defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86))
223DECLASM(void) ASMNopPause(void);
224#else
225DECLINLINE(void) ASMNopPause(void)
226{
227# if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
228# if RT_INLINE_ASM_GNU_STYLE
229 __asm__ __volatile__(".byte 0xf3,0x90\n\t");
230# else
231 __asm {
232 _emit 0f3h
233 _emit 090h
234 }
235# endif
236# else
237 /* dummy */
238# endif
239}
240#endif
241
242
243/**
244 * Atomically Exchange an unsigned 8-bit value, ordered.
245 *
246 * @returns Current *pu8 value
247 * @param pu8 Pointer to the 8-bit variable to update.
248 * @param u8 The 8-bit value to assign to *pu8.
249 */
250#if RT_INLINE_ASM_EXTERNAL
251DECLASM(uint8_t) ASMAtomicXchgU8(volatile uint8_t *pu8, uint8_t u8);
252#else
253DECLINLINE(uint8_t) ASMAtomicXchgU8(volatile uint8_t *pu8, uint8_t u8)
254{
255# if RT_INLINE_ASM_GNU_STYLE
256 __asm__ __volatile__("xchgb %0, %1\n\t"
257 : "=m" (*pu8),
258 "=q" (u8) /* =r - busted on g++ (GCC) 3.4.4 20050721 (Red Hat 3.4.4-2) */
259 : "1" (u8),
260 "m" (*pu8));
261# else
262 __asm
263 {
264# ifdef RT_ARCH_AMD64
265 mov rdx, [pu8]
266 mov al, [u8]
267 xchg [rdx], al
268 mov [u8], al
269# else
270 mov edx, [pu8]
271 mov al, [u8]
272 xchg [edx], al
273 mov [u8], al
274# endif
275 }
276# endif
277 return u8;
278}
279#endif
280
281
282/**
283 * Atomically Exchange a signed 8-bit value, ordered.
284 *
285 * @returns Current *pu8 value
286 * @param pi8 Pointer to the 8-bit variable to update.
287 * @param i8 The 8-bit value to assign to *pi8.
288 */
289DECLINLINE(int8_t) ASMAtomicXchgS8(volatile int8_t *pi8, int8_t i8)
290{
291 return (int8_t)ASMAtomicXchgU8((volatile uint8_t *)pi8, (uint8_t)i8);
292}
293
294
295/**
296 * Atomically Exchange a bool value, ordered.
297 *
298 * @returns Current *pf value
299 * @param pf Pointer to the 8-bit variable to update.
300 * @param f The 8-bit value to assign to *pi8.
301 */
302DECLINLINE(bool) ASMAtomicXchgBool(volatile bool *pf, bool f)
303{
304#ifdef _MSC_VER
305 return !!ASMAtomicXchgU8((volatile uint8_t *)pf, (uint8_t)f);
306#else
307 return (bool)ASMAtomicXchgU8((volatile uint8_t *)pf, (uint8_t)f);
308#endif
309}
310
311
312/**
313 * Atomically Exchange an unsigned 16-bit value, ordered.
314 *
315 * @returns Current *pu16 value
316 * @param pu16 Pointer to the 16-bit variable to update.
317 * @param u16 The 16-bit value to assign to *pu16.
318 */
319#if RT_INLINE_ASM_EXTERNAL
320DECLASM(uint16_t) ASMAtomicXchgU16(volatile uint16_t *pu16, uint16_t u16);
321#else
322DECLINLINE(uint16_t) ASMAtomicXchgU16(volatile uint16_t *pu16, uint16_t u16)
323{
324# if RT_INLINE_ASM_GNU_STYLE
325 __asm__ __volatile__("xchgw %0, %1\n\t"
326 : "=m" (*pu16),
327 "=r" (u16)
328 : "1" (u16),
329 "m" (*pu16));
330# else
331 __asm
332 {
333# ifdef RT_ARCH_AMD64
334 mov rdx, [pu16]
335 mov ax, [u16]
336 xchg [rdx], ax
337 mov [u16], ax
338# else
339 mov edx, [pu16]
340 mov ax, [u16]
341 xchg [edx], ax
342 mov [u16], ax
343# endif
344 }
345# endif
346 return u16;
347}
348#endif
349
350
351/**
352 * Atomically Exchange a signed 16-bit value, ordered.
353 *
354 * @returns Current *pu16 value
355 * @param pi16 Pointer to the 16-bit variable to update.
356 * @param i16 The 16-bit value to assign to *pi16.
357 */
358DECLINLINE(int16_t) ASMAtomicXchgS16(volatile int16_t *pi16, int16_t i16)
359{
360 return (int16_t)ASMAtomicXchgU16((volatile uint16_t *)pi16, (uint16_t)i16);
361}
362
363
364/**
365 * Atomically Exchange an unsigned 32-bit value, ordered.
366 *
367 * @returns Current *pu32 value
368 * @param pu32 Pointer to the 32-bit variable to update.
369 * @param u32 The 32-bit value to assign to *pu32.
370 */
371#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
372DECLASM(uint32_t) ASMAtomicXchgU32(volatile uint32_t *pu32, uint32_t u32);
373#else
374DECLINLINE(uint32_t) ASMAtomicXchgU32(volatile uint32_t *pu32, uint32_t u32)
375{
376# if RT_INLINE_ASM_GNU_STYLE
377 __asm__ __volatile__("xchgl %0, %1\n\t"
378 : "=m" (*pu32),
379 "=r" (u32)
380 : "1" (u32),
381 "m" (*pu32));
382
383# elif RT_INLINE_ASM_USES_INTRIN
384 u32 = _InterlockedExchange((long *)pu32, u32);
385
386# else
387 __asm
388 {
389# ifdef RT_ARCH_AMD64
390 mov rdx, [pu32]
391 mov eax, u32
392 xchg [rdx], eax
393 mov [u32], eax
394# else
395 mov edx, [pu32]
396 mov eax, u32
397 xchg [edx], eax
398 mov [u32], eax
399# endif
400 }
401# endif
402 return u32;
403}
404#endif
405
406
407/**
408 * Atomically Exchange a signed 32-bit value, ordered.
409 *
410 * @returns Current *pu32 value
411 * @param pi32 Pointer to the 32-bit variable to update.
412 * @param i32 The 32-bit value to assign to *pi32.
413 */
414DECLINLINE(int32_t) ASMAtomicXchgS32(volatile int32_t *pi32, int32_t i32)
415{
416 return (int32_t)ASMAtomicXchgU32((volatile uint32_t *)pi32, (uint32_t)i32);
417}
418
419
420/**
421 * Atomically Exchange an unsigned 64-bit value, ordered.
422 *
423 * @returns Current *pu64 value
424 * @param pu64 Pointer to the 64-bit variable to update.
425 * @param u64 The 64-bit value to assign to *pu64.
426 */
427#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
428 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
429DECLASM(uint64_t) ASMAtomicXchgU64(volatile uint64_t *pu64, uint64_t u64);
430#else
431DECLINLINE(uint64_t) ASMAtomicXchgU64(volatile uint64_t *pu64, uint64_t u64)
432{
433# if defined(RT_ARCH_AMD64)
434# if RT_INLINE_ASM_USES_INTRIN
435 u64 = _InterlockedExchange64((__int64 *)pu64, u64);
436
437# elif RT_INLINE_ASM_GNU_STYLE
438 __asm__ __volatile__("xchgq %0, %1\n\t"
439 : "=m" (*pu64),
440 "=r" (u64)
441 : "1" (u64),
442 "m" (*pu64));
443# else
444 __asm
445 {
446 mov rdx, [pu64]
447 mov rax, [u64]
448 xchg [rdx], rax
449 mov [u64], rax
450 }
451# endif
452# else /* !RT_ARCH_AMD64 */
453# if RT_INLINE_ASM_GNU_STYLE
454# if defined(PIC) || defined(__PIC__)
455 uint32_t u32EBX = (uint32_t)u64;
456 __asm__ __volatile__(/*"xchgl %%esi, %5\n\t"*/
457 "xchgl %%ebx, %3\n\t"
458 "1:\n\t"
459 "lock; cmpxchg8b (%5)\n\t"
460 "jnz 1b\n\t"
461 "movl %3, %%ebx\n\t"
462 /*"xchgl %%esi, %5\n\t"*/
463 : "=A" (u64),
464 "=m" (*pu64)
465 : "0" (*pu64),
466 "m" ( u32EBX ),
467 "c" ( (uint32_t)(u64 >> 32) ),
468 "S" (pu64));
469# else /* !PIC */
470 __asm__ __volatile__("1:\n\t"
471 "lock; cmpxchg8b %1\n\t"
472 "jnz 1b\n\t"
473 : "=A" (u64),
474 "=m" (*pu64)
475 : "0" (*pu64),
476 "b" ( (uint32_t)u64 ),
477 "c" ( (uint32_t)(u64 >> 32) ));
478# endif
479# else
480 __asm
481 {
482 mov ebx, dword ptr [u64]
483 mov ecx, dword ptr [u64 + 4]
484 mov edi, pu64
485 mov eax, dword ptr [edi]
486 mov edx, dword ptr [edi + 4]
487 retry:
488 lock cmpxchg8b [edi]
489 jnz retry
490 mov dword ptr [u64], eax
491 mov dword ptr [u64 + 4], edx
492 }
493# endif
494# endif /* !RT_ARCH_AMD64 */
495 return u64;
496}
497#endif
498
499
500/**
501 * Atomically Exchange an signed 64-bit value, ordered.
502 *
503 * @returns Current *pi64 value
504 * @param pi64 Pointer to the 64-bit variable to update.
505 * @param i64 The 64-bit value to assign to *pi64.
506 */
507DECLINLINE(int64_t) ASMAtomicXchgS64(volatile int64_t *pi64, int64_t i64)
508{
509 return (int64_t)ASMAtomicXchgU64((volatile uint64_t *)pi64, (uint64_t)i64);
510}
511
512
513/**
514 * Atomically Exchange a pointer value, ordered.
515 *
516 * @returns Current *ppv value
517 * @param ppv Pointer to the pointer variable to update.
518 * @param pv The pointer value to assign to *ppv.
519 */
520DECLINLINE(void *) ASMAtomicXchgPtr(void * volatile *ppv, const void *pv)
521{
522#if ARCH_BITS == 32
523 return (void *)ASMAtomicXchgU32((volatile uint32_t *)(void *)ppv, (uint32_t)pv);
524#elif ARCH_BITS == 64
525 return (void *)ASMAtomicXchgU64((volatile uint64_t *)(void *)ppv, (uint64_t)pv);
526#else
527# error "ARCH_BITS is bogus"
528#endif
529}
530
531
532/**
533 * Convenience macro for avoiding the annoying casting with ASMAtomicXchgPtr.
534 *
535 * @returns Current *pv value
536 * @param ppv Pointer to the pointer variable to update.
537 * @param pv The pointer value to assign to *ppv.
538 * @param Type The type of *ppv, sans volatile.
539 */
540#ifdef __GNUC__
541# define ASMAtomicXchgPtrT(ppv, pv, Type) \
542 __extension__ \
543 ({\
544 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
545 Type const pvTypeChecked = (pv); \
546 Type pvTypeCheckedRet = (__typeof__(*(ppv))) ASMAtomicXchgPtr((void * volatile *)ppvTypeChecked, (void *)pvTypeChecked); \
547 pvTypeCheckedRet; \
548 })
549#else
550# define ASMAtomicXchgPtrT(ppv, pv, Type) \
551 (Type)ASMAtomicXchgPtr((void * volatile *)(ppv), (void *)(pv))
552#endif
553
554
555/**
556 * Atomically Exchange a raw-mode context pointer value, ordered.
557 *
558 * @returns Current *ppv value
559 * @param ppvRC Pointer to the pointer variable to update.
560 * @param pvRC The pointer value to assign to *ppv.
561 */
562DECLINLINE(RTRCPTR) ASMAtomicXchgRCPtr(RTRCPTR volatile *ppvRC, RTRCPTR pvRC)
563{
564 return (RTRCPTR)ASMAtomicXchgU32((uint32_t volatile *)(void *)ppvRC, (uint32_t)pvRC);
565}
566
567
568/**
569 * Atomically Exchange a ring-0 pointer value, ordered.
570 *
571 * @returns Current *ppv value
572 * @param ppvR0 Pointer to the pointer variable to update.
573 * @param pvR0 The pointer value to assign to *ppv.
574 */
575DECLINLINE(RTR0PTR) ASMAtomicXchgR0Ptr(RTR0PTR volatile *ppvR0, RTR0PTR pvR0)
576{
577#if R0_ARCH_BITS == 32
578 return (RTR0PTR)ASMAtomicXchgU32((volatile uint32_t *)(void *)ppvR0, (uint32_t)pvR0);
579#elif R0_ARCH_BITS == 64
580 return (RTR0PTR)ASMAtomicXchgU64((volatile uint64_t *)(void *)ppvR0, (uint64_t)pvR0);
581#else
582# error "R0_ARCH_BITS is bogus"
583#endif
584}
585
586
587/**
588 * Atomically Exchange a ring-3 pointer value, ordered.
589 *
590 * @returns Current *ppv value
591 * @param ppvR3 Pointer to the pointer variable to update.
592 * @param pvR3 The pointer value to assign to *ppv.
593 */
594DECLINLINE(RTR3PTR) ASMAtomicXchgR3Ptr(RTR3PTR volatile *ppvR3, RTR3PTR pvR3)
595{
596#if R3_ARCH_BITS == 32
597 return (RTR3PTR)ASMAtomicXchgU32((volatile uint32_t *)(void *)ppvR3, (uint32_t)pvR3);
598#elif R3_ARCH_BITS == 64
599 return (RTR3PTR)ASMAtomicXchgU64((volatile uint64_t *)(void *)ppvR3, (uint64_t)pvR3);
600#else
601# error "R3_ARCH_BITS is bogus"
602#endif
603}
604
605
606/** @def ASMAtomicXchgHandle
607 * Atomically Exchange a typical IPRT handle value, ordered.
608 *
609 * @param ph Pointer to the value to update.
610 * @param hNew The new value to assigned to *pu.
611 * @param phRes Where to store the current *ph value.
612 *
613 * @remarks This doesn't currently work for all handles (like RTFILE).
614 */
615#if HC_ARCH_BITS == 32
616# define ASMAtomicXchgHandle(ph, hNew, phRes) \
617 do { \
618 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
619 AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
620 *(uint32_t *)(phRes) = ASMAtomicXchgU32((uint32_t volatile *)(ph), (const uint32_t)(hNew)); \
621 } while (0)
622#elif HC_ARCH_BITS == 64
623# define ASMAtomicXchgHandle(ph, hNew, phRes) \
624 do { \
625 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
626 AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
627 *(uint64_t *)(phRes) = ASMAtomicXchgU64((uint64_t volatile *)(ph), (const uint64_t)(hNew)); \
628 } while (0)
629#else
630# error HC_ARCH_BITS
631#endif
632
633
634/**
635 * Atomically Exchange a value which size might differ
636 * between platforms or compilers, ordered.
637 *
638 * @param pu Pointer to the variable to update.
639 * @param uNew The value to assign to *pu.
640 * @todo This is busted as its missing the result argument.
641 */
642#define ASMAtomicXchgSize(pu, uNew) \
643 do { \
644 switch (sizeof(*(pu))) { \
645 case 1: ASMAtomicXchgU8((volatile uint8_t *)(void *)(pu), (uint8_t)(uNew)); break; \
646 case 2: ASMAtomicXchgU16((volatile uint16_t *)(void *)(pu), (uint16_t)(uNew)); break; \
647 case 4: ASMAtomicXchgU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew)); break; \
648 case 8: ASMAtomicXchgU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew)); break; \
649 default: AssertMsgFailed(("ASMAtomicXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
650 } \
651 } while (0)
652
653/**
654 * Atomically Exchange a value which size might differ
655 * between platforms or compilers, ordered.
656 *
657 * @param pu Pointer to the variable to update.
658 * @param uNew The value to assign to *pu.
659 * @param puRes Where to store the current *pu value.
660 */
661#define ASMAtomicXchgSizeCorrect(pu, uNew, puRes) \
662 do { \
663 switch (sizeof(*(pu))) { \
664 case 1: *(uint8_t *)(puRes) = ASMAtomicXchgU8((volatile uint8_t *)(void *)(pu), (uint8_t)(uNew)); break; \
665 case 2: *(uint16_t *)(puRes) = ASMAtomicXchgU16((volatile uint16_t *)(void *)(pu), (uint16_t)(uNew)); break; \
666 case 4: *(uint32_t *)(puRes) = ASMAtomicXchgU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew)); break; \
667 case 8: *(uint64_t *)(puRes) = ASMAtomicXchgU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew)); break; \
668 default: AssertMsgFailed(("ASMAtomicXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
669 } \
670 } while (0)
671
672
673
674/**
675 * Atomically Compare and Exchange an unsigned 8-bit value, ordered.
676 *
677 * @returns true if xchg was done.
678 * @returns false if xchg wasn't done.
679 *
680 * @param pu8 Pointer to the value to update.
681 * @param u8New The new value to assigned to *pu8.
682 * @param u8Old The old value to *pu8 compare with.
683 */
684#if RT_INLINE_ASM_EXTERNAL || !RT_INLINE_ASM_GNU_STYLE
685DECLASM(bool) ASMAtomicCmpXchgU8(volatile uint8_t *pu8, const uint8_t u8New, const uint8_t u8Old);
686#else
687DECLINLINE(bool) ASMAtomicCmpXchgU8(volatile uint8_t *pu8, const uint8_t u8New, uint8_t u8Old)
688{
689 uint8_t u8Ret;
690 __asm__ __volatile__("lock; cmpxchgb %3, %0\n\t"
691 "setz %1\n\t"
692 : "=m" (*pu8),
693 "=qm" (u8Ret),
694 "=a" (u8Old)
695 : "q" (u8New),
696 "2" (u8Old),
697 "m" (*pu8));
698 return (bool)u8Ret;
699}
700#endif
701
702
703/**
704 * Atomically Compare and Exchange a signed 8-bit value, ordered.
705 *
706 * @returns true if xchg was done.
707 * @returns false if xchg wasn't done.
708 *
709 * @param pi8 Pointer to the value to update.
710 * @param i8New The new value to assigned to *pi8.
711 * @param i8Old The old value to *pi8 compare with.
712 */
713DECLINLINE(bool) ASMAtomicCmpXchgS8(volatile int8_t *pi8, const int8_t i8New, const int8_t i8Old)
714{
715 return ASMAtomicCmpXchgU8((volatile uint8_t *)pi8, (const uint8_t)i8New, (const uint8_t)i8Old);
716}
717
718
719/**
720 * Atomically Compare and Exchange a bool value, ordered.
721 *
722 * @returns true if xchg was done.
723 * @returns false if xchg wasn't done.
724 *
725 * @param pf Pointer to the value to update.
726 * @param fNew The new value to assigned to *pf.
727 * @param fOld The old value to *pf compare with.
728 */
729DECLINLINE(bool) ASMAtomicCmpXchgBool(volatile bool *pf, const bool fNew, const bool fOld)
730{
731 return ASMAtomicCmpXchgU8((volatile uint8_t *)pf, (const uint8_t)fNew, (const uint8_t)fOld);
732}
733
734
735/**
736 * Atomically Compare and Exchange an unsigned 32-bit value, ordered.
737 *
738 * @returns true if xchg was done.
739 * @returns false if xchg wasn't done.
740 *
741 * @param pu32 Pointer to the value to update.
742 * @param u32New The new value to assigned to *pu32.
743 * @param u32Old The old value to *pu32 compare with.
744 */
745#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
746DECLASM(bool) ASMAtomicCmpXchgU32(volatile uint32_t *pu32, const uint32_t u32New, const uint32_t u32Old);
747#else
748DECLINLINE(bool) ASMAtomicCmpXchgU32(volatile uint32_t *pu32, const uint32_t u32New, uint32_t u32Old)
749{
750# if RT_INLINE_ASM_GNU_STYLE
751 uint8_t u8Ret;
752 __asm__ __volatile__("lock; cmpxchgl %3, %0\n\t"
753 "setz %1\n\t"
754 : "=m" (*pu32),
755 "=qm" (u8Ret),
756 "=a" (u32Old)
757 : "r" (u32New),
758 "2" (u32Old),
759 "m" (*pu32));
760 return (bool)u8Ret;
761
762# elif RT_INLINE_ASM_USES_INTRIN
763 return _InterlockedCompareExchange((long *)pu32, u32New, u32Old) == u32Old;
764
765# else
766 uint32_t u32Ret;
767 __asm
768 {
769# ifdef RT_ARCH_AMD64
770 mov rdx, [pu32]
771# else
772 mov edx, [pu32]
773# endif
774 mov eax, [u32Old]
775 mov ecx, [u32New]
776# ifdef RT_ARCH_AMD64
777 lock cmpxchg [rdx], ecx
778# else
779 lock cmpxchg [edx], ecx
780# endif
781 setz al
782 movzx eax, al
783 mov [u32Ret], eax
784 }
785 return !!u32Ret;
786# endif
787}
788#endif
789
790
791/**
792 * Atomically Compare and Exchange a signed 32-bit value, ordered.
793 *
794 * @returns true if xchg was done.
795 * @returns false if xchg wasn't done.
796 *
797 * @param pi32 Pointer to the value to update.
798 * @param i32New The new value to assigned to *pi32.
799 * @param i32Old The old value to *pi32 compare with.
800 */
801DECLINLINE(bool) ASMAtomicCmpXchgS32(volatile int32_t *pi32, const int32_t i32New, const int32_t i32Old)
802{
803 return ASMAtomicCmpXchgU32((volatile uint32_t *)pi32, (uint32_t)i32New, (uint32_t)i32Old);
804}
805
806
807/**
808 * Atomically Compare and exchange an unsigned 64-bit value, ordered.
809 *
810 * @returns true if xchg was done.
811 * @returns false if xchg wasn't done.
812 *
813 * @param pu64 Pointer to the 64-bit variable to update.
814 * @param u64New The 64-bit value to assign to *pu64.
815 * @param u64Old The value to compare with.
816 */
817#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
818 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
819DECLASM(bool) ASMAtomicCmpXchgU64(volatile uint64_t *pu64, const uint64_t u64New, const uint64_t u64Old);
820#else
821DECLINLINE(bool) ASMAtomicCmpXchgU64(volatile uint64_t *pu64, uint64_t u64New, uint64_t u64Old)
822{
823# if RT_INLINE_ASM_USES_INTRIN
824 return _InterlockedCompareExchange64((__int64 *)pu64, u64New, u64Old) == u64Old;
825
826# elif defined(RT_ARCH_AMD64)
827# if RT_INLINE_ASM_GNU_STYLE
828 uint8_t u8Ret;
829 __asm__ __volatile__("lock; cmpxchgq %3, %0\n\t"
830 "setz %1\n\t"
831 : "=m" (*pu64),
832 "=qm" (u8Ret),
833 "=a" (u64Old)
834 : "r" (u64New),
835 "2" (u64Old),
836 "m" (*pu64));
837 return (bool)u8Ret;
838# else
839 bool fRet;
840 __asm
841 {
842 mov rdx, [pu32]
843 mov rax, [u64Old]
844 mov rcx, [u64New]
845 lock cmpxchg [rdx], rcx
846 setz al
847 mov [fRet], al
848 }
849 return fRet;
850# endif
851# else /* !RT_ARCH_AMD64 */
852 uint32_t u32Ret;
853# if RT_INLINE_ASM_GNU_STYLE
854# if defined(PIC) || defined(__PIC__)
855 uint32_t u32EBX = (uint32_t)u64New;
856 uint32_t u32Spill;
857 __asm__ __volatile__("xchgl %%ebx, %4\n\t"
858 "lock; cmpxchg8b (%6)\n\t"
859 "setz %%al\n\t"
860 "movl %4, %%ebx\n\t"
861 "movzbl %%al, %%eax\n\t"
862 : "=a" (u32Ret),
863 "=d" (u32Spill),
864# if (__GNUC__ * 100 + __GNUC_MINOR__) >= 403
865 "+m" (*pu64)
866# else
867 "=m" (*pu64)
868# endif
869 : "A" (u64Old),
870 "m" ( u32EBX ),
871 "c" ( (uint32_t)(u64New >> 32) ),
872 "S" (pu64));
873# else /* !PIC */
874 uint32_t u32Spill;
875 __asm__ __volatile__("lock; cmpxchg8b %2\n\t"
876 "setz %%al\n\t"
877 "movzbl %%al, %%eax\n\t"
878 : "=a" (u32Ret),
879 "=d" (u32Spill),
880 "+m" (*pu64)
881 : "A" (u64Old),
882 "b" ( (uint32_t)u64New ),
883 "c" ( (uint32_t)(u64New >> 32) ));
884# endif
885 return (bool)u32Ret;
886# else
887 __asm
888 {
889 mov ebx, dword ptr [u64New]
890 mov ecx, dword ptr [u64New + 4]
891 mov edi, [pu64]
892 mov eax, dword ptr [u64Old]
893 mov edx, dword ptr [u64Old + 4]
894 lock cmpxchg8b [edi]
895 setz al
896 movzx eax, al
897 mov dword ptr [u32Ret], eax
898 }
899 return !!u32Ret;
900# endif
901# endif /* !RT_ARCH_AMD64 */
902}
903#endif
904
905
906/**
907 * Atomically Compare and exchange a signed 64-bit value, ordered.
908 *
909 * @returns true if xchg was done.
910 * @returns false if xchg wasn't done.
911 *
912 * @param pi64 Pointer to the 64-bit variable to update.
913 * @param i64 The 64-bit value to assign to *pu64.
914 * @param i64Old The value to compare with.
915 */
916DECLINLINE(bool) ASMAtomicCmpXchgS64(volatile int64_t *pi64, const int64_t i64, const int64_t i64Old)
917{
918 return ASMAtomicCmpXchgU64((volatile uint64_t *)pi64, (uint64_t)i64, (uint64_t)i64Old);
919}
920
921
922/**
923 * Atomically Compare and Exchange a pointer value, ordered.
924 *
925 * @returns true if xchg was done.
926 * @returns false if xchg wasn't done.
927 *
928 * @param ppv Pointer to the value to update.
929 * @param pvNew The new value to assigned to *ppv.
930 * @param pvOld The old value to *ppv compare with.
931 */
932DECLINLINE(bool) ASMAtomicCmpXchgPtrVoid(void * volatile *ppv, const void *pvNew, const void *pvOld)
933{
934#if ARCH_BITS == 32
935 return ASMAtomicCmpXchgU32((volatile uint32_t *)(void *)ppv, (uint32_t)pvNew, (uint32_t)pvOld);
936#elif ARCH_BITS == 64
937 return ASMAtomicCmpXchgU64((volatile uint64_t *)(void *)ppv, (uint64_t)pvNew, (uint64_t)pvOld);
938#else
939# error "ARCH_BITS is bogus"
940#endif
941}
942
943
944/**
945 * Atomically Compare and Exchange a pointer value, ordered.
946 *
947 * @returns true if xchg was done.
948 * @returns false if xchg wasn't done.
949 *
950 * @param ppv Pointer to the value to update.
951 * @param pvNew The new value to assigned to *ppv.
952 * @param pvOld The old value to *ppv compare with.
953 *
954 * @remarks This is relatively type safe on GCC platforms.
955 */
956#ifdef __GNUC__
957# define ASMAtomicCmpXchgPtr(ppv, pvNew, pvOld) \
958 __extension__ \
959 ({\
960 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
961 __typeof__(*(ppv)) const pvNewTypeChecked = (pvNew); \
962 __typeof__(*(ppv)) const pvOldTypeChecked = (pvOld); \
963 bool fMacroRet = ASMAtomicCmpXchgPtrVoid((void * volatile *)ppvTypeChecked, \
964 (void *)pvNewTypeChecked, (void *)pvOldTypeChecked); \
965 fMacroRet; \
966 })
967#else
968# define ASMAtomicCmpXchgPtr(ppv, pvNew, pvOld) \
969 ASMAtomicCmpXchgPtrVoid((void * volatile *)(ppv), (void *)(pvNew), (void *)(pvOld))
970#endif
971
972
973/** @def ASMAtomicCmpXchgHandle
974 * Atomically Compare and Exchange a typical IPRT handle value, ordered.
975 *
976 * @param ph Pointer to the value to update.
977 * @param hNew The new value to assigned to *pu.
978 * @param hOld The old value to *pu compare with.
979 * @param fRc Where to store the result.
980 *
981 * @remarks This doesn't currently work for all handles (like RTFILE).
982 */
983#if HC_ARCH_BITS == 32
984# define ASMAtomicCmpXchgHandle(ph, hNew, hOld, fRc) \
985 do { \
986 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
987 (fRc) = ASMAtomicCmpXchgU32((uint32_t volatile *)(ph), (const uint32_t)(hNew), (const uint32_t)(hOld)); \
988 } while (0)
989#elif HC_ARCH_BITS == 64
990# define ASMAtomicCmpXchgHandle(ph, hNew, hOld, fRc) \
991 do { \
992 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
993 (fRc) = ASMAtomicCmpXchgU64((uint64_t volatile *)(ph), (const uint64_t)(hNew), (const uint64_t)(hOld)); \
994 } while (0)
995#else
996# error HC_ARCH_BITS
997#endif
998
999
1000/** @def ASMAtomicCmpXchgSize
1001 * Atomically Compare and Exchange a value which size might differ
1002 * between platforms or compilers, ordered.
1003 *
1004 * @param pu Pointer to the value to update.
1005 * @param uNew The new value to assigned to *pu.
1006 * @param uOld The old value to *pu compare with.
1007 * @param fRc Where to store the result.
1008 */
1009#define ASMAtomicCmpXchgSize(pu, uNew, uOld, fRc) \
1010 do { \
1011 switch (sizeof(*(pu))) { \
1012 case 4: (fRc) = ASMAtomicCmpXchgU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew), (uint32_t)(uOld)); \
1013 break; \
1014 case 8: (fRc) = ASMAtomicCmpXchgU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew), (uint64_t)(uOld)); \
1015 break; \
1016 default: AssertMsgFailed(("ASMAtomicCmpXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
1017 (fRc) = false; \
1018 break; \
1019 } \
1020 } while (0)
1021
1022
1023/**
1024 * Atomically Compare and Exchange an unsigned 32-bit value, additionally
1025 * passes back old value, ordered.
1026 *
1027 * @returns true if xchg was done.
1028 * @returns false if xchg wasn't done.
1029 *
1030 * @param pu32 Pointer to the value to update.
1031 * @param u32New The new value to assigned to *pu32.
1032 * @param u32Old The old value to *pu32 compare with.
1033 * @param pu32Old Pointer store the old value at.
1034 */
1035#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1036DECLASM(bool) ASMAtomicCmpXchgExU32(volatile uint32_t *pu32, const uint32_t u32New, const uint32_t u32Old, uint32_t *pu32Old);
1037#else
1038DECLINLINE(bool) ASMAtomicCmpXchgExU32(volatile uint32_t *pu32, const uint32_t u32New, const uint32_t u32Old, uint32_t *pu32Old)
1039{
1040# if RT_INLINE_ASM_GNU_STYLE
1041 uint8_t u8Ret;
1042 __asm__ __volatile__("lock; cmpxchgl %3, %0\n\t"
1043 "setz %1\n\t"
1044 : "=m" (*pu32),
1045 "=qm" (u8Ret),
1046 "=a" (*pu32Old)
1047 : "r" (u32New),
1048 "a" (u32Old),
1049 "m" (*pu32));
1050 return (bool)u8Ret;
1051
1052# elif RT_INLINE_ASM_USES_INTRIN
1053 return (*pu32Old =_InterlockedCompareExchange((long *)pu32, u32New, u32Old)) == u32Old;
1054
1055# else
1056 uint32_t u32Ret;
1057 __asm
1058 {
1059# ifdef RT_ARCH_AMD64
1060 mov rdx, [pu32]
1061# else
1062 mov edx, [pu32]
1063# endif
1064 mov eax, [u32Old]
1065 mov ecx, [u32New]
1066# ifdef RT_ARCH_AMD64
1067 lock cmpxchg [rdx], ecx
1068 mov rdx, [pu32Old]
1069 mov [rdx], eax
1070# else
1071 lock cmpxchg [edx], ecx
1072 mov edx, [pu32Old]
1073 mov [edx], eax
1074# endif
1075 setz al
1076 movzx eax, al
1077 mov [u32Ret], eax
1078 }
1079 return !!u32Ret;
1080# endif
1081}
1082#endif
1083
1084
1085/**
1086 * Atomically Compare and Exchange a signed 32-bit value, additionally
1087 * passes back old value, ordered.
1088 *
1089 * @returns true if xchg was done.
1090 * @returns false if xchg wasn't done.
1091 *
1092 * @param pi32 Pointer to the value to update.
1093 * @param i32New The new value to assigned to *pi32.
1094 * @param i32Old The old value to *pi32 compare with.
1095 * @param pi32Old Pointer store the old value at.
1096 */
1097DECLINLINE(bool) ASMAtomicCmpXchgExS32(volatile int32_t *pi32, const int32_t i32New, const int32_t i32Old, int32_t *pi32Old)
1098{
1099 return ASMAtomicCmpXchgExU32((volatile uint32_t *)pi32, (uint32_t)i32New, (uint32_t)i32Old, (uint32_t *)pi32Old);
1100}
1101
1102
1103/**
1104 * Atomically Compare and exchange an unsigned 64-bit value, additionally
1105 * passing back old value, ordered.
1106 *
1107 * @returns true if xchg was done.
1108 * @returns false if xchg wasn't done.
1109 *
1110 * @param pu64 Pointer to the 64-bit variable to update.
1111 * @param u64New The 64-bit value to assign to *pu64.
1112 * @param u64Old The value to compare with.
1113 * @param pu64Old Pointer store the old value at.
1114 */
1115#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
1116 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
1117DECLASM(bool) ASMAtomicCmpXchgExU64(volatile uint64_t *pu64, const uint64_t u64New, const uint64_t u64Old, uint64_t *pu64Old);
1118#else
1119DECLINLINE(bool) ASMAtomicCmpXchgExU64(volatile uint64_t *pu64, const uint64_t u64New, const uint64_t u64Old, uint64_t *pu64Old)
1120{
1121# if RT_INLINE_ASM_USES_INTRIN
1122 return (*pu64Old =_InterlockedCompareExchange64((__int64 *)pu64, u64New, u64Old)) == u64Old;
1123
1124# elif defined(RT_ARCH_AMD64)
1125# if RT_INLINE_ASM_GNU_STYLE
1126 uint8_t u8Ret;
1127 __asm__ __volatile__("lock; cmpxchgq %3, %0\n\t"
1128 "setz %1\n\t"
1129 : "=m" (*pu64),
1130 "=qm" (u8Ret),
1131 "=a" (*pu64Old)
1132 : "r" (u64New),
1133 "a" (u64Old),
1134 "m" (*pu64));
1135 return (bool)u8Ret;
1136# else
1137 bool fRet;
1138 __asm
1139 {
1140 mov rdx, [pu32]
1141 mov rax, [u64Old]
1142 mov rcx, [u64New]
1143 lock cmpxchg [rdx], rcx
1144 mov rdx, [pu64Old]
1145 mov [rdx], rax
1146 setz al
1147 mov [fRet], al
1148 }
1149 return fRet;
1150# endif
1151# else /* !RT_ARCH_AMD64 */
1152# if RT_INLINE_ASM_GNU_STYLE
1153 uint64_t u64Ret;
1154# if defined(PIC) || defined(__PIC__)
1155 /* NB: this code uses a memory clobber description, because the clean
1156 * solution with an output value for *pu64 makes gcc run out of registers.
1157 * This will cause suboptimal code, and anyone with a better solution is
1158 * welcome to improve this. */
1159 __asm__ __volatile__("xchgl %%ebx, %1\n\t"
1160 "lock; cmpxchg8b %3\n\t"
1161 "xchgl %%ebx, %1\n\t"
1162 : "=A" (u64Ret)
1163 : "DS" ((uint32_t)u64New),
1164 "c" ((uint32_t)(u64New >> 32)),
1165 "m" (*pu64),
1166 "0" (u64Old)
1167 : "memory" );
1168# else /* !PIC */
1169 __asm__ __volatile__("lock; cmpxchg8b %4\n\t"
1170 : "=A" (u64Ret),
1171 "=m" (*pu64)
1172 : "b" ((uint32_t)u64New),
1173 "c" ((uint32_t)(u64New >> 32)),
1174 "m" (*pu64),
1175 "0" (u64Old));
1176# endif
1177 *pu64Old = u64Ret;
1178 return u64Ret == u64Old;
1179# else
1180 uint32_t u32Ret;
1181 __asm
1182 {
1183 mov ebx, dword ptr [u64New]
1184 mov ecx, dword ptr [u64New + 4]
1185 mov edi, [pu64]
1186 mov eax, dword ptr [u64Old]
1187 mov edx, dword ptr [u64Old + 4]
1188 lock cmpxchg8b [edi]
1189 mov ebx, [pu64Old]
1190 mov [ebx], eax
1191 setz al
1192 movzx eax, al
1193 add ebx, 4
1194 mov [ebx], edx
1195 mov dword ptr [u32Ret], eax
1196 }
1197 return !!u32Ret;
1198# endif
1199# endif /* !RT_ARCH_AMD64 */
1200}
1201#endif
1202
1203
1204/**
1205 * Atomically Compare and exchange a signed 64-bit value, additionally
1206 * passing back old value, ordered.
1207 *
1208 * @returns true if xchg was done.
1209 * @returns false if xchg wasn't done.
1210 *
1211 * @param pi64 Pointer to the 64-bit variable to update.
1212 * @param i64 The 64-bit value to assign to *pu64.
1213 * @param i64Old The value to compare with.
1214 * @param pi64Old Pointer store the old value at.
1215 */
1216DECLINLINE(bool) ASMAtomicCmpXchgExS64(volatile int64_t *pi64, const int64_t i64, const int64_t i64Old, int64_t *pi64Old)
1217{
1218 return ASMAtomicCmpXchgExU64((volatile uint64_t *)pi64, (uint64_t)i64, (uint64_t)i64Old, (uint64_t *)pi64Old);
1219}
1220
1221/** @def ASMAtomicCmpXchgExHandle
1222 * Atomically Compare and Exchange a typical IPRT handle value, ordered.
1223 *
1224 * @param ph Pointer to the value to update.
1225 * @param hNew The new value to assigned to *pu.
1226 * @param hOld The old value to *pu compare with.
1227 * @param fRc Where to store the result.
1228 * @param phOldVal Pointer to where to store the old value.
1229 *
1230 * @remarks This doesn't currently work for all handles (like RTFILE).
1231 */
1232#if HC_ARCH_BITS == 32
1233# define ASMAtomicCmpXchgExHandle(ph, hNew, hOld, fRc, phOldVal) \
1234 do { \
1235 AssertCompile(sizeof(*ph) == sizeof(uint32_t)); \
1236 AssertCompile(sizeof(*phOldVal) == sizeof(uint32_t)); \
1237 (fRc) = ASMAtomicCmpXchgExU32((volatile uint32_t *)(pu), (uint32_t)(uNew), (uint32_t)(uOld), (uint32_t *)(puOldVal)); \
1238 } while (0)
1239#elif HC_ARCH_BITS == 64
1240# define ASMAtomicCmpXchgExHandle(ph, hNew, hOld, fRc, phOldVal) \
1241 do { \
1242 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
1243 AssertCompile(sizeof(*(phOldVal)) == sizeof(uint64_t)); \
1244 (fRc) = ASMAtomicCmpXchgExU64((volatile uint64_t *)(pu), (uint64_t)(uNew), (uint64_t)(uOld), (uint64_t *)(puOldVal)); \
1245 } while (0)
1246#else
1247# error HC_ARCH_BITS
1248#endif
1249
1250
1251/** @def ASMAtomicCmpXchgExSize
1252 * Atomically Compare and Exchange a value which size might differ
1253 * between platforms or compilers. Additionally passes back old value.
1254 *
1255 * @param pu Pointer to the value to update.
1256 * @param uNew The new value to assigned to *pu.
1257 * @param uOld The old value to *pu compare with.
1258 * @param fRc Where to store the result.
1259 * @param puOldVal Pointer to where to store the old value.
1260 */
1261#define ASMAtomicCmpXchgExSize(pu, uNew, uOld, fRc, puOldVal) \
1262 do { \
1263 switch (sizeof(*(pu))) { \
1264 case 4: (fRc) = ASMAtomicCmpXchgExU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew), (uint32_t)(uOld), (uint32_t *)(uOldVal)); \
1265 break; \
1266 case 8: (fRc) = ASMAtomicCmpXchgExU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew), (uint64_t)(uOld), (uint64_t *)(uOldVal)); \
1267 break; \
1268 default: AssertMsgFailed(("ASMAtomicCmpXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
1269 (fRc) = false; \
1270 (uOldVal) = 0; \
1271 break; \
1272 } \
1273 } while (0)
1274
1275
1276/**
1277 * Atomically Compare and Exchange a pointer value, additionally
1278 * passing back old value, ordered.
1279 *
1280 * @returns true if xchg was done.
1281 * @returns false if xchg wasn't done.
1282 *
1283 * @param ppv Pointer to the value to update.
1284 * @param pvNew The new value to assigned to *ppv.
1285 * @param pvOld The old value to *ppv compare with.
1286 * @param ppvOld Pointer store the old value at.
1287 */
1288DECLINLINE(bool) ASMAtomicCmpXchgExPtrVoid(void * volatile *ppv, const void *pvNew, const void *pvOld, void **ppvOld)
1289{
1290#if ARCH_BITS == 32
1291 return ASMAtomicCmpXchgExU32((volatile uint32_t *)(void *)ppv, (uint32_t)pvNew, (uint32_t)pvOld, (uint32_t *)ppvOld);
1292#elif ARCH_BITS == 64
1293 return ASMAtomicCmpXchgExU64((volatile uint64_t *)(void *)ppv, (uint64_t)pvNew, (uint64_t)pvOld, (uint64_t *)ppvOld);
1294#else
1295# error "ARCH_BITS is bogus"
1296#endif
1297}
1298
1299
1300/**
1301 * Atomically Compare and Exchange a pointer value, additionally
1302 * passing back old value, ordered.
1303 *
1304 * @returns true if xchg was done.
1305 * @returns false if xchg wasn't done.
1306 *
1307 * @param ppv Pointer to the value to update.
1308 * @param pvNew The new value to assigned to *ppv.
1309 * @param pvOld The old value to *ppv compare with.
1310 * @param ppvOld Pointer store the old value at.
1311 *
1312 * @remarks This is relatively type safe on GCC platforms.
1313 */
1314#ifdef __GNUC__
1315# define ASMAtomicCmpXchgExPtr(ppv, pvNew, pvOld, ppvOld) \
1316 __extension__ \
1317 ({\
1318 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
1319 __typeof__(*(ppv)) const pvNewTypeChecked = (pvNew); \
1320 __typeof__(*(ppv)) const pvOldTypeChecked = (pvOld); \
1321 __typeof__(*(ppv)) * const ppvOldTypeChecked = (ppvOld); \
1322 bool fMacroRet = ASMAtomicCmpXchgExPtrVoid((void * volatile *)ppvTypeChecked, \
1323 (void *)pvNewTypeChecked, (void *)pvOldTypeChecked, \
1324 (void **)ppvOld); \
1325 fMacroRet; \
1326 })
1327#else
1328# define ASMAtomicCmpXchgExPtr(ppv, pvNew, pvOld, ppvOld) \
1329 ASMAtomicCmpXchgExPtrVoid((void * volatile *)(ppv), (void *)(pvNew), (void *)pvOld, (void **)ppvOld)
1330#endif
1331
1332
1333/**
1334 * Serialize Instruction.
1335 */
1336#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1337DECLASM(void) ASMSerializeInstruction(void);
1338#else
1339DECLINLINE(void) ASMSerializeInstruction(void)
1340{
1341# if RT_INLINE_ASM_GNU_STYLE
1342 RTCCUINTREG xAX = 0;
1343# ifdef RT_ARCH_AMD64
1344 __asm__ ("cpuid"
1345 : "=a" (xAX)
1346 : "0" (xAX)
1347 : "rbx", "rcx", "rdx");
1348# elif (defined(PIC) || defined(__PIC__)) && defined(__i386__)
1349 __asm__ ("push %%ebx\n\t"
1350 "cpuid\n\t"
1351 "pop %%ebx\n\t"
1352 : "=a" (xAX)
1353 : "0" (xAX)
1354 : "ecx", "edx");
1355# else
1356 __asm__ ("cpuid"
1357 : "=a" (xAX)
1358 : "0" (xAX)
1359 : "ebx", "ecx", "edx");
1360# endif
1361
1362# elif RT_INLINE_ASM_USES_INTRIN
1363 int aInfo[4];
1364 __cpuid(aInfo, 0);
1365
1366# else
1367 __asm
1368 {
1369 push ebx
1370 xor eax, eax
1371 cpuid
1372 pop ebx
1373 }
1374# endif
1375}
1376#endif
1377
1378
1379/**
1380 * Memory fence, waits for any pending writes and reads to complete.
1381 */
1382DECLINLINE(void) ASMMemoryFence(void)
1383{
1384 /** @todo use mfence? check if all cpus we care for support it. */
1385 uint32_t volatile u32;
1386 ASMAtomicXchgU32(&u32, 0);
1387}
1388
1389
1390/**
1391 * Write fence, waits for any pending writes to complete.
1392 */
1393DECLINLINE(void) ASMWriteFence(void)
1394{
1395 /** @todo use sfence? check if all cpus we care for support it. */
1396 ASMMemoryFence();
1397}
1398
1399
1400/**
1401 * Read fence, waits for any pending reads to complete.
1402 */
1403DECLINLINE(void) ASMReadFence(void)
1404{
1405 /** @todo use lfence? check if all cpus we care for support it. */
1406 ASMMemoryFence();
1407}
1408
1409
1410/**
1411 * Atomically reads an unsigned 8-bit value, ordered.
1412 *
1413 * @returns Current *pu8 value
1414 * @param pu8 Pointer to the 8-bit variable to read.
1415 */
1416DECLINLINE(uint8_t) ASMAtomicReadU8(volatile uint8_t *pu8)
1417{
1418 ASMMemoryFence();
1419 return *pu8; /* byte reads are atomic on x86 */
1420}
1421
1422
1423/**
1424 * Atomically reads an unsigned 8-bit value, unordered.
1425 *
1426 * @returns Current *pu8 value
1427 * @param pu8 Pointer to the 8-bit variable to read.
1428 */
1429DECLINLINE(uint8_t) ASMAtomicUoReadU8(volatile uint8_t *pu8)
1430{
1431 return *pu8; /* byte reads are atomic on x86 */
1432}
1433
1434
1435/**
1436 * Atomically reads a signed 8-bit value, ordered.
1437 *
1438 * @returns Current *pi8 value
1439 * @param pi8 Pointer to the 8-bit variable to read.
1440 */
1441DECLINLINE(int8_t) ASMAtomicReadS8(volatile int8_t *pi8)
1442{
1443 ASMMemoryFence();
1444 return *pi8; /* byte reads are atomic on x86 */
1445}
1446
1447
1448/**
1449 * Atomically reads a signed 8-bit value, unordered.
1450 *
1451 * @returns Current *pi8 value
1452 * @param pi8 Pointer to the 8-bit variable to read.
1453 */
1454DECLINLINE(int8_t) ASMAtomicUoReadS8(volatile int8_t *pi8)
1455{
1456 return *pi8; /* byte reads are atomic on x86 */
1457}
1458
1459
1460/**
1461 * Atomically reads an unsigned 16-bit value, ordered.
1462 *
1463 * @returns Current *pu16 value
1464 * @param pu16 Pointer to the 16-bit variable to read.
1465 */
1466DECLINLINE(uint16_t) ASMAtomicReadU16(volatile uint16_t *pu16)
1467{
1468 ASMMemoryFence();
1469 Assert(!((uintptr_t)pu16 & 1));
1470 return *pu16;
1471}
1472
1473
1474/**
1475 * Atomically reads an unsigned 16-bit value, unordered.
1476 *
1477 * @returns Current *pu16 value
1478 * @param pu16 Pointer to the 16-bit variable to read.
1479 */
1480DECLINLINE(uint16_t) ASMAtomicUoReadU16(volatile uint16_t *pu16)
1481{
1482 Assert(!((uintptr_t)pu16 & 1));
1483 return *pu16;
1484}
1485
1486
1487/**
1488 * Atomically reads a signed 16-bit value, ordered.
1489 *
1490 * @returns Current *pi16 value
1491 * @param pi16 Pointer to the 16-bit variable to read.
1492 */
1493DECLINLINE(int16_t) ASMAtomicReadS16(volatile int16_t *pi16)
1494{
1495 ASMMemoryFence();
1496 Assert(!((uintptr_t)pi16 & 1));
1497 return *pi16;
1498}
1499
1500
1501/**
1502 * Atomically reads a signed 16-bit value, unordered.
1503 *
1504 * @returns Current *pi16 value
1505 * @param pi16 Pointer to the 16-bit variable to read.
1506 */
1507DECLINLINE(int16_t) ASMAtomicUoReadS16(volatile int16_t *pi16)
1508{
1509 Assert(!((uintptr_t)pi16 & 1));
1510 return *pi16;
1511}
1512
1513
1514/**
1515 * Atomically reads an unsigned 32-bit value, ordered.
1516 *
1517 * @returns Current *pu32 value
1518 * @param pu32 Pointer to the 32-bit variable to read.
1519 */
1520DECLINLINE(uint32_t) ASMAtomicReadU32(volatile uint32_t *pu32)
1521{
1522 ASMMemoryFence();
1523 Assert(!((uintptr_t)pu32 & 3));
1524 return *pu32;
1525}
1526
1527
1528/**
1529 * Atomically reads an unsigned 32-bit value, unordered.
1530 *
1531 * @returns Current *pu32 value
1532 * @param pu32 Pointer to the 32-bit variable to read.
1533 */
1534DECLINLINE(uint32_t) ASMAtomicUoReadU32(volatile uint32_t *pu32)
1535{
1536 Assert(!((uintptr_t)pu32 & 3));
1537 return *pu32;
1538}
1539
1540
1541/**
1542 * Atomically reads a signed 32-bit value, ordered.
1543 *
1544 * @returns Current *pi32 value
1545 * @param pi32 Pointer to the 32-bit variable to read.
1546 */
1547DECLINLINE(int32_t) ASMAtomicReadS32(volatile int32_t *pi32)
1548{
1549 ASMMemoryFence();
1550 Assert(!((uintptr_t)pi32 & 3));
1551 return *pi32;
1552}
1553
1554
1555/**
1556 * Atomically reads a signed 32-bit value, unordered.
1557 *
1558 * @returns Current *pi32 value
1559 * @param pi32 Pointer to the 32-bit variable to read.
1560 */
1561DECLINLINE(int32_t) ASMAtomicUoReadS32(volatile int32_t *pi32)
1562{
1563 Assert(!((uintptr_t)pi32 & 3));
1564 return *pi32;
1565}
1566
1567
1568/**
1569 * Atomically reads an unsigned 64-bit value, ordered.
1570 *
1571 * @returns Current *pu64 value
1572 * @param pu64 Pointer to the 64-bit variable to read.
1573 * The memory pointed to must be writable.
1574 * @remark This will fault if the memory is read-only!
1575 */
1576#if (RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)) \
1577 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
1578DECLASM(uint64_t) ASMAtomicReadU64(volatile uint64_t *pu64);
1579#else
1580DECLINLINE(uint64_t) ASMAtomicReadU64(volatile uint64_t *pu64)
1581{
1582 uint64_t u64;
1583# ifdef RT_ARCH_AMD64
1584 Assert(!((uintptr_t)pu64 & 7));
1585/*# if RT_INLINE_ASM_GNU_STYLE
1586 __asm__ __volatile__( "mfence\n\t"
1587 "movq %1, %0\n\t"
1588 : "=r" (u64)
1589 : "m" (*pu64));
1590# else
1591 __asm
1592 {
1593 mfence
1594 mov rdx, [pu64]
1595 mov rax, [rdx]
1596 mov [u64], rax
1597 }
1598# endif*/
1599 ASMMemoryFence();
1600 u64 = *pu64;
1601# else /* !RT_ARCH_AMD64 */
1602# if RT_INLINE_ASM_GNU_STYLE
1603# if defined(PIC) || defined(__PIC__)
1604 uint32_t u32EBX = 0;
1605 Assert(!((uintptr_t)pu64 & 7));
1606 __asm__ __volatile__("xchgl %%ebx, %3\n\t"
1607 "lock; cmpxchg8b (%5)\n\t"
1608 "movl %3, %%ebx\n\t"
1609 : "=A" (u64),
1610# if (__GNUC__ * 100 + __GNUC_MINOR__) >= 403
1611 "+m" (*pu64)
1612# else
1613 "=m" (*pu64)
1614# endif
1615 : "0" (0ULL),
1616 "m" (u32EBX),
1617 "c" (0),
1618 "S" (pu64));
1619# else /* !PIC */
1620 __asm__ __volatile__("lock; cmpxchg8b %1\n\t"
1621 : "=A" (u64),
1622 "+m" (*pu64)
1623 : "0" (0ULL),
1624 "b" (0),
1625 "c" (0));
1626# endif
1627# else
1628 Assert(!((uintptr_t)pu64 & 7));
1629 __asm
1630 {
1631 xor eax, eax
1632 xor edx, edx
1633 mov edi, pu64
1634 xor ecx, ecx
1635 xor ebx, ebx
1636 lock cmpxchg8b [edi]
1637 mov dword ptr [u64], eax
1638 mov dword ptr [u64 + 4], edx
1639 }
1640# endif
1641# endif /* !RT_ARCH_AMD64 */
1642 return u64;
1643}
1644#endif
1645
1646
1647/**
1648 * Atomically reads an unsigned 64-bit value, unordered.
1649 *
1650 * @returns Current *pu64 value
1651 * @param pu64 Pointer to the 64-bit variable to read.
1652 * The memory pointed to must be writable.
1653 * @remark This will fault if the memory is read-only!
1654 */
1655#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
1656 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
1657DECLASM(uint64_t) ASMAtomicUoReadU64(volatile uint64_t *pu64);
1658#else
1659DECLINLINE(uint64_t) ASMAtomicUoReadU64(volatile uint64_t *pu64)
1660{
1661 uint64_t u64;
1662# ifdef RT_ARCH_AMD64
1663 Assert(!((uintptr_t)pu64 & 7));
1664/*# if RT_INLINE_ASM_GNU_STYLE
1665 Assert(!((uintptr_t)pu64 & 7));
1666 __asm__ __volatile__("movq %1, %0\n\t"
1667 : "=r" (u64)
1668 : "m" (*pu64));
1669# else
1670 __asm
1671 {
1672 mov rdx, [pu64]
1673 mov rax, [rdx]
1674 mov [u64], rax
1675 }
1676# endif */
1677 u64 = *pu64;
1678# else /* !RT_ARCH_AMD64 */
1679# if RT_INLINE_ASM_GNU_STYLE
1680# if defined(PIC) || defined(__PIC__)
1681 uint32_t u32EBX = 0;
1682 uint32_t u32Spill;
1683 Assert(!((uintptr_t)pu64 & 7));
1684 __asm__ __volatile__("xor %%eax,%%eax\n\t"
1685 "xor %%ecx,%%ecx\n\t"
1686 "xor %%edx,%%edx\n\t"
1687 "xchgl %%ebx, %3\n\t"
1688 "lock; cmpxchg8b (%4)\n\t"
1689 "movl %3, %%ebx\n\t"
1690 : "=A" (u64),
1691# if (__GNUC__ * 100 + __GNUC_MINOR__) >= 403
1692 "+m" (*pu64),
1693# else
1694 "=m" (*pu64),
1695# endif
1696 "=c" (u32Spill)
1697 : "m" (u32EBX),
1698 "S" (pu64));
1699# else /* !PIC */
1700 __asm__ __volatile__("lock; cmpxchg8b %1\n\t"
1701 : "=A" (u64),
1702 "+m" (*pu64)
1703 : "0" (0ULL),
1704 "b" (0),
1705 "c" (0));
1706# endif
1707# else
1708 Assert(!((uintptr_t)pu64 & 7));
1709 __asm
1710 {
1711 xor eax, eax
1712 xor edx, edx
1713 mov edi, pu64
1714 xor ecx, ecx
1715 xor ebx, ebx
1716 lock cmpxchg8b [edi]
1717 mov dword ptr [u64], eax
1718 mov dword ptr [u64 + 4], edx
1719 }
1720# endif
1721# endif /* !RT_ARCH_AMD64 */
1722 return u64;
1723}
1724#endif
1725
1726
1727/**
1728 * Atomically reads a signed 64-bit value, ordered.
1729 *
1730 * @returns Current *pi64 value
1731 * @param pi64 Pointer to the 64-bit variable to read.
1732 * The memory pointed to must be writable.
1733 * @remark This will fault if the memory is read-only!
1734 */
1735DECLINLINE(int64_t) ASMAtomicReadS64(volatile int64_t *pi64)
1736{
1737 return (int64_t)ASMAtomicReadU64((volatile uint64_t *)pi64);
1738}
1739
1740
1741/**
1742 * Atomically reads a signed 64-bit value, unordered.
1743 *
1744 * @returns Current *pi64 value
1745 * @param pi64 Pointer to the 64-bit variable to read.
1746 * The memory pointed to must be writable.
1747 * @remark This will fault if the memory is read-only!
1748 */
1749DECLINLINE(int64_t) ASMAtomicUoReadS64(volatile int64_t *pi64)
1750{
1751 return (int64_t)ASMAtomicUoReadU64((volatile uint64_t *)pi64);
1752}
1753
1754
1755/**
1756 * Atomically reads a pointer value, ordered.
1757 *
1758 * @returns Current *pv value
1759 * @param ppv Pointer to the pointer variable to read.
1760 *
1761 * @remarks Please use ASMAtomicReadPtrT, it provides better type safety and
1762 * requires less typing (no casts).
1763 */
1764DECLINLINE(void *) ASMAtomicReadPtr(void * volatile *ppv)
1765{
1766#if ARCH_BITS == 32
1767 return (void *)ASMAtomicReadU32((volatile uint32_t *)(void *)ppv);
1768#elif ARCH_BITS == 64
1769 return (void *)ASMAtomicReadU64((volatile uint64_t *)(void *)ppv);
1770#else
1771# error "ARCH_BITS is bogus"
1772#endif
1773}
1774
1775/**
1776 * Convenience macro for avoiding the annoying casting with ASMAtomicReadPtr.
1777 *
1778 * @returns Current *pv value
1779 * @param ppv Pointer to the pointer variable to read.
1780 * @param Type The type of *ppv, sans volatile.
1781 */
1782#ifdef __GNUC__
1783# define ASMAtomicReadPtrT(ppv, Type) \
1784 __extension__ \
1785 ({\
1786 __typeof__(*(ppv)) volatile *ppvTypeChecked = (ppv); \
1787 Type pvTypeChecked = (__typeof__(*(ppv))) ASMAtomicReadPtr((void * volatile *)ppvTypeChecked); \
1788 pvTypeChecked; \
1789 })
1790#else
1791# define ASMAtomicReadPtrT(ppv, Type) \
1792 (Type)ASMAtomicReadPtr((void * volatile *)(ppv))
1793#endif
1794
1795
1796/**
1797 * Atomically reads a pointer value, unordered.
1798 *
1799 * @returns Current *pv value
1800 * @param ppv Pointer to the pointer variable to read.
1801 *
1802 * @remarks Please use ASMAtomicUoReadPtrT, it provides better type safety and
1803 * requires less typing (no casts).
1804 */
1805DECLINLINE(void *) ASMAtomicUoReadPtr(void * volatile *ppv)
1806{
1807#if ARCH_BITS == 32
1808 return (void *)ASMAtomicUoReadU32((volatile uint32_t *)(void *)ppv);
1809#elif ARCH_BITS == 64
1810 return (void *)ASMAtomicUoReadU64((volatile uint64_t *)(void *)ppv);
1811#else
1812# error "ARCH_BITS is bogus"
1813#endif
1814}
1815
1816
1817/**
1818 * Convenience macro for avoiding the annoying casting with ASMAtomicUoReadPtr.
1819 *
1820 * @returns Current *pv value
1821 * @param ppv Pointer to the pointer variable to read.
1822 * @param Type The type of *ppv, sans volatile.
1823 */
1824#ifdef __GNUC__
1825# define ASMAtomicUoReadPtrT(ppv, Type) \
1826 __extension__ \
1827 ({\
1828 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
1829 Type pvTypeChecked = (__typeof__(*(ppv))) ASMAtomicUoReadPtr((void * volatile *)ppvTypeChecked); \
1830 pvTypeChecked; \
1831 })
1832#else
1833# define ASMAtomicUoReadPtrT(ppv, Type) \
1834 (Type)ASMAtomicUoReadPtr((void * volatile *)(ppv))
1835#endif
1836
1837
1838/**
1839 * Atomically reads a boolean value, ordered.
1840 *
1841 * @returns Current *pf value
1842 * @param pf Pointer to the boolean variable to read.
1843 */
1844DECLINLINE(bool) ASMAtomicReadBool(volatile bool *pf)
1845{
1846 ASMMemoryFence();
1847 return *pf; /* byte reads are atomic on x86 */
1848}
1849
1850
1851/**
1852 * Atomically reads a boolean value, unordered.
1853 *
1854 * @returns Current *pf value
1855 * @param pf Pointer to the boolean variable to read.
1856 */
1857DECLINLINE(bool) ASMAtomicUoReadBool(volatile bool *pf)
1858{
1859 return *pf; /* byte reads are atomic on x86 */
1860}
1861
1862
1863/**
1864 * Atomically read a typical IPRT handle value, ordered.
1865 *
1866 * @param ph Pointer to the handle variable to read.
1867 * @param phRes Where to store the result.
1868 *
1869 * @remarks This doesn't currently work for all handles (like RTFILE).
1870 */
1871#if HC_ARCH_BITS == 32
1872# define ASMAtomicReadHandle(ph, phRes) \
1873 do { \
1874 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
1875 AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
1876 *(uint32_t *)(phRes) = ASMAtomicReadU32((uint32_t volatile *)(ph)); \
1877 } while (0)
1878#elif HC_ARCH_BITS == 64
1879# define ASMAtomicReadHandle(ph, phRes) \
1880 do { \
1881 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
1882 AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
1883 *(uint64_t *)(phRes) = ASMAtomicReadU64((uint64_t volatile *)(ph)); \
1884 } while (0)
1885#else
1886# error HC_ARCH_BITS
1887#endif
1888
1889
1890/**
1891 * Atomically read a typical IPRT handle value, unordered.
1892 *
1893 * @param ph Pointer to the handle variable to read.
1894 * @param phRes Where to store the result.
1895 *
1896 * @remarks This doesn't currently work for all handles (like RTFILE).
1897 */
1898#if HC_ARCH_BITS == 32
1899# define ASMAtomicUoReadHandle(ph, phRes) \
1900 do { \
1901 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
1902 AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
1903 *(uint32_t *)(phRes) = ASMAtomicUoReadU32((uint32_t volatile *)(ph)); \
1904 } while (0)
1905#elif HC_ARCH_BITS == 64
1906# define ASMAtomicUoReadHandle(ph, phRes) \
1907 do { \
1908 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
1909 AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
1910 *(uint64_t *)(phRes) = ASMAtomicUoReadU64((uint64_t volatile *)(ph)); \
1911 } while (0)
1912#else
1913# error HC_ARCH_BITS
1914#endif
1915
1916
1917/**
1918 * Atomically read a value which size might differ
1919 * between platforms or compilers, ordered.
1920 *
1921 * @param pu Pointer to the variable to update.
1922 * @param puRes Where to store the result.
1923 */
1924#define ASMAtomicReadSize(pu, puRes) \
1925 do { \
1926 switch (sizeof(*(pu))) { \
1927 case 1: *(uint8_t *)(puRes) = ASMAtomicReadU8( (volatile uint8_t *)(void *)(pu)); break; \
1928 case 2: *(uint16_t *)(puRes) = ASMAtomicReadU16((volatile uint16_t *)(void *)(pu)); break; \
1929 case 4: *(uint32_t *)(puRes) = ASMAtomicReadU32((volatile uint32_t *)(void *)(pu)); break; \
1930 case 8: *(uint64_t *)(puRes) = ASMAtomicReadU64((volatile uint64_t *)(void *)(pu)); break; \
1931 default: AssertMsgFailed(("ASMAtomicReadSize: size %d is not supported\n", sizeof(*(pu)))); \
1932 } \
1933 } while (0)
1934
1935
1936/**
1937 * Atomically read a value which size might differ
1938 * between platforms or compilers, unordered.
1939 *
1940 * @param pu Pointer to the variable to read.
1941 * @param puRes Where to store the result.
1942 */
1943#define ASMAtomicUoReadSize(pu, puRes) \
1944 do { \
1945 switch (sizeof(*(pu))) { \
1946 case 1: *(uint8_t *)(puRes) = ASMAtomicUoReadU8( (volatile uint8_t *)(void *)(pu)); break; \
1947 case 2: *(uint16_t *)(puRes) = ASMAtomicUoReadU16((volatile uint16_t *)(void *)(pu)); break; \
1948 case 4: *(uint32_t *)(puRes) = ASMAtomicUoReadU32((volatile uint32_t *)(void *)(pu)); break; \
1949 case 8: *(uint64_t *)(puRes) = ASMAtomicUoReadU64((volatile uint64_t *)(void *)(pu)); break; \
1950 default: AssertMsgFailed(("ASMAtomicReadSize: size %d is not supported\n", sizeof(*(pu)))); \
1951 } \
1952 } while (0)
1953
1954
1955/**
1956 * Atomically writes an unsigned 8-bit value, ordered.
1957 *
1958 * @param pu8 Pointer to the 8-bit variable.
1959 * @param u8 The 8-bit value to assign to *pu8.
1960 */
1961DECLINLINE(void) ASMAtomicWriteU8(volatile uint8_t *pu8, uint8_t u8)
1962{
1963 ASMAtomicXchgU8(pu8, u8);
1964}
1965
1966
1967/**
1968 * Atomically writes an unsigned 8-bit value, unordered.
1969 *
1970 * @param pu8 Pointer to the 8-bit variable.
1971 * @param u8 The 8-bit value to assign to *pu8.
1972 */
1973DECLINLINE(void) ASMAtomicUoWriteU8(volatile uint8_t *pu8, uint8_t u8)
1974{
1975 *pu8 = u8; /* byte writes are atomic on x86 */
1976}
1977
1978
1979/**
1980 * Atomically writes a signed 8-bit value, ordered.
1981 *
1982 * @param pi8 Pointer to the 8-bit variable to read.
1983 * @param i8 The 8-bit value to assign to *pi8.
1984 */
1985DECLINLINE(void) ASMAtomicWriteS8(volatile int8_t *pi8, int8_t i8)
1986{
1987 ASMAtomicXchgS8(pi8, i8);
1988}
1989
1990
1991/**
1992 * Atomically writes a signed 8-bit value, unordered.
1993 *
1994 * @param pi8 Pointer to the 8-bit variable to read.
1995 * @param i8 The 8-bit value to assign to *pi8.
1996 */
1997DECLINLINE(void) ASMAtomicUoWriteS8(volatile int8_t *pi8, int8_t i8)
1998{
1999 *pi8 = i8; /* byte writes are atomic on x86 */
2000}
2001
2002
2003/**
2004 * Atomically writes an unsigned 16-bit value, ordered.
2005 *
2006 * @param pu16 Pointer to the 16-bit variable.
2007 * @param u16 The 16-bit value to assign to *pu16.
2008 */
2009DECLINLINE(void) ASMAtomicWriteU16(volatile uint16_t *pu16, uint16_t u16)
2010{
2011 ASMAtomicXchgU16(pu16, u16);
2012}
2013
2014
2015/**
2016 * Atomically writes an unsigned 16-bit value, unordered.
2017 *
2018 * @param pu16 Pointer to the 16-bit variable.
2019 * @param u16 The 16-bit value to assign to *pu16.
2020 */
2021DECLINLINE(void) ASMAtomicUoWriteU16(volatile uint16_t *pu16, uint16_t u16)
2022{
2023 Assert(!((uintptr_t)pu16 & 1));
2024 *pu16 = u16;
2025}
2026
2027
2028/**
2029 * Atomically writes a signed 16-bit value, ordered.
2030 *
2031 * @param pi16 Pointer to the 16-bit variable to read.
2032 * @param i16 The 16-bit value to assign to *pi16.
2033 */
2034DECLINLINE(void) ASMAtomicWriteS16(volatile int16_t *pi16, int16_t i16)
2035{
2036 ASMAtomicXchgS16(pi16, i16);
2037}
2038
2039
2040/**
2041 * Atomically writes a signed 16-bit value, unordered.
2042 *
2043 * @param pi16 Pointer to the 16-bit variable to read.
2044 * @param i16 The 16-bit value to assign to *pi16.
2045 */
2046DECLINLINE(void) ASMAtomicUoWriteS16(volatile int16_t *pi16, int16_t i16)
2047{
2048 Assert(!((uintptr_t)pi16 & 1));
2049 *pi16 = i16;
2050}
2051
2052
2053/**
2054 * Atomically writes an unsigned 32-bit value, ordered.
2055 *
2056 * @param pu32 Pointer to the 32-bit variable.
2057 * @param u32 The 32-bit value to assign to *pu32.
2058 */
2059DECLINLINE(void) ASMAtomicWriteU32(volatile uint32_t *pu32, uint32_t u32)
2060{
2061 ASMAtomicXchgU32(pu32, u32);
2062}
2063
2064
2065/**
2066 * Atomically writes an unsigned 32-bit value, unordered.
2067 *
2068 * @param pu32 Pointer to the 32-bit variable.
2069 * @param u32 The 32-bit value to assign to *pu32.
2070 */
2071DECLINLINE(void) ASMAtomicUoWriteU32(volatile uint32_t *pu32, uint32_t u32)
2072{
2073 Assert(!((uintptr_t)pu32 & 3));
2074 *pu32 = u32;
2075}
2076
2077
2078/**
2079 * Atomically writes a signed 32-bit value, ordered.
2080 *
2081 * @param pi32 Pointer to the 32-bit variable to read.
2082 * @param i32 The 32-bit value to assign to *pi32.
2083 */
2084DECLINLINE(void) ASMAtomicWriteS32(volatile int32_t *pi32, int32_t i32)
2085{
2086 ASMAtomicXchgS32(pi32, i32);
2087}
2088
2089
2090/**
2091 * Atomically writes a signed 32-bit value, unordered.
2092 *
2093 * @param pi32 Pointer to the 32-bit variable to read.
2094 * @param i32 The 32-bit value to assign to *pi32.
2095 */
2096DECLINLINE(void) ASMAtomicUoWriteS32(volatile int32_t *pi32, int32_t i32)
2097{
2098 Assert(!((uintptr_t)pi32 & 3));
2099 *pi32 = i32;
2100}
2101
2102
2103/**
2104 * Atomically writes an unsigned 64-bit value, ordered.
2105 *
2106 * @param pu64 Pointer to the 64-bit variable.
2107 * @param u64 The 64-bit value to assign to *pu64.
2108 */
2109DECLINLINE(void) ASMAtomicWriteU64(volatile uint64_t *pu64, uint64_t u64)
2110{
2111 ASMAtomicXchgU64(pu64, u64);
2112}
2113
2114
2115/**
2116 * Atomically writes an unsigned 64-bit value, unordered.
2117 *
2118 * @param pu64 Pointer to the 64-bit variable.
2119 * @param u64 The 64-bit value to assign to *pu64.
2120 */
2121DECLINLINE(void) ASMAtomicUoWriteU64(volatile uint64_t *pu64, uint64_t u64)
2122{
2123 Assert(!((uintptr_t)pu64 & 7));
2124#if ARCH_BITS == 64
2125 *pu64 = u64;
2126#else
2127 ASMAtomicXchgU64(pu64, u64);
2128#endif
2129}
2130
2131
2132/**
2133 * Atomically writes a signed 64-bit value, ordered.
2134 *
2135 * @param pi64 Pointer to the 64-bit variable.
2136 * @param i64 The 64-bit value to assign to *pi64.
2137 */
2138DECLINLINE(void) ASMAtomicWriteS64(volatile int64_t *pi64, int64_t i64)
2139{
2140 ASMAtomicXchgS64(pi64, i64);
2141}
2142
2143
2144/**
2145 * Atomically writes a signed 64-bit value, unordered.
2146 *
2147 * @param pi64 Pointer to the 64-bit variable.
2148 * @param i64 The 64-bit value to assign to *pi64.
2149 */
2150DECLINLINE(void) ASMAtomicUoWriteS64(volatile int64_t *pi64, int64_t i64)
2151{
2152 Assert(!((uintptr_t)pi64 & 7));
2153#if ARCH_BITS == 64
2154 *pi64 = i64;
2155#else
2156 ASMAtomicXchgS64(pi64, i64);
2157#endif
2158}
2159
2160
2161/**
2162 * Atomically writes a boolean value, unordered.
2163 *
2164 * @param pf Pointer to the boolean variable.
2165 * @param f The boolean value to assign to *pf.
2166 */
2167DECLINLINE(void) ASMAtomicWriteBool(volatile bool *pf, bool f)
2168{
2169 ASMAtomicWriteU8((uint8_t volatile *)pf, f);
2170}
2171
2172
2173/**
2174 * Atomically writes a boolean value, unordered.
2175 *
2176 * @param pf Pointer to the boolean variable.
2177 * @param f The boolean value to assign to *pf.
2178 */
2179DECLINLINE(void) ASMAtomicUoWriteBool(volatile bool *pf, bool f)
2180{
2181 *pf = f; /* byte writes are atomic on x86 */
2182}
2183
2184
2185/**
2186 * Atomically writes a pointer value, ordered.
2187 *
2188 * @param ppv Pointer to the pointer variable.
2189 * @param pv The pointer value to assign to *ppv.
2190 */
2191DECLINLINE(void) ASMAtomicWritePtrVoid(void * volatile *ppv, const void *pv)
2192{
2193#if ARCH_BITS == 32
2194 ASMAtomicWriteU32((volatile uint32_t *)(void *)ppv, (uint32_t)pv);
2195#elif ARCH_BITS == 64
2196 ASMAtomicWriteU64((volatile uint64_t *)(void *)ppv, (uint64_t)pv);
2197#else
2198# error "ARCH_BITS is bogus"
2199#endif
2200}
2201
2202
2203/**
2204 * Atomically writes a pointer value, ordered.
2205 *
2206 * @param ppv Pointer to the pointer variable.
2207 * @param pv The pointer value to assign to *ppv. If NULL use
2208 * ASMAtomicWriteNullPtr or you'll land in trouble.
2209 *
2210 * @remarks This is relatively type safe on GCC platforms when @a pv isn't
2211 * NULL.
2212 */
2213#ifdef __GNUC__
2214# define ASMAtomicWritePtr(ppv, pv) \
2215 do \
2216 { \
2217 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
2218 __typeof__(*(ppv)) const pvTypeChecked = (pv); \
2219 \
2220 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2221 AssertCompile(sizeof(pv) == sizeof(void *)); \
2222 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2223 \
2224 ASMAtomicWritePtrVoid((void * volatile *)(ppvTypeChecked), (void *)(pvTypeChecked)); \
2225 } while (0)
2226#else
2227# define ASMAtomicWritePtr(ppv, pv) \
2228 do \
2229 { \
2230 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2231 AssertCompile(sizeof(pv) == sizeof(void *)); \
2232 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2233 \
2234 ASMAtomicWritePtrVoid((void * volatile *)(ppv), (void *)(pv)); \
2235 } while (0)
2236#endif
2237
2238
2239/**
2240 * Atomically sets a pointer to NULL, ordered.
2241 *
2242 * @param ppv Pointer to the pointer variable that should be set to NULL.
2243 *
2244 * @remarks This is relatively type safe on GCC platforms.
2245 */
2246#ifdef __GNUC__
2247# define ASMAtomicWriteNullPtr(ppv) \
2248 do \
2249 { \
2250 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
2251 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2252 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2253 ASMAtomicWritePtrVoid((void * volatile *)(ppvTypeChecked), NULL); \
2254 } while (0)
2255#else
2256# define ASMAtomicWriteNullPtr(ppv) \
2257 do \
2258 { \
2259 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2260 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2261 ASMAtomicWritePtrVoid((void * volatile *)(ppv), NULL); \
2262 } while (0)
2263#endif
2264
2265
2266/**
2267 * Atomically writes a pointer value, unordered.
2268 *
2269 * @returns Current *pv value
2270 * @param ppv Pointer to the pointer variable.
2271 * @param pv The pointer value to assign to *ppv. If NULL use
2272 * ASMAtomicUoWriteNullPtr or you'll land in trouble.
2273 *
2274 * @remarks This is relatively type safe on GCC platforms when @a pv isn't
2275 * NULL.
2276 */
2277#ifdef __GNUC__
2278# define ASMAtomicUoWritePtr(ppv, pv) \
2279 do \
2280 { \
2281 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
2282 __typeof__(*(ppv)) const pvTypeChecked = (pv); \
2283 \
2284 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2285 AssertCompile(sizeof(pv) == sizeof(void *)); \
2286 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2287 \
2288 *(ppvTypeChecked) = pvTypeChecked; \
2289 } while (0)
2290#else
2291# define ASMAtomicUoWritePtr(ppv, pv) \
2292 do \
2293 { \
2294 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2295 AssertCompile(sizeof(pv) == sizeof(void *)); \
2296 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2297 *(ppv) = pv; \
2298 } while (0)
2299#endif
2300
2301
2302/**
2303 * Atomically sets a pointer to NULL, unordered.
2304 *
2305 * @param ppv Pointer to the pointer variable that should be set to NULL.
2306 *
2307 * @remarks This is relatively type safe on GCC platforms.
2308 */
2309#ifdef __GNUC__
2310# define ASMAtomicUoWriteNullPtr(ppv) \
2311 do \
2312 { \
2313 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
2314 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2315 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2316 *(ppvTypeChecked) = NULL; \
2317 } while (0)
2318#else
2319# define ASMAtomicUoWriteNullPtr(ppv) \
2320 do \
2321 { \
2322 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2323 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2324 *(ppv) = NULL; \
2325 } while (0)
2326#endif
2327
2328
2329/**
2330 * Atomically write a typical IPRT handle value, ordered.
2331 *
2332 * @param ph Pointer to the variable to update.
2333 * @param hNew The value to assign to *ph.
2334 *
2335 * @remarks This doesn't currently work for all handles (like RTFILE).
2336 */
2337#if HC_ARCH_BITS == 32
2338# define ASMAtomicWriteHandle(ph, hNew) \
2339 do { \
2340 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
2341 ASMAtomicWriteU32((uint32_t volatile *)(ph), (const uint32_t)(hNew)); \
2342 } while (0)
2343#elif HC_ARCH_BITS == 64
2344# define ASMAtomicWriteHandle(ph, hNew) \
2345 do { \
2346 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
2347 ASMAtomicWriteU64((uint64_t volatile *)(ph), (const uint64_t)(hNew)); \
2348 } while (0)
2349#else
2350# error HC_ARCH_BITS
2351#endif
2352
2353
2354/**
2355 * Atomically write a typical IPRT handle value, unordered.
2356 *
2357 * @param ph Pointer to the variable to update.
2358 * @param hNew The value to assign to *ph.
2359 *
2360 * @remarks This doesn't currently work for all handles (like RTFILE).
2361 */
2362#if HC_ARCH_BITS == 32
2363# define ASMAtomicUoWriteHandle(ph, hNew) \
2364 do { \
2365 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
2366 ASMAtomicUoWriteU32((uint32_t volatile *)(ph), (const uint32_t)hNew); \
2367 } while (0)
2368#elif HC_ARCH_BITS == 64
2369# define ASMAtomicUoWriteHandle(ph, hNew) \
2370 do { \
2371 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
2372 ASMAtomicUoWriteU64((uint64_t volatile *)(ph), (const uint64_t)hNew); \
2373 } while (0)
2374#else
2375# error HC_ARCH_BITS
2376#endif
2377
2378
2379/**
2380 * Atomically write a value which size might differ
2381 * between platforms or compilers, ordered.
2382 *
2383 * @param pu Pointer to the variable to update.
2384 * @param uNew The value to assign to *pu.
2385 */
2386#define ASMAtomicWriteSize(pu, uNew) \
2387 do { \
2388 switch (sizeof(*(pu))) { \
2389 case 1: ASMAtomicWriteU8( (volatile uint8_t *)(void *)(pu), (uint8_t )(uNew)); break; \
2390 case 2: ASMAtomicWriteU16((volatile uint16_t *)(void *)(pu), (uint16_t)(uNew)); break; \
2391 case 4: ASMAtomicWriteU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew)); break; \
2392 case 8: ASMAtomicWriteU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew)); break; \
2393 default: AssertMsgFailed(("ASMAtomicWriteSize: size %d is not supported\n", sizeof(*(pu)))); \
2394 } \
2395 } while (0)
2396
2397/**
2398 * Atomically write a value which size might differ
2399 * between platforms or compilers, unordered.
2400 *
2401 * @param pu Pointer to the variable to update.
2402 * @param uNew The value to assign to *pu.
2403 */
2404#define ASMAtomicUoWriteSize(pu, uNew) \
2405 do { \
2406 switch (sizeof(*(pu))) { \
2407 case 1: ASMAtomicUoWriteU8( (volatile uint8_t *)(void *)(pu), (uint8_t )(uNew)); break; \
2408 case 2: ASMAtomicUoWriteU16((volatile uint16_t *)(void *)(pu), (uint16_t)(uNew)); break; \
2409 case 4: ASMAtomicUoWriteU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew)); break; \
2410 case 8: ASMAtomicUoWriteU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew)); break; \
2411 default: AssertMsgFailed(("ASMAtomicWriteSize: size %d is not supported\n", sizeof(*(pu)))); \
2412 } \
2413 } while (0)
2414
2415
2416
2417/**
2418 * Atomically exchanges and adds to a 32-bit value, ordered.
2419 *
2420 * @returns The old value.
2421 * @param pu32 Pointer to the value.
2422 * @param u32 Number to add.
2423 */
2424#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2425DECLASM(uint32_t) ASMAtomicAddU32(uint32_t volatile *pu32, uint32_t u32);
2426#else
2427DECLINLINE(uint32_t) ASMAtomicAddU32(uint32_t volatile *pu32, uint32_t u32)
2428{
2429# if RT_INLINE_ASM_USES_INTRIN
2430 u32 = _InterlockedExchangeAdd((long *)pu32, u32);
2431 return u32;
2432
2433# elif RT_INLINE_ASM_GNU_STYLE
2434 __asm__ __volatile__("lock; xaddl %0, %1\n\t"
2435 : "=r" (u32),
2436 "=m" (*pu32)
2437 : "0" (u32),
2438 "m" (*pu32)
2439 : "memory");
2440 return u32;
2441# else
2442 __asm
2443 {
2444 mov eax, [u32]
2445# ifdef RT_ARCH_AMD64
2446 mov rdx, [pu32]
2447 lock xadd [rdx], eax
2448# else
2449 mov edx, [pu32]
2450 lock xadd [edx], eax
2451# endif
2452 mov [u32], eax
2453 }
2454 return u32;
2455# endif
2456}
2457#endif
2458
2459
2460/**
2461 * Atomically exchanges and adds to a signed 32-bit value, ordered.
2462 *
2463 * @returns The old value.
2464 * @param pi32 Pointer to the value.
2465 * @param i32 Number to add.
2466 */
2467DECLINLINE(int32_t) ASMAtomicAddS32(int32_t volatile *pi32, int32_t i32)
2468{
2469 return (int32_t)ASMAtomicAddU32((uint32_t volatile *)pi32, (uint32_t)i32);
2470}
2471
2472
2473/**
2474 * Atomically exchanges and adds to a 64-bit value, ordered.
2475 *
2476 * @returns The old value.
2477 * @param pu64 Pointer to the value.
2478 * @param u64 Number to add.
2479 */
2480#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2481DECLASM(uint64_t) ASMAtomicAddU64(uint64_t volatile *pu64, uint64_t u64);
2482#else
2483DECLINLINE(uint64_t) ASMAtomicAddU64(uint64_t volatile *pu64, uint64_t u64)
2484{
2485# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
2486 u64 = _InterlockedExchangeAdd64((__int64 *)pu64, u64);
2487 return u64;
2488
2489# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
2490 __asm__ __volatile__("lock; xaddq %0, %1\n\t"
2491 : "=r" (u64),
2492 "=m" (*pu64)
2493 : "0" (u64),
2494 "m" (*pu64)
2495 : "memory");
2496 return u64;
2497# else
2498 uint64_t u64Old;
2499 for (;;)
2500 {
2501 uint64_t u64New;
2502 u64Old = ASMAtomicUoReadU64(pu64);
2503 u64New = u64Old + u64;
2504 if (ASMAtomicCmpXchgU64(pu64, u64New, u64Old))
2505 break;
2506 ASMNopPause();
2507 }
2508 return u64Old;
2509# endif
2510}
2511#endif
2512
2513
2514/**
2515 * Atomically exchanges and adds to a signed 64-bit value, ordered.
2516 *
2517 * @returns The old value.
2518 * @param pi64 Pointer to the value.
2519 * @param i64 Number to add.
2520 */
2521DECLINLINE(int64_t) ASMAtomicAddS64(int64_t volatile *pi64, int64_t i64)
2522{
2523 return (int64_t)ASMAtomicAddU64((uint64_t volatile *)pi64, (uint64_t)i64);
2524}
2525
2526
2527/**
2528 * Atomically exchanges and subtracts to an unsigned 32-bit value, ordered.
2529 *
2530 * @returns The old value.
2531 * @param pu32 Pointer to the value.
2532 * @param u32 Number to subtract.
2533 */
2534DECLINLINE(uint32_t) ASMAtomicSubU32(uint32_t volatile *pu32, uint32_t u32)
2535{
2536 return ASMAtomicAddU32(pu32, (uint32_t)-(int32_t)u32);
2537}
2538
2539
2540/**
2541 * Atomically exchanges and subtracts to a signed 32-bit value, ordered.
2542 *
2543 * @returns The old value.
2544 * @param pi32 Pointer to the value.
2545 * @param i32 Number to subtract.
2546 */
2547DECLINLINE(int32_t) ASMAtomicSubS32(int32_t volatile *pi32, int32_t i32)
2548{
2549 return (int32_t)ASMAtomicAddU32((uint32_t volatile *)pi32, (uint32_t)-i32);
2550}
2551
2552
2553/**
2554 * Atomically exchanges and subtracts to an unsigned 64-bit value, ordered.
2555 *
2556 * @returns The old value.
2557 * @param pu64 Pointer to the value.
2558 * @param u64 Number to subtract.
2559 */
2560DECLINLINE(uint64_t) ASMAtomicSubU64(uint64_t volatile *pu64, uint64_t u64)
2561{
2562 return ASMAtomicAddU64(pu64, (uint64_t)-(int64_t)u64);
2563}
2564
2565
2566/**
2567 * Atomically exchanges and subtracts to a signed 64-bit value, ordered.
2568 *
2569 * @returns The old value.
2570 * @param pi64 Pointer to the value.
2571 * @param i64 Number to subtract.
2572 */
2573DECLINLINE(int64_t) ASMAtomicSubS64(int64_t volatile *pi64, int64_t i64)
2574{
2575 return (int64_t)ASMAtomicAddU64((uint64_t volatile *)pi64, (uint64_t)-i64);
2576}
2577
2578
2579/**
2580 * Atomically increment a 32-bit value, ordered.
2581 *
2582 * @returns The new value.
2583 * @param pu32 Pointer to the value to increment.
2584 */
2585#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2586DECLASM(uint32_t) ASMAtomicIncU32(uint32_t volatile *pu32);
2587#else
2588DECLINLINE(uint32_t) ASMAtomicIncU32(uint32_t volatile *pu32)
2589{
2590 uint32_t u32;
2591# if RT_INLINE_ASM_USES_INTRIN
2592 u32 = _InterlockedIncrement((long *)pu32);
2593 return u32;
2594
2595# elif RT_INLINE_ASM_GNU_STYLE
2596 __asm__ __volatile__("lock; xaddl %0, %1\n\t"
2597 : "=r" (u32),
2598 "=m" (*pu32)
2599 : "0" (1),
2600 "m" (*pu32)
2601 : "memory");
2602 return u32+1;
2603# else
2604 __asm
2605 {
2606 mov eax, 1
2607# ifdef RT_ARCH_AMD64
2608 mov rdx, [pu32]
2609 lock xadd [rdx], eax
2610# else
2611 mov edx, [pu32]
2612 lock xadd [edx], eax
2613# endif
2614 mov u32, eax
2615 }
2616 return u32+1;
2617# endif
2618}
2619#endif
2620
2621
2622/**
2623 * Atomically increment a signed 32-bit value, ordered.
2624 *
2625 * @returns The new value.
2626 * @param pi32 Pointer to the value to increment.
2627 */
2628DECLINLINE(int32_t) ASMAtomicIncS32(int32_t volatile *pi32)
2629{
2630 return (int32_t)ASMAtomicIncU32((uint32_t volatile *)pi32);
2631}
2632
2633
2634/**
2635 * Atomically increment a 64-bit value, ordered.
2636 *
2637 * @returns The new value.
2638 * @param pu64 Pointer to the value to increment.
2639 */
2640#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2641DECLASM(uint64_t) ASMAtomicIncU64(uint64_t volatile *pu64);
2642#else
2643DECLINLINE(uint64_t) ASMAtomicIncU64(uint64_t volatile *pu64)
2644{
2645# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
2646 uint64_t u64;
2647 u64 = _InterlockedIncrement64((__int64 *)pu64);
2648 return u64;
2649
2650# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
2651 uint64_t u64;
2652 __asm__ __volatile__("lock; xaddq %0, %1\n\t"
2653 : "=r" (u64),
2654 "=m" (*pu64)
2655 : "0" (1),
2656 "m" (*pu64)
2657 : "memory");
2658 return u64 + 1;
2659# else
2660 return ASMAtomicAddU64(pu64, 1) + 1;
2661# endif
2662}
2663#endif
2664
2665
2666/**
2667 * Atomically increment a signed 64-bit value, ordered.
2668 *
2669 * @returns The new value.
2670 * @param pi64 Pointer to the value to increment.
2671 */
2672DECLINLINE(int64_t) ASMAtomicIncS64(int64_t volatile *pi64)
2673{
2674 return (int64_t)ASMAtomicIncU64((uint64_t volatile *)pi64);
2675}
2676
2677
2678/**
2679 * Atomically decrement an unsigned 32-bit value, ordered.
2680 *
2681 * @returns The new value.
2682 * @param pu32 Pointer to the value to decrement.
2683 */
2684#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2685DECLASM(uint32_t) ASMAtomicDecU32(uint32_t volatile *pu32);
2686#else
2687DECLINLINE(uint32_t) ASMAtomicDecU32(uint32_t volatile *pu32)
2688{
2689 uint32_t u32;
2690# if RT_INLINE_ASM_USES_INTRIN
2691 u32 = _InterlockedDecrement((long *)pu32);
2692 return u32;
2693
2694# elif RT_INLINE_ASM_GNU_STYLE
2695 __asm__ __volatile__("lock; xaddl %0, %1\n\t"
2696 : "=r" (u32),
2697 "=m" (*pu32)
2698 : "0" (-1),
2699 "m" (*pu32)
2700 : "memory");
2701 return u32-1;
2702# else
2703 __asm
2704 {
2705 mov eax, -1
2706# ifdef RT_ARCH_AMD64
2707 mov rdx, [pu32]
2708 lock xadd [rdx], eax
2709# else
2710 mov edx, [pu32]
2711 lock xadd [edx], eax
2712# endif
2713 mov u32, eax
2714 }
2715 return u32-1;
2716# endif
2717}
2718#endif
2719
2720
2721/**
2722 * Atomically decrement a signed 32-bit value, ordered.
2723 *
2724 * @returns The new value.
2725 * @param pi32 Pointer to the value to decrement.
2726 */
2727DECLINLINE(int32_t) ASMAtomicDecS32(int32_t volatile *pi32)
2728{
2729 return (int32_t)ASMAtomicDecU32((uint32_t volatile *)pi32);
2730}
2731
2732
2733/**
2734 * Atomically decrement an unsigned 64-bit value, ordered.
2735 *
2736 * @returns The new value.
2737 * @param pu64 Pointer to the value to decrement.
2738 */
2739#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2740DECLASM(uint64_t) ASMAtomicDecU64(uint64_t volatile *pu64);
2741#else
2742DECLINLINE(uint64_t) ASMAtomicDecU64(uint64_t volatile *pu64)
2743{
2744# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
2745 uint64_t u64 = _InterlockedDecrement64((__int64 volatile *)pu64);
2746 return u64;
2747
2748# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
2749 uint64_t u64;
2750 __asm__ __volatile__("lock; xaddq %q0, %1\n\t"
2751 : "=r" (u64),
2752 "=m" (*pu64)
2753 : "0" (~(uint64_t)0),
2754 "m" (*pu64)
2755 : "memory");
2756 return u64-1;
2757# else
2758 return ASMAtomicAddU64(pu64, UINT64_MAX) - 1;
2759# endif
2760}
2761#endif
2762
2763
2764/**
2765 * Atomically decrement a signed 64-bit value, ordered.
2766 *
2767 * @returns The new value.
2768 * @param pi64 Pointer to the value to decrement.
2769 */
2770DECLINLINE(int64_t) ASMAtomicDecS64(int64_t volatile *pi64)
2771{
2772 return (int64_t)ASMAtomicDecU64((uint64_t volatile *)pi64);
2773}
2774
2775
2776/**
2777 * Atomically Or an unsigned 32-bit value, ordered.
2778 *
2779 * @param pu32 Pointer to the pointer variable to OR u32 with.
2780 * @param u32 The value to OR *pu32 with.
2781 */
2782#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2783DECLASM(void) ASMAtomicOrU32(uint32_t volatile *pu32, uint32_t u32);
2784#else
2785DECLINLINE(void) ASMAtomicOrU32(uint32_t volatile *pu32, uint32_t u32)
2786{
2787# if RT_INLINE_ASM_USES_INTRIN
2788 _InterlockedOr((long volatile *)pu32, (long)u32);
2789
2790# elif RT_INLINE_ASM_GNU_STYLE
2791 __asm__ __volatile__("lock; orl %1, %0\n\t"
2792 : "=m" (*pu32)
2793 : "ir" (u32),
2794 "m" (*pu32));
2795# else
2796 __asm
2797 {
2798 mov eax, [u32]
2799# ifdef RT_ARCH_AMD64
2800 mov rdx, [pu32]
2801 lock or [rdx], eax
2802# else
2803 mov edx, [pu32]
2804 lock or [edx], eax
2805# endif
2806 }
2807# endif
2808}
2809#endif
2810
2811
2812/**
2813 * Atomically Or a signed 32-bit value, ordered.
2814 *
2815 * @param pi32 Pointer to the pointer variable to OR u32 with.
2816 * @param i32 The value to OR *pu32 with.
2817 */
2818DECLINLINE(void) ASMAtomicOrS32(int32_t volatile *pi32, int32_t i32)
2819{
2820 ASMAtomicOrU32((uint32_t volatile *)pi32, i32);
2821}
2822
2823
2824/**
2825 * Atomically Or an unsigned 64-bit value, ordered.
2826 *
2827 * @param pu64 Pointer to the pointer variable to OR u64 with.
2828 * @param u64 The value to OR *pu64 with.
2829 */
2830#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2831DECLASM(void) ASMAtomicOrU64(uint64_t volatile *pu64, uint64_t u64);
2832#else
2833DECLINLINE(void) ASMAtomicOrU64(uint64_t volatile *pu64, uint64_t u64)
2834{
2835# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
2836 _InterlockedOr64((__int64 volatile *)pu64, (__int64)u64);
2837
2838# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
2839 __asm__ __volatile__("lock; orq %1, %q0\n\t"
2840 : "=m" (*pu64)
2841 : "r" (u64),
2842 "m" (*pu64));
2843# else
2844 for (;;)
2845 {
2846 uint64_t u64Old = ASMAtomicUoReadU64(pu64);
2847 uint64_t u64New = u64Old | u64;
2848 if (ASMAtomicCmpXchgU64(pu64, u64New, u64Old))
2849 break;
2850 ASMNopPause();
2851 }
2852# endif
2853}
2854#endif
2855
2856
2857/**
2858 * Atomically Or a signed 64-bit value, ordered.
2859 *
2860 * @param pi64 Pointer to the pointer variable to OR u64 with.
2861 * @param i64 The value to OR *pu64 with.
2862 */
2863DECLINLINE(void) ASMAtomicOrS64(int64_t volatile *pi64, int64_t i64)
2864{
2865 ASMAtomicOrU64((uint64_t volatile *)pi64, i64);
2866}
2867/**
2868 * Atomically And an unsigned 32-bit value, ordered.
2869 *
2870 * @param pu32 Pointer to the pointer variable to AND u32 with.
2871 * @param u32 The value to AND *pu32 with.
2872 */
2873#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2874DECLASM(void) ASMAtomicAndU32(uint32_t volatile *pu32, uint32_t u32);
2875#else
2876DECLINLINE(void) ASMAtomicAndU32(uint32_t volatile *pu32, uint32_t u32)
2877{
2878# if RT_INLINE_ASM_USES_INTRIN
2879 _InterlockedAnd((long volatile *)pu32, u32);
2880
2881# elif RT_INLINE_ASM_GNU_STYLE
2882 __asm__ __volatile__("lock; andl %1, %0\n\t"
2883 : "=m" (*pu32)
2884 : "ir" (u32),
2885 "m" (*pu32));
2886# else
2887 __asm
2888 {
2889 mov eax, [u32]
2890# ifdef RT_ARCH_AMD64
2891 mov rdx, [pu32]
2892 lock and [rdx], eax
2893# else
2894 mov edx, [pu32]
2895 lock and [edx], eax
2896# endif
2897 }
2898# endif
2899}
2900#endif
2901
2902
2903/**
2904 * Atomically And a signed 32-bit value, ordered.
2905 *
2906 * @param pi32 Pointer to the pointer variable to AND i32 with.
2907 * @param i32 The value to AND *pi32 with.
2908 */
2909DECLINLINE(void) ASMAtomicAndS32(int32_t volatile *pi32, int32_t i32)
2910{
2911 ASMAtomicAndU32((uint32_t volatile *)pi32, (uint32_t)i32);
2912}
2913
2914
2915/**
2916 * Atomically And an unsigned 64-bit value, ordered.
2917 *
2918 * @param pu64 Pointer to the pointer variable to AND u64 with.
2919 * @param u64 The value to AND *pu64 with.
2920 */
2921#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2922DECLASM(void) ASMAtomicAndU64(uint64_t volatile *pu64, uint64_t u64);
2923#else
2924DECLINLINE(void) ASMAtomicAndU64(uint64_t volatile *pu64, uint64_t u64)
2925{
2926# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
2927 _InterlockedAnd64((__int64 volatile *)pu64, u64);
2928
2929# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
2930 __asm__ __volatile__("lock; andq %1, %0\n\t"
2931 : "=m" (*pu64)
2932 : "r" (u64),
2933 "m" (*pu64));
2934# else
2935 for (;;)
2936 {
2937 uint64_t u64Old = ASMAtomicUoReadU64(pu64);
2938 uint64_t u64New = u64Old & u64;
2939 if (ASMAtomicCmpXchgU64(pu64, u64New, u64Old))
2940 break;
2941 ASMNopPause();
2942 }
2943# endif
2944}
2945#endif
2946
2947
2948/**
2949 * Atomically And a signed 64-bit value, ordered.
2950 *
2951 * @param pi64 Pointer to the pointer variable to AND i64 with.
2952 * @param i64 The value to AND *pi64 with.
2953 */
2954DECLINLINE(void) ASMAtomicAndS64(int64_t volatile *pi64, int64_t i64)
2955{
2956 ASMAtomicAndU64((uint64_t volatile *)pi64, (uint64_t)i64);
2957}
2958
2959
2960
2961/** @def RT_ASM_PAGE_SIZE
2962 * We try avoid dragging in iprt/param.h here.
2963 * @internal
2964 */
2965#if defined(RT_ARCH_SPARC64)
2966# define RT_ASM_PAGE_SIZE 0x2000
2967# if defined(PAGE_SIZE) && !defined(NT_INCLUDED)
2968# if PAGE_SIZE != 0x2000
2969# error "PAGE_SIZE is not 0x2000!"
2970# endif
2971# endif
2972#else
2973# define RT_ASM_PAGE_SIZE 0x1000
2974# if defined(PAGE_SIZE) && !defined(NT_INCLUDED)
2975# if PAGE_SIZE != 0x1000
2976# error "PAGE_SIZE is not 0x1000!"
2977# endif
2978# endif
2979#endif
2980
2981/**
2982 * Zeros a 4K memory page.
2983 *
2984 * @param pv Pointer to the memory block. This must be page aligned.
2985 */
2986#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2987DECLASM(void) ASMMemZeroPage(volatile void *pv);
2988# else
2989DECLINLINE(void) ASMMemZeroPage(volatile void *pv)
2990{
2991# if RT_INLINE_ASM_USES_INTRIN
2992# ifdef RT_ARCH_AMD64
2993 __stosq((unsigned __int64 *)pv, 0, RT_ASM_PAGE_SIZE / 8);
2994# else
2995 __stosd((unsigned long *)pv, 0, RT_ASM_PAGE_SIZE / 4);
2996# endif
2997
2998# elif RT_INLINE_ASM_GNU_STYLE
2999 RTCCUINTREG uDummy;
3000# ifdef RT_ARCH_AMD64
3001 __asm__ __volatile__("rep stosq"
3002 : "=D" (pv),
3003 "=c" (uDummy)
3004 : "0" (pv),
3005 "c" (RT_ASM_PAGE_SIZE >> 3),
3006 "a" (0)
3007 : "memory");
3008# else
3009 __asm__ __volatile__("rep stosl"
3010 : "=D" (pv),
3011 "=c" (uDummy)
3012 : "0" (pv),
3013 "c" (RT_ASM_PAGE_SIZE >> 2),
3014 "a" (0)
3015 : "memory");
3016# endif
3017# else
3018 __asm
3019 {
3020# ifdef RT_ARCH_AMD64
3021 xor rax, rax
3022 mov ecx, 0200h
3023 mov rdi, [pv]
3024 rep stosq
3025# else
3026 xor eax, eax
3027 mov ecx, 0400h
3028 mov edi, [pv]
3029 rep stosd
3030# endif
3031 }
3032# endif
3033}
3034# endif
3035
3036
3037/**
3038 * Zeros a memory block with a 32-bit aligned size.
3039 *
3040 * @param pv Pointer to the memory block.
3041 * @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
3042 */
3043#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3044DECLASM(void) ASMMemZero32(volatile void *pv, size_t cb);
3045#else
3046DECLINLINE(void) ASMMemZero32(volatile void *pv, size_t cb)
3047{
3048# if RT_INLINE_ASM_USES_INTRIN
3049# ifdef RT_ARCH_AMD64
3050 if (!(cb & 7))
3051 __stosq((unsigned __int64 *)pv, 0, cb / 8);
3052 else
3053# endif
3054 __stosd((unsigned long *)pv, 0, cb / 4);
3055
3056# elif RT_INLINE_ASM_GNU_STYLE
3057 __asm__ __volatile__("rep stosl"
3058 : "=D" (pv),
3059 "=c" (cb)
3060 : "0" (pv),
3061 "1" (cb >> 2),
3062 "a" (0)
3063 : "memory");
3064# else
3065 __asm
3066 {
3067 xor eax, eax
3068# ifdef RT_ARCH_AMD64
3069 mov rcx, [cb]
3070 shr rcx, 2
3071 mov rdi, [pv]
3072# else
3073 mov ecx, [cb]
3074 shr ecx, 2
3075 mov edi, [pv]
3076# endif
3077 rep stosd
3078 }
3079# endif
3080}
3081#endif
3082
3083
3084/**
3085 * Fills a memory block with a 32-bit aligned size.
3086 *
3087 * @param pv Pointer to the memory block.
3088 * @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
3089 * @param u32 The value to fill with.
3090 */
3091#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3092DECLASM(void) ASMMemFill32(volatile void *pv, size_t cb, uint32_t u32);
3093#else
3094DECLINLINE(void) ASMMemFill32(volatile void *pv, size_t cb, uint32_t u32)
3095{
3096# if RT_INLINE_ASM_USES_INTRIN
3097# ifdef RT_ARCH_AMD64
3098 if (!(cb & 7))
3099 __stosq((unsigned __int64 *)pv, RT_MAKE_U64(u32, u32), cb / 8);
3100 else
3101# endif
3102 __stosd((unsigned long *)pv, u32, cb / 4);
3103
3104# elif RT_INLINE_ASM_GNU_STYLE
3105 __asm__ __volatile__("rep stosl"
3106 : "=D" (pv),
3107 "=c" (cb)
3108 : "0" (pv),
3109 "1" (cb >> 2),
3110 "a" (u32)
3111 : "memory");
3112# else
3113 __asm
3114 {
3115# ifdef RT_ARCH_AMD64
3116 mov rcx, [cb]
3117 shr rcx, 2
3118 mov rdi, [pv]
3119# else
3120 mov ecx, [cb]
3121 shr ecx, 2
3122 mov edi, [pv]
3123# endif
3124 mov eax, [u32]
3125 rep stosd
3126 }
3127# endif
3128}
3129#endif
3130
3131
3132/**
3133 * Checks if a memory page is all zeros.
3134 *
3135 * @returns true / false.
3136 *
3137 * @param pvPage Pointer to the page. Must be aligned on 16 byte
3138 * boundrary
3139 */
3140DECLINLINE(bool) ASMMemIsZeroPage(void const *pvPage)
3141{
3142# if 0 /*RT_INLINE_ASM_GNU_STYLE - this is actually slower... */
3143 union { RTCCUINTREG r; bool f; } uAX;
3144 RTCCUINTREG xCX, xDI;
3145 Assert(!((uintptr_t)pvPage & 15));
3146 __asm__ __volatile__("repe; "
3147# ifdef RT_ARCH_AMD64
3148 "scasq\n\t"
3149# else
3150 "scasl\n\t"
3151# endif
3152 "setnc %%al\n\t"
3153 : "=&c" (xCX),
3154 "=&D" (xDI),
3155 "=&a" (uAX.r)
3156 : "mr" (pvPage),
3157# ifdef RT_ARCH_AMD64
3158 "0" (RT_ASM_PAGE_SIZE/8),
3159# else
3160 "0" (RT_ASM_PAGE_SIZE/4),
3161# endif
3162 "1" (pvPage),
3163 "2" (0));
3164 return uAX.f;
3165# else
3166 uintptr_t const *puPtr = (uintptr_t const *)pvPage;
3167 int cLeft = RT_ASM_PAGE_SIZE / sizeof(uintptr_t) / 8;
3168 Assert(!((uintptr_t)pvPage & 15));
3169 for (;;)
3170 {
3171 if (puPtr[0]) return false;
3172 if (puPtr[4]) return false;
3173
3174 if (puPtr[2]) return false;
3175 if (puPtr[6]) return false;
3176
3177 if (puPtr[1]) return false;
3178 if (puPtr[5]) return false;
3179
3180 if (puPtr[3]) return false;
3181 if (puPtr[7]) return false;
3182
3183 if (!--cLeft)
3184 return true;
3185 puPtr += 8;
3186 }
3187 return true;
3188# endif
3189}
3190
3191
3192/**
3193 * Checks if a memory block is filled with the specified byte.
3194 *
3195 * This is a sort of inverted memchr.
3196 *
3197 * @returns Pointer to the byte which doesn't equal u8.
3198 * @returns NULL if all equal to u8.
3199 *
3200 * @param pv Pointer to the memory block.
3201 * @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
3202 * @param u8 The value it's supposed to be filled with.
3203 *
3204 * @todo Fix name, it is a predicate function but it's not returning boolean!
3205 */
3206DECLINLINE(void *) ASMMemIsAll8(void const *pv, size_t cb, uint8_t u8)
3207{
3208/** @todo rewrite this in inline assembly? */
3209 uint8_t const *pb = (uint8_t const *)pv;
3210 for (; cb; cb--, pb++)
3211 if (RT_UNLIKELY(*pb != u8))
3212 return (void *)pb;
3213 return NULL;
3214}
3215
3216
3217/**
3218 * Checks if a memory block is filled with the specified 32-bit value.
3219 *
3220 * This is a sort of inverted memchr.
3221 *
3222 * @returns Pointer to the first value which doesn't equal u32.
3223 * @returns NULL if all equal to u32.
3224 *
3225 * @param pv Pointer to the memory block.
3226 * @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
3227 * @param u32 The value it's supposed to be filled with.
3228 *
3229 * @todo Fix name, it is a predicate function but it's not returning boolean!
3230 */
3231DECLINLINE(uint32_t *) ASMMemIsAllU32(void const *pv, size_t cb, uint32_t u32)
3232{
3233/** @todo rewrite this in inline assembly? */
3234 uint32_t const *pu32 = (uint32_t const *)pv;
3235 for (; cb; cb -= 4, pu32++)
3236 if (RT_UNLIKELY(*pu32 != u32))
3237 return (uint32_t *)pu32;
3238 return NULL;
3239}
3240
3241
3242/**
3243 * Probes a byte pointer for read access.
3244 *
3245 * While the function will not fault if the byte is not read accessible,
3246 * the idea is to do this in a safe place like before acquiring locks
3247 * and such like.
3248 *
3249 * Also, this functions guarantees that an eager compiler is not going
3250 * to optimize the probing away.
3251 *
3252 * @param pvByte Pointer to the byte.
3253 */
3254#if RT_INLINE_ASM_EXTERNAL
3255DECLASM(uint8_t) ASMProbeReadByte(const void *pvByte);
3256#else
3257DECLINLINE(uint8_t) ASMProbeReadByte(const void *pvByte)
3258{
3259 /** @todo verify that the compiler actually doesn't optimize this away. (intel & gcc) */
3260 uint8_t u8;
3261# if RT_INLINE_ASM_GNU_STYLE
3262 __asm__ __volatile__("movb (%1), %0\n\t"
3263 : "=r" (u8)
3264 : "r" (pvByte));
3265# else
3266 __asm
3267 {
3268# ifdef RT_ARCH_AMD64
3269 mov rax, [pvByte]
3270 mov al, [rax]
3271# else
3272 mov eax, [pvByte]
3273 mov al, [eax]
3274# endif
3275 mov [u8], al
3276 }
3277# endif
3278 return u8;
3279}
3280#endif
3281
3282/**
3283 * Probes a buffer for read access page by page.
3284 *
3285 * While the function will fault if the buffer is not fully read
3286 * accessible, the idea is to do this in a safe place like before
3287 * acquiring locks and such like.
3288 *
3289 * Also, this functions guarantees that an eager compiler is not going
3290 * to optimize the probing away.
3291 *
3292 * @param pvBuf Pointer to the buffer.
3293 * @param cbBuf The size of the buffer in bytes. Must be >= 1.
3294 */
3295DECLINLINE(void) ASMProbeReadBuffer(const void *pvBuf, size_t cbBuf)
3296{
3297 /** @todo verify that the compiler actually doesn't optimize this away. (intel & gcc) */
3298 /* the first byte */
3299 const uint8_t *pu8 = (const uint8_t *)pvBuf;
3300 ASMProbeReadByte(pu8);
3301
3302 /* the pages in between pages. */
3303 while (cbBuf > RT_ASM_PAGE_SIZE)
3304 {
3305 ASMProbeReadByte(pu8);
3306 cbBuf -= RT_ASM_PAGE_SIZE;
3307 pu8 += RT_ASM_PAGE_SIZE;
3308 }
3309
3310 /* the last byte */
3311 ASMProbeReadByte(pu8 + cbBuf - 1);
3312}
3313
3314
3315
3316/** @defgroup grp_inline_bits Bit Operations
3317 * @{
3318 */
3319
3320
3321/**
3322 * Sets a bit in a bitmap.
3323 *
3324 * @param pvBitmap Pointer to the bitmap. This should be 32-bit aligned.
3325 * @param iBit The bit to set.
3326 *
3327 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3328 * However, doing so will yield better performance as well as avoiding
3329 * traps accessing the last bits in the bitmap.
3330 */
3331#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3332DECLASM(void) ASMBitSet(volatile void *pvBitmap, int32_t iBit);
3333#else
3334DECLINLINE(void) ASMBitSet(volatile void *pvBitmap, int32_t iBit)
3335{
3336# if RT_INLINE_ASM_USES_INTRIN
3337 _bittestandset((long *)pvBitmap, iBit);
3338
3339# elif RT_INLINE_ASM_GNU_STYLE
3340 __asm__ __volatile__("btsl %1, %0"
3341 : "=m" (*(volatile long *)pvBitmap)
3342 : "Ir" (iBit),
3343 "m" (*(volatile long *)pvBitmap)
3344 : "memory");
3345# else
3346 __asm
3347 {
3348# ifdef RT_ARCH_AMD64
3349 mov rax, [pvBitmap]
3350 mov edx, [iBit]
3351 bts [rax], edx
3352# else
3353 mov eax, [pvBitmap]
3354 mov edx, [iBit]
3355 bts [eax], edx
3356# endif
3357 }
3358# endif
3359}
3360#endif
3361
3362
3363/**
3364 * Atomically sets a bit in a bitmap, ordered.
3365 *
3366 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3367 * the memory access isn't atomic!
3368 * @param iBit The bit to set.
3369 */
3370#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3371DECLASM(void) ASMAtomicBitSet(volatile void *pvBitmap, int32_t iBit);
3372#else
3373DECLINLINE(void) ASMAtomicBitSet(volatile void *pvBitmap, int32_t iBit)
3374{
3375 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3376# if RT_INLINE_ASM_USES_INTRIN
3377 _interlockedbittestandset((long *)pvBitmap, iBit);
3378# elif RT_INLINE_ASM_GNU_STYLE
3379 __asm__ __volatile__("lock; btsl %1, %0"
3380 : "=m" (*(volatile long *)pvBitmap)
3381 : "Ir" (iBit),
3382 "m" (*(volatile long *)pvBitmap)
3383 : "memory");
3384# else
3385 __asm
3386 {
3387# ifdef RT_ARCH_AMD64
3388 mov rax, [pvBitmap]
3389 mov edx, [iBit]
3390 lock bts [rax], edx
3391# else
3392 mov eax, [pvBitmap]
3393 mov edx, [iBit]
3394 lock bts [eax], edx
3395# endif
3396 }
3397# endif
3398}
3399#endif
3400
3401
3402/**
3403 * Clears a bit in a bitmap.
3404 *
3405 * @param pvBitmap Pointer to the bitmap.
3406 * @param iBit The bit to clear.
3407 *
3408 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3409 * However, doing so will yield better performance as well as avoiding
3410 * traps accessing the last bits in the bitmap.
3411 */
3412#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3413DECLASM(void) ASMBitClear(volatile void *pvBitmap, int32_t iBit);
3414#else
3415DECLINLINE(void) ASMBitClear(volatile void *pvBitmap, int32_t iBit)
3416{
3417# if RT_INLINE_ASM_USES_INTRIN
3418 _bittestandreset((long *)pvBitmap, iBit);
3419
3420# elif RT_INLINE_ASM_GNU_STYLE
3421 __asm__ __volatile__("btrl %1, %0"
3422 : "=m" (*(volatile long *)pvBitmap)
3423 : "Ir" (iBit),
3424 "m" (*(volatile long *)pvBitmap)
3425 : "memory");
3426# else
3427 __asm
3428 {
3429# ifdef RT_ARCH_AMD64
3430 mov rax, [pvBitmap]
3431 mov edx, [iBit]
3432 btr [rax], edx
3433# else
3434 mov eax, [pvBitmap]
3435 mov edx, [iBit]
3436 btr [eax], edx
3437# endif
3438 }
3439# endif
3440}
3441#endif
3442
3443
3444/**
3445 * Atomically clears a bit in a bitmap, ordered.
3446 *
3447 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3448 * the memory access isn't atomic!
3449 * @param iBit The bit to toggle set.
3450 * @remarks No memory barrier, take care on smp.
3451 */
3452#if RT_INLINE_ASM_EXTERNAL
3453DECLASM(void) ASMAtomicBitClear(volatile void *pvBitmap, int32_t iBit);
3454#else
3455DECLINLINE(void) ASMAtomicBitClear(volatile void *pvBitmap, int32_t iBit)
3456{
3457 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3458# if RT_INLINE_ASM_GNU_STYLE
3459 __asm__ __volatile__("lock; btrl %1, %0"
3460 : "=m" (*(volatile long *)pvBitmap)
3461 : "Ir" (iBit),
3462 "m" (*(volatile long *)pvBitmap)
3463 : "memory");
3464# else
3465 __asm
3466 {
3467# ifdef RT_ARCH_AMD64
3468 mov rax, [pvBitmap]
3469 mov edx, [iBit]
3470 lock btr [rax], edx
3471# else
3472 mov eax, [pvBitmap]
3473 mov edx, [iBit]
3474 lock btr [eax], edx
3475# endif
3476 }
3477# endif
3478}
3479#endif
3480
3481
3482/**
3483 * Toggles a bit in a bitmap.
3484 *
3485 * @param pvBitmap Pointer to the bitmap.
3486 * @param iBit The bit to toggle.
3487 *
3488 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3489 * However, doing so will yield better performance as well as avoiding
3490 * traps accessing the last bits in the bitmap.
3491 */
3492#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3493DECLASM(void) ASMBitToggle(volatile void *pvBitmap, int32_t iBit);
3494#else
3495DECLINLINE(void) ASMBitToggle(volatile void *pvBitmap, int32_t iBit)
3496{
3497# if RT_INLINE_ASM_USES_INTRIN
3498 _bittestandcomplement((long *)pvBitmap, iBit);
3499# elif RT_INLINE_ASM_GNU_STYLE
3500 __asm__ __volatile__("btcl %1, %0"
3501 : "=m" (*(volatile long *)pvBitmap)
3502 : "Ir" (iBit),
3503 "m" (*(volatile long *)pvBitmap)
3504 : "memory");
3505# else
3506 __asm
3507 {
3508# ifdef RT_ARCH_AMD64
3509 mov rax, [pvBitmap]
3510 mov edx, [iBit]
3511 btc [rax], edx
3512# else
3513 mov eax, [pvBitmap]
3514 mov edx, [iBit]
3515 btc [eax], edx
3516# endif
3517 }
3518# endif
3519}
3520#endif
3521
3522
3523/**
3524 * Atomically toggles a bit in a bitmap, ordered.
3525 *
3526 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3527 * the memory access isn't atomic!
3528 * @param iBit The bit to test and set.
3529 */
3530#if RT_INLINE_ASM_EXTERNAL
3531DECLASM(void) ASMAtomicBitToggle(volatile void *pvBitmap, int32_t iBit);
3532#else
3533DECLINLINE(void) ASMAtomicBitToggle(volatile void *pvBitmap, int32_t iBit)
3534{
3535 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3536# if RT_INLINE_ASM_GNU_STYLE
3537 __asm__ __volatile__("lock; btcl %1, %0"
3538 : "=m" (*(volatile long *)pvBitmap)
3539 : "Ir" (iBit),
3540 "m" (*(volatile long *)pvBitmap)
3541 : "memory");
3542# else
3543 __asm
3544 {
3545# ifdef RT_ARCH_AMD64
3546 mov rax, [pvBitmap]
3547 mov edx, [iBit]
3548 lock btc [rax], edx
3549# else
3550 mov eax, [pvBitmap]
3551 mov edx, [iBit]
3552 lock btc [eax], edx
3553# endif
3554 }
3555# endif
3556}
3557#endif
3558
3559
3560/**
3561 * Tests and sets a bit in a bitmap.
3562 *
3563 * @returns true if the bit was set.
3564 * @returns false if the bit was clear.
3565 *
3566 * @param pvBitmap Pointer to the bitmap.
3567 * @param iBit The bit to test and set.
3568 *
3569 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3570 * However, doing so will yield better performance as well as avoiding
3571 * traps accessing the last bits in the bitmap.
3572 */
3573#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3574DECLASM(bool) ASMBitTestAndSet(volatile void *pvBitmap, int32_t iBit);
3575#else
3576DECLINLINE(bool) ASMBitTestAndSet(volatile void *pvBitmap, int32_t iBit)
3577{
3578 union { bool f; uint32_t u32; uint8_t u8; } rc;
3579# if RT_INLINE_ASM_USES_INTRIN
3580 rc.u8 = _bittestandset((long *)pvBitmap, iBit);
3581
3582# elif RT_INLINE_ASM_GNU_STYLE
3583 __asm__ __volatile__("btsl %2, %1\n\t"
3584 "setc %b0\n\t"
3585 "andl $1, %0\n\t"
3586 : "=q" (rc.u32),
3587 "=m" (*(volatile long *)pvBitmap)
3588 : "Ir" (iBit),
3589 "m" (*(volatile long *)pvBitmap)
3590 : "memory");
3591# else
3592 __asm
3593 {
3594 mov edx, [iBit]
3595# ifdef RT_ARCH_AMD64
3596 mov rax, [pvBitmap]
3597 bts [rax], edx
3598# else
3599 mov eax, [pvBitmap]
3600 bts [eax], edx
3601# endif
3602 setc al
3603 and eax, 1
3604 mov [rc.u32], eax
3605 }
3606# endif
3607 return rc.f;
3608}
3609#endif
3610
3611
3612/**
3613 * Atomically tests and sets a bit in a bitmap, ordered.
3614 *
3615 * @returns true if the bit was set.
3616 * @returns false if the bit was clear.
3617 *
3618 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3619 * the memory access isn't atomic!
3620 * @param iBit The bit to set.
3621 */
3622#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3623DECLASM(bool) ASMAtomicBitTestAndSet(volatile void *pvBitmap, int32_t iBit);
3624#else
3625DECLINLINE(bool) ASMAtomicBitTestAndSet(volatile void *pvBitmap, int32_t iBit)
3626{
3627 union { bool f; uint32_t u32; uint8_t u8; } rc;
3628 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3629# if RT_INLINE_ASM_USES_INTRIN
3630 rc.u8 = _interlockedbittestandset((long *)pvBitmap, iBit);
3631# elif RT_INLINE_ASM_GNU_STYLE
3632 __asm__ __volatile__("lock; btsl %2, %1\n\t"
3633 "setc %b0\n\t"
3634 "andl $1, %0\n\t"
3635 : "=q" (rc.u32),
3636 "=m" (*(volatile long *)pvBitmap)
3637 : "Ir" (iBit),
3638 "m" (*(volatile long *)pvBitmap)
3639 : "memory");
3640# else
3641 __asm
3642 {
3643 mov edx, [iBit]
3644# ifdef RT_ARCH_AMD64
3645 mov rax, [pvBitmap]
3646 lock bts [rax], edx
3647# else
3648 mov eax, [pvBitmap]
3649 lock bts [eax], edx
3650# endif
3651 setc al
3652 and eax, 1
3653 mov [rc.u32], eax
3654 }
3655# endif
3656 return rc.f;
3657}
3658#endif
3659
3660
3661/**
3662 * Tests and clears a bit in a bitmap.
3663 *
3664 * @returns true if the bit was set.
3665 * @returns false if the bit was clear.
3666 *
3667 * @param pvBitmap Pointer to the bitmap.
3668 * @param iBit The bit to test and clear.
3669 *
3670 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3671 * However, doing so will yield better performance as well as avoiding
3672 * traps accessing the last bits in the bitmap.
3673 */
3674#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3675DECLASM(bool) ASMBitTestAndClear(volatile void *pvBitmap, int32_t iBit);
3676#else
3677DECLINLINE(bool) ASMBitTestAndClear(volatile void *pvBitmap, int32_t iBit)
3678{
3679 union { bool f; uint32_t u32; uint8_t u8; } rc;
3680# if RT_INLINE_ASM_USES_INTRIN
3681 rc.u8 = _bittestandreset((long *)pvBitmap, iBit);
3682
3683# elif RT_INLINE_ASM_GNU_STYLE
3684 __asm__ __volatile__("btrl %2, %1\n\t"
3685 "setc %b0\n\t"
3686 "andl $1, %0\n\t"
3687 : "=q" (rc.u32),
3688 "=m" (*(volatile long *)pvBitmap)
3689 : "Ir" (iBit),
3690 "m" (*(volatile long *)pvBitmap)
3691 : "memory");
3692# else
3693 __asm
3694 {
3695 mov edx, [iBit]
3696# ifdef RT_ARCH_AMD64
3697 mov rax, [pvBitmap]
3698 btr [rax], edx
3699# else
3700 mov eax, [pvBitmap]
3701 btr [eax], edx
3702# endif
3703 setc al
3704 and eax, 1
3705 mov [rc.u32], eax
3706 }
3707# endif
3708 return rc.f;
3709}
3710#endif
3711
3712
3713/**
3714 * Atomically tests and clears a bit in a bitmap, ordered.
3715 *
3716 * @returns true if the bit was set.
3717 * @returns false if the bit was clear.
3718 *
3719 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3720 * the memory access isn't atomic!
3721 * @param iBit The bit to test and clear.
3722 *
3723 * @remarks No memory barrier, take care on smp.
3724 */
3725#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3726DECLASM(bool) ASMAtomicBitTestAndClear(volatile void *pvBitmap, int32_t iBit);
3727#else
3728DECLINLINE(bool) ASMAtomicBitTestAndClear(volatile void *pvBitmap, int32_t iBit)
3729{
3730 union { bool f; uint32_t u32; uint8_t u8; } rc;
3731 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3732# if RT_INLINE_ASM_USES_INTRIN
3733 rc.u8 = _interlockedbittestandreset((long *)pvBitmap, iBit);
3734
3735# elif RT_INLINE_ASM_GNU_STYLE
3736 __asm__ __volatile__("lock; btrl %2, %1\n\t"
3737 "setc %b0\n\t"
3738 "andl $1, %0\n\t"
3739 : "=q" (rc.u32),
3740 "=m" (*(volatile long *)pvBitmap)
3741 : "Ir" (iBit),
3742 "m" (*(volatile long *)pvBitmap)
3743 : "memory");
3744# else
3745 __asm
3746 {
3747 mov edx, [iBit]
3748# ifdef RT_ARCH_AMD64
3749 mov rax, [pvBitmap]
3750 lock btr [rax], edx
3751# else
3752 mov eax, [pvBitmap]
3753 lock btr [eax], edx
3754# endif
3755 setc al
3756 and eax, 1
3757 mov [rc.u32], eax
3758 }
3759# endif
3760 return rc.f;
3761}
3762#endif
3763
3764
3765/**
3766 * Tests and toggles a bit in a bitmap.
3767 *
3768 * @returns true if the bit was set.
3769 * @returns false if the bit was clear.
3770 *
3771 * @param pvBitmap Pointer to the bitmap.
3772 * @param iBit The bit to test and toggle.
3773 *
3774 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3775 * However, doing so will yield better performance as well as avoiding
3776 * traps accessing the last bits in the bitmap.
3777 */
3778#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3779DECLASM(bool) ASMBitTestAndToggle(volatile void *pvBitmap, int32_t iBit);
3780#else
3781DECLINLINE(bool) ASMBitTestAndToggle(volatile void *pvBitmap, int32_t iBit)
3782{
3783 union { bool f; uint32_t u32; uint8_t u8; } rc;
3784# if RT_INLINE_ASM_USES_INTRIN
3785 rc.u8 = _bittestandcomplement((long *)pvBitmap, iBit);
3786
3787# elif RT_INLINE_ASM_GNU_STYLE
3788 __asm__ __volatile__("btcl %2, %1\n\t"
3789 "setc %b0\n\t"
3790 "andl $1, %0\n\t"
3791 : "=q" (rc.u32),
3792 "=m" (*(volatile long *)pvBitmap)
3793 : "Ir" (iBit),
3794 "m" (*(volatile long *)pvBitmap)
3795 : "memory");
3796# else
3797 __asm
3798 {
3799 mov edx, [iBit]
3800# ifdef RT_ARCH_AMD64
3801 mov rax, [pvBitmap]
3802 btc [rax], edx
3803# else
3804 mov eax, [pvBitmap]
3805 btc [eax], edx
3806# endif
3807 setc al
3808 and eax, 1
3809 mov [rc.u32], eax
3810 }
3811# endif
3812 return rc.f;
3813}
3814#endif
3815
3816
3817/**
3818 * Atomically tests and toggles a bit in a bitmap, ordered.
3819 *
3820 * @returns true if the bit was set.
3821 * @returns false if the bit was clear.
3822 *
3823 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3824 * the memory access isn't atomic!
3825 * @param iBit The bit to test and toggle.
3826 */
3827#if RT_INLINE_ASM_EXTERNAL
3828DECLASM(bool) ASMAtomicBitTestAndToggle(volatile void *pvBitmap, int32_t iBit);
3829#else
3830DECLINLINE(bool) ASMAtomicBitTestAndToggle(volatile void *pvBitmap, int32_t iBit)
3831{
3832 union { bool f; uint32_t u32; uint8_t u8; } rc;
3833 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3834# if RT_INLINE_ASM_GNU_STYLE
3835 __asm__ __volatile__("lock; btcl %2, %1\n\t"
3836 "setc %b0\n\t"
3837 "andl $1, %0\n\t"
3838 : "=q" (rc.u32),
3839 "=m" (*(volatile long *)pvBitmap)
3840 : "Ir" (iBit),
3841 "m" (*(volatile long *)pvBitmap)
3842 : "memory");
3843# else
3844 __asm
3845 {
3846 mov edx, [iBit]
3847# ifdef RT_ARCH_AMD64
3848 mov rax, [pvBitmap]
3849 lock btc [rax], edx
3850# else
3851 mov eax, [pvBitmap]
3852 lock btc [eax], edx
3853# endif
3854 setc al
3855 and eax, 1
3856 mov [rc.u32], eax
3857 }
3858# endif
3859 return rc.f;
3860}
3861#endif
3862
3863
3864/**
3865 * Tests if a bit in a bitmap is set.
3866 *
3867 * @returns true if the bit is set.
3868 * @returns false if the bit is clear.
3869 *
3870 * @param pvBitmap Pointer to the bitmap.
3871 * @param iBit The bit to test.
3872 *
3873 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3874 * However, doing so will yield better performance as well as avoiding
3875 * traps accessing the last bits in the bitmap.
3876 */
3877#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3878DECLASM(bool) ASMBitTest(const volatile void *pvBitmap, int32_t iBit);
3879#else
3880DECLINLINE(bool) ASMBitTest(const volatile void *pvBitmap, int32_t iBit)
3881{
3882 union { bool f; uint32_t u32; uint8_t u8; } rc;
3883# if RT_INLINE_ASM_USES_INTRIN
3884 rc.u32 = _bittest((long *)pvBitmap, iBit);
3885# elif RT_INLINE_ASM_GNU_STYLE
3886
3887 __asm__ __volatile__("btl %2, %1\n\t"
3888 "setc %b0\n\t"
3889 "andl $1, %0\n\t"
3890 : "=q" (rc.u32)
3891 : "m" (*(const volatile long *)pvBitmap),
3892 "Ir" (iBit)
3893 : "memory");
3894# else
3895 __asm
3896 {
3897 mov edx, [iBit]
3898# ifdef RT_ARCH_AMD64
3899 mov rax, [pvBitmap]
3900 bt [rax], edx
3901# else
3902 mov eax, [pvBitmap]
3903 bt [eax], edx
3904# endif
3905 setc al
3906 and eax, 1
3907 mov [rc.u32], eax
3908 }
3909# endif
3910 return rc.f;
3911}
3912#endif
3913
3914
3915/**
3916 * Clears a bit range within a bitmap.
3917 *
3918 * @param pvBitmap Pointer to the bitmap.
3919 * @param iBitStart The First bit to clear.
3920 * @param iBitEnd The first bit not to clear.
3921 */
3922DECLINLINE(void) ASMBitClearRange(volatile void *pvBitmap, int32_t iBitStart, int32_t iBitEnd)
3923{
3924 if (iBitStart < iBitEnd)
3925 {
3926 volatile uint32_t *pu32 = (volatile uint32_t *)pvBitmap + (iBitStart >> 5);
3927 int iStart = iBitStart & ~31;
3928 int iEnd = iBitEnd & ~31;
3929 if (iStart == iEnd)
3930 *pu32 &= ((1 << (iBitStart & 31)) - 1) | ~((1 << (iBitEnd & 31)) - 1);
3931 else
3932 {
3933 /* bits in first dword. */
3934 if (iBitStart & 31)
3935 {
3936 *pu32 &= (1 << (iBitStart & 31)) - 1;
3937 pu32++;
3938 iBitStart = iStart + 32;
3939 }
3940
3941 /* whole dword. */
3942 if (iBitStart != iEnd)
3943 ASMMemZero32(pu32, (iEnd - iBitStart) >> 3);
3944
3945 /* bits in last dword. */
3946 if (iBitEnd & 31)
3947 {
3948 pu32 = (volatile uint32_t *)pvBitmap + (iBitEnd >> 5);
3949 *pu32 &= ~((1 << (iBitEnd & 31)) - 1);
3950 }
3951 }
3952 }
3953}
3954
3955
3956/**
3957 * Sets a bit range within a bitmap.
3958 *
3959 * @param pvBitmap Pointer to the bitmap.
3960 * @param iBitStart The First bit to set.
3961 * @param iBitEnd The first bit not to set.
3962 */
3963DECLINLINE(void) ASMBitSetRange(volatile void *pvBitmap, int32_t iBitStart, int32_t iBitEnd)
3964{
3965 if (iBitStart < iBitEnd)
3966 {
3967 volatile uint32_t *pu32 = (volatile uint32_t *)pvBitmap + (iBitStart >> 5);
3968 int iStart = iBitStart & ~31;
3969 int iEnd = iBitEnd & ~31;
3970 if (iStart == iEnd)
3971 *pu32 |= ((1 << (iBitEnd - iBitStart)) - 1) << iBitStart;
3972 else
3973 {
3974 /* bits in first dword. */
3975 if (iBitStart & 31)
3976 {
3977 *pu32 |= ~((1 << (iBitStart & 31)) - 1);
3978 pu32++;
3979 iBitStart = iStart + 32;
3980 }
3981
3982 /* whole dword. */
3983 if (iBitStart != iEnd)
3984 ASMMemFill32(pu32, (iEnd - iBitStart) >> 3, ~0);
3985
3986 /* bits in last dword. */
3987 if (iBitEnd & 31)
3988 {
3989 pu32 = (volatile uint32_t *)pvBitmap + (iBitEnd >> 5);
3990 *pu32 |= (1 << (iBitEnd & 31)) - 1;
3991 }
3992 }
3993 }
3994}
3995
3996
3997/**
3998 * Finds the first clear bit in a bitmap.
3999 *
4000 * @returns Index of the first zero bit.
4001 * @returns -1 if no clear bit was found.
4002 * @param pvBitmap Pointer to the bitmap.
4003 * @param cBits The number of bits in the bitmap. Multiple of 32.
4004 */
4005#if RT_INLINE_ASM_EXTERNAL
4006DECLASM(int) ASMBitFirstClear(const volatile void *pvBitmap, uint32_t cBits);
4007#else
4008DECLINLINE(int) ASMBitFirstClear(const volatile void *pvBitmap, uint32_t cBits)
4009{
4010 if (cBits)
4011 {
4012 int32_t iBit;
4013# if RT_INLINE_ASM_GNU_STYLE
4014 RTCCUINTREG uEAX, uECX, uEDI;
4015 cBits = RT_ALIGN_32(cBits, 32);
4016 __asm__ __volatile__("repe; scasl\n\t"
4017 "je 1f\n\t"
4018# ifdef RT_ARCH_AMD64
4019 "lea -4(%%rdi), %%rdi\n\t"
4020 "xorl (%%rdi), %%eax\n\t"
4021 "subq %5, %%rdi\n\t"
4022# else
4023 "lea -4(%%edi), %%edi\n\t"
4024 "xorl (%%edi), %%eax\n\t"
4025 "subl %5, %%edi\n\t"
4026# endif
4027 "shll $3, %%edi\n\t"
4028 "bsfl %%eax, %%edx\n\t"
4029 "addl %%edi, %%edx\n\t"
4030 "1:\t\n"
4031 : "=d" (iBit),
4032 "=&c" (uECX),
4033 "=&D" (uEDI),
4034 "=&a" (uEAX)
4035 : "0" (0xffffffff),
4036 "mr" (pvBitmap),
4037 "1" (cBits >> 5),
4038 "2" (pvBitmap),
4039 "3" (0xffffffff));
4040# else
4041 cBits = RT_ALIGN_32(cBits, 32);
4042 __asm
4043 {
4044# ifdef RT_ARCH_AMD64
4045 mov rdi, [pvBitmap]
4046 mov rbx, rdi
4047# else
4048 mov edi, [pvBitmap]
4049 mov ebx, edi
4050# endif
4051 mov edx, 0ffffffffh
4052 mov eax, edx
4053 mov ecx, [cBits]
4054 shr ecx, 5
4055 repe scasd
4056 je done
4057
4058# ifdef RT_ARCH_AMD64
4059 lea rdi, [rdi - 4]
4060 xor eax, [rdi]
4061 sub rdi, rbx
4062# else
4063 lea edi, [edi - 4]
4064 xor eax, [edi]
4065 sub edi, ebx
4066# endif
4067 shl edi, 3
4068 bsf edx, eax
4069 add edx, edi
4070 done:
4071 mov [iBit], edx
4072 }
4073# endif
4074 return iBit;
4075 }
4076 return -1;
4077}
4078#endif
4079
4080
4081/**
4082 * Finds the next clear bit in a bitmap.
4083 *
4084 * @returns Index of the first zero bit.
4085 * @returns -1 if no clear bit was found.
4086 * @param pvBitmap Pointer to the bitmap.
4087 * @param cBits The number of bits in the bitmap. Multiple of 32.
4088 * @param iBitPrev The bit returned from the last search.
4089 * The search will start at iBitPrev + 1.
4090 */
4091#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4092DECLASM(int) ASMBitNextClear(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev);
4093#else
4094DECLINLINE(int) ASMBitNextClear(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev)
4095{
4096 const volatile uint32_t *pau32Bitmap = (const volatile uint32_t *)pvBitmap;
4097 int iBit = ++iBitPrev & 31;
4098 if (iBit)
4099 {
4100 /*
4101 * Inspect the 32-bit word containing the unaligned bit.
4102 */
4103 uint32_t u32 = ~pau32Bitmap[iBitPrev / 32] >> iBit;
4104
4105# if RT_INLINE_ASM_USES_INTRIN
4106 unsigned long ulBit = 0;
4107 if (_BitScanForward(&ulBit, u32))
4108 return ulBit + iBitPrev;
4109# else
4110# if RT_INLINE_ASM_GNU_STYLE
4111 __asm__ __volatile__("bsf %1, %0\n\t"
4112 "jnz 1f\n\t"
4113 "movl $-1, %0\n\t"
4114 "1:\n\t"
4115 : "=r" (iBit)
4116 : "r" (u32));
4117# else
4118 __asm
4119 {
4120 mov edx, [u32]
4121 bsf eax, edx
4122 jnz done
4123 mov eax, 0ffffffffh
4124 done:
4125 mov [iBit], eax
4126 }
4127# endif
4128 if (iBit >= 0)
4129 return iBit + iBitPrev;
4130# endif
4131
4132 /*
4133 * Skip ahead and see if there is anything left to search.
4134 */
4135 iBitPrev |= 31;
4136 iBitPrev++;
4137 if (cBits <= (uint32_t)iBitPrev)
4138 return -1;
4139 }
4140
4141 /*
4142 * 32-bit aligned search, let ASMBitFirstClear do the dirty work.
4143 */
4144 iBit = ASMBitFirstClear(&pau32Bitmap[iBitPrev / 32], cBits - iBitPrev);
4145 if (iBit >= 0)
4146 iBit += iBitPrev;
4147 return iBit;
4148}
4149#endif
4150
4151
4152/**
4153 * Finds the first set bit in a bitmap.
4154 *
4155 * @returns Index of the first set bit.
4156 * @returns -1 if no clear bit was found.
4157 * @param pvBitmap Pointer to the bitmap.
4158 * @param cBits The number of bits in the bitmap. Multiple of 32.
4159 */
4160#if RT_INLINE_ASM_EXTERNAL
4161DECLASM(int) ASMBitFirstSet(const volatile void *pvBitmap, uint32_t cBits);
4162#else
4163DECLINLINE(int) ASMBitFirstSet(const volatile void *pvBitmap, uint32_t cBits)
4164{
4165 if (cBits)
4166 {
4167 int32_t iBit;
4168# if RT_INLINE_ASM_GNU_STYLE
4169 RTCCUINTREG uEAX, uECX, uEDI;
4170 cBits = RT_ALIGN_32(cBits, 32);
4171 __asm__ __volatile__("repe; scasl\n\t"
4172 "je 1f\n\t"
4173# ifdef RT_ARCH_AMD64
4174 "lea -4(%%rdi), %%rdi\n\t"
4175 "movl (%%rdi), %%eax\n\t"
4176 "subq %5, %%rdi\n\t"
4177# else
4178 "lea -4(%%edi), %%edi\n\t"
4179 "movl (%%edi), %%eax\n\t"
4180 "subl %5, %%edi\n\t"
4181# endif
4182 "shll $3, %%edi\n\t"
4183 "bsfl %%eax, %%edx\n\t"
4184 "addl %%edi, %%edx\n\t"
4185 "1:\t\n"
4186 : "=d" (iBit),
4187 "=&c" (uECX),
4188 "=&D" (uEDI),
4189 "=&a" (uEAX)
4190 : "0" (0xffffffff),
4191 "mr" (pvBitmap),
4192 "1" (cBits >> 5),
4193 "2" (pvBitmap),
4194 "3" (0));
4195# else
4196 cBits = RT_ALIGN_32(cBits, 32);
4197 __asm
4198 {
4199# ifdef RT_ARCH_AMD64
4200 mov rdi, [pvBitmap]
4201 mov rbx, rdi
4202# else
4203 mov edi, [pvBitmap]
4204 mov ebx, edi
4205# endif
4206 mov edx, 0ffffffffh
4207 xor eax, eax
4208 mov ecx, [cBits]
4209 shr ecx, 5
4210 repe scasd
4211 je done
4212# ifdef RT_ARCH_AMD64
4213 lea rdi, [rdi - 4]
4214 mov eax, [rdi]
4215 sub rdi, rbx
4216# else
4217 lea edi, [edi - 4]
4218 mov eax, [edi]
4219 sub edi, ebx
4220# endif
4221 shl edi, 3
4222 bsf edx, eax
4223 add edx, edi
4224 done:
4225 mov [iBit], edx
4226 }
4227# endif
4228 return iBit;
4229 }
4230 return -1;
4231}
4232#endif
4233
4234
4235/**
4236 * Finds the next set bit in a bitmap.
4237 *
4238 * @returns Index of the next set bit.
4239 * @returns -1 if no set bit was found.
4240 * @param pvBitmap Pointer to the bitmap.
4241 * @param cBits The number of bits in the bitmap. Multiple of 32.
4242 * @param iBitPrev The bit returned from the last search.
4243 * The search will start at iBitPrev + 1.
4244 */
4245#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4246DECLASM(int) ASMBitNextSet(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev);
4247#else
4248DECLINLINE(int) ASMBitNextSet(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev)
4249{
4250 const volatile uint32_t *pau32Bitmap = (const volatile uint32_t *)pvBitmap;
4251 int iBit = ++iBitPrev & 31;
4252 if (iBit)
4253 {
4254 /*
4255 * Inspect the 32-bit word containing the unaligned bit.
4256 */
4257 uint32_t u32 = pau32Bitmap[iBitPrev / 32] >> iBit;
4258
4259# if RT_INLINE_ASM_USES_INTRIN
4260 unsigned long ulBit = 0;
4261 if (_BitScanForward(&ulBit, u32))
4262 return ulBit + iBitPrev;
4263# else
4264# if RT_INLINE_ASM_GNU_STYLE
4265 __asm__ __volatile__("bsf %1, %0\n\t"
4266 "jnz 1f\n\t"
4267 "movl $-1, %0\n\t"
4268 "1:\n\t"
4269 : "=r" (iBit)
4270 : "r" (u32));
4271# else
4272 __asm
4273 {
4274 mov edx, [u32]
4275 bsf eax, edx
4276 jnz done
4277 mov eax, 0ffffffffh
4278 done:
4279 mov [iBit], eax
4280 }
4281# endif
4282 if (iBit >= 0)
4283 return iBit + iBitPrev;
4284# endif
4285
4286 /*
4287 * Skip ahead and see if there is anything left to search.
4288 */
4289 iBitPrev |= 31;
4290 iBitPrev++;
4291 if (cBits <= (uint32_t)iBitPrev)
4292 return -1;
4293 }
4294
4295 /*
4296 * 32-bit aligned search, let ASMBitFirstClear do the dirty work.
4297 */
4298 iBit = ASMBitFirstSet(&pau32Bitmap[iBitPrev / 32], cBits - iBitPrev);
4299 if (iBit >= 0)
4300 iBit += iBitPrev;
4301 return iBit;
4302}
4303#endif
4304
4305
4306/**
4307 * Finds the first bit which is set in the given 32-bit integer.
4308 * Bits are numbered from 1 (least significant) to 32.
4309 *
4310 * @returns index [1..32] of the first set bit.
4311 * @returns 0 if all bits are cleared.
4312 * @param u32 Integer to search for set bits.
4313 * @remark Similar to ffs() in BSD.
4314 */
4315#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4316DECLASM(unsigned) ASMBitFirstSetU32(uint32_t u32);
4317#else
4318DECLINLINE(unsigned) ASMBitFirstSetU32(uint32_t u32)
4319{
4320# if RT_INLINE_ASM_USES_INTRIN
4321 unsigned long iBit;
4322 if (_BitScanForward(&iBit, u32))
4323 iBit++;
4324 else
4325 iBit = 0;
4326# elif RT_INLINE_ASM_GNU_STYLE
4327 uint32_t iBit;
4328 __asm__ __volatile__("bsf %1, %0\n\t"
4329 "jnz 1f\n\t"
4330 "xorl %0, %0\n\t"
4331 "jmp 2f\n"
4332 "1:\n\t"
4333 "incl %0\n"
4334 "2:\n\t"
4335 : "=r" (iBit)
4336 : "rm" (u32));
4337# else
4338 uint32_t iBit;
4339 _asm
4340 {
4341 bsf eax, [u32]
4342 jnz found
4343 xor eax, eax
4344 jmp done
4345 found:
4346 inc eax
4347 done:
4348 mov [iBit], eax
4349 }
4350# endif
4351 return iBit;
4352}
4353#endif
4354
4355
4356/**
4357 * Finds the first bit which is set in the given 32-bit integer.
4358 * Bits are numbered from 1 (least significant) to 32.
4359 *
4360 * @returns index [1..32] of the first set bit.
4361 * @returns 0 if all bits are cleared.
4362 * @param i32 Integer to search for set bits.
4363 * @remark Similar to ffs() in BSD.
4364 */
4365DECLINLINE(unsigned) ASMBitFirstSetS32(int32_t i32)
4366{
4367 return ASMBitFirstSetU32((uint32_t)i32);
4368}
4369
4370
4371/**
4372 * Finds the last bit which is set in the given 32-bit integer.
4373 * Bits are numbered from 1 (least significant) to 32.
4374 *
4375 * @returns index [1..32] of the last set bit.
4376 * @returns 0 if all bits are cleared.
4377 * @param u32 Integer to search for set bits.
4378 * @remark Similar to fls() in BSD.
4379 */
4380#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4381DECLASM(unsigned) ASMBitLastSetU32(uint32_t u32);
4382#else
4383DECLINLINE(unsigned) ASMBitLastSetU32(uint32_t u32)
4384{
4385# if RT_INLINE_ASM_USES_INTRIN
4386 unsigned long iBit;
4387 if (_BitScanReverse(&iBit, u32))
4388 iBit++;
4389 else
4390 iBit = 0;
4391# elif RT_INLINE_ASM_GNU_STYLE
4392 uint32_t iBit;
4393 __asm__ __volatile__("bsrl %1, %0\n\t"
4394 "jnz 1f\n\t"
4395 "xorl %0, %0\n\t"
4396 "jmp 2f\n"
4397 "1:\n\t"
4398 "incl %0\n"
4399 "2:\n\t"
4400 : "=r" (iBit)
4401 : "rm" (u32));
4402# else
4403 uint32_t iBit;
4404 _asm
4405 {
4406 bsr eax, [u32]
4407 jnz found
4408 xor eax, eax
4409 jmp done
4410 found:
4411 inc eax
4412 done:
4413 mov [iBit], eax
4414 }
4415# endif
4416 return iBit;
4417}
4418#endif
4419
4420
4421/**
4422 * Finds the last bit which is set in the given 32-bit integer.
4423 * Bits are numbered from 1 (least significant) to 32.
4424 *
4425 * @returns index [1..32] of the last set bit.
4426 * @returns 0 if all bits are cleared.
4427 * @param i32 Integer to search for set bits.
4428 * @remark Similar to fls() in BSD.
4429 */
4430DECLINLINE(unsigned) ASMBitLastSetS32(int32_t i32)
4431{
4432 return ASMBitLastSetU32((uint32_t)i32);
4433}
4434
4435/**
4436 * Reverse the byte order of the given 16-bit integer.
4437 *
4438 * @returns Revert
4439 * @param u16 16-bit integer value.
4440 */
4441#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4442DECLASM(uint16_t) ASMByteSwapU16(uint16_t u16);
4443#else
4444DECLINLINE(uint16_t) ASMByteSwapU16(uint16_t u16)
4445{
4446# if RT_INLINE_ASM_USES_INTRIN
4447 u16 = _byteswap_ushort(u16);
4448# elif RT_INLINE_ASM_GNU_STYLE
4449 __asm__ ("rorw $8, %0" : "=r" (u16) : "0" (u16));
4450# else
4451 _asm
4452 {
4453 mov ax, [u16]
4454 ror ax, 8
4455 mov [u16], ax
4456 }
4457# endif
4458 return u16;
4459}
4460#endif
4461
4462
4463/**
4464 * Reverse the byte order of the given 32-bit integer.
4465 *
4466 * @returns Revert
4467 * @param u32 32-bit integer value.
4468 */
4469#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4470DECLASM(uint32_t) ASMByteSwapU32(uint32_t u32);
4471#else
4472DECLINLINE(uint32_t) ASMByteSwapU32(uint32_t u32)
4473{
4474# if RT_INLINE_ASM_USES_INTRIN
4475 u32 = _byteswap_ulong(u32);
4476# elif RT_INLINE_ASM_GNU_STYLE
4477 __asm__ ("bswapl %0" : "=r" (u32) : "0" (u32));
4478# else
4479 _asm
4480 {
4481 mov eax, [u32]
4482 bswap eax
4483 mov [u32], eax
4484 }
4485# endif
4486 return u32;
4487}
4488#endif
4489
4490
4491/**
4492 * Reverse the byte order of the given 64-bit integer.
4493 *
4494 * @returns Revert
4495 * @param u64 64-bit integer value.
4496 */
4497DECLINLINE(uint64_t) ASMByteSwapU64(uint64_t u64)
4498{
4499#if defined(RT_ARCH_AMD64) && RT_INLINE_ASM_USES_INTRIN
4500 u64 = _byteswap_uint64(u64);
4501#else
4502 u64 = (uint64_t)ASMByteSwapU32((uint32_t)u64) << 32
4503 | (uint64_t)ASMByteSwapU32((uint32_t)(u64 >> 32));
4504#endif
4505 return u64;
4506}
4507
4508
4509/** @} */
4510
4511
4512/** @} */
4513
4514/* KLUDGE: Play safe for now as I cannot test all solaris and linux usages. */
4515#if !defined(__cplusplus) && !defined(DEBUG)
4516# if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
4517# include <iprt/asm-amd64-x86.h>
4518# endif
4519# include <iprt/asm-math.h>
4520#endif
4521
4522#endif
4523
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette