VirtualBox

source: vbox/trunk/include/iprt/asm.h@ 34435

Last change on this file since 34435 was 33540, checked in by vboxsync, 14 years ago

*: spelling fixes, thanks Timeless!
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 130.0 KB
Line 
1/** @file
2 * IPRT - Assembly Functions.
3 */
4
5/*
6 * Copyright (C) 2006-2010 Oracle Corporation
7 *
8 * This file is part of VirtualBox Open Source Edition (OSE), as
9 * available from http://www.virtualbox.org. This file is free software;
10 * you can redistribute it and/or modify it under the terms of the GNU
11 * General Public License (GPL) as published by the Free Software
12 * Foundation, in version 2 as it comes in the "COPYING" file of the
13 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
14 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
15 *
16 * The contents of this file may alternatively be used under the terms
17 * of the Common Development and Distribution License Version 1.0
18 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
19 * VirtualBox OSE distribution, in which case the provisions of the
20 * CDDL are applicable instead of those of the GPL.
21 *
22 * You may elect to license modified versions of this file under the
23 * terms and conditions of either the GPL or the CDDL or both.
24 */
25
26#ifndef ___iprt_asm_h
27#define ___iprt_asm_h
28
29#include <iprt/cdefs.h>
30#include <iprt/types.h>
31#include <iprt/assert.h>
32/** @def RT_INLINE_ASM_USES_INTRIN
33 * Defined as 1 if we're using a _MSC_VER 1400.
34 * Otherwise defined as 0.
35 */
36
37/* Solaris 10 header ugliness */
38#ifdef u
39# undef u
40#endif
41
42#if defined(_MSC_VER) && RT_INLINE_ASM_USES_INTRIN
43# include <intrin.h>
44 /* Emit the intrinsics at all optimization levels. */
45# pragma intrinsic(_ReadWriteBarrier)
46# pragma intrinsic(__cpuid)
47# pragma intrinsic(__stosd)
48# pragma intrinsic(__stosw)
49# pragma intrinsic(__stosb)
50# pragma intrinsic(_BitScanForward)
51# pragma intrinsic(_BitScanReverse)
52# pragma intrinsic(_bittest)
53# pragma intrinsic(_bittestandset)
54# pragma intrinsic(_bittestandreset)
55# pragma intrinsic(_bittestandcomplement)
56# pragma intrinsic(_byteswap_ushort)
57# pragma intrinsic(_byteswap_ulong)
58# pragma intrinsic(_interlockedbittestandset)
59# pragma intrinsic(_interlockedbittestandreset)
60# pragma intrinsic(_InterlockedAnd)
61# pragma intrinsic(_InterlockedOr)
62# pragma intrinsic(_InterlockedIncrement)
63# pragma intrinsic(_InterlockedDecrement)
64# pragma intrinsic(_InterlockedExchange)
65# pragma intrinsic(_InterlockedExchangeAdd)
66# pragma intrinsic(_InterlockedCompareExchange)
67# pragma intrinsic(_InterlockedCompareExchange64)
68# ifdef RT_ARCH_AMD64
69# pragma intrinsic(__stosq)
70# pragma intrinsic(_byteswap_uint64)
71# pragma intrinsic(_InterlockedExchange64)
72# pragma intrinsic(_InterlockedExchangeAdd64)
73# pragma intrinsic(_InterlockedAnd64)
74# pragma intrinsic(_InterlockedOr64)
75# pragma intrinsic(_InterlockedIncrement64)
76# pragma intrinsic(_InterlockedDecrement64)
77# endif
78#endif
79
80
81/** @defgroup grp_rt_asm ASM - Assembly Routines
82 * @ingroup grp_rt
83 *
84 * @remarks The difference between ordered and unordered atomic operations are that
85 * the former will complete outstanding reads and writes before continuing
86 * while the latter doesn't make any promisses about the order. Ordered
87 * operations doesn't, it seems, make any 100% promise wrt to whether
88 * the operation will complete before any subsequent memory access.
89 * (please, correct if wrong.)
90 *
91 * ASMAtomicSomething operations are all ordered, while ASMAtomicUoSomething
92 * are unordered (note the Uo).
93 *
94 * @remarks Some remarks about __volatile__: Without this keyword gcc is allowed to reorder
95 * or even optimize assembler instructions away. For instance, in the following code
96 * the second rdmsr instruction is optimized away because gcc treats that instruction
97 * as deterministic:
98 *
99 * @code
100 * static inline uint64_t rdmsr_low(int idx)
101 * {
102 * uint32_t low;
103 * __asm__ ("rdmsr" : "=a"(low) : "c"(idx) : "edx");
104 * }
105 * ...
106 * uint32_t msr1 = rdmsr_low(1);
107 * foo(msr1);
108 * msr1 = rdmsr_low(1);
109 * bar(msr1);
110 * @endcode
111 *
112 * The input parameter of rdmsr_low is the same for both calls and therefore gcc will
113 * use the result of the first call as input parameter for bar() as well. For rdmsr this
114 * is not acceptable as this instruction is _not_ deterministic. This applies to reading
115 * machine status information in general.
116 *
117 * @{
118 */
119
120
121/** @def RT_INLINE_ASM_GCC_4_3_X_X86
122 * Used to work around some 4.3.x register allocation issues in this version of
123 * the compiler. So far this workaround is still required for 4.4 and 4.5. */
124#ifdef __GNUC__
125# define RT_INLINE_ASM_GCC_4_3_X_X86 (__GNUC__ == 4 && __GNUC_MINOR__ >= 3 && defined(__i386__))
126#endif
127#ifndef RT_INLINE_ASM_GCC_4_3_X_X86
128# define RT_INLINE_ASM_GCC_4_3_X_X86 0
129#endif
130
131/** @def RT_INLINE_DONT_USE_CMPXCHG8B
132 * i686-apple-darwin9-gcc-4.0.1 (GCC) 4.0.1 (Apple Inc. build 5493) screws up
133 * RTSemRWRequestWrite semsemrw-lockless-generic.cpp in release builds. PIC
134 * mode, x86.
135 *
136 * Some gcc 4.3.x versions may have register allocation issues with cmpxchg8b
137 * when in PIC mode on x86.
138 */
139#ifndef RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
140# define RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC \
141 ( (defined(PIC) || defined(__PIC__)) \
142 && defined(RT_ARCH_X86) \
143 && ( RT_INLINE_ASM_GCC_4_3_X_X86 \
144 || defined(RT_OS_DARWIN)) )
145#endif
146
147
148/** @def ASMReturnAddress
149 * Gets the return address of the current (or calling if you like) function or method.
150 */
151#ifdef _MSC_VER
152# ifdef __cplusplus
153extern "C"
154# endif
155void * _ReturnAddress(void);
156# pragma intrinsic(_ReturnAddress)
157# define ASMReturnAddress() _ReturnAddress()
158#elif defined(__GNUC__) || defined(DOXYGEN_RUNNING)
159# define ASMReturnAddress() __builtin_return_address(0)
160#else
161# error "Unsupported compiler."
162#endif
163
164
165/**
166 * Compiler memory barrier.
167 *
168 * Ensure that the compiler does not use any cached (register/tmp stack) memory
169 * values or any outstanding writes when returning from this function.
170 *
171 * This function must be used if non-volatile data is modified by a
172 * device or the VMM. Typical cases are port access, MMIO access,
173 * trapping instruction, etc.
174 */
175#if RT_INLINE_ASM_GNU_STYLE
176# define ASMCompilerBarrier() do { __asm__ __volatile__("" : : : "memory"); } while (0)
177#elif RT_INLINE_ASM_USES_INTRIN
178# define ASMCompilerBarrier() do { _ReadWriteBarrier(); } while (0)
179#else /* 2003 should have _ReadWriteBarrier() but I guess we're at 2002 level then... */
180DECLINLINE(void) ASMCompilerBarrier(void)
181{
182 __asm
183 {
184 }
185}
186#endif
187
188
189/** @def ASMBreakpoint
190 * Debugger Breakpoint.
191 * @remark In the gnu world we add a nop instruction after the int3 to
192 * force gdb to remain at the int3 source line.
193 * @remark The L4 kernel will try make sense of the breakpoint, thus the jmp.
194 * @internal
195 */
196#if RT_INLINE_ASM_GNU_STYLE
197# if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
198# ifndef __L4ENV__
199# define ASMBreakpoint() do { __asm__ __volatile__("int3\n\tnop"); } while (0)
200# else
201# define ASMBreakpoint() do { __asm__ __volatile__("int3; jmp 1f; 1:"); } while (0)
202# endif
203# elif defined(RT_ARCH_SPARC64)
204# define ASMBreakpoint() do { __asm__ __volatile__("illtrap 0\n\t") } while (0) /** @todo Sparc64: this is just a wild guess. */
205# elif defined(RT_ARCH_SPARC)
206# define ASMBreakpoint() do { __asm__ __volatile__("unimp 0\n\t"); } while (0) /** @todo Sparc: this is just a wild guess (same as Sparc64, just different name). */
207# else
208# error "PORTME"
209# endif
210#else
211# define ASMBreakpoint() __debugbreak()
212#endif
213
214
215/**
216 * Spinloop hint for platforms that have these, empty function on the other
217 * platforms.
218 *
219 * x86 & AMD64: The PAUSE variant of NOP for helping hyperthreaded CPUs detecting
220 * spin locks.
221 */
222#if RT_INLINE_ASM_EXTERNAL && (defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86))
223DECLASM(void) ASMNopPause(void);
224#else
225DECLINLINE(void) ASMNopPause(void)
226{
227# if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
228# if RT_INLINE_ASM_GNU_STYLE
229 __asm__ __volatile__(".byte 0xf3,0x90\n\t");
230# else
231 __asm {
232 _emit 0f3h
233 _emit 090h
234 }
235# endif
236# else
237 /* dummy */
238# endif
239}
240#endif
241
242
243/**
244 * Atomically Exchange an unsigned 8-bit value, ordered.
245 *
246 * @returns Current *pu8 value
247 * @param pu8 Pointer to the 8-bit variable to update.
248 * @param u8 The 8-bit value to assign to *pu8.
249 */
250#if RT_INLINE_ASM_EXTERNAL
251DECLASM(uint8_t) ASMAtomicXchgU8(volatile uint8_t *pu8, uint8_t u8);
252#else
253DECLINLINE(uint8_t) ASMAtomicXchgU8(volatile uint8_t *pu8, uint8_t u8)
254{
255# if RT_INLINE_ASM_GNU_STYLE
256 __asm__ __volatile__("xchgb %0, %1\n\t"
257 : "=m" (*pu8),
258 "=q" (u8) /* =r - busted on g++ (GCC) 3.4.4 20050721 (Red Hat 3.4.4-2) */
259 : "1" (u8),
260 "m" (*pu8));
261# else
262 __asm
263 {
264# ifdef RT_ARCH_AMD64
265 mov rdx, [pu8]
266 mov al, [u8]
267 xchg [rdx], al
268 mov [u8], al
269# else
270 mov edx, [pu8]
271 mov al, [u8]
272 xchg [edx], al
273 mov [u8], al
274# endif
275 }
276# endif
277 return u8;
278}
279#endif
280
281
282/**
283 * Atomically Exchange a signed 8-bit value, ordered.
284 *
285 * @returns Current *pu8 value
286 * @param pi8 Pointer to the 8-bit variable to update.
287 * @param i8 The 8-bit value to assign to *pi8.
288 */
289DECLINLINE(int8_t) ASMAtomicXchgS8(volatile int8_t *pi8, int8_t i8)
290{
291 return (int8_t)ASMAtomicXchgU8((volatile uint8_t *)pi8, (uint8_t)i8);
292}
293
294
295/**
296 * Atomically Exchange a bool value, ordered.
297 *
298 * @returns Current *pf value
299 * @param pf Pointer to the 8-bit variable to update.
300 * @param f The 8-bit value to assign to *pi8.
301 */
302DECLINLINE(bool) ASMAtomicXchgBool(volatile bool *pf, bool f)
303{
304#ifdef _MSC_VER
305 return !!ASMAtomicXchgU8((volatile uint8_t *)pf, (uint8_t)f);
306#else
307 return (bool)ASMAtomicXchgU8((volatile uint8_t *)pf, (uint8_t)f);
308#endif
309}
310
311
312/**
313 * Atomically Exchange an unsigned 16-bit value, ordered.
314 *
315 * @returns Current *pu16 value
316 * @param pu16 Pointer to the 16-bit variable to update.
317 * @param u16 The 16-bit value to assign to *pu16.
318 */
319#if RT_INLINE_ASM_EXTERNAL
320DECLASM(uint16_t) ASMAtomicXchgU16(volatile uint16_t *pu16, uint16_t u16);
321#else
322DECLINLINE(uint16_t) ASMAtomicXchgU16(volatile uint16_t *pu16, uint16_t u16)
323{
324# if RT_INLINE_ASM_GNU_STYLE
325 __asm__ __volatile__("xchgw %0, %1\n\t"
326 : "=m" (*pu16),
327 "=r" (u16)
328 : "1" (u16),
329 "m" (*pu16));
330# else
331 __asm
332 {
333# ifdef RT_ARCH_AMD64
334 mov rdx, [pu16]
335 mov ax, [u16]
336 xchg [rdx], ax
337 mov [u16], ax
338# else
339 mov edx, [pu16]
340 mov ax, [u16]
341 xchg [edx], ax
342 mov [u16], ax
343# endif
344 }
345# endif
346 return u16;
347}
348#endif
349
350
351/**
352 * Atomically Exchange a signed 16-bit value, ordered.
353 *
354 * @returns Current *pu16 value
355 * @param pi16 Pointer to the 16-bit variable to update.
356 * @param i16 The 16-bit value to assign to *pi16.
357 */
358DECLINLINE(int16_t) ASMAtomicXchgS16(volatile int16_t *pi16, int16_t i16)
359{
360 return (int16_t)ASMAtomicXchgU16((volatile uint16_t *)pi16, (uint16_t)i16);
361}
362
363
364/**
365 * Atomically Exchange an unsigned 32-bit value, ordered.
366 *
367 * @returns Current *pu32 value
368 * @param pu32 Pointer to the 32-bit variable to update.
369 * @param u32 The 32-bit value to assign to *pu32.
370 */
371#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
372DECLASM(uint32_t) ASMAtomicXchgU32(volatile uint32_t *pu32, uint32_t u32);
373#else
374DECLINLINE(uint32_t) ASMAtomicXchgU32(volatile uint32_t *pu32, uint32_t u32)
375{
376# if RT_INLINE_ASM_GNU_STYLE
377 __asm__ __volatile__("xchgl %0, %1\n\t"
378 : "=m" (*pu32),
379 "=r" (u32)
380 : "1" (u32),
381 "m" (*pu32));
382
383# elif RT_INLINE_ASM_USES_INTRIN
384 u32 = _InterlockedExchange((long *)pu32, u32);
385
386# else
387 __asm
388 {
389# ifdef RT_ARCH_AMD64
390 mov rdx, [pu32]
391 mov eax, u32
392 xchg [rdx], eax
393 mov [u32], eax
394# else
395 mov edx, [pu32]
396 mov eax, u32
397 xchg [edx], eax
398 mov [u32], eax
399# endif
400 }
401# endif
402 return u32;
403}
404#endif
405
406
407/**
408 * Atomically Exchange a signed 32-bit value, ordered.
409 *
410 * @returns Current *pu32 value
411 * @param pi32 Pointer to the 32-bit variable to update.
412 * @param i32 The 32-bit value to assign to *pi32.
413 */
414DECLINLINE(int32_t) ASMAtomicXchgS32(volatile int32_t *pi32, int32_t i32)
415{
416 return (int32_t)ASMAtomicXchgU32((volatile uint32_t *)pi32, (uint32_t)i32);
417}
418
419
420/**
421 * Atomically Exchange an unsigned 64-bit value, ordered.
422 *
423 * @returns Current *pu64 value
424 * @param pu64 Pointer to the 64-bit variable to update.
425 * @param u64 The 64-bit value to assign to *pu64.
426 */
427#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
428 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
429DECLASM(uint64_t) ASMAtomicXchgU64(volatile uint64_t *pu64, uint64_t u64);
430#else
431DECLINLINE(uint64_t) ASMAtomicXchgU64(volatile uint64_t *pu64, uint64_t u64)
432{
433# if defined(RT_ARCH_AMD64)
434# if RT_INLINE_ASM_USES_INTRIN
435 u64 = _InterlockedExchange64((__int64 *)pu64, u64);
436
437# elif RT_INLINE_ASM_GNU_STYLE
438 __asm__ __volatile__("xchgq %0, %1\n\t"
439 : "=m" (*pu64),
440 "=r" (u64)
441 : "1" (u64),
442 "m" (*pu64));
443# else
444 __asm
445 {
446 mov rdx, [pu64]
447 mov rax, [u64]
448 xchg [rdx], rax
449 mov [u64], rax
450 }
451# endif
452# else /* !RT_ARCH_AMD64 */
453# if RT_INLINE_ASM_GNU_STYLE
454# if defined(PIC) || defined(__PIC__)
455 uint32_t u32EBX = (uint32_t)u64;
456 __asm__ __volatile__(/*"xchgl %%esi, %5\n\t"*/
457 "xchgl %%ebx, %3\n\t"
458 "1:\n\t"
459 "lock; cmpxchg8b (%5)\n\t"
460 "jnz 1b\n\t"
461 "movl %3, %%ebx\n\t"
462 /*"xchgl %%esi, %5\n\t"*/
463 : "=A" (u64),
464 "=m" (*pu64)
465 : "0" (*pu64),
466 "m" ( u32EBX ),
467 "c" ( (uint32_t)(u64 >> 32) ),
468 "S" (pu64));
469# else /* !PIC */
470 __asm__ __volatile__("1:\n\t"
471 "lock; cmpxchg8b %1\n\t"
472 "jnz 1b\n\t"
473 : "=A" (u64),
474 "=m" (*pu64)
475 : "0" (*pu64),
476 "b" ( (uint32_t)u64 ),
477 "c" ( (uint32_t)(u64 >> 32) ));
478# endif
479# else
480 __asm
481 {
482 mov ebx, dword ptr [u64]
483 mov ecx, dword ptr [u64 + 4]
484 mov edi, pu64
485 mov eax, dword ptr [edi]
486 mov edx, dword ptr [edi + 4]
487 retry:
488 lock cmpxchg8b [edi]
489 jnz retry
490 mov dword ptr [u64], eax
491 mov dword ptr [u64 + 4], edx
492 }
493# endif
494# endif /* !RT_ARCH_AMD64 */
495 return u64;
496}
497#endif
498
499
500/**
501 * Atomically Exchange an signed 64-bit value, ordered.
502 *
503 * @returns Current *pi64 value
504 * @param pi64 Pointer to the 64-bit variable to update.
505 * @param i64 The 64-bit value to assign to *pi64.
506 */
507DECLINLINE(int64_t) ASMAtomicXchgS64(volatile int64_t *pi64, int64_t i64)
508{
509 return (int64_t)ASMAtomicXchgU64((volatile uint64_t *)pi64, (uint64_t)i64);
510}
511
512
513/**
514 * Atomically Exchange a pointer value, ordered.
515 *
516 * @returns Current *ppv value
517 * @param ppv Pointer to the pointer variable to update.
518 * @param pv The pointer value to assign to *ppv.
519 */
520DECLINLINE(void *) ASMAtomicXchgPtr(void * volatile *ppv, const void *pv)
521{
522#if ARCH_BITS == 32
523 return (void *)ASMAtomicXchgU32((volatile uint32_t *)(void *)ppv, (uint32_t)pv);
524#elif ARCH_BITS == 64
525 return (void *)ASMAtomicXchgU64((volatile uint64_t *)(void *)ppv, (uint64_t)pv);
526#else
527# error "ARCH_BITS is bogus"
528#endif
529}
530
531
532/**
533 * Convenience macro for avoiding the annoying casting with ASMAtomicXchgPtr.
534 *
535 * @returns Current *pv value
536 * @param ppv Pointer to the pointer variable to update.
537 * @param pv The pointer value to assign to *ppv.
538 * @param Type The type of *ppv, sans volatile.
539 */
540#ifdef __GNUC__
541# define ASMAtomicXchgPtrT(ppv, pv, Type) \
542 __extension__ \
543 ({\
544 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
545 Type const pvTypeChecked = (pv); \
546 Type pvTypeCheckedRet = (__typeof__(*(ppv))) ASMAtomicXchgPtr((void * volatile *)ppvTypeChecked, (void *)pvTypeChecked); \
547 pvTypeCheckedRet; \
548 })
549#else
550# define ASMAtomicXchgPtrT(ppv, pv, Type) \
551 (Type)ASMAtomicXchgPtr((void * volatile *)(ppv), (void *)(pv))
552#endif
553
554
555/**
556 * Atomically Exchange a raw-mode context pointer value, ordered.
557 *
558 * @returns Current *ppv value
559 * @param ppvRC Pointer to the pointer variable to update.
560 * @param pvRC The pointer value to assign to *ppv.
561 */
562DECLINLINE(RTRCPTR) ASMAtomicXchgRCPtr(RTRCPTR volatile *ppvRC, RTRCPTR pvRC)
563{
564 return (RTRCPTR)ASMAtomicXchgU32((uint32_t volatile *)(void *)ppvRC, (uint32_t)pvRC);
565}
566
567
568/**
569 * Atomically Exchange a ring-0 pointer value, ordered.
570 *
571 * @returns Current *ppv value
572 * @param ppvR0 Pointer to the pointer variable to update.
573 * @param pvR0 The pointer value to assign to *ppv.
574 */
575DECLINLINE(RTR0PTR) ASMAtomicXchgR0Ptr(RTR0PTR volatile *ppvR0, RTR0PTR pvR0)
576{
577#if R0_ARCH_BITS == 32
578 return (RTR0PTR)ASMAtomicXchgU32((volatile uint32_t *)(void *)ppvR0, (uint32_t)pvR0);
579#elif R0_ARCH_BITS == 64
580 return (RTR0PTR)ASMAtomicXchgU64((volatile uint64_t *)(void *)ppvR0, (uint64_t)pvR0);
581#else
582# error "R0_ARCH_BITS is bogus"
583#endif
584}
585
586
587/**
588 * Atomically Exchange a ring-3 pointer value, ordered.
589 *
590 * @returns Current *ppv value
591 * @param ppvR3 Pointer to the pointer variable to update.
592 * @param pvR3 The pointer value to assign to *ppv.
593 */
594DECLINLINE(RTR3PTR) ASMAtomicXchgR3Ptr(RTR3PTR volatile *ppvR3, RTR3PTR pvR3)
595{
596#if R3_ARCH_BITS == 32
597 return (RTR3PTR)ASMAtomicXchgU32((volatile uint32_t *)(void *)ppvR3, (uint32_t)pvR3);
598#elif R3_ARCH_BITS == 64
599 return (RTR3PTR)ASMAtomicXchgU64((volatile uint64_t *)(void *)ppvR3, (uint64_t)pvR3);
600#else
601# error "R3_ARCH_BITS is bogus"
602#endif
603}
604
605
606/** @def ASMAtomicXchgHandle
607 * Atomically Exchange a typical IPRT handle value, ordered.
608 *
609 * @param ph Pointer to the value to update.
610 * @param hNew The new value to assigned to *pu.
611 * @param phRes Where to store the current *ph value.
612 *
613 * @remarks This doesn't currently work for all handles (like RTFILE).
614 */
615#if HC_ARCH_BITS == 32
616# define ASMAtomicXchgHandle(ph, hNew, phRes) \
617 do { \
618 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
619 AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
620 *(uint32_t *)(phRes) = ASMAtomicXchgU32((uint32_t volatile *)(ph), (const uint32_t)(hNew)); \
621 } while (0)
622#elif HC_ARCH_BITS == 64
623# define ASMAtomicXchgHandle(ph, hNew, phRes) \
624 do { \
625 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
626 AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
627 *(uint64_t *)(phRes) = ASMAtomicXchgU64((uint64_t volatile *)(ph), (const uint64_t)(hNew)); \
628 } while (0)
629#else
630# error HC_ARCH_BITS
631#endif
632
633
634/**
635 * Atomically Exchange a value which size might differ
636 * between platforms or compilers, ordered.
637 *
638 * @param pu Pointer to the variable to update.
639 * @param uNew The value to assign to *pu.
640 * @todo This is busted as its missing the result argument.
641 */
642#define ASMAtomicXchgSize(pu, uNew) \
643 do { \
644 switch (sizeof(*(pu))) { \
645 case 1: ASMAtomicXchgU8((volatile uint8_t *)(void *)(pu), (uint8_t)(uNew)); break; \
646 case 2: ASMAtomicXchgU16((volatile uint16_t *)(void *)(pu), (uint16_t)(uNew)); break; \
647 case 4: ASMAtomicXchgU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew)); break; \
648 case 8: ASMAtomicXchgU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew)); break; \
649 default: AssertMsgFailed(("ASMAtomicXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
650 } \
651 } while (0)
652
653/**
654 * Atomically Exchange a value which size might differ
655 * between platforms or compilers, ordered.
656 *
657 * @param pu Pointer to the variable to update.
658 * @param uNew The value to assign to *pu.
659 * @param puRes Where to store the current *pu value.
660 */
661#define ASMAtomicXchgSizeCorrect(pu, uNew, puRes) \
662 do { \
663 switch (sizeof(*(pu))) { \
664 case 1: *(uint8_t *)(puRes) = ASMAtomicXchgU8((volatile uint8_t *)(void *)(pu), (uint8_t)(uNew)); break; \
665 case 2: *(uint16_t *)(puRes) = ASMAtomicXchgU16((volatile uint16_t *)(void *)(pu), (uint16_t)(uNew)); break; \
666 case 4: *(uint32_t *)(puRes) = ASMAtomicXchgU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew)); break; \
667 case 8: *(uint64_t *)(puRes) = ASMAtomicXchgU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew)); break; \
668 default: AssertMsgFailed(("ASMAtomicXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
669 } \
670 } while (0)
671
672
673
674/**
675 * Atomically Compare and Exchange an unsigned 8-bit value, ordered.
676 *
677 * @returns true if xchg was done.
678 * @returns false if xchg wasn't done.
679 *
680 * @param pu8 Pointer to the value to update.
681 * @param u8New The new value to assigned to *pu8.
682 * @param u8Old The old value to *pu8 compare with.
683 */
684#if RT_INLINE_ASM_EXTERNAL || !RT_INLINE_ASM_GNU_STYLE
685DECLASM(bool) ASMAtomicCmpXchgU8(volatile uint8_t *pu8, const uint8_t u8New, const uint8_t u8Old);
686#else
687DECLINLINE(bool) ASMAtomicCmpXchgU8(volatile uint8_t *pu8, const uint8_t u8New, uint8_t u8Old)
688{
689 uint8_t u8Ret;
690 __asm__ __volatile__("lock; cmpxchgb %3, %0\n\t"
691 "setz %1\n\t"
692 : "=m" (*pu8),
693 "=qm" (u8Ret),
694 "=a" (u8Old)
695 : "q" (u8New),
696 "2" (u8Old),
697 "m" (*pu8));
698 return (bool)u8Ret;
699}
700#endif
701
702
703/**
704 * Atomically Compare and Exchange a signed 8-bit value, ordered.
705 *
706 * @returns true if xchg was done.
707 * @returns false if xchg wasn't done.
708 *
709 * @param pi8 Pointer to the value to update.
710 * @param i8New The new value to assigned to *pi8.
711 * @param i8Old The old value to *pi8 compare with.
712 */
713DECLINLINE(bool) ASMAtomicCmpXchgS8(volatile int8_t *pi8, const int8_t i8New, const int8_t i8Old)
714{
715 return ASMAtomicCmpXchgU8((volatile uint8_t *)pi8, (const uint8_t)i8New, (const uint8_t)i8Old);
716}
717
718
719/**
720 * Atomically Compare and Exchange a bool value, ordered.
721 *
722 * @returns true if xchg was done.
723 * @returns false if xchg wasn't done.
724 *
725 * @param pf Pointer to the value to update.
726 * @param fNew The new value to assigned to *pf.
727 * @param fOld The old value to *pf compare with.
728 */
729DECLINLINE(bool) ASMAtomicCmpXchgBool(volatile bool *pf, const bool fNew, const bool fOld)
730{
731 return ASMAtomicCmpXchgU8((volatile uint8_t *)pf, (const uint8_t)fNew, (const uint8_t)fOld);
732}
733
734
735/**
736 * Atomically Compare and Exchange an unsigned 32-bit value, ordered.
737 *
738 * @returns true if xchg was done.
739 * @returns false if xchg wasn't done.
740 *
741 * @param pu32 Pointer to the value to update.
742 * @param u32New The new value to assigned to *pu32.
743 * @param u32Old The old value to *pu32 compare with.
744 */
745#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
746DECLASM(bool) ASMAtomicCmpXchgU32(volatile uint32_t *pu32, const uint32_t u32New, const uint32_t u32Old);
747#else
748DECLINLINE(bool) ASMAtomicCmpXchgU32(volatile uint32_t *pu32, const uint32_t u32New, uint32_t u32Old)
749{
750# if RT_INLINE_ASM_GNU_STYLE
751 uint8_t u8Ret;
752 __asm__ __volatile__("lock; cmpxchgl %3, %0\n\t"
753 "setz %1\n\t"
754 : "=m" (*pu32),
755 "=qm" (u8Ret),
756 "=a" (u32Old)
757 : "r" (u32New),
758 "2" (u32Old),
759 "m" (*pu32));
760 return (bool)u8Ret;
761
762# elif RT_INLINE_ASM_USES_INTRIN
763 return _InterlockedCompareExchange((long *)pu32, u32New, u32Old) == u32Old;
764
765# else
766 uint32_t u32Ret;
767 __asm
768 {
769# ifdef RT_ARCH_AMD64
770 mov rdx, [pu32]
771# else
772 mov edx, [pu32]
773# endif
774 mov eax, [u32Old]
775 mov ecx, [u32New]
776# ifdef RT_ARCH_AMD64
777 lock cmpxchg [rdx], ecx
778# else
779 lock cmpxchg [edx], ecx
780# endif
781 setz al
782 movzx eax, al
783 mov [u32Ret], eax
784 }
785 return !!u32Ret;
786# endif
787}
788#endif
789
790
791/**
792 * Atomically Compare and Exchange a signed 32-bit value, ordered.
793 *
794 * @returns true if xchg was done.
795 * @returns false if xchg wasn't done.
796 *
797 * @param pi32 Pointer to the value to update.
798 * @param i32New The new value to assigned to *pi32.
799 * @param i32Old The old value to *pi32 compare with.
800 */
801DECLINLINE(bool) ASMAtomicCmpXchgS32(volatile int32_t *pi32, const int32_t i32New, const int32_t i32Old)
802{
803 return ASMAtomicCmpXchgU32((volatile uint32_t *)pi32, (uint32_t)i32New, (uint32_t)i32Old);
804}
805
806
807/**
808 * Atomically Compare and exchange an unsigned 64-bit value, ordered.
809 *
810 * @returns true if xchg was done.
811 * @returns false if xchg wasn't done.
812 *
813 * @param pu64 Pointer to the 64-bit variable to update.
814 * @param u64New The 64-bit value to assign to *pu64.
815 * @param u64Old The value to compare with.
816 */
817#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
818 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
819DECLASM(bool) ASMAtomicCmpXchgU64(volatile uint64_t *pu64, const uint64_t u64New, const uint64_t u64Old);
820#else
821DECLINLINE(bool) ASMAtomicCmpXchgU64(volatile uint64_t *pu64, uint64_t u64New, uint64_t u64Old)
822{
823# if RT_INLINE_ASM_USES_INTRIN
824 return _InterlockedCompareExchange64((__int64 *)pu64, u64New, u64Old) == u64Old;
825
826# elif defined(RT_ARCH_AMD64)
827# if RT_INLINE_ASM_GNU_STYLE
828 uint8_t u8Ret;
829 __asm__ __volatile__("lock; cmpxchgq %3, %0\n\t"
830 "setz %1\n\t"
831 : "=m" (*pu64),
832 "=qm" (u8Ret),
833 "=a" (u64Old)
834 : "r" (u64New),
835 "2" (u64Old),
836 "m" (*pu64));
837 return (bool)u8Ret;
838# else
839 bool fRet;
840 __asm
841 {
842 mov rdx, [pu32]
843 mov rax, [u64Old]
844 mov rcx, [u64New]
845 lock cmpxchg [rdx], rcx
846 setz al
847 mov [fRet], al
848 }
849 return fRet;
850# endif
851# else /* !RT_ARCH_AMD64 */
852 uint32_t u32Ret;
853# if RT_INLINE_ASM_GNU_STYLE
854# if defined(PIC) || defined(__PIC__)
855 uint32_t u32EBX = (uint32_t)u64New;
856 uint32_t u32Spill;
857 __asm__ __volatile__("xchgl %%ebx, %4\n\t"
858 "lock; cmpxchg8b (%6)\n\t"
859 "setz %%al\n\t"
860 "movl %4, %%ebx\n\t"
861 "movzbl %%al, %%eax\n\t"
862 : "=a" (u32Ret),
863 "=d" (u32Spill),
864# if (__GNUC__ * 100 + __GNUC_MINOR__) >= 403
865 "+m" (*pu64)
866# else
867 "=m" (*pu64)
868# endif
869 : "A" (u64Old),
870 "m" ( u32EBX ),
871 "c" ( (uint32_t)(u64New >> 32) ),
872 "S" (pu64));
873# else /* !PIC */
874 uint32_t u32Spill;
875 __asm__ __volatile__("lock; cmpxchg8b %2\n\t"
876 "setz %%al\n\t"
877 "movzbl %%al, %%eax\n\t"
878 : "=a" (u32Ret),
879 "=d" (u32Spill),
880 "+m" (*pu64)
881 : "A" (u64Old),
882 "b" ( (uint32_t)u64New ),
883 "c" ( (uint32_t)(u64New >> 32) ));
884# endif
885 return (bool)u32Ret;
886# else
887 __asm
888 {
889 mov ebx, dword ptr [u64New]
890 mov ecx, dword ptr [u64New + 4]
891 mov edi, [pu64]
892 mov eax, dword ptr [u64Old]
893 mov edx, dword ptr [u64Old + 4]
894 lock cmpxchg8b [edi]
895 setz al
896 movzx eax, al
897 mov dword ptr [u32Ret], eax
898 }
899 return !!u32Ret;
900# endif
901# endif /* !RT_ARCH_AMD64 */
902}
903#endif
904
905
906/**
907 * Atomically Compare and exchange a signed 64-bit value, ordered.
908 *
909 * @returns true if xchg was done.
910 * @returns false if xchg wasn't done.
911 *
912 * @param pi64 Pointer to the 64-bit variable to update.
913 * @param i64 The 64-bit value to assign to *pu64.
914 * @param i64Old The value to compare with.
915 */
916DECLINLINE(bool) ASMAtomicCmpXchgS64(volatile int64_t *pi64, const int64_t i64, const int64_t i64Old)
917{
918 return ASMAtomicCmpXchgU64((volatile uint64_t *)pi64, (uint64_t)i64, (uint64_t)i64Old);
919}
920
921
922/**
923 * Atomically Compare and Exchange a pointer value, ordered.
924 *
925 * @returns true if xchg was done.
926 * @returns false if xchg wasn't done.
927 *
928 * @param ppv Pointer to the value to update.
929 * @param pvNew The new value to assigned to *ppv.
930 * @param pvOld The old value to *ppv compare with.
931 */
932DECLINLINE(bool) ASMAtomicCmpXchgPtrVoid(void * volatile *ppv, const void *pvNew, const void *pvOld)
933{
934#if ARCH_BITS == 32
935 return ASMAtomicCmpXchgU32((volatile uint32_t *)(void *)ppv, (uint32_t)pvNew, (uint32_t)pvOld);
936#elif ARCH_BITS == 64
937 return ASMAtomicCmpXchgU64((volatile uint64_t *)(void *)ppv, (uint64_t)pvNew, (uint64_t)pvOld);
938#else
939# error "ARCH_BITS is bogus"
940#endif
941}
942
943
944/**
945 * Atomically Compare and Exchange a pointer value, ordered.
946 *
947 * @returns true if xchg was done.
948 * @returns false if xchg wasn't done.
949 *
950 * @param ppv Pointer to the value to update.
951 * @param pvNew The new value to assigned to *ppv.
952 * @param pvOld The old value to *ppv compare with.
953 *
954 * @remarks This is relatively type safe on GCC platforms.
955 */
956#ifdef __GNUC__
957# define ASMAtomicCmpXchgPtr(ppv, pvNew, pvOld) \
958 __extension__ \
959 ({\
960 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
961 __typeof__(*(ppv)) const pvNewTypeChecked = (pvNew); \
962 __typeof__(*(ppv)) const pvOldTypeChecked = (pvOld); \
963 bool fMacroRet = ASMAtomicCmpXchgPtrVoid((void * volatile *)ppvTypeChecked, \
964 (void *)pvNewTypeChecked, (void *)pvOldTypeChecked); \
965 fMacroRet; \
966 })
967#else
968# define ASMAtomicCmpXchgPtr(ppv, pvNew, pvOld) \
969 ASMAtomicCmpXchgPtrVoid((void * volatile *)(ppv), (void *)(pvNew), (void *)(pvOld))
970#endif
971
972
973/** @def ASMAtomicCmpXchgHandle
974 * Atomically Compare and Exchange a typical IPRT handle value, ordered.
975 *
976 * @param ph Pointer to the value to update.
977 * @param hNew The new value to assigned to *pu.
978 * @param hOld The old value to *pu compare with.
979 * @param fRc Where to store the result.
980 *
981 * @remarks This doesn't currently work for all handles (like RTFILE).
982 */
983#if HC_ARCH_BITS == 32
984# define ASMAtomicCmpXchgHandle(ph, hNew, hOld, fRc) \
985 do { \
986 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
987 (fRc) = ASMAtomicCmpXchgU32((uint32_t volatile *)(ph), (const uint32_t)(hNew), (const uint32_t)(hOld)); \
988 } while (0)
989#elif HC_ARCH_BITS == 64
990# define ASMAtomicCmpXchgHandle(ph, hNew, hOld, fRc) \
991 do { \
992 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
993 (fRc) = ASMAtomicCmpXchgU64((uint64_t volatile *)(ph), (const uint64_t)(hNew), (const uint64_t)(hOld)); \
994 } while (0)
995#else
996# error HC_ARCH_BITS
997#endif
998
999
1000/** @def ASMAtomicCmpXchgSize
1001 * Atomically Compare and Exchange a value which size might differ
1002 * between platforms or compilers, ordered.
1003 *
1004 * @param pu Pointer to the value to update.
1005 * @param uNew The new value to assigned to *pu.
1006 * @param uOld The old value to *pu compare with.
1007 * @param fRc Where to store the result.
1008 */
1009#define ASMAtomicCmpXchgSize(pu, uNew, uOld, fRc) \
1010 do { \
1011 switch (sizeof(*(pu))) { \
1012 case 4: (fRc) = ASMAtomicCmpXchgU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew), (uint32_t)(uOld)); \
1013 break; \
1014 case 8: (fRc) = ASMAtomicCmpXchgU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew), (uint64_t)(uOld)); \
1015 break; \
1016 default: AssertMsgFailed(("ASMAtomicCmpXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
1017 (fRc) = false; \
1018 break; \
1019 } \
1020 } while (0)
1021
1022
1023/**
1024 * Atomically Compare and Exchange an unsigned 32-bit value, additionally
1025 * passes back old value, ordered.
1026 *
1027 * @returns true if xchg was done.
1028 * @returns false if xchg wasn't done.
1029 *
1030 * @param pu32 Pointer to the value to update.
1031 * @param u32New The new value to assigned to *pu32.
1032 * @param u32Old The old value to *pu32 compare with.
1033 * @param pu32Old Pointer store the old value at.
1034 */
1035#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1036DECLASM(bool) ASMAtomicCmpXchgExU32(volatile uint32_t *pu32, const uint32_t u32New, const uint32_t u32Old, uint32_t *pu32Old);
1037#else
1038DECLINLINE(bool) ASMAtomicCmpXchgExU32(volatile uint32_t *pu32, const uint32_t u32New, const uint32_t u32Old, uint32_t *pu32Old)
1039{
1040# if RT_INLINE_ASM_GNU_STYLE
1041 uint8_t u8Ret;
1042 __asm__ __volatile__("lock; cmpxchgl %3, %0\n\t"
1043 "setz %1\n\t"
1044 : "=m" (*pu32),
1045 "=qm" (u8Ret),
1046 "=a" (*pu32Old)
1047 : "r" (u32New),
1048 "a" (u32Old),
1049 "m" (*pu32));
1050 return (bool)u8Ret;
1051
1052# elif RT_INLINE_ASM_USES_INTRIN
1053 return (*pu32Old =_InterlockedCompareExchange((long *)pu32, u32New, u32Old)) == u32Old;
1054
1055# else
1056 uint32_t u32Ret;
1057 __asm
1058 {
1059# ifdef RT_ARCH_AMD64
1060 mov rdx, [pu32]
1061# else
1062 mov edx, [pu32]
1063# endif
1064 mov eax, [u32Old]
1065 mov ecx, [u32New]
1066# ifdef RT_ARCH_AMD64
1067 lock cmpxchg [rdx], ecx
1068 mov rdx, [pu32Old]
1069 mov [rdx], eax
1070# else
1071 lock cmpxchg [edx], ecx
1072 mov edx, [pu32Old]
1073 mov [edx], eax
1074# endif
1075 setz al
1076 movzx eax, al
1077 mov [u32Ret], eax
1078 }
1079 return !!u32Ret;
1080# endif
1081}
1082#endif
1083
1084
1085/**
1086 * Atomically Compare and Exchange a signed 32-bit value, additionally
1087 * passes back old value, ordered.
1088 *
1089 * @returns true if xchg was done.
1090 * @returns false if xchg wasn't done.
1091 *
1092 * @param pi32 Pointer to the value to update.
1093 * @param i32New The new value to assigned to *pi32.
1094 * @param i32Old The old value to *pi32 compare with.
1095 * @param pi32Old Pointer store the old value at.
1096 */
1097DECLINLINE(bool) ASMAtomicCmpXchgExS32(volatile int32_t *pi32, const int32_t i32New, const int32_t i32Old, int32_t *pi32Old)
1098{
1099 return ASMAtomicCmpXchgExU32((volatile uint32_t *)pi32, (uint32_t)i32New, (uint32_t)i32Old, (uint32_t *)pi32Old);
1100}
1101
1102
1103/**
1104 * Atomically Compare and exchange an unsigned 64-bit value, additionally
1105 * passing back old value, ordered.
1106 *
1107 * @returns true if xchg was done.
1108 * @returns false if xchg wasn't done.
1109 *
1110 * @param pu64 Pointer to the 64-bit variable to update.
1111 * @param u64New The 64-bit value to assign to *pu64.
1112 * @param u64Old The value to compare with.
1113 * @param pu64Old Pointer store the old value at.
1114 */
1115#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
1116 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
1117DECLASM(bool) ASMAtomicCmpXchgExU64(volatile uint64_t *pu64, const uint64_t u64New, const uint64_t u64Old, uint64_t *pu64Old);
1118#else
1119DECLINLINE(bool) ASMAtomicCmpXchgExU64(volatile uint64_t *pu64, const uint64_t u64New, const uint64_t u64Old, uint64_t *pu64Old)
1120{
1121# if RT_INLINE_ASM_USES_INTRIN
1122 return (*pu64Old =_InterlockedCompareExchange64((__int64 *)pu64, u64New, u64Old)) == u64Old;
1123
1124# elif defined(RT_ARCH_AMD64)
1125# if RT_INLINE_ASM_GNU_STYLE
1126 uint8_t u8Ret;
1127 __asm__ __volatile__("lock; cmpxchgq %3, %0\n\t"
1128 "setz %1\n\t"
1129 : "=m" (*pu64),
1130 "=qm" (u8Ret),
1131 "=a" (*pu64Old)
1132 : "r" (u64New),
1133 "a" (u64Old),
1134 "m" (*pu64));
1135 return (bool)u8Ret;
1136# else
1137 bool fRet;
1138 __asm
1139 {
1140 mov rdx, [pu32]
1141 mov rax, [u64Old]
1142 mov rcx, [u64New]
1143 lock cmpxchg [rdx], rcx
1144 mov rdx, [pu64Old]
1145 mov [rdx], rax
1146 setz al
1147 mov [fRet], al
1148 }
1149 return fRet;
1150# endif
1151# else /* !RT_ARCH_AMD64 */
1152# if RT_INLINE_ASM_GNU_STYLE
1153 uint64_t u64Ret;
1154# if defined(PIC) || defined(__PIC__)
1155 /* NB: this code uses a memory clobber description, because the clean
1156 * solution with an output value for *pu64 makes gcc run out of registers.
1157 * This will cause suboptimal code, and anyone with a better solution is
1158 * welcome to improve this. */
1159 __asm__ __volatile__("xchgl %%ebx, %1\n\t"
1160 "lock; cmpxchg8b %3\n\t"
1161 "xchgl %%ebx, %1\n\t"
1162 : "=A" (u64Ret)
1163 : "DS" ((uint32_t)u64New),
1164 "c" ((uint32_t)(u64New >> 32)),
1165 "m" (*pu64),
1166 "0" (u64Old)
1167 : "memory" );
1168# else /* !PIC */
1169 __asm__ __volatile__("lock; cmpxchg8b %4\n\t"
1170 : "=A" (u64Ret),
1171 "=m" (*pu64)
1172 : "b" ((uint32_t)u64New),
1173 "c" ((uint32_t)(u64New >> 32)),
1174 "m" (*pu64),
1175 "0" (u64Old));
1176# endif
1177 *pu64Old = u64Ret;
1178 return u64Ret == u64Old;
1179# else
1180 uint32_t u32Ret;
1181 __asm
1182 {
1183 mov ebx, dword ptr [u64New]
1184 mov ecx, dword ptr [u64New + 4]
1185 mov edi, [pu64]
1186 mov eax, dword ptr [u64Old]
1187 mov edx, dword ptr [u64Old + 4]
1188 lock cmpxchg8b [edi]
1189 mov ebx, [pu64Old]
1190 mov [ebx], eax
1191 setz al
1192 movzx eax, al
1193 add ebx, 4
1194 mov [ebx], edx
1195 mov dword ptr [u32Ret], eax
1196 }
1197 return !!u32Ret;
1198# endif
1199# endif /* !RT_ARCH_AMD64 */
1200}
1201#endif
1202
1203
1204/**
1205 * Atomically Compare and exchange a signed 64-bit value, additionally
1206 * passing back old value, ordered.
1207 *
1208 * @returns true if xchg was done.
1209 * @returns false if xchg wasn't done.
1210 *
1211 * @param pi64 Pointer to the 64-bit variable to update.
1212 * @param i64 The 64-bit value to assign to *pu64.
1213 * @param i64Old The value to compare with.
1214 * @param pi64Old Pointer store the old value at.
1215 */
1216DECLINLINE(bool) ASMAtomicCmpXchgExS64(volatile int64_t *pi64, const int64_t i64, const int64_t i64Old, int64_t *pi64Old)
1217{
1218 return ASMAtomicCmpXchgExU64((volatile uint64_t *)pi64, (uint64_t)i64, (uint64_t)i64Old, (uint64_t *)pi64Old);
1219}
1220
1221/** @def ASMAtomicCmpXchgExHandle
1222 * Atomically Compare and Exchange a typical IPRT handle value, ordered.
1223 *
1224 * @param ph Pointer to the value to update.
1225 * @param hNew The new value to assigned to *pu.
1226 * @param hOld The old value to *pu compare with.
1227 * @param fRc Where to store the result.
1228 * @param phOldVal Pointer to where to store the old value.
1229 *
1230 * @remarks This doesn't currently work for all handles (like RTFILE).
1231 */
1232#if HC_ARCH_BITS == 32
1233# define ASMAtomicCmpXchgExHandle(ph, hNew, hOld, fRc, phOldVal) \
1234 do { \
1235 AssertCompile(sizeof(*ph) == sizeof(uint32_t)); \
1236 AssertCompile(sizeof(*phOldVal) == sizeof(uint32_t)); \
1237 (fRc) = ASMAtomicCmpXchgExU32((volatile uint32_t *)(pu), (uint32_t)(uNew), (uint32_t)(uOld), (uint32_t *)(puOldVal)); \
1238 } while (0)
1239#elif HC_ARCH_BITS == 64
1240# define ASMAtomicCmpXchgExHandle(ph, hNew, hOld, fRc, phOldVal) \
1241 do { \
1242 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
1243 AssertCompile(sizeof(*(phOldVal)) == sizeof(uint64_t)); \
1244 (fRc) = ASMAtomicCmpXchgExU64((volatile uint64_t *)(pu), (uint64_t)(uNew), (uint64_t)(uOld), (uint64_t *)(puOldVal)); \
1245 } while (0)
1246#else
1247# error HC_ARCH_BITS
1248#endif
1249
1250
1251/** @def ASMAtomicCmpXchgExSize
1252 * Atomically Compare and Exchange a value which size might differ
1253 * between platforms or compilers. Additionally passes back old value.
1254 *
1255 * @param pu Pointer to the value to update.
1256 * @param uNew The new value to assigned to *pu.
1257 * @param uOld The old value to *pu compare with.
1258 * @param fRc Where to store the result.
1259 * @param puOldVal Pointer to where to store the old value.
1260 */
1261#define ASMAtomicCmpXchgExSize(pu, uNew, uOld, fRc, puOldVal) \
1262 do { \
1263 switch (sizeof(*(pu))) { \
1264 case 4: (fRc) = ASMAtomicCmpXchgExU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew), (uint32_t)(uOld), (uint32_t *)(uOldVal)); \
1265 break; \
1266 case 8: (fRc) = ASMAtomicCmpXchgExU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew), (uint64_t)(uOld), (uint64_t *)(uOldVal)); \
1267 break; \
1268 default: AssertMsgFailed(("ASMAtomicCmpXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
1269 (fRc) = false; \
1270 (uOldVal) = 0; \
1271 break; \
1272 } \
1273 } while (0)
1274
1275
1276/**
1277 * Atomically Compare and Exchange a pointer value, additionally
1278 * passing back old value, ordered.
1279 *
1280 * @returns true if xchg was done.
1281 * @returns false if xchg wasn't done.
1282 *
1283 * @param ppv Pointer to the value to update.
1284 * @param pvNew The new value to assigned to *ppv.
1285 * @param pvOld The old value to *ppv compare with.
1286 * @param ppvOld Pointer store the old value at.
1287 */
1288DECLINLINE(bool) ASMAtomicCmpXchgExPtrVoid(void * volatile *ppv, const void *pvNew, const void *pvOld, void **ppvOld)
1289{
1290#if ARCH_BITS == 32
1291 return ASMAtomicCmpXchgExU32((volatile uint32_t *)(void *)ppv, (uint32_t)pvNew, (uint32_t)pvOld, (uint32_t *)ppvOld);
1292#elif ARCH_BITS == 64
1293 return ASMAtomicCmpXchgExU64((volatile uint64_t *)(void *)ppv, (uint64_t)pvNew, (uint64_t)pvOld, (uint64_t *)ppvOld);
1294#else
1295# error "ARCH_BITS is bogus"
1296#endif
1297}
1298
1299
1300/**
1301 * Atomically Compare and Exchange a pointer value, additionally
1302 * passing back old value, ordered.
1303 *
1304 * @returns true if xchg was done.
1305 * @returns false if xchg wasn't done.
1306 *
1307 * @param ppv Pointer to the value to update.
1308 * @param pvNew The new value to assigned to *ppv.
1309 * @param pvOld The old value to *ppv compare with.
1310 * @param ppvOld Pointer store the old value at.
1311 *
1312 * @remarks This is relatively type safe on GCC platforms.
1313 */
1314#ifdef __GNUC__
1315# define ASMAtomicCmpXchgExPtr(ppv, pvNew, pvOld, ppvOld) \
1316 __extension__ \
1317 ({\
1318 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
1319 __typeof__(*(ppv)) const pvNewTypeChecked = (pvNew); \
1320 __typeof__(*(ppv)) const pvOldTypeChecked = (pvOld); \
1321 __typeof__(*(ppv)) * const ppvOldTypeChecked = (ppvOld); \
1322 bool fMacroRet = ASMAtomicCmpXchgExPtrVoid((void * volatile *)ppvTypeChecked, \
1323 (void *)pvNewTypeChecked, (void *)pvOldTypeChecked, \
1324 (void **)ppvOldTypeChecked); \
1325 fMacroRet; \
1326 })
1327#else
1328# define ASMAtomicCmpXchgExPtr(ppv, pvNew, pvOld, ppvOld) \
1329 ASMAtomicCmpXchgExPtrVoid((void * volatile *)(ppv), (void *)(pvNew), (void *)(pvOld), (void **)(ppvOld))
1330#endif
1331
1332
1333/**
1334 * Serialize Instruction.
1335 */
1336#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1337DECLASM(void) ASMSerializeInstruction(void);
1338#else
1339DECLINLINE(void) ASMSerializeInstruction(void)
1340{
1341# if RT_INLINE_ASM_GNU_STYLE
1342 RTCCUINTREG xAX = 0;
1343# ifdef RT_ARCH_AMD64
1344 __asm__ ("cpuid"
1345 : "=a" (xAX)
1346 : "0" (xAX)
1347 : "rbx", "rcx", "rdx");
1348# elif (defined(PIC) || defined(__PIC__)) && defined(__i386__)
1349 __asm__ ("push %%ebx\n\t"
1350 "cpuid\n\t"
1351 "pop %%ebx\n\t"
1352 : "=a" (xAX)
1353 : "0" (xAX)
1354 : "ecx", "edx");
1355# else
1356 __asm__ ("cpuid"
1357 : "=a" (xAX)
1358 : "0" (xAX)
1359 : "ebx", "ecx", "edx");
1360# endif
1361
1362# elif RT_INLINE_ASM_USES_INTRIN
1363 int aInfo[4];
1364 __cpuid(aInfo, 0);
1365
1366# else
1367 __asm
1368 {
1369 push ebx
1370 xor eax, eax
1371 cpuid
1372 pop ebx
1373 }
1374# endif
1375}
1376#endif
1377
1378
1379/**
1380 * Memory fence, waits for any pending writes and reads to complete.
1381 */
1382DECLINLINE(void) ASMMemoryFence(void)
1383{
1384 /** @todo use mfence? check if all cpus we care for support it. */
1385 uint32_t volatile u32;
1386 ASMAtomicXchgU32(&u32, 0);
1387}
1388
1389
1390/**
1391 * Write fence, waits for any pending writes to complete.
1392 */
1393DECLINLINE(void) ASMWriteFence(void)
1394{
1395 /** @todo use sfence? check if all cpus we care for support it. */
1396 ASMMemoryFence();
1397}
1398
1399
1400/**
1401 * Read fence, waits for any pending reads to complete.
1402 */
1403DECLINLINE(void) ASMReadFence(void)
1404{
1405 /** @todo use lfence? check if all cpus we care for support it. */
1406 ASMMemoryFence();
1407}
1408
1409
1410/**
1411 * Atomically reads an unsigned 8-bit value, ordered.
1412 *
1413 * @returns Current *pu8 value
1414 * @param pu8 Pointer to the 8-bit variable to read.
1415 */
1416DECLINLINE(uint8_t) ASMAtomicReadU8(volatile uint8_t *pu8)
1417{
1418 ASMMemoryFence();
1419 return *pu8; /* byte reads are atomic on x86 */
1420}
1421
1422
1423/**
1424 * Atomically reads an unsigned 8-bit value, unordered.
1425 *
1426 * @returns Current *pu8 value
1427 * @param pu8 Pointer to the 8-bit variable to read.
1428 */
1429DECLINLINE(uint8_t) ASMAtomicUoReadU8(volatile uint8_t *pu8)
1430{
1431 return *pu8; /* byte reads are atomic on x86 */
1432}
1433
1434
1435/**
1436 * Atomically reads a signed 8-bit value, ordered.
1437 *
1438 * @returns Current *pi8 value
1439 * @param pi8 Pointer to the 8-bit variable to read.
1440 */
1441DECLINLINE(int8_t) ASMAtomicReadS8(volatile int8_t *pi8)
1442{
1443 ASMMemoryFence();
1444 return *pi8; /* byte reads are atomic on x86 */
1445}
1446
1447
1448/**
1449 * Atomically reads a signed 8-bit value, unordered.
1450 *
1451 * @returns Current *pi8 value
1452 * @param pi8 Pointer to the 8-bit variable to read.
1453 */
1454DECLINLINE(int8_t) ASMAtomicUoReadS8(volatile int8_t *pi8)
1455{
1456 return *pi8; /* byte reads are atomic on x86 */
1457}
1458
1459
1460/**
1461 * Atomically reads an unsigned 16-bit value, ordered.
1462 *
1463 * @returns Current *pu16 value
1464 * @param pu16 Pointer to the 16-bit variable to read.
1465 */
1466DECLINLINE(uint16_t) ASMAtomicReadU16(volatile uint16_t *pu16)
1467{
1468 ASMMemoryFence();
1469 Assert(!((uintptr_t)pu16 & 1));
1470 return *pu16;
1471}
1472
1473
1474/**
1475 * Atomically reads an unsigned 16-bit value, unordered.
1476 *
1477 * @returns Current *pu16 value
1478 * @param pu16 Pointer to the 16-bit variable to read.
1479 */
1480DECLINLINE(uint16_t) ASMAtomicUoReadU16(volatile uint16_t *pu16)
1481{
1482 Assert(!((uintptr_t)pu16 & 1));
1483 return *pu16;
1484}
1485
1486
1487/**
1488 * Atomically reads a signed 16-bit value, ordered.
1489 *
1490 * @returns Current *pi16 value
1491 * @param pi16 Pointer to the 16-bit variable to read.
1492 */
1493DECLINLINE(int16_t) ASMAtomicReadS16(volatile int16_t *pi16)
1494{
1495 ASMMemoryFence();
1496 Assert(!((uintptr_t)pi16 & 1));
1497 return *pi16;
1498}
1499
1500
1501/**
1502 * Atomically reads a signed 16-bit value, unordered.
1503 *
1504 * @returns Current *pi16 value
1505 * @param pi16 Pointer to the 16-bit variable to read.
1506 */
1507DECLINLINE(int16_t) ASMAtomicUoReadS16(volatile int16_t *pi16)
1508{
1509 Assert(!((uintptr_t)pi16 & 1));
1510 return *pi16;
1511}
1512
1513
1514/**
1515 * Atomically reads an unsigned 32-bit value, ordered.
1516 *
1517 * @returns Current *pu32 value
1518 * @param pu32 Pointer to the 32-bit variable to read.
1519 */
1520DECLINLINE(uint32_t) ASMAtomicReadU32(volatile uint32_t *pu32)
1521{
1522 ASMMemoryFence();
1523 Assert(!((uintptr_t)pu32 & 3));
1524 return *pu32;
1525}
1526
1527
1528/**
1529 * Atomically reads an unsigned 32-bit value, unordered.
1530 *
1531 * @returns Current *pu32 value
1532 * @param pu32 Pointer to the 32-bit variable to read.
1533 */
1534DECLINLINE(uint32_t) ASMAtomicUoReadU32(volatile uint32_t *pu32)
1535{
1536 Assert(!((uintptr_t)pu32 & 3));
1537 return *pu32;
1538}
1539
1540
1541/**
1542 * Atomically reads a signed 32-bit value, ordered.
1543 *
1544 * @returns Current *pi32 value
1545 * @param pi32 Pointer to the 32-bit variable to read.
1546 */
1547DECLINLINE(int32_t) ASMAtomicReadS32(volatile int32_t *pi32)
1548{
1549 ASMMemoryFence();
1550 Assert(!((uintptr_t)pi32 & 3));
1551 return *pi32;
1552}
1553
1554
1555/**
1556 * Atomically reads a signed 32-bit value, unordered.
1557 *
1558 * @returns Current *pi32 value
1559 * @param pi32 Pointer to the 32-bit variable to read.
1560 */
1561DECLINLINE(int32_t) ASMAtomicUoReadS32(volatile int32_t *pi32)
1562{
1563 Assert(!((uintptr_t)pi32 & 3));
1564 return *pi32;
1565}
1566
1567
1568/**
1569 * Atomically reads an unsigned 64-bit value, ordered.
1570 *
1571 * @returns Current *pu64 value
1572 * @param pu64 Pointer to the 64-bit variable to read.
1573 * The memory pointed to must be writable.
1574 * @remark This will fault if the memory is read-only!
1575 */
1576#if (RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)) \
1577 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
1578DECLASM(uint64_t) ASMAtomicReadU64(volatile uint64_t *pu64);
1579#else
1580DECLINLINE(uint64_t) ASMAtomicReadU64(volatile uint64_t *pu64)
1581{
1582 uint64_t u64;
1583# ifdef RT_ARCH_AMD64
1584 Assert(!((uintptr_t)pu64 & 7));
1585/*# if RT_INLINE_ASM_GNU_STYLE
1586 __asm__ __volatile__( "mfence\n\t"
1587 "movq %1, %0\n\t"
1588 : "=r" (u64)
1589 : "m" (*pu64));
1590# else
1591 __asm
1592 {
1593 mfence
1594 mov rdx, [pu64]
1595 mov rax, [rdx]
1596 mov [u64], rax
1597 }
1598# endif*/
1599 ASMMemoryFence();
1600 u64 = *pu64;
1601# else /* !RT_ARCH_AMD64 */
1602# if RT_INLINE_ASM_GNU_STYLE
1603# if defined(PIC) || defined(__PIC__)
1604 uint32_t u32EBX = 0;
1605 Assert(!((uintptr_t)pu64 & 7));
1606 __asm__ __volatile__("xchgl %%ebx, %3\n\t"
1607 "lock; cmpxchg8b (%5)\n\t"
1608 "movl %3, %%ebx\n\t"
1609 : "=A" (u64),
1610# if (__GNUC__ * 100 + __GNUC_MINOR__) >= 403
1611 "+m" (*pu64)
1612# else
1613 "=m" (*pu64)
1614# endif
1615 : "0" (0ULL),
1616 "m" (u32EBX),
1617 "c" (0),
1618 "S" (pu64));
1619# else /* !PIC */
1620 __asm__ __volatile__("lock; cmpxchg8b %1\n\t"
1621 : "=A" (u64),
1622 "+m" (*pu64)
1623 : "0" (0ULL),
1624 "b" (0),
1625 "c" (0));
1626# endif
1627# else
1628 Assert(!((uintptr_t)pu64 & 7));
1629 __asm
1630 {
1631 xor eax, eax
1632 xor edx, edx
1633 mov edi, pu64
1634 xor ecx, ecx
1635 xor ebx, ebx
1636 lock cmpxchg8b [edi]
1637 mov dword ptr [u64], eax
1638 mov dword ptr [u64 + 4], edx
1639 }
1640# endif
1641# endif /* !RT_ARCH_AMD64 */
1642 return u64;
1643}
1644#endif
1645
1646
1647/**
1648 * Atomically reads an unsigned 64-bit value, unordered.
1649 *
1650 * @returns Current *pu64 value
1651 * @param pu64 Pointer to the 64-bit variable to read.
1652 * The memory pointed to must be writable.
1653 * @remark This will fault if the memory is read-only!
1654 */
1655#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
1656 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
1657DECLASM(uint64_t) ASMAtomicUoReadU64(volatile uint64_t *pu64);
1658#else
1659DECLINLINE(uint64_t) ASMAtomicUoReadU64(volatile uint64_t *pu64)
1660{
1661 uint64_t u64;
1662# ifdef RT_ARCH_AMD64
1663 Assert(!((uintptr_t)pu64 & 7));
1664/*# if RT_INLINE_ASM_GNU_STYLE
1665 Assert(!((uintptr_t)pu64 & 7));
1666 __asm__ __volatile__("movq %1, %0\n\t"
1667 : "=r" (u64)
1668 : "m" (*pu64));
1669# else
1670 __asm
1671 {
1672 mov rdx, [pu64]
1673 mov rax, [rdx]
1674 mov [u64], rax
1675 }
1676# endif */
1677 u64 = *pu64;
1678# else /* !RT_ARCH_AMD64 */
1679# if RT_INLINE_ASM_GNU_STYLE
1680# if defined(PIC) || defined(__PIC__)
1681 uint32_t u32EBX = 0;
1682 uint32_t u32Spill;
1683 Assert(!((uintptr_t)pu64 & 7));
1684 __asm__ __volatile__("xor %%eax,%%eax\n\t"
1685 "xor %%ecx,%%ecx\n\t"
1686 "xor %%edx,%%edx\n\t"
1687 "xchgl %%ebx, %3\n\t"
1688 "lock; cmpxchg8b (%4)\n\t"
1689 "movl %3, %%ebx\n\t"
1690 : "=A" (u64),
1691# if (__GNUC__ * 100 + __GNUC_MINOR__) >= 403
1692 "+m" (*pu64),
1693# else
1694 "=m" (*pu64),
1695# endif
1696 "=c" (u32Spill)
1697 : "m" (u32EBX),
1698 "S" (pu64));
1699# else /* !PIC */
1700 __asm__ __volatile__("lock; cmpxchg8b %1\n\t"
1701 : "=A" (u64),
1702 "+m" (*pu64)
1703 : "0" (0ULL),
1704 "b" (0),
1705 "c" (0));
1706# endif
1707# else
1708 Assert(!((uintptr_t)pu64 & 7));
1709 __asm
1710 {
1711 xor eax, eax
1712 xor edx, edx
1713 mov edi, pu64
1714 xor ecx, ecx
1715 xor ebx, ebx
1716 lock cmpxchg8b [edi]
1717 mov dword ptr [u64], eax
1718 mov dword ptr [u64 + 4], edx
1719 }
1720# endif
1721# endif /* !RT_ARCH_AMD64 */
1722 return u64;
1723}
1724#endif
1725
1726
1727/**
1728 * Atomically reads a signed 64-bit value, ordered.
1729 *
1730 * @returns Current *pi64 value
1731 * @param pi64 Pointer to the 64-bit variable to read.
1732 * The memory pointed to must be writable.
1733 * @remark This will fault if the memory is read-only!
1734 */
1735DECLINLINE(int64_t) ASMAtomicReadS64(volatile int64_t *pi64)
1736{
1737 return (int64_t)ASMAtomicReadU64((volatile uint64_t *)pi64);
1738}
1739
1740
1741/**
1742 * Atomically reads a signed 64-bit value, unordered.
1743 *
1744 * @returns Current *pi64 value
1745 * @param pi64 Pointer to the 64-bit variable to read.
1746 * The memory pointed to must be writable.
1747 * @remark This will fault if the memory is read-only!
1748 */
1749DECLINLINE(int64_t) ASMAtomicUoReadS64(volatile int64_t *pi64)
1750{
1751 return (int64_t)ASMAtomicUoReadU64((volatile uint64_t *)pi64);
1752}
1753
1754
1755/**
1756 * Atomically reads a pointer value, ordered.
1757 *
1758 * @returns Current *pv value
1759 * @param ppv Pointer to the pointer variable to read.
1760 *
1761 * @remarks Please use ASMAtomicReadPtrT, it provides better type safety and
1762 * requires less typing (no casts).
1763 */
1764DECLINLINE(void *) ASMAtomicReadPtr(void * volatile *ppv)
1765{
1766#if ARCH_BITS == 32
1767 return (void *)ASMAtomicReadU32((volatile uint32_t *)(void *)ppv);
1768#elif ARCH_BITS == 64
1769 return (void *)ASMAtomicReadU64((volatile uint64_t *)(void *)ppv);
1770#else
1771# error "ARCH_BITS is bogus"
1772#endif
1773}
1774
1775/**
1776 * Convenience macro for avoiding the annoying casting with ASMAtomicReadPtr.
1777 *
1778 * @returns Current *pv value
1779 * @param ppv Pointer to the pointer variable to read.
1780 * @param Type The type of *ppv, sans volatile.
1781 */
1782#ifdef __GNUC__
1783# define ASMAtomicReadPtrT(ppv, Type) \
1784 __extension__ \
1785 ({\
1786 __typeof__(*(ppv)) volatile *ppvTypeChecked = (ppv); \
1787 Type pvTypeChecked = (__typeof__(*(ppv))) ASMAtomicReadPtr((void * volatile *)ppvTypeChecked); \
1788 pvTypeChecked; \
1789 })
1790#else
1791# define ASMAtomicReadPtrT(ppv, Type) \
1792 (Type)ASMAtomicReadPtr((void * volatile *)(ppv))
1793#endif
1794
1795
1796/**
1797 * Atomically reads a pointer value, unordered.
1798 *
1799 * @returns Current *pv value
1800 * @param ppv Pointer to the pointer variable to read.
1801 *
1802 * @remarks Please use ASMAtomicUoReadPtrT, it provides better type safety and
1803 * requires less typing (no casts).
1804 */
1805DECLINLINE(void *) ASMAtomicUoReadPtr(void * volatile *ppv)
1806{
1807#if ARCH_BITS == 32
1808 return (void *)ASMAtomicUoReadU32((volatile uint32_t *)(void *)ppv);
1809#elif ARCH_BITS == 64
1810 return (void *)ASMAtomicUoReadU64((volatile uint64_t *)(void *)ppv);
1811#else
1812# error "ARCH_BITS is bogus"
1813#endif
1814}
1815
1816
1817/**
1818 * Convenience macro for avoiding the annoying casting with ASMAtomicUoReadPtr.
1819 *
1820 * @returns Current *pv value
1821 * @param ppv Pointer to the pointer variable to read.
1822 * @param Type The type of *ppv, sans volatile.
1823 */
1824#ifdef __GNUC__
1825# define ASMAtomicUoReadPtrT(ppv, Type) \
1826 __extension__ \
1827 ({\
1828 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
1829 Type pvTypeChecked = (__typeof__(*(ppv))) ASMAtomicUoReadPtr((void * volatile *)ppvTypeChecked); \
1830 pvTypeChecked; \
1831 })
1832#else
1833# define ASMAtomicUoReadPtrT(ppv, Type) \
1834 (Type)ASMAtomicUoReadPtr((void * volatile *)(ppv))
1835#endif
1836
1837
1838/**
1839 * Atomically reads a boolean value, ordered.
1840 *
1841 * @returns Current *pf value
1842 * @param pf Pointer to the boolean variable to read.
1843 */
1844DECLINLINE(bool) ASMAtomicReadBool(volatile bool *pf)
1845{
1846 ASMMemoryFence();
1847 return *pf; /* byte reads are atomic on x86 */
1848}
1849
1850
1851/**
1852 * Atomically reads a boolean value, unordered.
1853 *
1854 * @returns Current *pf value
1855 * @param pf Pointer to the boolean variable to read.
1856 */
1857DECLINLINE(bool) ASMAtomicUoReadBool(volatile bool *pf)
1858{
1859 return *pf; /* byte reads are atomic on x86 */
1860}
1861
1862
1863/**
1864 * Atomically read a typical IPRT handle value, ordered.
1865 *
1866 * @param ph Pointer to the handle variable to read.
1867 * @param phRes Where to store the result.
1868 *
1869 * @remarks This doesn't currently work for all handles (like RTFILE).
1870 */
1871#if HC_ARCH_BITS == 32
1872# define ASMAtomicReadHandle(ph, phRes) \
1873 do { \
1874 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
1875 AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
1876 *(uint32_t *)(phRes) = ASMAtomicReadU32((uint32_t volatile *)(ph)); \
1877 } while (0)
1878#elif HC_ARCH_BITS == 64
1879# define ASMAtomicReadHandle(ph, phRes) \
1880 do { \
1881 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
1882 AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
1883 *(uint64_t *)(phRes) = ASMAtomicReadU64((uint64_t volatile *)(ph)); \
1884 } while (0)
1885#else
1886# error HC_ARCH_BITS
1887#endif
1888
1889
1890/**
1891 * Atomically read a typical IPRT handle value, unordered.
1892 *
1893 * @param ph Pointer to the handle variable to read.
1894 * @param phRes Where to store the result.
1895 *
1896 * @remarks This doesn't currently work for all handles (like RTFILE).
1897 */
1898#if HC_ARCH_BITS == 32
1899# define ASMAtomicUoReadHandle(ph, phRes) \
1900 do { \
1901 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
1902 AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
1903 *(uint32_t *)(phRes) = ASMAtomicUoReadU32((uint32_t volatile *)(ph)); \
1904 } while (0)
1905#elif HC_ARCH_BITS == 64
1906# define ASMAtomicUoReadHandle(ph, phRes) \
1907 do { \
1908 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
1909 AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
1910 *(uint64_t *)(phRes) = ASMAtomicUoReadU64((uint64_t volatile *)(ph)); \
1911 } while (0)
1912#else
1913# error HC_ARCH_BITS
1914#endif
1915
1916
1917/**
1918 * Atomically read a value which size might differ
1919 * between platforms or compilers, ordered.
1920 *
1921 * @param pu Pointer to the variable to update.
1922 * @param puRes Where to store the result.
1923 */
1924#define ASMAtomicReadSize(pu, puRes) \
1925 do { \
1926 switch (sizeof(*(pu))) { \
1927 case 1: *(uint8_t *)(puRes) = ASMAtomicReadU8( (volatile uint8_t *)(void *)(pu)); break; \
1928 case 2: *(uint16_t *)(puRes) = ASMAtomicReadU16((volatile uint16_t *)(void *)(pu)); break; \
1929 case 4: *(uint32_t *)(puRes) = ASMAtomicReadU32((volatile uint32_t *)(void *)(pu)); break; \
1930 case 8: *(uint64_t *)(puRes) = ASMAtomicReadU64((volatile uint64_t *)(void *)(pu)); break; \
1931 default: AssertMsgFailed(("ASMAtomicReadSize: size %d is not supported\n", sizeof(*(pu)))); \
1932 } \
1933 } while (0)
1934
1935
1936/**
1937 * Atomically read a value which size might differ
1938 * between platforms or compilers, unordered.
1939 *
1940 * @param pu Pointer to the variable to read.
1941 * @param puRes Where to store the result.
1942 */
1943#define ASMAtomicUoReadSize(pu, puRes) \
1944 do { \
1945 switch (sizeof(*(pu))) { \
1946 case 1: *(uint8_t *)(puRes) = ASMAtomicUoReadU8( (volatile uint8_t *)(void *)(pu)); break; \
1947 case 2: *(uint16_t *)(puRes) = ASMAtomicUoReadU16((volatile uint16_t *)(void *)(pu)); break; \
1948 case 4: *(uint32_t *)(puRes) = ASMAtomicUoReadU32((volatile uint32_t *)(void *)(pu)); break; \
1949 case 8: *(uint64_t *)(puRes) = ASMAtomicUoReadU64((volatile uint64_t *)(void *)(pu)); break; \
1950 default: AssertMsgFailed(("ASMAtomicReadSize: size %d is not supported\n", sizeof(*(pu)))); \
1951 } \
1952 } while (0)
1953
1954
1955/**
1956 * Atomically writes an unsigned 8-bit value, ordered.
1957 *
1958 * @param pu8 Pointer to the 8-bit variable.
1959 * @param u8 The 8-bit value to assign to *pu8.
1960 */
1961DECLINLINE(void) ASMAtomicWriteU8(volatile uint8_t *pu8, uint8_t u8)
1962{
1963 ASMAtomicXchgU8(pu8, u8);
1964}
1965
1966
1967/**
1968 * Atomically writes an unsigned 8-bit value, unordered.
1969 *
1970 * @param pu8 Pointer to the 8-bit variable.
1971 * @param u8 The 8-bit value to assign to *pu8.
1972 */
1973DECLINLINE(void) ASMAtomicUoWriteU8(volatile uint8_t *pu8, uint8_t u8)
1974{
1975 *pu8 = u8; /* byte writes are atomic on x86 */
1976}
1977
1978
1979/**
1980 * Atomically writes a signed 8-bit value, ordered.
1981 *
1982 * @param pi8 Pointer to the 8-bit variable to read.
1983 * @param i8 The 8-bit value to assign to *pi8.
1984 */
1985DECLINLINE(void) ASMAtomicWriteS8(volatile int8_t *pi8, int8_t i8)
1986{
1987 ASMAtomicXchgS8(pi8, i8);
1988}
1989
1990
1991/**
1992 * Atomically writes a signed 8-bit value, unordered.
1993 *
1994 * @param pi8 Pointer to the 8-bit variable to read.
1995 * @param i8 The 8-bit value to assign to *pi8.
1996 */
1997DECLINLINE(void) ASMAtomicUoWriteS8(volatile int8_t *pi8, int8_t i8)
1998{
1999 *pi8 = i8; /* byte writes are atomic on x86 */
2000}
2001
2002
2003/**
2004 * Atomically writes an unsigned 16-bit value, ordered.
2005 *
2006 * @param pu16 Pointer to the 16-bit variable.
2007 * @param u16 The 16-bit value to assign to *pu16.
2008 */
2009DECLINLINE(void) ASMAtomicWriteU16(volatile uint16_t *pu16, uint16_t u16)
2010{
2011 ASMAtomicXchgU16(pu16, u16);
2012}
2013
2014
2015/**
2016 * Atomically writes an unsigned 16-bit value, unordered.
2017 *
2018 * @param pu16 Pointer to the 16-bit variable.
2019 * @param u16 The 16-bit value to assign to *pu16.
2020 */
2021DECLINLINE(void) ASMAtomicUoWriteU16(volatile uint16_t *pu16, uint16_t u16)
2022{
2023 Assert(!((uintptr_t)pu16 & 1));
2024 *pu16 = u16;
2025}
2026
2027
2028/**
2029 * Atomically writes a signed 16-bit value, ordered.
2030 *
2031 * @param pi16 Pointer to the 16-bit variable to read.
2032 * @param i16 The 16-bit value to assign to *pi16.
2033 */
2034DECLINLINE(void) ASMAtomicWriteS16(volatile int16_t *pi16, int16_t i16)
2035{
2036 ASMAtomicXchgS16(pi16, i16);
2037}
2038
2039
2040/**
2041 * Atomically writes a signed 16-bit value, unordered.
2042 *
2043 * @param pi16 Pointer to the 16-bit variable to read.
2044 * @param i16 The 16-bit value to assign to *pi16.
2045 */
2046DECLINLINE(void) ASMAtomicUoWriteS16(volatile int16_t *pi16, int16_t i16)
2047{
2048 Assert(!((uintptr_t)pi16 & 1));
2049 *pi16 = i16;
2050}
2051
2052
2053/**
2054 * Atomically writes an unsigned 32-bit value, ordered.
2055 *
2056 * @param pu32 Pointer to the 32-bit variable.
2057 * @param u32 The 32-bit value to assign to *pu32.
2058 */
2059DECLINLINE(void) ASMAtomicWriteU32(volatile uint32_t *pu32, uint32_t u32)
2060{
2061 ASMAtomicXchgU32(pu32, u32);
2062}
2063
2064
2065/**
2066 * Atomically writes an unsigned 32-bit value, unordered.
2067 *
2068 * @param pu32 Pointer to the 32-bit variable.
2069 * @param u32 The 32-bit value to assign to *pu32.
2070 */
2071DECLINLINE(void) ASMAtomicUoWriteU32(volatile uint32_t *pu32, uint32_t u32)
2072{
2073 Assert(!((uintptr_t)pu32 & 3));
2074 *pu32 = u32;
2075}
2076
2077
2078/**
2079 * Atomically writes a signed 32-bit value, ordered.
2080 *
2081 * @param pi32 Pointer to the 32-bit variable to read.
2082 * @param i32 The 32-bit value to assign to *pi32.
2083 */
2084DECLINLINE(void) ASMAtomicWriteS32(volatile int32_t *pi32, int32_t i32)
2085{
2086 ASMAtomicXchgS32(pi32, i32);
2087}
2088
2089
2090/**
2091 * Atomically writes a signed 32-bit value, unordered.
2092 *
2093 * @param pi32 Pointer to the 32-bit variable to read.
2094 * @param i32 The 32-bit value to assign to *pi32.
2095 */
2096DECLINLINE(void) ASMAtomicUoWriteS32(volatile int32_t *pi32, int32_t i32)
2097{
2098 Assert(!((uintptr_t)pi32 & 3));
2099 *pi32 = i32;
2100}
2101
2102
2103/**
2104 * Atomically writes an unsigned 64-bit value, ordered.
2105 *
2106 * @param pu64 Pointer to the 64-bit variable.
2107 * @param u64 The 64-bit value to assign to *pu64.
2108 */
2109DECLINLINE(void) ASMAtomicWriteU64(volatile uint64_t *pu64, uint64_t u64)
2110{
2111 ASMAtomicXchgU64(pu64, u64);
2112}
2113
2114
2115/**
2116 * Atomically writes an unsigned 64-bit value, unordered.
2117 *
2118 * @param pu64 Pointer to the 64-bit variable.
2119 * @param u64 The 64-bit value to assign to *pu64.
2120 */
2121DECLINLINE(void) ASMAtomicUoWriteU64(volatile uint64_t *pu64, uint64_t u64)
2122{
2123 Assert(!((uintptr_t)pu64 & 7));
2124#if ARCH_BITS == 64
2125 *pu64 = u64;
2126#else
2127 ASMAtomicXchgU64(pu64, u64);
2128#endif
2129}
2130
2131
2132/**
2133 * Atomically writes a signed 64-bit value, ordered.
2134 *
2135 * @param pi64 Pointer to the 64-bit variable.
2136 * @param i64 The 64-bit value to assign to *pi64.
2137 */
2138DECLINLINE(void) ASMAtomicWriteS64(volatile int64_t *pi64, int64_t i64)
2139{
2140 ASMAtomicXchgS64(pi64, i64);
2141}
2142
2143
2144/**
2145 * Atomically writes a signed 64-bit value, unordered.
2146 *
2147 * @param pi64 Pointer to the 64-bit variable.
2148 * @param i64 The 64-bit value to assign to *pi64.
2149 */
2150DECLINLINE(void) ASMAtomicUoWriteS64(volatile int64_t *pi64, int64_t i64)
2151{
2152 Assert(!((uintptr_t)pi64 & 7));
2153#if ARCH_BITS == 64
2154 *pi64 = i64;
2155#else
2156 ASMAtomicXchgS64(pi64, i64);
2157#endif
2158}
2159
2160
2161/**
2162 * Atomically writes a boolean value, unordered.
2163 *
2164 * @param pf Pointer to the boolean variable.
2165 * @param f The boolean value to assign to *pf.
2166 */
2167DECLINLINE(void) ASMAtomicWriteBool(volatile bool *pf, bool f)
2168{
2169 ASMAtomicWriteU8((uint8_t volatile *)pf, f);
2170}
2171
2172
2173/**
2174 * Atomically writes a boolean value, unordered.
2175 *
2176 * @param pf Pointer to the boolean variable.
2177 * @param f The boolean value to assign to *pf.
2178 */
2179DECLINLINE(void) ASMAtomicUoWriteBool(volatile bool *pf, bool f)
2180{
2181 *pf = f; /* byte writes are atomic on x86 */
2182}
2183
2184
2185/**
2186 * Atomically writes a pointer value, ordered.
2187 *
2188 * @param ppv Pointer to the pointer variable.
2189 * @param pv The pointer value to assign to *ppv.
2190 */
2191DECLINLINE(void) ASMAtomicWritePtrVoid(void * volatile *ppv, const void *pv)
2192{
2193#if ARCH_BITS == 32
2194 ASMAtomicWriteU32((volatile uint32_t *)(void *)ppv, (uint32_t)pv);
2195#elif ARCH_BITS == 64
2196 ASMAtomicWriteU64((volatile uint64_t *)(void *)ppv, (uint64_t)pv);
2197#else
2198# error "ARCH_BITS is bogus"
2199#endif
2200}
2201
2202
2203/**
2204 * Atomically writes a pointer value, ordered.
2205 *
2206 * @param ppv Pointer to the pointer variable.
2207 * @param pv The pointer value to assign to *ppv. If NULL use
2208 * ASMAtomicWriteNullPtr or you'll land in trouble.
2209 *
2210 * @remarks This is relatively type safe on GCC platforms when @a pv isn't
2211 * NULL.
2212 */
2213#ifdef __GNUC__
2214# define ASMAtomicWritePtr(ppv, pv) \
2215 do \
2216 { \
2217 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
2218 __typeof__(*(ppv)) const pvTypeChecked = (pv); \
2219 \
2220 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2221 AssertCompile(sizeof(pv) == sizeof(void *)); \
2222 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2223 \
2224 ASMAtomicWritePtrVoid((void * volatile *)(ppvTypeChecked), (void *)(pvTypeChecked)); \
2225 } while (0)
2226#else
2227# define ASMAtomicWritePtr(ppv, pv) \
2228 do \
2229 { \
2230 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2231 AssertCompile(sizeof(pv) == sizeof(void *)); \
2232 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2233 \
2234 ASMAtomicWritePtrVoid((void * volatile *)(ppv), (void *)(pv)); \
2235 } while (0)
2236#endif
2237
2238
2239/**
2240 * Atomically sets a pointer to NULL, ordered.
2241 *
2242 * @param ppv Pointer to the pointer variable that should be set to NULL.
2243 *
2244 * @remarks This is relatively type safe on GCC platforms.
2245 */
2246#ifdef __GNUC__
2247# define ASMAtomicWriteNullPtr(ppv) \
2248 do \
2249 { \
2250 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
2251 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2252 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2253 ASMAtomicWritePtrVoid((void * volatile *)(ppvTypeChecked), NULL); \
2254 } while (0)
2255#else
2256# define ASMAtomicWriteNullPtr(ppv) \
2257 do \
2258 { \
2259 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2260 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2261 ASMAtomicWritePtrVoid((void * volatile *)(ppv), NULL); \
2262 } while (0)
2263#endif
2264
2265
2266/**
2267 * Atomically writes a pointer value, unordered.
2268 *
2269 * @returns Current *pv value
2270 * @param ppv Pointer to the pointer variable.
2271 * @param pv The pointer value to assign to *ppv. If NULL use
2272 * ASMAtomicUoWriteNullPtr or you'll land in trouble.
2273 *
2274 * @remarks This is relatively type safe on GCC platforms when @a pv isn't
2275 * NULL.
2276 */
2277#ifdef __GNUC__
2278# define ASMAtomicUoWritePtr(ppv, pv) \
2279 do \
2280 { \
2281 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
2282 __typeof__(*(ppv)) const pvTypeChecked = (pv); \
2283 \
2284 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2285 AssertCompile(sizeof(pv) == sizeof(void *)); \
2286 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2287 \
2288 *(ppvTypeChecked) = pvTypeChecked; \
2289 } while (0)
2290#else
2291# define ASMAtomicUoWritePtr(ppv, pv) \
2292 do \
2293 { \
2294 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2295 AssertCompile(sizeof(pv) == sizeof(void *)); \
2296 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2297 *(ppv) = pv; \
2298 } while (0)
2299#endif
2300
2301
2302/**
2303 * Atomically sets a pointer to NULL, unordered.
2304 *
2305 * @param ppv Pointer to the pointer variable that should be set to NULL.
2306 *
2307 * @remarks This is relatively type safe on GCC platforms.
2308 */
2309#ifdef __GNUC__
2310# define ASMAtomicUoWriteNullPtr(ppv) \
2311 do \
2312 { \
2313 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
2314 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2315 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2316 *(ppvTypeChecked) = NULL; \
2317 } while (0)
2318#else
2319# define ASMAtomicUoWriteNullPtr(ppv) \
2320 do \
2321 { \
2322 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2323 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2324 *(ppv) = NULL; \
2325 } while (0)
2326#endif
2327
2328
2329/**
2330 * Atomically write a typical IPRT handle value, ordered.
2331 *
2332 * @param ph Pointer to the variable to update.
2333 * @param hNew The value to assign to *ph.
2334 *
2335 * @remarks This doesn't currently work for all handles (like RTFILE).
2336 */
2337#if HC_ARCH_BITS == 32
2338# define ASMAtomicWriteHandle(ph, hNew) \
2339 do { \
2340 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
2341 ASMAtomicWriteU32((uint32_t volatile *)(ph), (const uint32_t)(hNew)); \
2342 } while (0)
2343#elif HC_ARCH_BITS == 64
2344# define ASMAtomicWriteHandle(ph, hNew) \
2345 do { \
2346 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
2347 ASMAtomicWriteU64((uint64_t volatile *)(ph), (const uint64_t)(hNew)); \
2348 } while (0)
2349#else
2350# error HC_ARCH_BITS
2351#endif
2352
2353
2354/**
2355 * Atomically write a typical IPRT handle value, unordered.
2356 *
2357 * @param ph Pointer to the variable to update.
2358 * @param hNew The value to assign to *ph.
2359 *
2360 * @remarks This doesn't currently work for all handles (like RTFILE).
2361 */
2362#if HC_ARCH_BITS == 32
2363# define ASMAtomicUoWriteHandle(ph, hNew) \
2364 do { \
2365 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
2366 ASMAtomicUoWriteU32((uint32_t volatile *)(ph), (const uint32_t)hNew); \
2367 } while (0)
2368#elif HC_ARCH_BITS == 64
2369# define ASMAtomicUoWriteHandle(ph, hNew) \
2370 do { \
2371 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
2372 ASMAtomicUoWriteU64((uint64_t volatile *)(ph), (const uint64_t)hNew); \
2373 } while (0)
2374#else
2375# error HC_ARCH_BITS
2376#endif
2377
2378
2379/**
2380 * Atomically write a value which size might differ
2381 * between platforms or compilers, ordered.
2382 *
2383 * @param pu Pointer to the variable to update.
2384 * @param uNew The value to assign to *pu.
2385 */
2386#define ASMAtomicWriteSize(pu, uNew) \
2387 do { \
2388 switch (sizeof(*(pu))) { \
2389 case 1: ASMAtomicWriteU8( (volatile uint8_t *)(void *)(pu), (uint8_t )(uNew)); break; \
2390 case 2: ASMAtomicWriteU16((volatile uint16_t *)(void *)(pu), (uint16_t)(uNew)); break; \
2391 case 4: ASMAtomicWriteU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew)); break; \
2392 case 8: ASMAtomicWriteU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew)); break; \
2393 default: AssertMsgFailed(("ASMAtomicWriteSize: size %d is not supported\n", sizeof(*(pu)))); \
2394 } \
2395 } while (0)
2396
2397/**
2398 * Atomically write a value which size might differ
2399 * between platforms or compilers, unordered.
2400 *
2401 * @param pu Pointer to the variable to update.
2402 * @param uNew The value to assign to *pu.
2403 */
2404#define ASMAtomicUoWriteSize(pu, uNew) \
2405 do { \
2406 switch (sizeof(*(pu))) { \
2407 case 1: ASMAtomicUoWriteU8( (volatile uint8_t *)(void *)(pu), (uint8_t )(uNew)); break; \
2408 case 2: ASMAtomicUoWriteU16((volatile uint16_t *)(void *)(pu), (uint16_t)(uNew)); break; \
2409 case 4: ASMAtomicUoWriteU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew)); break; \
2410 case 8: ASMAtomicUoWriteU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew)); break; \
2411 default: AssertMsgFailed(("ASMAtomicWriteSize: size %d is not supported\n", sizeof(*(pu)))); \
2412 } \
2413 } while (0)
2414
2415
2416
2417/**
2418 * Atomically exchanges and adds to a 32-bit value, ordered.
2419 *
2420 * @returns The old value.
2421 * @param pu32 Pointer to the value.
2422 * @param u32 Number to add.
2423 */
2424#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2425DECLASM(uint32_t) ASMAtomicAddU32(uint32_t volatile *pu32, uint32_t u32);
2426#else
2427DECLINLINE(uint32_t) ASMAtomicAddU32(uint32_t volatile *pu32, uint32_t u32)
2428{
2429# if RT_INLINE_ASM_USES_INTRIN
2430 u32 = _InterlockedExchangeAdd((long *)pu32, u32);
2431 return u32;
2432
2433# elif RT_INLINE_ASM_GNU_STYLE
2434 __asm__ __volatile__("lock; xaddl %0, %1\n\t"
2435 : "=r" (u32),
2436 "=m" (*pu32)
2437 : "0" (u32),
2438 "m" (*pu32)
2439 : "memory");
2440 return u32;
2441# else
2442 __asm
2443 {
2444 mov eax, [u32]
2445# ifdef RT_ARCH_AMD64
2446 mov rdx, [pu32]
2447 lock xadd [rdx], eax
2448# else
2449 mov edx, [pu32]
2450 lock xadd [edx], eax
2451# endif
2452 mov [u32], eax
2453 }
2454 return u32;
2455# endif
2456}
2457#endif
2458
2459
2460/**
2461 * Atomically exchanges and adds to a signed 32-bit value, ordered.
2462 *
2463 * @returns The old value.
2464 * @param pi32 Pointer to the value.
2465 * @param i32 Number to add.
2466 */
2467DECLINLINE(int32_t) ASMAtomicAddS32(int32_t volatile *pi32, int32_t i32)
2468{
2469 return (int32_t)ASMAtomicAddU32((uint32_t volatile *)pi32, (uint32_t)i32);
2470}
2471
2472
2473/**
2474 * Atomically exchanges and adds to a 64-bit value, ordered.
2475 *
2476 * @returns The old value.
2477 * @param pu64 Pointer to the value.
2478 * @param u64 Number to add.
2479 */
2480#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2481DECLASM(uint64_t) ASMAtomicAddU64(uint64_t volatile *pu64, uint64_t u64);
2482#else
2483DECLINLINE(uint64_t) ASMAtomicAddU64(uint64_t volatile *pu64, uint64_t u64)
2484{
2485# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
2486 u64 = _InterlockedExchangeAdd64((__int64 *)pu64, u64);
2487 return u64;
2488
2489# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
2490 __asm__ __volatile__("lock; xaddq %0, %1\n\t"
2491 : "=r" (u64),
2492 "=m" (*pu64)
2493 : "0" (u64),
2494 "m" (*pu64)
2495 : "memory");
2496 return u64;
2497# else
2498 uint64_t u64Old;
2499 for (;;)
2500 {
2501 uint64_t u64New;
2502 u64Old = ASMAtomicUoReadU64(pu64);
2503 u64New = u64Old + u64;
2504 if (ASMAtomicCmpXchgU64(pu64, u64New, u64Old))
2505 break;
2506 ASMNopPause();
2507 }
2508 return u64Old;
2509# endif
2510}
2511#endif
2512
2513
2514/**
2515 * Atomically exchanges and adds to a signed 64-bit value, ordered.
2516 *
2517 * @returns The old value.
2518 * @param pi64 Pointer to the value.
2519 * @param i64 Number to add.
2520 */
2521DECLINLINE(int64_t) ASMAtomicAddS64(int64_t volatile *pi64, int64_t i64)
2522{
2523 return (int64_t)ASMAtomicAddU64((uint64_t volatile *)pi64, (uint64_t)i64);
2524}
2525
2526
2527/**
2528 * Atomically exchanges and adds a value which size might differ between
2529 * platforms or compilers, ordered.
2530 *
2531 * @param pu Pointer to the variable to update.
2532 * @param uNew The value to add to *pu.
2533 * @param puOld Where to store the old value.
2534 */
2535#define ASMAtomicAddSize(pu, uNew, puOld) \
2536 do { \
2537 switch (sizeof(*(pu))) { \
2538 case 4: *(uint32_t *)(puOld) = ASMAtomicAddU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew)); break; \
2539 case 8: *(uint64_t *)(puOld) = ASMAtomicAddU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew)); break; \
2540 default: AssertMsgFailed(("ASMAtomicAddSize: size %d is not supported\n", sizeof(*(pu)))); \
2541 } \
2542 } while (0)
2543
2544
2545/**
2546 * Atomically exchanges and subtracts to an unsigned 32-bit value, ordered.
2547 *
2548 * @returns The old value.
2549 * @param pu32 Pointer to the value.
2550 * @param u32 Number to subtract.
2551 */
2552DECLINLINE(uint32_t) ASMAtomicSubU32(uint32_t volatile *pu32, uint32_t u32)
2553{
2554 return ASMAtomicAddU32(pu32, (uint32_t)-(int32_t)u32);
2555}
2556
2557
2558/**
2559 * Atomically exchanges and subtracts to a signed 32-bit value, ordered.
2560 *
2561 * @returns The old value.
2562 * @param pi32 Pointer to the value.
2563 * @param i32 Number to subtract.
2564 */
2565DECLINLINE(int32_t) ASMAtomicSubS32(int32_t volatile *pi32, int32_t i32)
2566{
2567 return (int32_t)ASMAtomicAddU32((uint32_t volatile *)pi32, (uint32_t)-i32);
2568}
2569
2570
2571/**
2572 * Atomically exchanges and subtracts to an unsigned 64-bit value, ordered.
2573 *
2574 * @returns The old value.
2575 * @param pu64 Pointer to the value.
2576 * @param u64 Number to subtract.
2577 */
2578DECLINLINE(uint64_t) ASMAtomicSubU64(uint64_t volatile *pu64, uint64_t u64)
2579{
2580 return ASMAtomicAddU64(pu64, (uint64_t)-(int64_t)u64);
2581}
2582
2583
2584/**
2585 * Atomically exchanges and subtracts to a signed 64-bit value, ordered.
2586 *
2587 * @returns The old value.
2588 * @param pi64 Pointer to the value.
2589 * @param i64 Number to subtract.
2590 */
2591DECLINLINE(int64_t) ASMAtomicSubS64(int64_t volatile *pi64, int64_t i64)
2592{
2593 return (int64_t)ASMAtomicAddU64((uint64_t volatile *)pi64, (uint64_t)-i64);
2594}
2595
2596/**
2597 * Atomically exchanges and subtracts a value which size might differ between
2598 * platforms or compilers, ordered.
2599 *
2600 * @param pu Pointer to the variable to update.
2601 * @param uNew The value to subtract to *pu.
2602 * @param puOld Where to store the old value.
2603 */
2604#define ASMAtomicSubSize(pu, uNew, puOld) \
2605 do { \
2606 switch (sizeof(*(pu))) { \
2607 case 4: *(uint32_t *)(puOld) = ASMAtomicSubU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew)); break; \
2608 case 8: *(uint64_t *)(puOld) = ASMAtomicSubU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew)); break; \
2609 default: AssertMsgFailed(("ASMAtomicSubSize: size %d is not supported\n", sizeof(*(pu)))); \
2610 } \
2611 } while (0)
2612
2613
2614/**
2615 * Atomically increment a 32-bit value, ordered.
2616 *
2617 * @returns The new value.
2618 * @param pu32 Pointer to the value to increment.
2619 */
2620#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2621DECLASM(uint32_t) ASMAtomicIncU32(uint32_t volatile *pu32);
2622#else
2623DECLINLINE(uint32_t) ASMAtomicIncU32(uint32_t volatile *pu32)
2624{
2625 uint32_t u32;
2626# if RT_INLINE_ASM_USES_INTRIN
2627 u32 = _InterlockedIncrement((long *)pu32);
2628 return u32;
2629
2630# elif RT_INLINE_ASM_GNU_STYLE
2631 __asm__ __volatile__("lock; xaddl %0, %1\n\t"
2632 : "=r" (u32),
2633 "=m" (*pu32)
2634 : "0" (1),
2635 "m" (*pu32)
2636 : "memory");
2637 return u32+1;
2638# else
2639 __asm
2640 {
2641 mov eax, 1
2642# ifdef RT_ARCH_AMD64
2643 mov rdx, [pu32]
2644 lock xadd [rdx], eax
2645# else
2646 mov edx, [pu32]
2647 lock xadd [edx], eax
2648# endif
2649 mov u32, eax
2650 }
2651 return u32+1;
2652# endif
2653}
2654#endif
2655
2656
2657/**
2658 * Atomically increment a signed 32-bit value, ordered.
2659 *
2660 * @returns The new value.
2661 * @param pi32 Pointer to the value to increment.
2662 */
2663DECLINLINE(int32_t) ASMAtomicIncS32(int32_t volatile *pi32)
2664{
2665 return (int32_t)ASMAtomicIncU32((uint32_t volatile *)pi32);
2666}
2667
2668
2669/**
2670 * Atomically increment a 64-bit value, ordered.
2671 *
2672 * @returns The new value.
2673 * @param pu64 Pointer to the value to increment.
2674 */
2675#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2676DECLASM(uint64_t) ASMAtomicIncU64(uint64_t volatile *pu64);
2677#else
2678DECLINLINE(uint64_t) ASMAtomicIncU64(uint64_t volatile *pu64)
2679{
2680# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
2681 uint64_t u64;
2682 u64 = _InterlockedIncrement64((__int64 *)pu64);
2683 return u64;
2684
2685# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
2686 uint64_t u64;
2687 __asm__ __volatile__("lock; xaddq %0, %1\n\t"
2688 : "=r" (u64),
2689 "=m" (*pu64)
2690 : "0" (1),
2691 "m" (*pu64)
2692 : "memory");
2693 return u64 + 1;
2694# else
2695 return ASMAtomicAddU64(pu64, 1) + 1;
2696# endif
2697}
2698#endif
2699
2700
2701/**
2702 * Atomically increment a signed 64-bit value, ordered.
2703 *
2704 * @returns The new value.
2705 * @param pi64 Pointer to the value to increment.
2706 */
2707DECLINLINE(int64_t) ASMAtomicIncS64(int64_t volatile *pi64)
2708{
2709 return (int64_t)ASMAtomicIncU64((uint64_t volatile *)pi64);
2710}
2711
2712
2713/**
2714 * Atomically decrement an unsigned 32-bit value, ordered.
2715 *
2716 * @returns The new value.
2717 * @param pu32 Pointer to the value to decrement.
2718 */
2719#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2720DECLASM(uint32_t) ASMAtomicDecU32(uint32_t volatile *pu32);
2721#else
2722DECLINLINE(uint32_t) ASMAtomicDecU32(uint32_t volatile *pu32)
2723{
2724 uint32_t u32;
2725# if RT_INLINE_ASM_USES_INTRIN
2726 u32 = _InterlockedDecrement((long *)pu32);
2727 return u32;
2728
2729# elif RT_INLINE_ASM_GNU_STYLE
2730 __asm__ __volatile__("lock; xaddl %0, %1\n\t"
2731 : "=r" (u32),
2732 "=m" (*pu32)
2733 : "0" (-1),
2734 "m" (*pu32)
2735 : "memory");
2736 return u32-1;
2737# else
2738 __asm
2739 {
2740 mov eax, -1
2741# ifdef RT_ARCH_AMD64
2742 mov rdx, [pu32]
2743 lock xadd [rdx], eax
2744# else
2745 mov edx, [pu32]
2746 lock xadd [edx], eax
2747# endif
2748 mov u32, eax
2749 }
2750 return u32-1;
2751# endif
2752}
2753#endif
2754
2755
2756/**
2757 * Atomically decrement a signed 32-bit value, ordered.
2758 *
2759 * @returns The new value.
2760 * @param pi32 Pointer to the value to decrement.
2761 */
2762DECLINLINE(int32_t) ASMAtomicDecS32(int32_t volatile *pi32)
2763{
2764 return (int32_t)ASMAtomicDecU32((uint32_t volatile *)pi32);
2765}
2766
2767
2768/**
2769 * Atomically decrement an unsigned 64-bit value, ordered.
2770 *
2771 * @returns The new value.
2772 * @param pu64 Pointer to the value to decrement.
2773 */
2774#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2775DECLASM(uint64_t) ASMAtomicDecU64(uint64_t volatile *pu64);
2776#else
2777DECLINLINE(uint64_t) ASMAtomicDecU64(uint64_t volatile *pu64)
2778{
2779# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
2780 uint64_t u64 = _InterlockedDecrement64((__int64 volatile *)pu64);
2781 return u64;
2782
2783# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
2784 uint64_t u64;
2785 __asm__ __volatile__("lock; xaddq %q0, %1\n\t"
2786 : "=r" (u64),
2787 "=m" (*pu64)
2788 : "0" (~(uint64_t)0),
2789 "m" (*pu64)
2790 : "memory");
2791 return u64-1;
2792# else
2793 return ASMAtomicAddU64(pu64, UINT64_MAX) - 1;
2794# endif
2795}
2796#endif
2797
2798
2799/**
2800 * Atomically decrement a signed 64-bit value, ordered.
2801 *
2802 * @returns The new value.
2803 * @param pi64 Pointer to the value to decrement.
2804 */
2805DECLINLINE(int64_t) ASMAtomicDecS64(int64_t volatile *pi64)
2806{
2807 return (int64_t)ASMAtomicDecU64((uint64_t volatile *)pi64);
2808}
2809
2810
2811/**
2812 * Atomically Or an unsigned 32-bit value, ordered.
2813 *
2814 * @param pu32 Pointer to the pointer variable to OR u32 with.
2815 * @param u32 The value to OR *pu32 with.
2816 */
2817#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2818DECLASM(void) ASMAtomicOrU32(uint32_t volatile *pu32, uint32_t u32);
2819#else
2820DECLINLINE(void) ASMAtomicOrU32(uint32_t volatile *pu32, uint32_t u32)
2821{
2822# if RT_INLINE_ASM_USES_INTRIN
2823 _InterlockedOr((long volatile *)pu32, (long)u32);
2824
2825# elif RT_INLINE_ASM_GNU_STYLE
2826 __asm__ __volatile__("lock; orl %1, %0\n\t"
2827 : "=m" (*pu32)
2828 : "ir" (u32),
2829 "m" (*pu32));
2830# else
2831 __asm
2832 {
2833 mov eax, [u32]
2834# ifdef RT_ARCH_AMD64
2835 mov rdx, [pu32]
2836 lock or [rdx], eax
2837# else
2838 mov edx, [pu32]
2839 lock or [edx], eax
2840# endif
2841 }
2842# endif
2843}
2844#endif
2845
2846
2847/**
2848 * Atomically Or a signed 32-bit value, ordered.
2849 *
2850 * @param pi32 Pointer to the pointer variable to OR u32 with.
2851 * @param i32 The value to OR *pu32 with.
2852 */
2853DECLINLINE(void) ASMAtomicOrS32(int32_t volatile *pi32, int32_t i32)
2854{
2855 ASMAtomicOrU32((uint32_t volatile *)pi32, i32);
2856}
2857
2858
2859/**
2860 * Atomically Or an unsigned 64-bit value, ordered.
2861 *
2862 * @param pu64 Pointer to the pointer variable to OR u64 with.
2863 * @param u64 The value to OR *pu64 with.
2864 */
2865#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2866DECLASM(void) ASMAtomicOrU64(uint64_t volatile *pu64, uint64_t u64);
2867#else
2868DECLINLINE(void) ASMAtomicOrU64(uint64_t volatile *pu64, uint64_t u64)
2869{
2870# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
2871 _InterlockedOr64((__int64 volatile *)pu64, (__int64)u64);
2872
2873# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
2874 __asm__ __volatile__("lock; orq %1, %q0\n\t"
2875 : "=m" (*pu64)
2876 : "r" (u64),
2877 "m" (*pu64));
2878# else
2879 for (;;)
2880 {
2881 uint64_t u64Old = ASMAtomicUoReadU64(pu64);
2882 uint64_t u64New = u64Old | u64;
2883 if (ASMAtomicCmpXchgU64(pu64, u64New, u64Old))
2884 break;
2885 ASMNopPause();
2886 }
2887# endif
2888}
2889#endif
2890
2891
2892/**
2893 * Atomically Or a signed 64-bit value, ordered.
2894 *
2895 * @param pi64 Pointer to the pointer variable to OR u64 with.
2896 * @param i64 The value to OR *pu64 with.
2897 */
2898DECLINLINE(void) ASMAtomicOrS64(int64_t volatile *pi64, int64_t i64)
2899{
2900 ASMAtomicOrU64((uint64_t volatile *)pi64, i64);
2901}
2902/**
2903 * Atomically And an unsigned 32-bit value, ordered.
2904 *
2905 * @param pu32 Pointer to the pointer variable to AND u32 with.
2906 * @param u32 The value to AND *pu32 with.
2907 */
2908#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2909DECLASM(void) ASMAtomicAndU32(uint32_t volatile *pu32, uint32_t u32);
2910#else
2911DECLINLINE(void) ASMAtomicAndU32(uint32_t volatile *pu32, uint32_t u32)
2912{
2913# if RT_INLINE_ASM_USES_INTRIN
2914 _InterlockedAnd((long volatile *)pu32, u32);
2915
2916# elif RT_INLINE_ASM_GNU_STYLE
2917 __asm__ __volatile__("lock; andl %1, %0\n\t"
2918 : "=m" (*pu32)
2919 : "ir" (u32),
2920 "m" (*pu32));
2921# else
2922 __asm
2923 {
2924 mov eax, [u32]
2925# ifdef RT_ARCH_AMD64
2926 mov rdx, [pu32]
2927 lock and [rdx], eax
2928# else
2929 mov edx, [pu32]
2930 lock and [edx], eax
2931# endif
2932 }
2933# endif
2934}
2935#endif
2936
2937
2938/**
2939 * Atomically And a signed 32-bit value, ordered.
2940 *
2941 * @param pi32 Pointer to the pointer variable to AND i32 with.
2942 * @param i32 The value to AND *pi32 with.
2943 */
2944DECLINLINE(void) ASMAtomicAndS32(int32_t volatile *pi32, int32_t i32)
2945{
2946 ASMAtomicAndU32((uint32_t volatile *)pi32, (uint32_t)i32);
2947}
2948
2949
2950/**
2951 * Atomically And an unsigned 64-bit value, ordered.
2952 *
2953 * @param pu64 Pointer to the pointer variable to AND u64 with.
2954 * @param u64 The value to AND *pu64 with.
2955 */
2956#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2957DECLASM(void) ASMAtomicAndU64(uint64_t volatile *pu64, uint64_t u64);
2958#else
2959DECLINLINE(void) ASMAtomicAndU64(uint64_t volatile *pu64, uint64_t u64)
2960{
2961# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
2962 _InterlockedAnd64((__int64 volatile *)pu64, u64);
2963
2964# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
2965 __asm__ __volatile__("lock; andq %1, %0\n\t"
2966 : "=m" (*pu64)
2967 : "r" (u64),
2968 "m" (*pu64));
2969# else
2970 for (;;)
2971 {
2972 uint64_t u64Old = ASMAtomicUoReadU64(pu64);
2973 uint64_t u64New = u64Old & u64;
2974 if (ASMAtomicCmpXchgU64(pu64, u64New, u64Old))
2975 break;
2976 ASMNopPause();
2977 }
2978# endif
2979}
2980#endif
2981
2982
2983/**
2984 * Atomically And a signed 64-bit value, ordered.
2985 *
2986 * @param pi64 Pointer to the pointer variable to AND i64 with.
2987 * @param i64 The value to AND *pi64 with.
2988 */
2989DECLINLINE(void) ASMAtomicAndS64(int64_t volatile *pi64, int64_t i64)
2990{
2991 ASMAtomicAndU64((uint64_t volatile *)pi64, (uint64_t)i64);
2992}
2993
2994
2995
2996/** @def RT_ASM_PAGE_SIZE
2997 * We try avoid dragging in iprt/param.h here.
2998 * @internal
2999 */
3000#if defined(RT_ARCH_SPARC64)
3001# define RT_ASM_PAGE_SIZE 0x2000
3002# if defined(PAGE_SIZE) && !defined(NT_INCLUDED)
3003# if PAGE_SIZE != 0x2000
3004# error "PAGE_SIZE is not 0x2000!"
3005# endif
3006# endif
3007#else
3008# define RT_ASM_PAGE_SIZE 0x1000
3009# if defined(PAGE_SIZE) && !defined(NT_INCLUDED)
3010# if PAGE_SIZE != 0x1000
3011# error "PAGE_SIZE is not 0x1000!"
3012# endif
3013# endif
3014#endif
3015
3016/**
3017 * Zeros a 4K memory page.
3018 *
3019 * @param pv Pointer to the memory block. This must be page aligned.
3020 */
3021#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3022DECLASM(void) ASMMemZeroPage(volatile void *pv);
3023# else
3024DECLINLINE(void) ASMMemZeroPage(volatile void *pv)
3025{
3026# if RT_INLINE_ASM_USES_INTRIN
3027# ifdef RT_ARCH_AMD64
3028 __stosq((unsigned __int64 *)pv, 0, RT_ASM_PAGE_SIZE / 8);
3029# else
3030 __stosd((unsigned long *)pv, 0, RT_ASM_PAGE_SIZE / 4);
3031# endif
3032
3033# elif RT_INLINE_ASM_GNU_STYLE
3034 RTCCUINTREG uDummy;
3035# ifdef RT_ARCH_AMD64
3036 __asm__ __volatile__("rep stosq"
3037 : "=D" (pv),
3038 "=c" (uDummy)
3039 : "0" (pv),
3040 "c" (RT_ASM_PAGE_SIZE >> 3),
3041 "a" (0)
3042 : "memory");
3043# else
3044 __asm__ __volatile__("rep stosl"
3045 : "=D" (pv),
3046 "=c" (uDummy)
3047 : "0" (pv),
3048 "c" (RT_ASM_PAGE_SIZE >> 2),
3049 "a" (0)
3050 : "memory");
3051# endif
3052# else
3053 __asm
3054 {
3055# ifdef RT_ARCH_AMD64
3056 xor rax, rax
3057 mov ecx, 0200h
3058 mov rdi, [pv]
3059 rep stosq
3060# else
3061 xor eax, eax
3062 mov ecx, 0400h
3063 mov edi, [pv]
3064 rep stosd
3065# endif
3066 }
3067# endif
3068}
3069# endif
3070
3071
3072/**
3073 * Zeros a memory block with a 32-bit aligned size.
3074 *
3075 * @param pv Pointer to the memory block.
3076 * @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
3077 */
3078#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3079DECLASM(void) ASMMemZero32(volatile void *pv, size_t cb);
3080#else
3081DECLINLINE(void) ASMMemZero32(volatile void *pv, size_t cb)
3082{
3083# if RT_INLINE_ASM_USES_INTRIN
3084# ifdef RT_ARCH_AMD64
3085 if (!(cb & 7))
3086 __stosq((unsigned __int64 *)pv, 0, cb / 8);
3087 else
3088# endif
3089 __stosd((unsigned long *)pv, 0, cb / 4);
3090
3091# elif RT_INLINE_ASM_GNU_STYLE
3092 __asm__ __volatile__("rep stosl"
3093 : "=D" (pv),
3094 "=c" (cb)
3095 : "0" (pv),
3096 "1" (cb >> 2),
3097 "a" (0)
3098 : "memory");
3099# else
3100 __asm
3101 {
3102 xor eax, eax
3103# ifdef RT_ARCH_AMD64
3104 mov rcx, [cb]
3105 shr rcx, 2
3106 mov rdi, [pv]
3107# else
3108 mov ecx, [cb]
3109 shr ecx, 2
3110 mov edi, [pv]
3111# endif
3112 rep stosd
3113 }
3114# endif
3115}
3116#endif
3117
3118
3119/**
3120 * Fills a memory block with a 32-bit aligned size.
3121 *
3122 * @param pv Pointer to the memory block.
3123 * @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
3124 * @param u32 The value to fill with.
3125 */
3126#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3127DECLASM(void) ASMMemFill32(volatile void *pv, size_t cb, uint32_t u32);
3128#else
3129DECLINLINE(void) ASMMemFill32(volatile void *pv, size_t cb, uint32_t u32)
3130{
3131# if RT_INLINE_ASM_USES_INTRIN
3132# ifdef RT_ARCH_AMD64
3133 if (!(cb & 7))
3134 __stosq((unsigned __int64 *)pv, RT_MAKE_U64(u32, u32), cb / 8);
3135 else
3136# endif
3137 __stosd((unsigned long *)pv, u32, cb / 4);
3138
3139# elif RT_INLINE_ASM_GNU_STYLE
3140 __asm__ __volatile__("rep stosl"
3141 : "=D" (pv),
3142 "=c" (cb)
3143 : "0" (pv),
3144 "1" (cb >> 2),
3145 "a" (u32)
3146 : "memory");
3147# else
3148 __asm
3149 {
3150# ifdef RT_ARCH_AMD64
3151 mov rcx, [cb]
3152 shr rcx, 2
3153 mov rdi, [pv]
3154# else
3155 mov ecx, [cb]
3156 shr ecx, 2
3157 mov edi, [pv]
3158# endif
3159 mov eax, [u32]
3160 rep stosd
3161 }
3162# endif
3163}
3164#endif
3165
3166
3167/**
3168 * Checks if a memory page is all zeros.
3169 *
3170 * @returns true / false.
3171 *
3172 * @param pvPage Pointer to the page. Must be aligned on 16 byte
3173 * boundary
3174 */
3175DECLINLINE(bool) ASMMemIsZeroPage(void const *pvPage)
3176{
3177# if 0 /*RT_INLINE_ASM_GNU_STYLE - this is actually slower... */
3178 union { RTCCUINTREG r; bool f; } uAX;
3179 RTCCUINTREG xCX, xDI;
3180 Assert(!((uintptr_t)pvPage & 15));
3181 __asm__ __volatile__("repe; "
3182# ifdef RT_ARCH_AMD64
3183 "scasq\n\t"
3184# else
3185 "scasl\n\t"
3186# endif
3187 "setnc %%al\n\t"
3188 : "=&c" (xCX),
3189 "=&D" (xDI),
3190 "=&a" (uAX.r)
3191 : "mr" (pvPage),
3192# ifdef RT_ARCH_AMD64
3193 "0" (RT_ASM_PAGE_SIZE/8),
3194# else
3195 "0" (RT_ASM_PAGE_SIZE/4),
3196# endif
3197 "1" (pvPage),
3198 "2" (0));
3199 return uAX.f;
3200# else
3201 uintptr_t const *puPtr = (uintptr_t const *)pvPage;
3202 int cLeft = RT_ASM_PAGE_SIZE / sizeof(uintptr_t) / 8;
3203 Assert(!((uintptr_t)pvPage & 15));
3204 for (;;)
3205 {
3206 if (puPtr[0]) return false;
3207 if (puPtr[4]) return false;
3208
3209 if (puPtr[2]) return false;
3210 if (puPtr[6]) return false;
3211
3212 if (puPtr[1]) return false;
3213 if (puPtr[5]) return false;
3214
3215 if (puPtr[3]) return false;
3216 if (puPtr[7]) return false;
3217
3218 if (!--cLeft)
3219 return true;
3220 puPtr += 8;
3221 }
3222 return true;
3223# endif
3224}
3225
3226
3227/**
3228 * Checks if a memory block is filled with the specified byte.
3229 *
3230 * This is a sort of inverted memchr.
3231 *
3232 * @returns Pointer to the byte which doesn't equal u8.
3233 * @returns NULL if all equal to u8.
3234 *
3235 * @param pv Pointer to the memory block.
3236 * @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
3237 * @param u8 The value it's supposed to be filled with.
3238 *
3239 * @todo Fix name, it is a predicate function but it's not returning boolean!
3240 */
3241DECLINLINE(void *) ASMMemIsAll8(void const *pv, size_t cb, uint8_t u8)
3242{
3243/** @todo rewrite this in inline assembly? */
3244 uint8_t const *pb = (uint8_t const *)pv;
3245 for (; cb; cb--, pb++)
3246 if (RT_UNLIKELY(*pb != u8))
3247 return (void *)pb;
3248 return NULL;
3249}
3250
3251
3252/**
3253 * Checks if a memory block is filled with the specified 32-bit value.
3254 *
3255 * This is a sort of inverted memchr.
3256 *
3257 * @returns Pointer to the first value which doesn't equal u32.
3258 * @returns NULL if all equal to u32.
3259 *
3260 * @param pv Pointer to the memory block.
3261 * @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
3262 * @param u32 The value it's supposed to be filled with.
3263 *
3264 * @todo Fix name, it is a predicate function but it's not returning boolean!
3265 */
3266DECLINLINE(uint32_t *) ASMMemIsAllU32(void const *pv, size_t cb, uint32_t u32)
3267{
3268/** @todo rewrite this in inline assembly? */
3269 uint32_t const *pu32 = (uint32_t const *)pv;
3270 for (; cb; cb -= 4, pu32++)
3271 if (RT_UNLIKELY(*pu32 != u32))
3272 return (uint32_t *)pu32;
3273 return NULL;
3274}
3275
3276
3277/**
3278 * Probes a byte pointer for read access.
3279 *
3280 * While the function will not fault if the byte is not read accessible,
3281 * the idea is to do this in a safe place like before acquiring locks
3282 * and such like.
3283 *
3284 * Also, this functions guarantees that an eager compiler is not going
3285 * to optimize the probing away.
3286 *
3287 * @param pvByte Pointer to the byte.
3288 */
3289#if RT_INLINE_ASM_EXTERNAL
3290DECLASM(uint8_t) ASMProbeReadByte(const void *pvByte);
3291#else
3292DECLINLINE(uint8_t) ASMProbeReadByte(const void *pvByte)
3293{
3294 /** @todo verify that the compiler actually doesn't optimize this away. (intel & gcc) */
3295 uint8_t u8;
3296# if RT_INLINE_ASM_GNU_STYLE
3297 __asm__ __volatile__("movb (%1), %0\n\t"
3298 : "=r" (u8)
3299 : "r" (pvByte));
3300# else
3301 __asm
3302 {
3303# ifdef RT_ARCH_AMD64
3304 mov rax, [pvByte]
3305 mov al, [rax]
3306# else
3307 mov eax, [pvByte]
3308 mov al, [eax]
3309# endif
3310 mov [u8], al
3311 }
3312# endif
3313 return u8;
3314}
3315#endif
3316
3317/**
3318 * Probes a buffer for read access page by page.
3319 *
3320 * While the function will fault if the buffer is not fully read
3321 * accessible, the idea is to do this in a safe place like before
3322 * acquiring locks and such like.
3323 *
3324 * Also, this functions guarantees that an eager compiler is not going
3325 * to optimize the probing away.
3326 *
3327 * @param pvBuf Pointer to the buffer.
3328 * @param cbBuf The size of the buffer in bytes. Must be >= 1.
3329 */
3330DECLINLINE(void) ASMProbeReadBuffer(const void *pvBuf, size_t cbBuf)
3331{
3332 /** @todo verify that the compiler actually doesn't optimize this away. (intel & gcc) */
3333 /* the first byte */
3334 const uint8_t *pu8 = (const uint8_t *)pvBuf;
3335 ASMProbeReadByte(pu8);
3336
3337 /* the pages in between pages. */
3338 while (cbBuf > RT_ASM_PAGE_SIZE)
3339 {
3340 ASMProbeReadByte(pu8);
3341 cbBuf -= RT_ASM_PAGE_SIZE;
3342 pu8 += RT_ASM_PAGE_SIZE;
3343 }
3344
3345 /* the last byte */
3346 ASMProbeReadByte(pu8 + cbBuf - 1);
3347}
3348
3349
3350
3351/** @defgroup grp_inline_bits Bit Operations
3352 * @{
3353 */
3354
3355
3356/**
3357 * Sets a bit in a bitmap.
3358 *
3359 * @param pvBitmap Pointer to the bitmap. This should be 32-bit aligned.
3360 * @param iBit The bit to set.
3361 *
3362 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3363 * However, doing so will yield better performance as well as avoiding
3364 * traps accessing the last bits in the bitmap.
3365 */
3366#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3367DECLASM(void) ASMBitSet(volatile void *pvBitmap, int32_t iBit);
3368#else
3369DECLINLINE(void) ASMBitSet(volatile void *pvBitmap, int32_t iBit)
3370{
3371# if RT_INLINE_ASM_USES_INTRIN
3372 _bittestandset((long *)pvBitmap, iBit);
3373
3374# elif RT_INLINE_ASM_GNU_STYLE
3375 __asm__ __volatile__("btsl %1, %0"
3376 : "=m" (*(volatile long *)pvBitmap)
3377 : "Ir" (iBit),
3378 "m" (*(volatile long *)pvBitmap)
3379 : "memory");
3380# else
3381 __asm
3382 {
3383# ifdef RT_ARCH_AMD64
3384 mov rax, [pvBitmap]
3385 mov edx, [iBit]
3386 bts [rax], edx
3387# else
3388 mov eax, [pvBitmap]
3389 mov edx, [iBit]
3390 bts [eax], edx
3391# endif
3392 }
3393# endif
3394}
3395#endif
3396
3397
3398/**
3399 * Atomically sets a bit in a bitmap, ordered.
3400 *
3401 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3402 * the memory access isn't atomic!
3403 * @param iBit The bit to set.
3404 */
3405#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3406DECLASM(void) ASMAtomicBitSet(volatile void *pvBitmap, int32_t iBit);
3407#else
3408DECLINLINE(void) ASMAtomicBitSet(volatile void *pvBitmap, int32_t iBit)
3409{
3410 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3411# if RT_INLINE_ASM_USES_INTRIN
3412 _interlockedbittestandset((long *)pvBitmap, iBit);
3413# elif RT_INLINE_ASM_GNU_STYLE
3414 __asm__ __volatile__("lock; btsl %1, %0"
3415 : "=m" (*(volatile long *)pvBitmap)
3416 : "Ir" (iBit),
3417 "m" (*(volatile long *)pvBitmap)
3418 : "memory");
3419# else
3420 __asm
3421 {
3422# ifdef RT_ARCH_AMD64
3423 mov rax, [pvBitmap]
3424 mov edx, [iBit]
3425 lock bts [rax], edx
3426# else
3427 mov eax, [pvBitmap]
3428 mov edx, [iBit]
3429 lock bts [eax], edx
3430# endif
3431 }
3432# endif
3433}
3434#endif
3435
3436
3437/**
3438 * Clears a bit in a bitmap.
3439 *
3440 * @param pvBitmap Pointer to the bitmap.
3441 * @param iBit The bit to clear.
3442 *
3443 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3444 * However, doing so will yield better performance as well as avoiding
3445 * traps accessing the last bits in the bitmap.
3446 */
3447#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3448DECLASM(void) ASMBitClear(volatile void *pvBitmap, int32_t iBit);
3449#else
3450DECLINLINE(void) ASMBitClear(volatile void *pvBitmap, int32_t iBit)
3451{
3452# if RT_INLINE_ASM_USES_INTRIN
3453 _bittestandreset((long *)pvBitmap, iBit);
3454
3455# elif RT_INLINE_ASM_GNU_STYLE
3456 __asm__ __volatile__("btrl %1, %0"
3457 : "=m" (*(volatile long *)pvBitmap)
3458 : "Ir" (iBit),
3459 "m" (*(volatile long *)pvBitmap)
3460 : "memory");
3461# else
3462 __asm
3463 {
3464# ifdef RT_ARCH_AMD64
3465 mov rax, [pvBitmap]
3466 mov edx, [iBit]
3467 btr [rax], edx
3468# else
3469 mov eax, [pvBitmap]
3470 mov edx, [iBit]
3471 btr [eax], edx
3472# endif
3473 }
3474# endif
3475}
3476#endif
3477
3478
3479/**
3480 * Atomically clears a bit in a bitmap, ordered.
3481 *
3482 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3483 * the memory access isn't atomic!
3484 * @param iBit The bit to toggle set.
3485 * @remarks No memory barrier, take care on smp.
3486 */
3487#if RT_INLINE_ASM_EXTERNAL
3488DECLASM(void) ASMAtomicBitClear(volatile void *pvBitmap, int32_t iBit);
3489#else
3490DECLINLINE(void) ASMAtomicBitClear(volatile void *pvBitmap, int32_t iBit)
3491{
3492 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3493# if RT_INLINE_ASM_GNU_STYLE
3494 __asm__ __volatile__("lock; btrl %1, %0"
3495 : "=m" (*(volatile long *)pvBitmap)
3496 : "Ir" (iBit),
3497 "m" (*(volatile long *)pvBitmap)
3498 : "memory");
3499# else
3500 __asm
3501 {
3502# ifdef RT_ARCH_AMD64
3503 mov rax, [pvBitmap]
3504 mov edx, [iBit]
3505 lock btr [rax], edx
3506# else
3507 mov eax, [pvBitmap]
3508 mov edx, [iBit]
3509 lock btr [eax], edx
3510# endif
3511 }
3512# endif
3513}
3514#endif
3515
3516
3517/**
3518 * Toggles a bit in a bitmap.
3519 *
3520 * @param pvBitmap Pointer to the bitmap.
3521 * @param iBit The bit to toggle.
3522 *
3523 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3524 * However, doing so will yield better performance as well as avoiding
3525 * traps accessing the last bits in the bitmap.
3526 */
3527#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3528DECLASM(void) ASMBitToggle(volatile void *pvBitmap, int32_t iBit);
3529#else
3530DECLINLINE(void) ASMBitToggle(volatile void *pvBitmap, int32_t iBit)
3531{
3532# if RT_INLINE_ASM_USES_INTRIN
3533 _bittestandcomplement((long *)pvBitmap, iBit);
3534# elif RT_INLINE_ASM_GNU_STYLE
3535 __asm__ __volatile__("btcl %1, %0"
3536 : "=m" (*(volatile long *)pvBitmap)
3537 : "Ir" (iBit),
3538 "m" (*(volatile long *)pvBitmap)
3539 : "memory");
3540# else
3541 __asm
3542 {
3543# ifdef RT_ARCH_AMD64
3544 mov rax, [pvBitmap]
3545 mov edx, [iBit]
3546 btc [rax], edx
3547# else
3548 mov eax, [pvBitmap]
3549 mov edx, [iBit]
3550 btc [eax], edx
3551# endif
3552 }
3553# endif
3554}
3555#endif
3556
3557
3558/**
3559 * Atomically toggles a bit in a bitmap, ordered.
3560 *
3561 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3562 * the memory access isn't atomic!
3563 * @param iBit The bit to test and set.
3564 */
3565#if RT_INLINE_ASM_EXTERNAL
3566DECLASM(void) ASMAtomicBitToggle(volatile void *pvBitmap, int32_t iBit);
3567#else
3568DECLINLINE(void) ASMAtomicBitToggle(volatile void *pvBitmap, int32_t iBit)
3569{
3570 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3571# if RT_INLINE_ASM_GNU_STYLE
3572 __asm__ __volatile__("lock; btcl %1, %0"
3573 : "=m" (*(volatile long *)pvBitmap)
3574 : "Ir" (iBit),
3575 "m" (*(volatile long *)pvBitmap)
3576 : "memory");
3577# else
3578 __asm
3579 {
3580# ifdef RT_ARCH_AMD64
3581 mov rax, [pvBitmap]
3582 mov edx, [iBit]
3583 lock btc [rax], edx
3584# else
3585 mov eax, [pvBitmap]
3586 mov edx, [iBit]
3587 lock btc [eax], edx
3588# endif
3589 }
3590# endif
3591}
3592#endif
3593
3594
3595/**
3596 * Tests and sets a bit in a bitmap.
3597 *
3598 * @returns true if the bit was set.
3599 * @returns false if the bit was clear.
3600 *
3601 * @param pvBitmap Pointer to the bitmap.
3602 * @param iBit The bit to test and set.
3603 *
3604 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3605 * However, doing so will yield better performance as well as avoiding
3606 * traps accessing the last bits in the bitmap.
3607 */
3608#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3609DECLASM(bool) ASMBitTestAndSet(volatile void *pvBitmap, int32_t iBit);
3610#else
3611DECLINLINE(bool) ASMBitTestAndSet(volatile void *pvBitmap, int32_t iBit)
3612{
3613 union { bool f; uint32_t u32; uint8_t u8; } rc;
3614# if RT_INLINE_ASM_USES_INTRIN
3615 rc.u8 = _bittestandset((long *)pvBitmap, iBit);
3616
3617# elif RT_INLINE_ASM_GNU_STYLE
3618 __asm__ __volatile__("btsl %2, %1\n\t"
3619 "setc %b0\n\t"
3620 "andl $1, %0\n\t"
3621 : "=q" (rc.u32),
3622 "=m" (*(volatile long *)pvBitmap)
3623 : "Ir" (iBit),
3624 "m" (*(volatile long *)pvBitmap)
3625 : "memory");
3626# else
3627 __asm
3628 {
3629 mov edx, [iBit]
3630# ifdef RT_ARCH_AMD64
3631 mov rax, [pvBitmap]
3632 bts [rax], edx
3633# else
3634 mov eax, [pvBitmap]
3635 bts [eax], edx
3636# endif
3637 setc al
3638 and eax, 1
3639 mov [rc.u32], eax
3640 }
3641# endif
3642 return rc.f;
3643}
3644#endif
3645
3646
3647/**
3648 * Atomically tests and sets a bit in a bitmap, ordered.
3649 *
3650 * @returns true if the bit was set.
3651 * @returns false if the bit was clear.
3652 *
3653 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3654 * the memory access isn't atomic!
3655 * @param iBit The bit to set.
3656 */
3657#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3658DECLASM(bool) ASMAtomicBitTestAndSet(volatile void *pvBitmap, int32_t iBit);
3659#else
3660DECLINLINE(bool) ASMAtomicBitTestAndSet(volatile void *pvBitmap, int32_t iBit)
3661{
3662 union { bool f; uint32_t u32; uint8_t u8; } rc;
3663 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3664# if RT_INLINE_ASM_USES_INTRIN
3665 rc.u8 = _interlockedbittestandset((long *)pvBitmap, iBit);
3666# elif RT_INLINE_ASM_GNU_STYLE
3667 __asm__ __volatile__("lock; btsl %2, %1\n\t"
3668 "setc %b0\n\t"
3669 "andl $1, %0\n\t"
3670 : "=q" (rc.u32),
3671 "=m" (*(volatile long *)pvBitmap)
3672 : "Ir" (iBit),
3673 "m" (*(volatile long *)pvBitmap)
3674 : "memory");
3675# else
3676 __asm
3677 {
3678 mov edx, [iBit]
3679# ifdef RT_ARCH_AMD64
3680 mov rax, [pvBitmap]
3681 lock bts [rax], edx
3682# else
3683 mov eax, [pvBitmap]
3684 lock bts [eax], edx
3685# endif
3686 setc al
3687 and eax, 1
3688 mov [rc.u32], eax
3689 }
3690# endif
3691 return rc.f;
3692}
3693#endif
3694
3695
3696/**
3697 * Tests and clears a bit in a bitmap.
3698 *
3699 * @returns true if the bit was set.
3700 * @returns false if the bit was clear.
3701 *
3702 * @param pvBitmap Pointer to the bitmap.
3703 * @param iBit The bit to test and clear.
3704 *
3705 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3706 * However, doing so will yield better performance as well as avoiding
3707 * traps accessing the last bits in the bitmap.
3708 */
3709#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3710DECLASM(bool) ASMBitTestAndClear(volatile void *pvBitmap, int32_t iBit);
3711#else
3712DECLINLINE(bool) ASMBitTestAndClear(volatile void *pvBitmap, int32_t iBit)
3713{
3714 union { bool f; uint32_t u32; uint8_t u8; } rc;
3715# if RT_INLINE_ASM_USES_INTRIN
3716 rc.u8 = _bittestandreset((long *)pvBitmap, iBit);
3717
3718# elif RT_INLINE_ASM_GNU_STYLE
3719 __asm__ __volatile__("btrl %2, %1\n\t"
3720 "setc %b0\n\t"
3721 "andl $1, %0\n\t"
3722 : "=q" (rc.u32),
3723 "=m" (*(volatile long *)pvBitmap)
3724 : "Ir" (iBit),
3725 "m" (*(volatile long *)pvBitmap)
3726 : "memory");
3727# else
3728 __asm
3729 {
3730 mov edx, [iBit]
3731# ifdef RT_ARCH_AMD64
3732 mov rax, [pvBitmap]
3733 btr [rax], edx
3734# else
3735 mov eax, [pvBitmap]
3736 btr [eax], edx
3737# endif
3738 setc al
3739 and eax, 1
3740 mov [rc.u32], eax
3741 }
3742# endif
3743 return rc.f;
3744}
3745#endif
3746
3747
3748/**
3749 * Atomically tests and clears a bit in a bitmap, ordered.
3750 *
3751 * @returns true if the bit was set.
3752 * @returns false if the bit was clear.
3753 *
3754 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3755 * the memory access isn't atomic!
3756 * @param iBit The bit to test and clear.
3757 *
3758 * @remarks No memory barrier, take care on smp.
3759 */
3760#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3761DECLASM(bool) ASMAtomicBitTestAndClear(volatile void *pvBitmap, int32_t iBit);
3762#else
3763DECLINLINE(bool) ASMAtomicBitTestAndClear(volatile void *pvBitmap, int32_t iBit)
3764{
3765 union { bool f; uint32_t u32; uint8_t u8; } rc;
3766 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3767# if RT_INLINE_ASM_USES_INTRIN
3768 rc.u8 = _interlockedbittestandreset((long *)pvBitmap, iBit);
3769
3770# elif RT_INLINE_ASM_GNU_STYLE
3771 __asm__ __volatile__("lock; btrl %2, %1\n\t"
3772 "setc %b0\n\t"
3773 "andl $1, %0\n\t"
3774 : "=q" (rc.u32),
3775 "=m" (*(volatile long *)pvBitmap)
3776 : "Ir" (iBit),
3777 "m" (*(volatile long *)pvBitmap)
3778 : "memory");
3779# else
3780 __asm
3781 {
3782 mov edx, [iBit]
3783# ifdef RT_ARCH_AMD64
3784 mov rax, [pvBitmap]
3785 lock btr [rax], edx
3786# else
3787 mov eax, [pvBitmap]
3788 lock btr [eax], edx
3789# endif
3790 setc al
3791 and eax, 1
3792 mov [rc.u32], eax
3793 }
3794# endif
3795 return rc.f;
3796}
3797#endif
3798
3799
3800/**
3801 * Tests and toggles a bit in a bitmap.
3802 *
3803 * @returns true if the bit was set.
3804 * @returns false if the bit was clear.
3805 *
3806 * @param pvBitmap Pointer to the bitmap.
3807 * @param iBit The bit to test and toggle.
3808 *
3809 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3810 * However, doing so will yield better performance as well as avoiding
3811 * traps accessing the last bits in the bitmap.
3812 */
3813#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3814DECLASM(bool) ASMBitTestAndToggle(volatile void *pvBitmap, int32_t iBit);
3815#else
3816DECLINLINE(bool) ASMBitTestAndToggle(volatile void *pvBitmap, int32_t iBit)
3817{
3818 union { bool f; uint32_t u32; uint8_t u8; } rc;
3819# if RT_INLINE_ASM_USES_INTRIN
3820 rc.u8 = _bittestandcomplement((long *)pvBitmap, iBit);
3821
3822# elif RT_INLINE_ASM_GNU_STYLE
3823 __asm__ __volatile__("btcl %2, %1\n\t"
3824 "setc %b0\n\t"
3825 "andl $1, %0\n\t"
3826 : "=q" (rc.u32),
3827 "=m" (*(volatile long *)pvBitmap)
3828 : "Ir" (iBit),
3829 "m" (*(volatile long *)pvBitmap)
3830 : "memory");
3831# else
3832 __asm
3833 {
3834 mov edx, [iBit]
3835# ifdef RT_ARCH_AMD64
3836 mov rax, [pvBitmap]
3837 btc [rax], edx
3838# else
3839 mov eax, [pvBitmap]
3840 btc [eax], edx
3841# endif
3842 setc al
3843 and eax, 1
3844 mov [rc.u32], eax
3845 }
3846# endif
3847 return rc.f;
3848}
3849#endif
3850
3851
3852/**
3853 * Atomically tests and toggles a bit in a bitmap, ordered.
3854 *
3855 * @returns true if the bit was set.
3856 * @returns false if the bit was clear.
3857 *
3858 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3859 * the memory access isn't atomic!
3860 * @param iBit The bit to test and toggle.
3861 */
3862#if RT_INLINE_ASM_EXTERNAL
3863DECLASM(bool) ASMAtomicBitTestAndToggle(volatile void *pvBitmap, int32_t iBit);
3864#else
3865DECLINLINE(bool) ASMAtomicBitTestAndToggle(volatile void *pvBitmap, int32_t iBit)
3866{
3867 union { bool f; uint32_t u32; uint8_t u8; } rc;
3868 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3869# if RT_INLINE_ASM_GNU_STYLE
3870 __asm__ __volatile__("lock; btcl %2, %1\n\t"
3871 "setc %b0\n\t"
3872 "andl $1, %0\n\t"
3873 : "=q" (rc.u32),
3874 "=m" (*(volatile long *)pvBitmap)
3875 : "Ir" (iBit),
3876 "m" (*(volatile long *)pvBitmap)
3877 : "memory");
3878# else
3879 __asm
3880 {
3881 mov edx, [iBit]
3882# ifdef RT_ARCH_AMD64
3883 mov rax, [pvBitmap]
3884 lock btc [rax], edx
3885# else
3886 mov eax, [pvBitmap]
3887 lock btc [eax], edx
3888# endif
3889 setc al
3890 and eax, 1
3891 mov [rc.u32], eax
3892 }
3893# endif
3894 return rc.f;
3895}
3896#endif
3897
3898
3899/**
3900 * Tests if a bit in a bitmap is set.
3901 *
3902 * @returns true if the bit is set.
3903 * @returns false if the bit is clear.
3904 *
3905 * @param pvBitmap Pointer to the bitmap.
3906 * @param iBit The bit to test.
3907 *
3908 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3909 * However, doing so will yield better performance as well as avoiding
3910 * traps accessing the last bits in the bitmap.
3911 */
3912#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3913DECLASM(bool) ASMBitTest(const volatile void *pvBitmap, int32_t iBit);
3914#else
3915DECLINLINE(bool) ASMBitTest(const volatile void *pvBitmap, int32_t iBit)
3916{
3917 union { bool f; uint32_t u32; uint8_t u8; } rc;
3918# if RT_INLINE_ASM_USES_INTRIN
3919 rc.u32 = _bittest((long *)pvBitmap, iBit);
3920# elif RT_INLINE_ASM_GNU_STYLE
3921
3922 __asm__ __volatile__("btl %2, %1\n\t"
3923 "setc %b0\n\t"
3924 "andl $1, %0\n\t"
3925 : "=q" (rc.u32)
3926 : "m" (*(const volatile long *)pvBitmap),
3927 "Ir" (iBit)
3928 : "memory");
3929# else
3930 __asm
3931 {
3932 mov edx, [iBit]
3933# ifdef RT_ARCH_AMD64
3934 mov rax, [pvBitmap]
3935 bt [rax], edx
3936# else
3937 mov eax, [pvBitmap]
3938 bt [eax], edx
3939# endif
3940 setc al
3941 and eax, 1
3942 mov [rc.u32], eax
3943 }
3944# endif
3945 return rc.f;
3946}
3947#endif
3948
3949
3950/**
3951 * Clears a bit range within a bitmap.
3952 *
3953 * @param pvBitmap Pointer to the bitmap.
3954 * @param iBitStart The First bit to clear.
3955 * @param iBitEnd The first bit not to clear.
3956 */
3957DECLINLINE(void) ASMBitClearRange(volatile void *pvBitmap, int32_t iBitStart, int32_t iBitEnd)
3958{
3959 if (iBitStart < iBitEnd)
3960 {
3961 volatile uint32_t *pu32 = (volatile uint32_t *)pvBitmap + (iBitStart >> 5);
3962 int iStart = iBitStart & ~31;
3963 int iEnd = iBitEnd & ~31;
3964 if (iStart == iEnd)
3965 *pu32 &= ((1 << (iBitStart & 31)) - 1) | ~((1 << (iBitEnd & 31)) - 1);
3966 else
3967 {
3968 /* bits in first dword. */
3969 if (iBitStart & 31)
3970 {
3971 *pu32 &= (1 << (iBitStart & 31)) - 1;
3972 pu32++;
3973 iBitStart = iStart + 32;
3974 }
3975
3976 /* whole dword. */
3977 if (iBitStart != iEnd)
3978 ASMMemZero32(pu32, (iEnd - iBitStart) >> 3);
3979
3980 /* bits in last dword. */
3981 if (iBitEnd & 31)
3982 {
3983 pu32 = (volatile uint32_t *)pvBitmap + (iBitEnd >> 5);
3984 *pu32 &= ~((1 << (iBitEnd & 31)) - 1);
3985 }
3986 }
3987 }
3988}
3989
3990
3991/**
3992 * Sets a bit range within a bitmap.
3993 *
3994 * @param pvBitmap Pointer to the bitmap.
3995 * @param iBitStart The First bit to set.
3996 * @param iBitEnd The first bit not to set.
3997 */
3998DECLINLINE(void) ASMBitSetRange(volatile void *pvBitmap, int32_t iBitStart, int32_t iBitEnd)
3999{
4000 if (iBitStart < iBitEnd)
4001 {
4002 volatile uint32_t *pu32 = (volatile uint32_t *)pvBitmap + (iBitStart >> 5);
4003 int iStart = iBitStart & ~31;
4004 int iEnd = iBitEnd & ~31;
4005 if (iStart == iEnd)
4006 *pu32 |= ((1 << (iBitEnd - iBitStart)) - 1) << iBitStart;
4007 else
4008 {
4009 /* bits in first dword. */
4010 if (iBitStart & 31)
4011 {
4012 *pu32 |= ~((1 << (iBitStart & 31)) - 1);
4013 pu32++;
4014 iBitStart = iStart + 32;
4015 }
4016
4017 /* whole dword. */
4018 if (iBitStart != iEnd)
4019 ASMMemFill32(pu32, (iEnd - iBitStart) >> 3, ~0);
4020
4021 /* bits in last dword. */
4022 if (iBitEnd & 31)
4023 {
4024 pu32 = (volatile uint32_t *)pvBitmap + (iBitEnd >> 5);
4025 *pu32 |= (1 << (iBitEnd & 31)) - 1;
4026 }
4027 }
4028 }
4029}
4030
4031
4032/**
4033 * Finds the first clear bit in a bitmap.
4034 *
4035 * @returns Index of the first zero bit.
4036 * @returns -1 if no clear bit was found.
4037 * @param pvBitmap Pointer to the bitmap.
4038 * @param cBits The number of bits in the bitmap. Multiple of 32.
4039 */
4040#if RT_INLINE_ASM_EXTERNAL
4041DECLASM(int) ASMBitFirstClear(const volatile void *pvBitmap, uint32_t cBits);
4042#else
4043DECLINLINE(int) ASMBitFirstClear(const volatile void *pvBitmap, uint32_t cBits)
4044{
4045 if (cBits)
4046 {
4047 int32_t iBit;
4048# if RT_INLINE_ASM_GNU_STYLE
4049 RTCCUINTREG uEAX, uECX, uEDI;
4050 cBits = RT_ALIGN_32(cBits, 32);
4051 __asm__ __volatile__("repe; scasl\n\t"
4052 "je 1f\n\t"
4053# ifdef RT_ARCH_AMD64
4054 "lea -4(%%rdi), %%rdi\n\t"
4055 "xorl (%%rdi), %%eax\n\t"
4056 "subq %5, %%rdi\n\t"
4057# else
4058 "lea -4(%%edi), %%edi\n\t"
4059 "xorl (%%edi), %%eax\n\t"
4060 "subl %5, %%edi\n\t"
4061# endif
4062 "shll $3, %%edi\n\t"
4063 "bsfl %%eax, %%edx\n\t"
4064 "addl %%edi, %%edx\n\t"
4065 "1:\t\n"
4066 : "=d" (iBit),
4067 "=&c" (uECX),
4068 "=&D" (uEDI),
4069 "=&a" (uEAX)
4070 : "0" (0xffffffff),
4071 "mr" (pvBitmap),
4072 "1" (cBits >> 5),
4073 "2" (pvBitmap),
4074 "3" (0xffffffff));
4075# else
4076 cBits = RT_ALIGN_32(cBits, 32);
4077 __asm
4078 {
4079# ifdef RT_ARCH_AMD64
4080 mov rdi, [pvBitmap]
4081 mov rbx, rdi
4082# else
4083 mov edi, [pvBitmap]
4084 mov ebx, edi
4085# endif
4086 mov edx, 0ffffffffh
4087 mov eax, edx
4088 mov ecx, [cBits]
4089 shr ecx, 5
4090 repe scasd
4091 je done
4092
4093# ifdef RT_ARCH_AMD64
4094 lea rdi, [rdi - 4]
4095 xor eax, [rdi]
4096 sub rdi, rbx
4097# else
4098 lea edi, [edi - 4]
4099 xor eax, [edi]
4100 sub edi, ebx
4101# endif
4102 shl edi, 3
4103 bsf edx, eax
4104 add edx, edi
4105 done:
4106 mov [iBit], edx
4107 }
4108# endif
4109 return iBit;
4110 }
4111 return -1;
4112}
4113#endif
4114
4115
4116/**
4117 * Finds the next clear bit in a bitmap.
4118 *
4119 * @returns Index of the first zero bit.
4120 * @returns -1 if no clear bit was found.
4121 * @param pvBitmap Pointer to the bitmap.
4122 * @param cBits The number of bits in the bitmap. Multiple of 32.
4123 * @param iBitPrev The bit returned from the last search.
4124 * The search will start at iBitPrev + 1.
4125 */
4126#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4127DECLASM(int) ASMBitNextClear(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev);
4128#else
4129DECLINLINE(int) ASMBitNextClear(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev)
4130{
4131 const volatile uint32_t *pau32Bitmap = (const volatile uint32_t *)pvBitmap;
4132 int iBit = ++iBitPrev & 31;
4133 if (iBit)
4134 {
4135 /*
4136 * Inspect the 32-bit word containing the unaligned bit.
4137 */
4138 uint32_t u32 = ~pau32Bitmap[iBitPrev / 32] >> iBit;
4139
4140# if RT_INLINE_ASM_USES_INTRIN
4141 unsigned long ulBit = 0;
4142 if (_BitScanForward(&ulBit, u32))
4143 return ulBit + iBitPrev;
4144# else
4145# if RT_INLINE_ASM_GNU_STYLE
4146 __asm__ __volatile__("bsf %1, %0\n\t"
4147 "jnz 1f\n\t"
4148 "movl $-1, %0\n\t"
4149 "1:\n\t"
4150 : "=r" (iBit)
4151 : "r" (u32));
4152# else
4153 __asm
4154 {
4155 mov edx, [u32]
4156 bsf eax, edx
4157 jnz done
4158 mov eax, 0ffffffffh
4159 done:
4160 mov [iBit], eax
4161 }
4162# endif
4163 if (iBit >= 0)
4164 return iBit + iBitPrev;
4165# endif
4166
4167 /*
4168 * Skip ahead and see if there is anything left to search.
4169 */
4170 iBitPrev |= 31;
4171 iBitPrev++;
4172 if (cBits <= (uint32_t)iBitPrev)
4173 return -1;
4174 }
4175
4176 /*
4177 * 32-bit aligned search, let ASMBitFirstClear do the dirty work.
4178 */
4179 iBit = ASMBitFirstClear(&pau32Bitmap[iBitPrev / 32], cBits - iBitPrev);
4180 if (iBit >= 0)
4181 iBit += iBitPrev;
4182 return iBit;
4183}
4184#endif
4185
4186
4187/**
4188 * Finds the first set bit in a bitmap.
4189 *
4190 * @returns Index of the first set bit.
4191 * @returns -1 if no clear bit was found.
4192 * @param pvBitmap Pointer to the bitmap.
4193 * @param cBits The number of bits in the bitmap. Multiple of 32.
4194 */
4195#if RT_INLINE_ASM_EXTERNAL
4196DECLASM(int) ASMBitFirstSet(const volatile void *pvBitmap, uint32_t cBits);
4197#else
4198DECLINLINE(int) ASMBitFirstSet(const volatile void *pvBitmap, uint32_t cBits)
4199{
4200 if (cBits)
4201 {
4202 int32_t iBit;
4203# if RT_INLINE_ASM_GNU_STYLE
4204 RTCCUINTREG uEAX, uECX, uEDI;
4205 cBits = RT_ALIGN_32(cBits, 32);
4206 __asm__ __volatile__("repe; scasl\n\t"
4207 "je 1f\n\t"
4208# ifdef RT_ARCH_AMD64
4209 "lea -4(%%rdi), %%rdi\n\t"
4210 "movl (%%rdi), %%eax\n\t"
4211 "subq %5, %%rdi\n\t"
4212# else
4213 "lea -4(%%edi), %%edi\n\t"
4214 "movl (%%edi), %%eax\n\t"
4215 "subl %5, %%edi\n\t"
4216# endif
4217 "shll $3, %%edi\n\t"
4218 "bsfl %%eax, %%edx\n\t"
4219 "addl %%edi, %%edx\n\t"
4220 "1:\t\n"
4221 : "=d" (iBit),
4222 "=&c" (uECX),
4223 "=&D" (uEDI),
4224 "=&a" (uEAX)
4225 : "0" (0xffffffff),
4226 "mr" (pvBitmap),
4227 "1" (cBits >> 5),
4228 "2" (pvBitmap),
4229 "3" (0));
4230# else
4231 cBits = RT_ALIGN_32(cBits, 32);
4232 __asm
4233 {
4234# ifdef RT_ARCH_AMD64
4235 mov rdi, [pvBitmap]
4236 mov rbx, rdi
4237# else
4238 mov edi, [pvBitmap]
4239 mov ebx, edi
4240# endif
4241 mov edx, 0ffffffffh
4242 xor eax, eax
4243 mov ecx, [cBits]
4244 shr ecx, 5
4245 repe scasd
4246 je done
4247# ifdef RT_ARCH_AMD64
4248 lea rdi, [rdi - 4]
4249 mov eax, [rdi]
4250 sub rdi, rbx
4251# else
4252 lea edi, [edi - 4]
4253 mov eax, [edi]
4254 sub edi, ebx
4255# endif
4256 shl edi, 3
4257 bsf edx, eax
4258 add edx, edi
4259 done:
4260 mov [iBit], edx
4261 }
4262# endif
4263 return iBit;
4264 }
4265 return -1;
4266}
4267#endif
4268
4269
4270/**
4271 * Finds the next set bit in a bitmap.
4272 *
4273 * @returns Index of the next set bit.
4274 * @returns -1 if no set bit was found.
4275 * @param pvBitmap Pointer to the bitmap.
4276 * @param cBits The number of bits in the bitmap. Multiple of 32.
4277 * @param iBitPrev The bit returned from the last search.
4278 * The search will start at iBitPrev + 1.
4279 */
4280#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4281DECLASM(int) ASMBitNextSet(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev);
4282#else
4283DECLINLINE(int) ASMBitNextSet(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev)
4284{
4285 const volatile uint32_t *pau32Bitmap = (const volatile uint32_t *)pvBitmap;
4286 int iBit = ++iBitPrev & 31;
4287 if (iBit)
4288 {
4289 /*
4290 * Inspect the 32-bit word containing the unaligned bit.
4291 */
4292 uint32_t u32 = pau32Bitmap[iBitPrev / 32] >> iBit;
4293
4294# if RT_INLINE_ASM_USES_INTRIN
4295 unsigned long ulBit = 0;
4296 if (_BitScanForward(&ulBit, u32))
4297 return ulBit + iBitPrev;
4298# else
4299# if RT_INLINE_ASM_GNU_STYLE
4300 __asm__ __volatile__("bsf %1, %0\n\t"
4301 "jnz 1f\n\t"
4302 "movl $-1, %0\n\t"
4303 "1:\n\t"
4304 : "=r" (iBit)
4305 : "r" (u32));
4306# else
4307 __asm
4308 {
4309 mov edx, [u32]
4310 bsf eax, edx
4311 jnz done
4312 mov eax, 0ffffffffh
4313 done:
4314 mov [iBit], eax
4315 }
4316# endif
4317 if (iBit >= 0)
4318 return iBit + iBitPrev;
4319# endif
4320
4321 /*
4322 * Skip ahead and see if there is anything left to search.
4323 */
4324 iBitPrev |= 31;
4325 iBitPrev++;
4326 if (cBits <= (uint32_t)iBitPrev)
4327 return -1;
4328 }
4329
4330 /*
4331 * 32-bit aligned search, let ASMBitFirstClear do the dirty work.
4332 */
4333 iBit = ASMBitFirstSet(&pau32Bitmap[iBitPrev / 32], cBits - iBitPrev);
4334 if (iBit >= 0)
4335 iBit += iBitPrev;
4336 return iBit;
4337}
4338#endif
4339
4340
4341/**
4342 * Finds the first bit which is set in the given 32-bit integer.
4343 * Bits are numbered from 1 (least significant) to 32.
4344 *
4345 * @returns index [1..32] of the first set bit.
4346 * @returns 0 if all bits are cleared.
4347 * @param u32 Integer to search for set bits.
4348 * @remark Similar to ffs() in BSD.
4349 */
4350#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4351DECLASM(unsigned) ASMBitFirstSetU32(uint32_t u32);
4352#else
4353DECLINLINE(unsigned) ASMBitFirstSetU32(uint32_t u32)
4354{
4355# if RT_INLINE_ASM_USES_INTRIN
4356 unsigned long iBit;
4357 if (_BitScanForward(&iBit, u32))
4358 iBit++;
4359 else
4360 iBit = 0;
4361# elif RT_INLINE_ASM_GNU_STYLE
4362 uint32_t iBit;
4363 __asm__ __volatile__("bsf %1, %0\n\t"
4364 "jnz 1f\n\t"
4365 "xorl %0, %0\n\t"
4366 "jmp 2f\n"
4367 "1:\n\t"
4368 "incl %0\n"
4369 "2:\n\t"
4370 : "=r" (iBit)
4371 : "rm" (u32));
4372# else
4373 uint32_t iBit;
4374 _asm
4375 {
4376 bsf eax, [u32]
4377 jnz found
4378 xor eax, eax
4379 jmp done
4380 found:
4381 inc eax
4382 done:
4383 mov [iBit], eax
4384 }
4385# endif
4386 return iBit;
4387}
4388#endif
4389
4390
4391/**
4392 * Finds the first bit which is set in the given 32-bit integer.
4393 * Bits are numbered from 1 (least significant) to 32.
4394 *
4395 * @returns index [1..32] of the first set bit.
4396 * @returns 0 if all bits are cleared.
4397 * @param i32 Integer to search for set bits.
4398 * @remark Similar to ffs() in BSD.
4399 */
4400DECLINLINE(unsigned) ASMBitFirstSetS32(int32_t i32)
4401{
4402 return ASMBitFirstSetU32((uint32_t)i32);
4403}
4404
4405
4406/**
4407 * Finds the last bit which is set in the given 32-bit integer.
4408 * Bits are numbered from 1 (least significant) to 32.
4409 *
4410 * @returns index [1..32] of the last set bit.
4411 * @returns 0 if all bits are cleared.
4412 * @param u32 Integer to search for set bits.
4413 * @remark Similar to fls() in BSD.
4414 */
4415#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4416DECLASM(unsigned) ASMBitLastSetU32(uint32_t u32);
4417#else
4418DECLINLINE(unsigned) ASMBitLastSetU32(uint32_t u32)
4419{
4420# if RT_INLINE_ASM_USES_INTRIN
4421 unsigned long iBit;
4422 if (_BitScanReverse(&iBit, u32))
4423 iBit++;
4424 else
4425 iBit = 0;
4426# elif RT_INLINE_ASM_GNU_STYLE
4427 uint32_t iBit;
4428 __asm__ __volatile__("bsrl %1, %0\n\t"
4429 "jnz 1f\n\t"
4430 "xorl %0, %0\n\t"
4431 "jmp 2f\n"
4432 "1:\n\t"
4433 "incl %0\n"
4434 "2:\n\t"
4435 : "=r" (iBit)
4436 : "rm" (u32));
4437# else
4438 uint32_t iBit;
4439 _asm
4440 {
4441 bsr eax, [u32]
4442 jnz found
4443 xor eax, eax
4444 jmp done
4445 found:
4446 inc eax
4447 done:
4448 mov [iBit], eax
4449 }
4450# endif
4451 return iBit;
4452}
4453#endif
4454
4455
4456/**
4457 * Finds the last bit which is set in the given 32-bit integer.
4458 * Bits are numbered from 1 (least significant) to 32.
4459 *
4460 * @returns index [1..32] of the last set bit.
4461 * @returns 0 if all bits are cleared.
4462 * @param i32 Integer to search for set bits.
4463 * @remark Similar to fls() in BSD.
4464 */
4465DECLINLINE(unsigned) ASMBitLastSetS32(int32_t i32)
4466{
4467 return ASMBitLastSetU32((uint32_t)i32);
4468}
4469
4470/**
4471 * Reverse the byte order of the given 16-bit integer.
4472 *
4473 * @returns Revert
4474 * @param u16 16-bit integer value.
4475 */
4476#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4477DECLASM(uint16_t) ASMByteSwapU16(uint16_t u16);
4478#else
4479DECLINLINE(uint16_t) ASMByteSwapU16(uint16_t u16)
4480{
4481# if RT_INLINE_ASM_USES_INTRIN
4482 u16 = _byteswap_ushort(u16);
4483# elif RT_INLINE_ASM_GNU_STYLE
4484 __asm__ ("rorw $8, %0" : "=r" (u16) : "0" (u16));
4485# else
4486 _asm
4487 {
4488 mov ax, [u16]
4489 ror ax, 8
4490 mov [u16], ax
4491 }
4492# endif
4493 return u16;
4494}
4495#endif
4496
4497
4498/**
4499 * Reverse the byte order of the given 32-bit integer.
4500 *
4501 * @returns Revert
4502 * @param u32 32-bit integer value.
4503 */
4504#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4505DECLASM(uint32_t) ASMByteSwapU32(uint32_t u32);
4506#else
4507DECLINLINE(uint32_t) ASMByteSwapU32(uint32_t u32)
4508{
4509# if RT_INLINE_ASM_USES_INTRIN
4510 u32 = _byteswap_ulong(u32);
4511# elif RT_INLINE_ASM_GNU_STYLE
4512 __asm__ ("bswapl %0" : "=r" (u32) : "0" (u32));
4513# else
4514 _asm
4515 {
4516 mov eax, [u32]
4517 bswap eax
4518 mov [u32], eax
4519 }
4520# endif
4521 return u32;
4522}
4523#endif
4524
4525
4526/**
4527 * Reverse the byte order of the given 64-bit integer.
4528 *
4529 * @returns Revert
4530 * @param u64 64-bit integer value.
4531 */
4532DECLINLINE(uint64_t) ASMByteSwapU64(uint64_t u64)
4533{
4534#if defined(RT_ARCH_AMD64) && RT_INLINE_ASM_USES_INTRIN
4535 u64 = _byteswap_uint64(u64);
4536#else
4537 u64 = (uint64_t)ASMByteSwapU32((uint32_t)u64) << 32
4538 | (uint64_t)ASMByteSwapU32((uint32_t)(u64 >> 32));
4539#endif
4540 return u64;
4541}
4542
4543
4544/** @} */
4545
4546
4547/** @} */
4548
4549#endif
4550
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette