asm.h@ 63688

Last change on this file since 63688 was 63688, checked in by vboxsync, 8 years ago
asm.h: here as well
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 158.3 KB

Line
1	/** @file
2	* IPRT - Assembly Functions.
3	*/
4
5	/*
6	* Copyright (C) 2006-2016 Oracle Corporation
7	*
8	* This file is part of VirtualBox Open Source Edition (OSE), as
9	* available from http://www.virtualbox.org. This file is free software;
10	* you can redistribute it and/or modify it under the terms of the GNU
11	* General Public License (GPL) as published by the Free Software
12	* Foundation, in version 2 as it comes in the "COPYING" file of the
13	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
14	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
15	*
16	* The contents of this file may alternatively be used under the terms
17	* of the Common Development and Distribution License Version 1.0
18	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
19	* VirtualBox OSE distribution, in which case the provisions of the
20	* CDDL are applicable instead of those of the GPL.
21	*
22	* You may elect to license modified versions of this file under the
23	* terms and conditions of either the GPL or the CDDL or both.
24	*/
25
26	#ifndef ___iprt_asm_h
27	#define ___iprt_asm_h
28
29	#include <iprt/cdefs.h>
30	#include <iprt/types.h>
31	#include <iprt/assert.h>
32	/** @def RT_INLINE_ASM_USES_INTRIN
33	* Defined as 1 if we're using a _MSC_VER 1400.
34	* Otherwise defined as 0.
35	*/
36
37	/* Solaris 10 header ugliness */
38	#ifdef u
39	# undef u
40	#endif
41
42	#if defined(_MSC_VER) && RT_INLINE_ASM_USES_INTRIN
43	# pragma warning(push)
44	# pragma warning(disable:4668) /* Several incorrect __cplusplus uses. */
45	# pragma warning(disable:4255) /* Incorrect __slwpcb prototype. */
46	# include <intrin.h>
47	# pragma warning(pop)
48	/* Emit the intrinsics at all optimization levels. */
49	# pragma intrinsic(_ReadWriteBarrier)
50	# pragma intrinsic(__cpuid)
51	# pragma intrinsic(__stosd)
52	# pragma intrinsic(__stosw)
53	# pragma intrinsic(__stosb)
54	# pragma intrinsic(_BitScanForward)
55	# pragma intrinsic(_BitScanReverse)
56	# pragma intrinsic(_bittest)
57	# pragma intrinsic(_bittestandset)
58	# pragma intrinsic(_bittestandreset)
59	# pragma intrinsic(_bittestandcomplement)
60	# pragma intrinsic(_byteswap_ushort)
61	# pragma intrinsic(_byteswap_ulong)
62	# pragma intrinsic(_interlockedbittestandset)
63	# pragma intrinsic(_interlockedbittestandreset)
64	# pragma intrinsic(_InterlockedAnd)
65	# pragma intrinsic(_InterlockedOr)
66	# pragma intrinsic(_InterlockedIncrement)
67	# pragma intrinsic(_InterlockedDecrement)
68	# pragma intrinsic(_InterlockedExchange)
69	# pragma intrinsic(_InterlockedExchangeAdd)
70	# pragma intrinsic(_InterlockedCompareExchange)
71	# pragma intrinsic(_InterlockedCompareExchange64)
72	# pragma intrinsic(_rotl)
73	# pragma intrinsic(_rotr)
74	# pragma intrinsic(_rotl64)
75	# pragma intrinsic(_rotr64)
76	# ifdef RT_ARCH_AMD64
77	# pragma intrinsic(__stosq)
78	# pragma intrinsic(_byteswap_uint64)
79	# pragma intrinsic(_InterlockedExchange64)
80	# pragma intrinsic(_InterlockedExchangeAdd64)
81	# pragma intrinsic(_InterlockedAnd64)
82	# pragma intrinsic(_InterlockedOr64)
83	# pragma intrinsic(_InterlockedIncrement64)
84	# pragma intrinsic(_InterlockedDecrement64)
85	# endif
86	#endif
87
88	/*
89	* Include #pragma aux definitions for Watcom C/C++.
90	*/
91	#if defined(__WATCOMC__) && ARCH_BITS == 16 && defined(RT_ARCH_X86)
92	# include "asm-watcom-x86-16.h"
93	#elif defined(__WATCOMC__) && ARCH_BITS == 32 && defined(RT_ARCH_X86)
94	# include "asm-watcom-x86-32.h"
95	#endif
96
97
98
99	/** @defgroup grp_rt_asm ASM - Assembly Routines
100	* @ingroup grp_rt
101	*
102	* @remarks The difference between ordered and unordered atomic operations are that
103	* the former will complete outstanding reads and writes before continuing
104	* while the latter doesn't make any promises about the order. Ordered
105	* operations doesn't, it seems, make any 100% promise wrt to whether
106	* the operation will complete before any subsequent memory access.
107	* (please, correct if wrong.)
108	*
109	* ASMAtomicSomething operations are all ordered, while ASMAtomicUoSomething
110	* are unordered (note the Uo).
111	*
112	* @remarks Some remarks about __volatile__: Without this keyword gcc is allowed to reorder
113	* or even optimize assembler instructions away. For instance, in the following code
114	* the second rdmsr instruction is optimized away because gcc treats that instruction
115	* as deterministic:
116	*
117	* @code
118	* static inline uint64_t rdmsr_low(int idx)
119	* {
120	* uint32_t low;
121	* __asm__ ("rdmsr" : "=a"(low) : "c"(idx) : "edx");
122	* }
123	* ...
124	* uint32_t msr1 = rdmsr_low(1);
125	* foo(msr1);
126	* msr1 = rdmsr_low(1);
127	* bar(msr1);
128	* @endcode
129	*
130	* The input parameter of rdmsr_low is the same for both calls and therefore gcc will
131	* use the result of the first call as input parameter for bar() as well. For rdmsr this
132	* is not acceptable as this instruction is _not_ deterministic. This applies to reading
133	* machine status information in general.
134	*
135	* @{
136	*/
137
138
139	/** @def RT_INLINE_ASM_GCC_4_3_X_X86
140	* Used to work around some 4.3.x register allocation issues in this version of
141	* the compiler. So far this workaround is still required for 4.4 and 4.5 but
142	* definitely not for 5.x */
143	#define RT_INLINE_ASM_GCC_4_3_X_X86 (RT_GNUC_PREREQ(4, 3) && !RT_GNUC_PREREQ(5, 0) && defined(__i386__))
144	#ifndef RT_INLINE_ASM_GCC_4_3_X_X86
145	# define RT_INLINE_ASM_GCC_4_3_X_X86 0
146	#endif
147
148	/** @def RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
149	* i686-apple-darwin9-gcc-4.0.1 (GCC) 4.0.1 (Apple Inc. build 5493) screws up
150	* RTSemRWRequestWrite semsemrw-lockless-generic.cpp in release builds. PIC
151	* mode, x86.
152	*
153	* Some gcc 4.3.x versions may have register allocation issues with cmpxchg8b
154	* when in PIC mode on x86.
155	*/
156	#ifndef RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
157	# if defined(DOXYGEN_RUNNING) \|\| defined(__WATCOMC__) /* Watcom has trouble with the expression below */
158	# define RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC 1
159	# elif defined(_MSC_VER) /* Visual C++ has trouble too, but it'll only tell us when C4688 is enabled. */
160	# define RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC 0
161	# else
162	# define RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC \
163	( (defined(PIC) \|\| defined(__PIC__)) \
164	&& defined(RT_ARCH_X86) \
165	&& ( RT_INLINE_ASM_GCC_4_3_X_X86 \
166	\|\| defined(RT_OS_DARWIN)) )
167	# endif
168	#endif
169
170
171	/** @def ASMReturnAddress
172	* Gets the return address of the current (or calling if you like) function or method.
173	*/
174	#ifdef _MSC_VER
175	# ifdef __cplusplus
176	extern "C"
177	# endif
178	void * _ReturnAddress(void);
179	# pragma intrinsic(_ReturnAddress)
180	# define ASMReturnAddress() _ReturnAddress()
181	#elif defined(__GNUC__) \|\| defined(DOXYGEN_RUNNING)
182	# define ASMReturnAddress() __builtin_return_address(0)
183	#elif defined(__WATCOMC__)
184	# define ASMReturnAddress() Watcom_does_not_appear_to_have_intrinsic_return_address_function()
185	#else
186	# error "Unsupported compiler."
187	#endif
188
189
190	/**
191	* Compiler memory barrier.
192	*
193	* Ensure that the compiler does not use any cached (register/tmp stack) memory
194	* values or any outstanding writes when returning from this function.
195	*
196	* This function must be used if non-volatile data is modified by a
197	* device or the VMM. Typical cases are port access, MMIO access,
198	* trapping instruction, etc.
199	*/
200	#if RT_INLINE_ASM_GNU_STYLE
201	# define ASMCompilerBarrier() do { __asm__ __volatile__("" : : : "memory"); } while (0)
202	#elif RT_INLINE_ASM_USES_INTRIN
203	# define ASMCompilerBarrier() do { _ReadWriteBarrier(); } while (0)
204	#elif defined(__WATCOMC__)
205	void ASMCompilerBarrier(void);
206	#else /* 2003 should have _ReadWriteBarrier() but I guess we're at 2002 level then... */
207	DECLINLINE(void) ASMCompilerBarrier(void)
208	{
209	__asm
210	{
211	}
212	}
213	#endif
214
215
216	/** @def ASMBreakpoint
217	* Debugger Breakpoint.
218	* @deprecated Use RT_BREAKPOINT instead.
219	* @internal
220	*/
221	#define ASMBreakpoint() RT_BREAKPOINT()
222
223
224	/**
225	* Spinloop hint for platforms that have these, empty function on the other
226	* platforms.
227	*
228	* x86 & AMD64: The PAUSE variant of NOP for helping hyperthreaded CPUs detecting
229	* spin locks.
230	*/
231	#if RT_INLINE_ASM_EXTERNAL && (defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86))
232	DECLASM(void) ASMNopPause(void);
233	#else
234	DECLINLINE(void) ASMNopPause(void)
235	{
236	# if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
237	# if RT_INLINE_ASM_GNU_STYLE
238	__asm__ __volatile__(".byte 0xf3,0x90\n\t");
239	# else
240	__asm {
241	_emit 0f3h
242	_emit 090h
243	}
244	# endif
245	# else
246	/* dummy */
247	# endif
248	}
249	#endif
250
251
252	/**
253	* Atomically Exchange an unsigned 8-bit value, ordered.
254	*
255	* @returns Current *pu8 value
256	* @param pu8 Pointer to the 8-bit variable to update.
257	* @param u8 The 8-bit value to assign to *pu8.
258	*/
259	#if RT_INLINE_ASM_EXTERNAL
260	DECLASM(uint8_t) ASMAtomicXchgU8(volatile uint8_t *pu8, uint8_t u8);
261	#else
262	DECLINLINE(uint8_t) ASMAtomicXchgU8(volatile uint8_t *pu8, uint8_t u8)
263	{
264	# if RT_INLINE_ASM_GNU_STYLE
265	__asm__ __volatile__("xchgb %0, %1\n\t"
266	: "=m" (*pu8),
267	"=q" (u8) /* =r - busted on g++ (GCC) 3.4.4 20050721 (Red Hat 3.4.4-2) */
268	: "1" (u8),
269	"m" (*pu8));
270	# else
271	__asm
272	{
273	# ifdef RT_ARCH_AMD64
274	mov rdx, [pu8]
275	mov al, [u8]
276	xchg [rdx], al
277	mov [u8], al
278	# else
279	mov edx, [pu8]
280	mov al, [u8]
281	xchg [edx], al
282	mov [u8], al
283	# endif
284	}
285	# endif
286	return u8;
287	}
288	#endif
289
290
291	/**
292	* Atomically Exchange a signed 8-bit value, ordered.
293	*
294	* @returns Current *pu8 value
295	* @param pi8 Pointer to the 8-bit variable to update.
296	* @param i8 The 8-bit value to assign to *pi8.
297	*/
298	DECLINLINE(int8_t) ASMAtomicXchgS8(volatile int8_t *pi8, int8_t i8)
299	{
300	return (int8_t)ASMAtomicXchgU8((volatile uint8_t *)pi8, (uint8_t)i8);
301	}
302
303
304	/**
305	* Atomically Exchange a bool value, ordered.
306	*
307	* @returns Current *pf value
308	* @param pf Pointer to the 8-bit variable to update.
309	* @param f The 8-bit value to assign to *pi8.
310	*/
311	DECLINLINE(bool) ASMAtomicXchgBool(volatile bool *pf, bool f)
312	{
313	#ifdef _MSC_VER
314	return !!ASMAtomicXchgU8((volatile uint8_t *)pf, (uint8_t)f);
315	#else
316	return (bool)ASMAtomicXchgU8((volatile uint8_t *)pf, (uint8_t)f);
317	#endif
318	}
319
320
321	/**
322	* Atomically Exchange an unsigned 16-bit value, ordered.
323	*
324	* @returns Current *pu16 value
325	* @param pu16 Pointer to the 16-bit variable to update.
326	* @param u16 The 16-bit value to assign to *pu16.
327	*/
328	#if RT_INLINE_ASM_EXTERNAL
329	DECLASM(uint16_t) ASMAtomicXchgU16(volatile uint16_t *pu16, uint16_t u16);
330	#else
331	DECLINLINE(uint16_t) ASMAtomicXchgU16(volatile uint16_t *pu16, uint16_t u16)
332	{
333	# if RT_INLINE_ASM_GNU_STYLE
334	__asm__ __volatile__("xchgw %0, %1\n\t"
335	: "=m" (*pu16),
336	"=r" (u16)
337	: "1" (u16),
338	"m" (*pu16));
339	# else
340	__asm
341	{
342	# ifdef RT_ARCH_AMD64
343	mov rdx, [pu16]
344	mov ax, [u16]
345	xchg [rdx], ax
346	mov [u16], ax
347	# else
348	mov edx, [pu16]
349	mov ax, [u16]
350	xchg [edx], ax
351	mov [u16], ax
352	# endif
353	}
354	# endif
355	return u16;
356	}
357	#endif
358
359
360	/**
361	* Atomically Exchange a signed 16-bit value, ordered.
362	*
363	* @returns Current *pu16 value
364	* @param pi16 Pointer to the 16-bit variable to update.
365	* @param i16 The 16-bit value to assign to *pi16.
366	*/
367	DECLINLINE(int16_t) ASMAtomicXchgS16(volatile int16_t *pi16, int16_t i16)
368	{
369	return (int16_t)ASMAtomicXchgU16((volatile uint16_t *)pi16, (uint16_t)i16);
370	}
371
372
373	/**
374	* Atomically Exchange an unsigned 32-bit value, ordered.
375	*
376	* @returns Current *pu32 value
377	* @param pu32 Pointer to the 32-bit variable to update.
378	* @param u32 The 32-bit value to assign to *pu32.
379	*
380	* @remarks Does not work on 286 and earlier.
381	*/
382	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
383	DECLASM(uint32_t) ASMAtomicXchgU32(volatile uint32_t *pu32, uint32_t u32);
384	#else
385	DECLINLINE(uint32_t) ASMAtomicXchgU32(volatile uint32_t *pu32, uint32_t u32)
386	{
387	# if RT_INLINE_ASM_GNU_STYLE
388	__asm__ __volatile__("xchgl %0, %1\n\t"
389	: "=m" (*pu32),
390	"=r" (u32)
391	: "1" (u32),
392	"m" (*pu32));
393
394	# elif RT_INLINE_ASM_USES_INTRIN
395	u32 = _InterlockedExchange((long *)pu32, u32);
396
397	# else
398	__asm
399	{
400	# ifdef RT_ARCH_AMD64
401	mov rdx, [pu32]
402	mov eax, u32
403	xchg [rdx], eax
404	mov [u32], eax
405	# else
406	mov edx, [pu32]
407	mov eax, u32
408	xchg [edx], eax
409	mov [u32], eax
410	# endif
411	}
412	# endif
413	return u32;
414	}
415	#endif
416
417
418	/**
419	* Atomically Exchange a signed 32-bit value, ordered.
420	*
421	* @returns Current *pu32 value
422	* @param pi32 Pointer to the 32-bit variable to update.
423	* @param i32 The 32-bit value to assign to *pi32.
424	*/
425	DECLINLINE(int32_t) ASMAtomicXchgS32(volatile int32_t *pi32, int32_t i32)
426	{
427	return (int32_t)ASMAtomicXchgU32((volatile uint32_t *)pi32, (uint32_t)i32);
428	}
429
430
431	/**
432	* Atomically Exchange an unsigned 64-bit value, ordered.
433	*
434	* @returns Current *pu64 value
435	* @param pu64 Pointer to the 64-bit variable to update.
436	* @param u64 The 64-bit value to assign to *pu64.
437	*
438	* @remarks Works on 32-bit x86 CPUs starting with Pentium.
439	*/
440	#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
441	\|\| RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
442	DECLASM(uint64_t) ASMAtomicXchgU64(volatile uint64_t *pu64, uint64_t u64);
443	#else
444	DECLINLINE(uint64_t) ASMAtomicXchgU64(volatile uint64_t *pu64, uint64_t u64)
445	{
446	# if defined(RT_ARCH_AMD64)
447	# if RT_INLINE_ASM_USES_INTRIN
448	u64 = _InterlockedExchange64((__int64 *)pu64, u64);
449
450	# elif RT_INLINE_ASM_GNU_STYLE
451	__asm__ __volatile__("xchgq %0, %1\n\t"
452	: "=m" (*pu64),
453	"=r" (u64)
454	: "1" (u64),
455	"m" (*pu64));
456	# else
457	__asm
458	{
459	mov rdx, [pu64]
460	mov rax, [u64]
461	xchg [rdx], rax
462	mov [u64], rax
463	}
464	# endif
465	# else /* !RT_ARCH_AMD64 */
466	# if RT_INLINE_ASM_GNU_STYLE
467	# if defined(PIC) \|\| defined(__PIC__)
468	uint32_t u32EBX = (uint32_t)u64;
469	__asm__ __volatile__(/"xchgl %%esi, %5\n\t"/
470	"xchgl %%ebx, %3\n\t"
471	"1:\n\t"
472	"lock; cmpxchg8b (%5)\n\t"
473	"jnz 1b\n\t"
474	"movl %3, %%ebx\n\t"
475	/"xchgl %%esi, %5\n\t"/
476	: "=A" (u64),
477	"=m" (*pu64)
478	: "0" (*pu64),
479	"m" ( u32EBX ),
480	"c" ( (uint32_t)(u64 >> 32) ),
481	"S" (pu64));
482	# else /* !PIC */
483	__asm__ __volatile__("1:\n\t"
484	"lock; cmpxchg8b %1\n\t"
485	"jnz 1b\n\t"
486	: "=A" (u64),
487	"=m" (*pu64)
488	: "0" (*pu64),
489	"b" ( (uint32_t)u64 ),
490	"c" ( (uint32_t)(u64 >> 32) ));
491	# endif
492	# else
493	__asm
494	{
495	mov ebx, dword ptr [u64]
496	mov ecx, dword ptr [u64 + 4]
497	mov edi, pu64
498	mov eax, dword ptr [edi]
499	mov edx, dword ptr [edi + 4]
500	retry:
501	lock cmpxchg8b [edi]
502	jnz retry
503	mov dword ptr [u64], eax
504	mov dword ptr [u64 + 4], edx
505	}
506	# endif
507	# endif /* !RT_ARCH_AMD64 */
508	return u64;
509	}
510	#endif
511
512
513	/**
514	* Atomically Exchange an signed 64-bit value, ordered.
515	*
516	* @returns Current *pi64 value
517	* @param pi64 Pointer to the 64-bit variable to update.
518	* @param i64 The 64-bit value to assign to *pi64.
519	*/
520	DECLINLINE(int64_t) ASMAtomicXchgS64(volatile int64_t *pi64, int64_t i64)
521	{
522	return (int64_t)ASMAtomicXchgU64((volatile uint64_t *)pi64, (uint64_t)i64);
523	}
524
525
526	/**
527	* Atomically Exchange a pointer value, ordered.
528	*
529	* @returns Current *ppv value
530	* @param ppv Pointer to the pointer variable to update.
531	* @param pv The pointer value to assign to *ppv.
532	*/
533	DECLINLINE(void ) ASMAtomicXchgPtr(void volatile ppv, const void pv)
534	{
535	#if ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
536	return (void )ASMAtomicXchgU32((volatile uint32_t )(void *)ppv, (uint32_t)pv);
537	#elif ARCH_BITS == 64
538	return (void )ASMAtomicXchgU64((volatile uint64_t )(void *)ppv, (uint64_t)pv);
539	#else
540	# error "ARCH_BITS is bogus"
541	#endif
542	}
543
544
545	/**
546	* Convenience macro for avoiding the annoying casting with ASMAtomicXchgPtr.
547	*
548	* @returns Current *pv value
549	* @param ppv Pointer to the pointer variable to update.
550	* @param pv The pointer value to assign to *ppv.
551	* @param Type The type of *ppv, sans volatile.
552	*/
553	#ifdef __GNUC__
554	# define ASMAtomicXchgPtrT(ppv, pv, Type) \
555	__extension__ \
556	({\
557	__typeof__((ppv)) volatile const ppvTypeChecked = (ppv); \
558	Type const pvTypeChecked = (pv); \
559	Type pvTypeCheckedRet = (__typeof__((ppv))) ASMAtomicXchgPtr((void volatile )ppvTypeChecked, (void )pvTypeChecked); \
560	pvTypeCheckedRet; \
561	})
562	#else
563	# define ASMAtomicXchgPtrT(ppv, pv, Type) \
564	(Type)ASMAtomicXchgPtr((void * volatile )(ppv), (void )(pv))
565	#endif
566
567
568	/**
569	* Atomically Exchange a raw-mode context pointer value, ordered.
570	*
571	* @returns Current *ppv value
572	* @param ppvRC Pointer to the pointer variable to update.
573	* @param pvRC The pointer value to assign to *ppv.
574	*/
575	DECLINLINE(RTRCPTR) ASMAtomicXchgRCPtr(RTRCPTR volatile *ppvRC, RTRCPTR pvRC)
576	{
577	return (RTRCPTR)ASMAtomicXchgU32((uint32_t volatile )(void )ppvRC, (uint32_t)pvRC);
578	}
579
580
581	/**
582	* Atomically Exchange a ring-0 pointer value, ordered.
583	*
584	* @returns Current *ppv value
585	* @param ppvR0 Pointer to the pointer variable to update.
586	* @param pvR0 The pointer value to assign to *ppv.
587	*/
588	DECLINLINE(RTR0PTR) ASMAtomicXchgR0Ptr(RTR0PTR volatile *ppvR0, RTR0PTR pvR0)
589	{
590	#if R0_ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
591	return (RTR0PTR)ASMAtomicXchgU32((volatile uint32_t )(void )ppvR0, (uint32_t)pvR0);
592	#elif R0_ARCH_BITS == 64
593	return (RTR0PTR)ASMAtomicXchgU64((volatile uint64_t )(void )ppvR0, (uint64_t)pvR0);
594	#else
595	# error "R0_ARCH_BITS is bogus"
596	#endif
597	}
598
599
600	/**
601	* Atomically Exchange a ring-3 pointer value, ordered.
602	*
603	* @returns Current *ppv value
604	* @param ppvR3 Pointer to the pointer variable to update.
605	* @param pvR3 The pointer value to assign to *ppv.
606	*/
607	DECLINLINE(RTR3PTR) ASMAtomicXchgR3Ptr(RTR3PTR volatile *ppvR3, RTR3PTR pvR3)
608	{
609	#if R3_ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
610	return (RTR3PTR)ASMAtomicXchgU32((volatile uint32_t )(void )ppvR3, (uint32_t)pvR3);
611	#elif R3_ARCH_BITS == 64
612	return (RTR3PTR)ASMAtomicXchgU64((volatile uint64_t )(void )ppvR3, (uint64_t)pvR3);
613	#else
614	# error "R3_ARCH_BITS is bogus"
615	#endif
616	}
617
618
619	/** @def ASMAtomicXchgHandle
620	* Atomically Exchange a typical IPRT handle value, ordered.
621	*
622	* @param ph Pointer to the value to update.
623	* @param hNew The new value to assigned to *pu.
624	* @param phRes Where to store the current *ph value.
625	*
626	* @remarks This doesn't currently work for all handles (like RTFILE).
627	*/
628	#if HC_ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
629	# define ASMAtomicXchgHandle(ph, hNew, phRes) \
630	do { \
631	AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
632	AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
633	(uint32_t )(phRes) = ASMAtomicXchgU32((uint32_t volatile *)(ph), (const uint32_t)(hNew)); \
634	} while (0)
635	#elif HC_ARCH_BITS == 64
636	# define ASMAtomicXchgHandle(ph, hNew, phRes) \
637	do { \
638	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
639	AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
640	(uint64_t )(phRes) = ASMAtomicXchgU64((uint64_t volatile *)(ph), (const uint64_t)(hNew)); \
641	} while (0)
642	#else
643	# error HC_ARCH_BITS
644	#endif
645
646
647	/**
648	* Atomically Exchange a value which size might differ
649	* between platforms or compilers, ordered.
650	*
651	* @param pu Pointer to the variable to update.
652	* @param uNew The value to assign to *pu.
653	* @todo This is busted as its missing the result argument.
654	*/
655	#define ASMAtomicXchgSize(pu, uNew) \
656	do { \
657	switch (sizeof(*(pu))) { \
658	case 1: ASMAtomicXchgU8((volatile uint8_t )(void )(pu), (uint8_t)(uNew)); break; \
659	case 2: ASMAtomicXchgU16((volatile uint16_t )(void )(pu), (uint16_t)(uNew)); break; \
660	case 4: ASMAtomicXchgU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew)); break; \
661	case 8: ASMAtomicXchgU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew)); break; \
662	default: AssertMsgFailed(("ASMAtomicXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
663	} \
664	} while (0)
665
666	/**
667	* Atomically Exchange a value which size might differ
668	* between platforms or compilers, ordered.
669	*
670	* @param pu Pointer to the variable to update.
671	* @param uNew The value to assign to *pu.
672	* @param puRes Where to store the current *pu value.
673	*/
674	#define ASMAtomicXchgSizeCorrect(pu, uNew, puRes) \
675	do { \
676	switch (sizeof(*(pu))) { \
677	case 1: (uint8_t )(puRes) = ASMAtomicXchgU8((volatile uint8_t )(void )(pu), (uint8_t)(uNew)); break; \
678	case 2: (uint16_t )(puRes) = ASMAtomicXchgU16((volatile uint16_t )(void )(pu), (uint16_t)(uNew)); break; \
679	case 4: (uint32_t )(puRes) = ASMAtomicXchgU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew)); break; \
680	case 8: (uint64_t )(puRes) = ASMAtomicXchgU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew)); break; \
681	default: AssertMsgFailed(("ASMAtomicXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
682	} \
683	} while (0)
684
685
686
687	/**
688	* Atomically Compare and Exchange an unsigned 8-bit value, ordered.
689	*
690	* @returns true if xchg was done.
691	* @returns false if xchg wasn't done.
692	*
693	* @param pu8 Pointer to the value to update.
694	* @param u8New The new value to assigned to *pu8.
695	* @param u8Old The old value to *pu8 compare with.
696	*
697	* @remarks x86: Requires a 486 or later.
698	*/
699	#if RT_INLINE_ASM_EXTERNAL \|\| !RT_INLINE_ASM_GNU_STYLE
700	DECLASM(bool) ASMAtomicCmpXchgU8(volatile uint8_t *pu8, const uint8_t u8New, const uint8_t u8Old);
701	#else
702	DECLINLINE(bool) ASMAtomicCmpXchgU8(volatile uint8_t *pu8, const uint8_t u8New, uint8_t u8Old)
703	{
704	uint8_t u8Ret;
705	__asm__ __volatile__("lock; cmpxchgb %3, %0\n\t"
706	"setz %1\n\t"
707	: "=m" (*pu8),
708	"=qm" (u8Ret),
709	"=a" (u8Old)
710	: "q" (u8New),
711	"2" (u8Old),
712	"m" (*pu8));
713	return (bool)u8Ret;
714	}
715	#endif
716
717
718	/**
719	* Atomically Compare and Exchange a signed 8-bit value, ordered.
720	*
721	* @returns true if xchg was done.
722	* @returns false if xchg wasn't done.
723	*
724	* @param pi8 Pointer to the value to update.
725	* @param i8New The new value to assigned to *pi8.
726	* @param i8Old The old value to *pi8 compare with.
727	*
728	* @remarks x86: Requires a 486 or later.
729	*/
730	DECLINLINE(bool) ASMAtomicCmpXchgS8(volatile int8_t *pi8, const int8_t i8New, const int8_t i8Old)
731	{
732	return ASMAtomicCmpXchgU8((volatile uint8_t *)pi8, (const uint8_t)i8New, (const uint8_t)i8Old);
733	}
734
735
736	/**
737	* Atomically Compare and Exchange a bool value, ordered.
738	*
739	* @returns true if xchg was done.
740	* @returns false if xchg wasn't done.
741	*
742	* @param pf Pointer to the value to update.
743	* @param fNew The new value to assigned to *pf.
744	* @param fOld The old value to *pf compare with.
745	*
746	* @remarks x86: Requires a 486 or later.
747	*/
748	DECLINLINE(bool) ASMAtomicCmpXchgBool(volatile bool *pf, const bool fNew, const bool fOld)
749	{
750	return ASMAtomicCmpXchgU8((volatile uint8_t *)pf, (const uint8_t)fNew, (const uint8_t)fOld);
751	}
752
753
754	/**
755	* Atomically Compare and Exchange an unsigned 32-bit value, ordered.
756	*
757	* @returns true if xchg was done.
758	* @returns false if xchg wasn't done.
759	*
760	* @param pu32 Pointer to the value to update.
761	* @param u32New The new value to assigned to *pu32.
762	* @param u32Old The old value to *pu32 compare with.
763	*
764	* @remarks x86: Requires a 486 or later.
765	*/
766	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
767	DECLASM(bool) ASMAtomicCmpXchgU32(volatile uint32_t *pu32, const uint32_t u32New, const uint32_t u32Old);
768	#else
769	DECLINLINE(bool) ASMAtomicCmpXchgU32(volatile uint32_t *pu32, const uint32_t u32New, uint32_t u32Old)
770	{
771	# if RT_INLINE_ASM_GNU_STYLE
772	uint8_t u8Ret;
773	__asm__ __volatile__("lock; cmpxchgl %3, %0\n\t"
774	"setz %1\n\t"
775	: "=m" (*pu32),
776	"=qm" (u8Ret),
777	"=a" (u32Old)
778	: "r" (u32New),
779	"2" (u32Old),
780	"m" (*pu32));
781	return (bool)u8Ret;
782
783	# elif RT_INLINE_ASM_USES_INTRIN
784	return (uint32_t)_InterlockedCompareExchange((long *)pu32, u32New, u32Old) == u32Old;
785
786	# else
787	uint32_t u32Ret;
788	__asm
789	{
790	# ifdef RT_ARCH_AMD64
791	mov rdx, [pu32]
792	# else
793	mov edx, [pu32]
794	# endif
795	mov eax, [u32Old]
796	mov ecx, [u32New]
797	# ifdef RT_ARCH_AMD64
798	lock cmpxchg [rdx], ecx
799	# else
800	lock cmpxchg [edx], ecx
801	# endif
802	setz al
803	movzx eax, al
804	mov [u32Ret], eax
805	}
806	return !!u32Ret;
807	# endif
808	}
809	#endif
810
811
812	/**
813	* Atomically Compare and Exchange a signed 32-bit value, ordered.
814	*
815	* @returns true if xchg was done.
816	* @returns false if xchg wasn't done.
817	*
818	* @param pi32 Pointer to the value to update.
819	* @param i32New The new value to assigned to *pi32.
820	* @param i32Old The old value to *pi32 compare with.
821	*
822	* @remarks x86: Requires a 486 or later.
823	*/
824	DECLINLINE(bool) ASMAtomicCmpXchgS32(volatile int32_t *pi32, const int32_t i32New, const int32_t i32Old)
825	{
826	return ASMAtomicCmpXchgU32((volatile uint32_t *)pi32, (uint32_t)i32New, (uint32_t)i32Old);
827	}
828
829
830	/**
831	* Atomically Compare and exchange an unsigned 64-bit value, ordered.
832	*
833	* @returns true if xchg was done.
834	* @returns false if xchg wasn't done.
835	*
836	* @param pu64 Pointer to the 64-bit variable to update.
837	* @param u64New The 64-bit value to assign to *pu64.
838	* @param u64Old The value to compare with.
839	*
840	* @remarks x86: Requires a Pentium or later.
841	*/
842	#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
843	\|\| RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
844	DECLASM(bool) ASMAtomicCmpXchgU64(volatile uint64_t *pu64, const uint64_t u64New, const uint64_t u64Old);
845	#else
846	DECLINLINE(bool) ASMAtomicCmpXchgU64(volatile uint64_t *pu64, uint64_t u64New, uint64_t u64Old)
847	{
848	# if RT_INLINE_ASM_USES_INTRIN
849	return (uint64_t)_InterlockedCompareExchange64((__int64 *)pu64, u64New, u64Old) == u64Old;
850
851	# elif defined(RT_ARCH_AMD64)
852	# if RT_INLINE_ASM_GNU_STYLE
853	uint8_t u8Ret;
854	__asm__ __volatile__("lock; cmpxchgq %3, %0\n\t"
855	"setz %1\n\t"
856	: "=m" (*pu64),
857	"=qm" (u8Ret),
858	"=a" (u64Old)
859	: "r" (u64New),
860	"2" (u64Old),
861	"m" (*pu64));
862	return (bool)u8Ret;
863	# else
864	bool fRet;
865	__asm
866	{
867	mov rdx, [pu32]
868	mov rax, [u64Old]
869	mov rcx, [u64New]
870	lock cmpxchg [rdx], rcx
871	setz al
872	mov [fRet], al
873	}
874	return fRet;
875	# endif
876	# else /* !RT_ARCH_AMD64 */
877	uint32_t u32Ret;
878	# if RT_INLINE_ASM_GNU_STYLE
879	# if defined(PIC) \|\| defined(__PIC__)
880	uint32_t u32EBX = (uint32_t)u64New;
881	uint32_t u32Spill;
882	__asm__ __volatile__("xchgl %%ebx, %4\n\t"
883	"lock; cmpxchg8b (%6)\n\t"
884	"setz %%al\n\t"
885	"movl %4, %%ebx\n\t"
886	"movzbl %%al, %%eax\n\t"
887	: "=a" (u32Ret),
888	"=d" (u32Spill),
889	# if RT_GNUC_PREREQ(4, 3)
890	"+m" (*pu64)
891	# else
892	"=m" (*pu64)
893	# endif
894	: "A" (u64Old),
895	"m" ( u32EBX ),
896	"c" ( (uint32_t)(u64New >> 32) ),
897	"S" (pu64));
898	# else /* !PIC */
899	uint32_t u32Spill;
900	__asm__ __volatile__("lock; cmpxchg8b %2\n\t"
901	"setz %%al\n\t"
902	"movzbl %%al, %%eax\n\t"
903	: "=a" (u32Ret),
904	"=d" (u32Spill),
905	"+m" (*pu64)
906	: "A" (u64Old),
907	"b" ( (uint32_t)u64New ),
908	"c" ( (uint32_t)(u64New >> 32) ));
909	# endif
910	return (bool)u32Ret;
911	# else
912	__asm
913	{
914	mov ebx, dword ptr [u64New]
915	mov ecx, dword ptr [u64New + 4]
916	mov edi, [pu64]
917	mov eax, dword ptr [u64Old]
918	mov edx, dword ptr [u64Old + 4]
919	lock cmpxchg8b [edi]
920	setz al
921	movzx eax, al
922	mov dword ptr [u32Ret], eax
923	}
924	return !!u32Ret;
925	# endif
926	# endif /* !RT_ARCH_AMD64 */
927	}
928	#endif
929
930
931	/**
932	* Atomically Compare and exchange a signed 64-bit value, ordered.
933	*
934	* @returns true if xchg was done.
935	* @returns false if xchg wasn't done.
936	*
937	* @param pi64 Pointer to the 64-bit variable to update.
938	* @param i64 The 64-bit value to assign to *pu64.
939	* @param i64Old The value to compare with.
940	*
941	* @remarks x86: Requires a Pentium or later.
942	*/
943	DECLINLINE(bool) ASMAtomicCmpXchgS64(volatile int64_t *pi64, const int64_t i64, const int64_t i64Old)
944	{
945	return ASMAtomicCmpXchgU64((volatile uint64_t *)pi64, (uint64_t)i64, (uint64_t)i64Old);
946	}
947
948
949	/**
950	* Atomically Compare and Exchange a pointer value, ordered.
951	*
952	* @returns true if xchg was done.
953	* @returns false if xchg wasn't done.
954	*
955	* @param ppv Pointer to the value to update.
956	* @param pvNew The new value to assigned to *ppv.
957	* @param pvOld The old value to *ppv compare with.
958	*
959	* @remarks x86: Requires a 486 or later.
960	*/
961	DECLINLINE(bool) ASMAtomicCmpXchgPtrVoid(void * volatile ppv, const void pvNew, const void *pvOld)
962	{
963	#if ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
964	return ASMAtomicCmpXchgU32((volatile uint32_t )(void )ppv, (uint32_t)pvNew, (uint32_t)pvOld);
965	#elif ARCH_BITS == 64
966	return ASMAtomicCmpXchgU64((volatile uint64_t )(void )ppv, (uint64_t)pvNew, (uint64_t)pvOld);
967	#else
968	# error "ARCH_BITS is bogus"
969	#endif
970	}
971
972
973	/**
974	* Atomically Compare and Exchange a pointer value, ordered.
975	*
976	* @returns true if xchg was done.
977	* @returns false if xchg wasn't done.
978	*
979	* @param ppv Pointer to the value to update.
980	* @param pvNew The new value to assigned to *ppv.
981	* @param pvOld The old value to *ppv compare with.
982	*
983	* @remarks This is relatively type safe on GCC platforms.
984	* @remarks x86: Requires a 486 or later.
985	*/
986	#ifdef __GNUC__
987	# define ASMAtomicCmpXchgPtr(ppv, pvNew, pvOld) \
988	__extension__ \
989	({\
990	__typeof__((ppv)) volatile const ppvTypeChecked = (ppv); \
991	__typeof__(*(ppv)) const pvNewTypeChecked = (pvNew); \
992	__typeof__(*(ppv)) const pvOldTypeChecked = (pvOld); \
993	bool fMacroRet = ASMAtomicCmpXchgPtrVoid((void * volatile *)ppvTypeChecked, \
994	(void )pvNewTypeChecked, (void )pvOldTypeChecked); \
995	fMacroRet; \
996	})
997	#else
998	# define ASMAtomicCmpXchgPtr(ppv, pvNew, pvOld) \
999	ASMAtomicCmpXchgPtrVoid((void * volatile )(ppv), (void )(pvNew), (void *)(pvOld))
1000	#endif
1001
1002
1003	/** @def ASMAtomicCmpXchgHandle
1004	* Atomically Compare and Exchange a typical IPRT handle value, ordered.
1005	*
1006	* @param ph Pointer to the value to update.
1007	* @param hNew The new value to assigned to *pu.
1008	* @param hOld The old value to *pu compare with.
1009	* @param fRc Where to store the result.
1010	*
1011	* @remarks This doesn't currently work for all handles (like RTFILE).
1012	* @remarks x86: Requires a 486 or later.
1013	*/
1014	#if HC_ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
1015	# define ASMAtomicCmpXchgHandle(ph, hNew, hOld, fRc) \
1016	do { \
1017	AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
1018	(fRc) = ASMAtomicCmpXchgU32((uint32_t volatile *)(ph), (const uint32_t)(hNew), (const uint32_t)(hOld)); \
1019	} while (0)
1020	#elif HC_ARCH_BITS == 64
1021	# define ASMAtomicCmpXchgHandle(ph, hNew, hOld, fRc) \
1022	do { \
1023	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
1024	(fRc) = ASMAtomicCmpXchgU64((uint64_t volatile *)(ph), (const uint64_t)(hNew), (const uint64_t)(hOld)); \
1025	} while (0)
1026	#else
1027	# error HC_ARCH_BITS
1028	#endif
1029
1030
1031	/** @def ASMAtomicCmpXchgSize
1032	* Atomically Compare and Exchange a value which size might differ
1033	* between platforms or compilers, ordered.
1034	*
1035	* @param pu Pointer to the value to update.
1036	* @param uNew The new value to assigned to *pu.
1037	* @param uOld The old value to *pu compare with.
1038	* @param fRc Where to store the result.
1039	*
1040	* @remarks x86: Requires a 486 or later.
1041	*/
1042	#define ASMAtomicCmpXchgSize(pu, uNew, uOld, fRc) \
1043	do { \
1044	switch (sizeof(*(pu))) { \
1045	case 4: (fRc) = ASMAtomicCmpXchgU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew), (uint32_t)(uOld)); \
1046	break; \
1047	case 8: (fRc) = ASMAtomicCmpXchgU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew), (uint64_t)(uOld)); \
1048	break; \
1049	default: AssertMsgFailed(("ASMAtomicCmpXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
1050	(fRc) = false; \
1051	break; \
1052	} \
1053	} while (0)
1054
1055
1056	/**
1057	* Atomically Compare and Exchange an unsigned 32-bit value, additionally
1058	* passes back old value, ordered.
1059	*
1060	* @returns true if xchg was done.
1061	* @returns false if xchg wasn't done.
1062	*
1063	* @param pu32 Pointer to the value to update.
1064	* @param u32New The new value to assigned to *pu32.
1065	* @param u32Old The old value to *pu32 compare with.
1066	* @param pu32Old Pointer store the old value at.
1067	*
1068	* @remarks x86: Requires a 486 or later.
1069	*/
1070	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1071	DECLASM(bool) ASMAtomicCmpXchgExU32(volatile uint32_t pu32, const uint32_t u32New, const uint32_t u32Old, uint32_t pu32Old);
1072	#else
1073	DECLINLINE(bool) ASMAtomicCmpXchgExU32(volatile uint32_t pu32, const uint32_t u32New, const uint32_t u32Old, uint32_t pu32Old)
1074	{
1075	# if RT_INLINE_ASM_GNU_STYLE
1076	uint8_t u8Ret;
1077	__asm__ __volatile__("lock; cmpxchgl %3, %0\n\t"
1078	"setz %1\n\t"
1079	: "=m" (*pu32),
1080	"=qm" (u8Ret),
1081	"=a" (*pu32Old)
1082	: "r" (u32New),
1083	"a" (u32Old),
1084	"m" (*pu32));
1085	return (bool)u8Ret;
1086
1087	# elif RT_INLINE_ASM_USES_INTRIN
1088	return (pu32Old =_InterlockedCompareExchange((long )pu32, u32New, u32Old)) == u32Old;
1089
1090	# else
1091	uint32_t u32Ret;
1092	__asm
1093	{
1094	# ifdef RT_ARCH_AMD64
1095	mov rdx, [pu32]
1096	# else
1097	mov edx, [pu32]
1098	# endif
1099	mov eax, [u32Old]
1100	mov ecx, [u32New]
1101	# ifdef RT_ARCH_AMD64
1102	lock cmpxchg [rdx], ecx
1103	mov rdx, [pu32Old]
1104	mov [rdx], eax
1105	# else
1106	lock cmpxchg [edx], ecx
1107	mov edx, [pu32Old]
1108	mov [edx], eax
1109	# endif
1110	setz al
1111	movzx eax, al
1112	mov [u32Ret], eax
1113	}
1114	return !!u32Ret;
1115	# endif
1116	}
1117	#endif
1118
1119
1120	/**
1121	* Atomically Compare and Exchange a signed 32-bit value, additionally
1122	* passes back old value, ordered.
1123	*
1124	* @returns true if xchg was done.
1125	* @returns false if xchg wasn't done.
1126	*
1127	* @param pi32 Pointer to the value to update.
1128	* @param i32New The new value to assigned to *pi32.
1129	* @param i32Old The old value to *pi32 compare with.
1130	* @param pi32Old Pointer store the old value at.
1131	*
1132	* @remarks x86: Requires a 486 or later.
1133	*/
1134	DECLINLINE(bool) ASMAtomicCmpXchgExS32(volatile int32_t pi32, const int32_t i32New, const int32_t i32Old, int32_t pi32Old)
1135	{
1136	return ASMAtomicCmpXchgExU32((volatile uint32_t )pi32, (uint32_t)i32New, (uint32_t)i32Old, (uint32_t )pi32Old);
1137	}
1138
1139
1140	/**
1141	* Atomically Compare and exchange an unsigned 64-bit value, additionally
1142	* passing back old value, ordered.
1143	*
1144	* @returns true if xchg was done.
1145	* @returns false if xchg wasn't done.
1146	*
1147	* @param pu64 Pointer to the 64-bit variable to update.
1148	* @param u64New The 64-bit value to assign to *pu64.
1149	* @param u64Old The value to compare with.
1150	* @param pu64Old Pointer store the old value at.
1151	*
1152	* @remarks x86: Requires a Pentium or later.
1153	*/
1154	#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
1155	\|\| RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
1156	DECLASM(bool) ASMAtomicCmpXchgExU64(volatile uint64_t pu64, const uint64_t u64New, const uint64_t u64Old, uint64_t pu64Old);
1157	#else
1158	DECLINLINE(bool) ASMAtomicCmpXchgExU64(volatile uint64_t pu64, const uint64_t u64New, const uint64_t u64Old, uint64_t pu64Old)
1159	{
1160	# if RT_INLINE_ASM_USES_INTRIN
1161	return (pu64Old =_InterlockedCompareExchange64((__int64 )pu64, u64New, u64Old)) == u64Old;
1162
1163	# elif defined(RT_ARCH_AMD64)
1164	# if RT_INLINE_ASM_GNU_STYLE
1165	uint8_t u8Ret;
1166	__asm__ __volatile__("lock; cmpxchgq %3, %0\n\t"
1167	"setz %1\n\t"
1168	: "=m" (*pu64),
1169	"=qm" (u8Ret),
1170	"=a" (*pu64Old)
1171	: "r" (u64New),
1172	"a" (u64Old),
1173	"m" (*pu64));
1174	return (bool)u8Ret;
1175	# else
1176	bool fRet;
1177	__asm
1178	{
1179	mov rdx, [pu32]
1180	mov rax, [u64Old]
1181	mov rcx, [u64New]
1182	lock cmpxchg [rdx], rcx
1183	mov rdx, [pu64Old]
1184	mov [rdx], rax
1185	setz al
1186	mov [fRet], al
1187	}
1188	return fRet;
1189	# endif
1190	# else /* !RT_ARCH_AMD64 */
1191	# if RT_INLINE_ASM_GNU_STYLE
1192	uint64_t u64Ret;
1193	# if defined(PIC) \|\| defined(__PIC__)
1194	/* NB: this code uses a memory clobber description, because the clean
1195	* solution with an output value for *pu64 makes gcc run out of registers.
1196	* This will cause suboptimal code, and anyone with a better solution is
1197	* welcome to improve this. */
1198	__asm__ __volatile__("xchgl %%ebx, %1\n\t"
1199	"lock; cmpxchg8b %3\n\t"
1200	"xchgl %%ebx, %1\n\t"
1201	: "=A" (u64Ret)
1202	: "DS" ((uint32_t)u64New),
1203	"c" ((uint32_t)(u64New >> 32)),
1204	"m" (*pu64),
1205	"0" (u64Old)
1206	: "memory" );
1207	# else /* !PIC */
1208	__asm__ __volatile__("lock; cmpxchg8b %4\n\t"
1209	: "=A" (u64Ret),
1210	"=m" (*pu64)
1211	: "b" ((uint32_t)u64New),
1212	"c" ((uint32_t)(u64New >> 32)),
1213	"m" (*pu64),
1214	"0" (u64Old));
1215	# endif
1216	*pu64Old = u64Ret;
1217	return u64Ret == u64Old;
1218	# else
1219	uint32_t u32Ret;
1220	__asm
1221	{
1222	mov ebx, dword ptr [u64New]
1223	mov ecx, dword ptr [u64New + 4]
1224	mov edi, [pu64]
1225	mov eax, dword ptr [u64Old]
1226	mov edx, dword ptr [u64Old + 4]
1227	lock cmpxchg8b [edi]
1228	mov ebx, [pu64Old]
1229	mov [ebx], eax
1230	setz al
1231	movzx eax, al
1232	add ebx, 4
1233	mov [ebx], edx
1234	mov dword ptr [u32Ret], eax
1235	}
1236	return !!u32Ret;
1237	# endif
1238	# endif /* !RT_ARCH_AMD64 */
1239	}
1240	#endif
1241
1242
1243	/**
1244	* Atomically Compare and exchange a signed 64-bit value, additionally
1245	* passing back old value, ordered.
1246	*
1247	* @returns true if xchg was done.
1248	* @returns false if xchg wasn't done.
1249	*
1250	* @param pi64 Pointer to the 64-bit variable to update.
1251	* @param i64 The 64-bit value to assign to *pu64.
1252	* @param i64Old The value to compare with.
1253	* @param pi64Old Pointer store the old value at.
1254	*
1255	* @remarks x86: Requires a Pentium or later.
1256	*/
1257	DECLINLINE(bool) ASMAtomicCmpXchgExS64(volatile int64_t pi64, const int64_t i64, const int64_t i64Old, int64_t pi64Old)
1258	{
1259	return ASMAtomicCmpXchgExU64((volatile uint64_t )pi64, (uint64_t)i64, (uint64_t)i64Old, (uint64_t )pi64Old);
1260	}
1261
1262	/** @def ASMAtomicCmpXchgExHandle
1263	* Atomically Compare and Exchange a typical IPRT handle value, ordered.
1264	*
1265	* @param ph Pointer to the value to update.
1266	* @param hNew The new value to assigned to *pu.
1267	* @param hOld The old value to *pu compare with.
1268	* @param fRc Where to store the result.
1269	* @param phOldVal Pointer to where to store the old value.
1270	*
1271	* @remarks This doesn't currently work for all handles (like RTFILE).
1272	*/
1273	#if HC_ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
1274	# define ASMAtomicCmpXchgExHandle(ph, hNew, hOld, fRc, phOldVal) \
1275	do { \
1276	AssertCompile(sizeof(*ph) == sizeof(uint32_t)); \
1277	AssertCompile(sizeof(*phOldVal) == sizeof(uint32_t)); \
1278	(fRc) = ASMAtomicCmpXchgExU32((volatile uint32_t )(pu), (uint32_t)(uNew), (uint32_t)(uOld), (uint32_t )(puOldVal)); \
1279	} while (0)
1280	#elif HC_ARCH_BITS == 64
1281	# define ASMAtomicCmpXchgExHandle(ph, hNew, hOld, fRc, phOldVal) \
1282	do { \
1283	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
1284	AssertCompile(sizeof(*(phOldVal)) == sizeof(uint64_t)); \
1285	(fRc) = ASMAtomicCmpXchgExU64((volatile uint64_t )(pu), (uint64_t)(uNew), (uint64_t)(uOld), (uint64_t )(puOldVal)); \
1286	} while (0)
1287	#else
1288	# error HC_ARCH_BITS
1289	#endif
1290
1291
1292	/** @def ASMAtomicCmpXchgExSize
1293	* Atomically Compare and Exchange a value which size might differ
1294	* between platforms or compilers. Additionally passes back old value.
1295	*
1296	* @param pu Pointer to the value to update.
1297	* @param uNew The new value to assigned to *pu.
1298	* @param uOld The old value to *pu compare with.
1299	* @param fRc Where to store the result.
1300	* @param puOldVal Pointer to where to store the old value.
1301	*
1302	* @remarks x86: Requires a 486 or later.
1303	*/
1304	#define ASMAtomicCmpXchgExSize(pu, uNew, uOld, fRc, puOldVal) \
1305	do { \
1306	switch (sizeof(*(pu))) { \
1307	case 4: (fRc) = ASMAtomicCmpXchgExU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew), (uint32_t)(uOld), (uint32_t *)(uOldVal)); \
1308	break; \
1309	case 8: (fRc) = ASMAtomicCmpXchgExU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew), (uint64_t)(uOld), (uint64_t *)(uOldVal)); \
1310	break; \
1311	default: AssertMsgFailed(("ASMAtomicCmpXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
1312	(fRc) = false; \
1313	(uOldVal) = 0; \
1314	break; \
1315	} \
1316	} while (0)
1317
1318
1319	/**
1320	* Atomically Compare and Exchange a pointer value, additionally
1321	* passing back old value, ordered.
1322	*
1323	* @returns true if xchg was done.
1324	* @returns false if xchg wasn't done.
1325	*
1326	* @param ppv Pointer to the value to update.
1327	* @param pvNew The new value to assigned to *ppv.
1328	* @param pvOld The old value to *ppv compare with.
1329	* @param ppvOld Pointer store the old value at.
1330	*
1331	* @remarks x86: Requires a 486 or later.
1332	*/
1333	DECLINLINE(bool) ASMAtomicCmpXchgExPtrVoid(void * volatile ppv, const void pvNew, const void pvOld, void *ppvOld)
1334	{
1335	#if ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
1336	return ASMAtomicCmpXchgExU32((volatile uint32_t )(void )ppv, (uint32_t)pvNew, (uint32_t)pvOld, (uint32_t *)ppvOld);
1337	#elif ARCH_BITS == 64
1338	return ASMAtomicCmpXchgExU64((volatile uint64_t )(void )ppv, (uint64_t)pvNew, (uint64_t)pvOld, (uint64_t *)ppvOld);
1339	#else
1340	# error "ARCH_BITS is bogus"
1341	#endif
1342	}
1343
1344
1345	/**
1346	* Atomically Compare and Exchange a pointer value, additionally
1347	* passing back old value, ordered.
1348	*
1349	* @returns true if xchg was done.
1350	* @returns false if xchg wasn't done.
1351	*
1352	* @param ppv Pointer to the value to update.
1353	* @param pvNew The new value to assigned to *ppv.
1354	* @param pvOld The old value to *ppv compare with.
1355	* @param ppvOld Pointer store the old value at.
1356	*
1357	* @remarks This is relatively type safe on GCC platforms.
1358	* @remarks x86: Requires a 486 or later.
1359	*/
1360	#ifdef __GNUC__
1361	# define ASMAtomicCmpXchgExPtr(ppv, pvNew, pvOld, ppvOld) \
1362	__extension__ \
1363	({\
1364	__typeof__((ppv)) volatile const ppvTypeChecked = (ppv); \
1365	__typeof__(*(ppv)) const pvNewTypeChecked = (pvNew); \
1366	__typeof__(*(ppv)) const pvOldTypeChecked = (pvOld); \
1367	__typeof__((ppv)) const ppvOldTypeChecked = (ppvOld); \
1368	bool fMacroRet = ASMAtomicCmpXchgExPtrVoid((void * volatile *)ppvTypeChecked, \
1369	(void )pvNewTypeChecked, (void )pvOldTypeChecked, \
1370	(void **)ppvOldTypeChecked); \
1371	fMacroRet; \
1372	})
1373	#else
1374	# define ASMAtomicCmpXchgExPtr(ppv, pvNew, pvOld, ppvOld) \
1375	ASMAtomicCmpXchgExPtrVoid((void * volatile )(ppv), (void )(pvNew), (void )(pvOld), (void *)(ppvOld))
1376	#endif
1377
1378
1379	/**
1380	* Virtualization unfriendly serializing instruction, always exits.
1381	*/
1382	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1383	DECLASM(void) ASMSerializeInstructionCpuId(void);
1384	#else
1385	DECLINLINE(void) ASMSerializeInstructionCpuId(void)
1386	{
1387	# if RT_INLINE_ASM_GNU_STYLE
1388	RTCCUINTREG xAX = 0;
1389	# ifdef RT_ARCH_AMD64
1390	__asm__ __volatile__ ("cpuid"
1391	: "=a" (xAX)
1392	: "0" (xAX)
1393	: "rbx", "rcx", "rdx", "memory");
1394	# elif (defined(PIC) \|\| defined(__PIC__)) && defined(__i386__)
1395	__asm__ __volatile__ ("push %%ebx\n\t"
1396	"cpuid\n\t"
1397	"pop %%ebx\n\t"
1398	: "=a" (xAX)
1399	: "0" (xAX)
1400	: "ecx", "edx", "memory");
1401	# else
1402	__asm__ __volatile__ ("cpuid"
1403	: "=a" (xAX)
1404	: "0" (xAX)
1405	: "ebx", "ecx", "edx", "memory");
1406	# endif
1407
1408	# elif RT_INLINE_ASM_USES_INTRIN
1409	int aInfo[4];
1410	_ReadWriteBarrier();
1411	__cpuid(aInfo, 0);
1412
1413	# else
1414	__asm
1415	{
1416	push ebx
1417	xor eax, eax
1418	cpuid
1419	pop ebx
1420	}
1421	# endif
1422	}
1423	#endif
1424
1425	/**
1426	* Virtualization friendly serializing instruction, though more expensive.
1427	*/
1428	#if RT_INLINE_ASM_EXTERNAL
1429	DECLASM(void) ASMSerializeInstructionIRet(void);
1430	#else
1431	DECLINLINE(void) ASMSerializeInstructionIRet(void)
1432	{
1433	# if RT_INLINE_ASM_GNU_STYLE
1434	# ifdef RT_ARCH_AMD64
1435	__asm__ __volatile__ ("movq %%rsp,%%r10\n\t"
1436	"subq $128, %%rsp\n\t" /redzone/
1437	"mov %%ss, %%eax\n\t"
1438	"pushq %%rax\n\t"
1439	"pushq %%r10\n\t"
1440	"pushfq\n\t"
1441	"movl %%cs, %%eax\n\t"
1442	"pushq %%rax\n\t"
1443	"leaq 1f(%%rip), %%rax\n\t"
1444	"pushq %%rax\n\t"
1445	"iretq\n\t"
1446	"1:\n\t"
1447	::: "rax", "r10", "memory");
1448	# else
1449	__asm__ __volatile__ ("pushfl\n\t"
1450	"pushl %%cs\n\t"
1451	"pushl $1f\n\t"
1452	"iretl\n\t"
1453	"1:\n\t"
1454	::: "memory");
1455	# endif
1456
1457	# else
1458	__asm
1459	{
1460	pushfd
1461	push cs
1462	push la_ret
1463	iretd
1464	la_ret:
1465	}
1466	# endif
1467	}
1468	#endif
1469
1470	/**
1471	* Virtualization friendlier serializing instruction, may still cause exits.
1472	*/
1473	#if RT_INLINE_ASM_EXTERNAL && RT_INLINE_ASM_USES_INTRIN < 15
1474	DECLASM(void) ASMSerializeInstructionRdTscp(void);
1475	#else
1476	DECLINLINE(void) ASMSerializeInstructionRdTscp(void)
1477	{
1478	# if RT_INLINE_ASM_GNU_STYLE
1479	/* rdtscp is not supported by ancient linux build VM of course :-( */
1480	# ifdef RT_ARCH_AMD64
1481	/__asm__ __volatile__("rdtscp\n\t" ::: "rax", "rdx, "rcx"); /
1482	__asm__ __volatile__(".byte 0x0f,0x01,0xf9\n\t" ::: "rax", "rdx", "rcx", "memory");
1483	# else
1484	/__asm__ __volatile__("rdtscp\n\t" ::: "eax", "edx, "ecx"); /
1485	__asm__ __volatile__(".byte 0x0f,0x01,0xf9\n\t" ::: "eax", "edx", "ecx", "memory");
1486	# endif
1487	# else
1488	# if RT_INLINE_ASM_USES_INTRIN >= 15
1489	uint32_t uIgnore;
1490	_ReadWriteBarrier();
1491	(void)__rdtscp(&uIgnore);
1492	(void)uIgnore;
1493	# else
1494	__asm
1495	{
1496	rdtscp
1497	}
1498	# endif
1499	# endif
1500	}
1501	#endif
1502
1503
1504	/**
1505	* Serialize Instruction.
1506	*/
1507	#if (defined(RT_ARCH_X86) && ARCH_BITS == 16) \|\| defined(IN_GUEST)
1508	# define ASMSerializeInstruction() ASMSerializeInstructionIRet()
1509	#else
1510	# define ASMSerializeInstruction() ASMSerializeInstructionCpuId()
1511	#endif
1512
1513
1514	/**
1515	* Memory fence, waits for any pending writes and reads to complete.
1516	*/
1517	DECLINLINE(void) ASMMemoryFence(void)
1518	{
1519	/** @todo use mfence? check if all cpus we care for support it. */
1520	#if ARCH_BITS == 16
1521	uint16_t volatile u16;
1522	ASMAtomicXchgU16(&u16, 0);
1523	#else
1524	uint32_t volatile u32;
1525	ASMAtomicXchgU32(&u32, 0);
1526	#endif
1527	}
1528
1529
1530	/**
1531	* Write fence, waits for any pending writes to complete.
1532	*/
1533	DECLINLINE(void) ASMWriteFence(void)
1534	{
1535	/** @todo use sfence? check if all cpus we care for support it. */
1536	ASMMemoryFence();
1537	}
1538
1539
1540	/**
1541	* Read fence, waits for any pending reads to complete.
1542	*/
1543	DECLINLINE(void) ASMReadFence(void)
1544	{
1545	/** @todo use lfence? check if all cpus we care for support it. */
1546	ASMMemoryFence();
1547	}
1548
1549
1550	/**
1551	* Atomically reads an unsigned 8-bit value, ordered.
1552	*
1553	* @returns Current *pu8 value
1554	* @param pu8 Pointer to the 8-bit variable to read.
1555	*/
1556	DECLINLINE(uint8_t) ASMAtomicReadU8(volatile uint8_t *pu8)
1557	{
1558	ASMMemoryFence();
1559	return pu8; / byte reads are atomic on x86 */
1560	}
1561
1562
1563	/**
1564	* Atomically reads an unsigned 8-bit value, unordered.
1565	*
1566	* @returns Current *pu8 value
1567	* @param pu8 Pointer to the 8-bit variable to read.
1568	*/
1569	DECLINLINE(uint8_t) ASMAtomicUoReadU8(volatile uint8_t *pu8)
1570	{
1571	return pu8; / byte reads are atomic on x86 */
1572	}
1573
1574
1575	/**
1576	* Atomically reads a signed 8-bit value, ordered.
1577	*
1578	* @returns Current *pi8 value
1579	* @param pi8 Pointer to the 8-bit variable to read.
1580	*/
1581	DECLINLINE(int8_t) ASMAtomicReadS8(volatile int8_t *pi8)
1582	{
1583	ASMMemoryFence();
1584	return pi8; / byte reads are atomic on x86 */
1585	}
1586
1587
1588	/**
1589	* Atomically reads a signed 8-bit value, unordered.
1590	*
1591	* @returns Current *pi8 value
1592	* @param pi8 Pointer to the 8-bit variable to read.
1593	*/
1594	DECLINLINE(int8_t) ASMAtomicUoReadS8(volatile int8_t *pi8)
1595	{
1596	return pi8; / byte reads are atomic on x86 */
1597	}
1598
1599
1600	/**
1601	* Atomically reads an unsigned 16-bit value, ordered.
1602	*
1603	* @returns Current *pu16 value
1604	* @param pu16 Pointer to the 16-bit variable to read.
1605	*/
1606	DECLINLINE(uint16_t) ASMAtomicReadU16(volatile uint16_t *pu16)
1607	{
1608	ASMMemoryFence();
1609	Assert(!((uintptr_t)pu16 & 1));
1610	return *pu16;
1611	}
1612
1613
1614	/**
1615	* Atomically reads an unsigned 16-bit value, unordered.
1616	*
1617	* @returns Current *pu16 value
1618	* @param pu16 Pointer to the 16-bit variable to read.
1619	*/
1620	DECLINLINE(uint16_t) ASMAtomicUoReadU16(volatile uint16_t *pu16)
1621	{
1622	Assert(!((uintptr_t)pu16 & 1));
1623	return *pu16;
1624	}
1625
1626
1627	/**
1628	* Atomically reads a signed 16-bit value, ordered.
1629	*
1630	* @returns Current *pi16 value
1631	* @param pi16 Pointer to the 16-bit variable to read.
1632	*/
1633	DECLINLINE(int16_t) ASMAtomicReadS16(volatile int16_t *pi16)
1634	{
1635	ASMMemoryFence();
1636	Assert(!((uintptr_t)pi16 & 1));
1637	return *pi16;
1638	}
1639
1640
1641	/**
1642	* Atomically reads a signed 16-bit value, unordered.
1643	*
1644	* @returns Current *pi16 value
1645	* @param pi16 Pointer to the 16-bit variable to read.
1646	*/
1647	DECLINLINE(int16_t) ASMAtomicUoReadS16(volatile int16_t *pi16)
1648	{
1649	Assert(!((uintptr_t)pi16 & 1));
1650	return *pi16;
1651	}
1652
1653
1654	/**
1655	* Atomically reads an unsigned 32-bit value, ordered.
1656	*
1657	* @returns Current *pu32 value
1658	* @param pu32 Pointer to the 32-bit variable to read.
1659	*/
1660	DECLINLINE(uint32_t) ASMAtomicReadU32(volatile uint32_t *pu32)
1661	{
1662	ASMMemoryFence();
1663	Assert(!((uintptr_t)pu32 & 3));
1664	#if ARCH_BITS == 16
1665	AssertFailed(); /** @todo 16-bit */
1666	#endif
1667	return *pu32;
1668	}
1669
1670
1671	/**
1672	* Atomically reads an unsigned 32-bit value, unordered.
1673	*
1674	* @returns Current *pu32 value
1675	* @param pu32 Pointer to the 32-bit variable to read.
1676	*/
1677	DECLINLINE(uint32_t) ASMAtomicUoReadU32(volatile uint32_t *pu32)
1678	{
1679	Assert(!((uintptr_t)pu32 & 3));
1680	#if ARCH_BITS == 16
1681	AssertFailed(); /** @todo 16-bit */
1682	#endif
1683	return *pu32;
1684	}
1685
1686
1687	/**
1688	* Atomically reads a signed 32-bit value, ordered.
1689	*
1690	* @returns Current *pi32 value
1691	* @param pi32 Pointer to the 32-bit variable to read.
1692	*/
1693	DECLINLINE(int32_t) ASMAtomicReadS32(volatile int32_t *pi32)
1694	{
1695	ASMMemoryFence();
1696	Assert(!((uintptr_t)pi32 & 3));
1697	#if ARCH_BITS == 16
1698	AssertFailed(); /** @todo 16-bit */
1699	#endif
1700	return *pi32;
1701	}
1702
1703
1704	/**
1705	* Atomically reads a signed 32-bit value, unordered.
1706	*
1707	* @returns Current *pi32 value
1708	* @param pi32 Pointer to the 32-bit variable to read.
1709	*/
1710	DECLINLINE(int32_t) ASMAtomicUoReadS32(volatile int32_t *pi32)
1711	{
1712	Assert(!((uintptr_t)pi32 & 3));
1713	#if ARCH_BITS == 16
1714	AssertFailed(); /** @todo 16-bit */
1715	#endif
1716	return *pi32;
1717	}
1718
1719
1720	/**
1721	* Atomically reads an unsigned 64-bit value, ordered.
1722	*
1723	* @returns Current *pu64 value
1724	* @param pu64 Pointer to the 64-bit variable to read.
1725	* The memory pointed to must be writable.
1726	*
1727	* @remarks This may fault if the memory is read-only!
1728	* @remarks x86: Requires a Pentium or later.
1729	*/
1730	#if (RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)) \
1731	\|\| RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
1732	DECLASM(uint64_t) ASMAtomicReadU64(volatile uint64_t *pu64);
1733	#else
1734	DECLINLINE(uint64_t) ASMAtomicReadU64(volatile uint64_t *pu64)
1735	{
1736	uint64_t u64;
1737	# ifdef RT_ARCH_AMD64
1738	Assert(!((uintptr_t)pu64 & 7));
1739	/*# if RT_INLINE_ASM_GNU_STYLE
1740	__asm__ __volatile__( "mfence\n\t"
1741	"movq %1, %0\n\t"
1742	: "=r" (u64)
1743	: "m" (*pu64));
1744	# else
1745	__asm
1746	{
1747	mfence
1748	mov rdx, [pu64]
1749	mov rax, [rdx]
1750	mov [u64], rax
1751	}
1752	# endif*/
1753	ASMMemoryFence();
1754	u64 = *pu64;
1755	# else /* !RT_ARCH_AMD64 */
1756	# if RT_INLINE_ASM_GNU_STYLE
1757	# if defined(PIC) \|\| defined(__PIC__)
1758	uint32_t u32EBX = 0;
1759	Assert(!((uintptr_t)pu64 & 7));
1760	__asm__ __volatile__("xchgl %%ebx, %3\n\t"
1761	"lock; cmpxchg8b (%5)\n\t"
1762	"movl %3, %%ebx\n\t"
1763	: "=A" (u64),
1764	# if RT_GNUC_PREREQ(4, 3)
1765	"+m" (*pu64)
1766	# else
1767	"=m" (*pu64)
1768	# endif
1769	: "0" (0ULL),
1770	"m" (u32EBX),
1771	"c" (0),
1772	"S" (pu64));
1773	# else /* !PIC */
1774	__asm__ __volatile__("lock; cmpxchg8b %1\n\t"
1775	: "=A" (u64),
1776	"+m" (*pu64)
1777	: "0" (0ULL),
1778	"b" (0),
1779	"c" (0));
1780	# endif
1781	# else
1782	Assert(!((uintptr_t)pu64 & 7));
1783	__asm
1784	{
1785	xor eax, eax
1786	xor edx, edx
1787	mov edi, pu64
1788	xor ecx, ecx
1789	xor ebx, ebx
1790	lock cmpxchg8b [edi]
1791	mov dword ptr [u64], eax
1792	mov dword ptr [u64 + 4], edx
1793	}
1794	# endif
1795	# endif /* !RT_ARCH_AMD64 */
1796	return u64;
1797	}
1798	#endif
1799
1800
1801	/**
1802	* Atomically reads an unsigned 64-bit value, unordered.
1803	*
1804	* @returns Current *pu64 value
1805	* @param pu64 Pointer to the 64-bit variable to read.
1806	* The memory pointed to must be writable.
1807	*
1808	* @remarks This may fault if the memory is read-only!
1809	* @remarks x86: Requires a Pentium or later.
1810	*/
1811	#if !defined(RT_ARCH_AMD64) \
1812	&& ( (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
1813	\|\| RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC)
1814	DECLASM(uint64_t) ASMAtomicUoReadU64(volatile uint64_t *pu64);
1815	#else
1816	DECLINLINE(uint64_t) ASMAtomicUoReadU64(volatile uint64_t *pu64)
1817	{
1818	uint64_t u64;
1819	# ifdef RT_ARCH_AMD64
1820	Assert(!((uintptr_t)pu64 & 7));
1821	/*# if RT_INLINE_ASM_GNU_STYLE
1822	Assert(!((uintptr_t)pu64 & 7));
1823	__asm__ __volatile__("movq %1, %0\n\t"
1824	: "=r" (u64)
1825	: "m" (*pu64));
1826	# else
1827	__asm
1828	{
1829	mov rdx, [pu64]
1830	mov rax, [rdx]
1831	mov [u64], rax
1832	}
1833	# endif */
1834	u64 = *pu64;
1835	# else /* !RT_ARCH_AMD64 */
1836	# if RT_INLINE_ASM_GNU_STYLE
1837	# if defined(PIC) \|\| defined(__PIC__)
1838	uint32_t u32EBX = 0;
1839	uint32_t u32Spill;
1840	Assert(!((uintptr_t)pu64 & 7));
1841	__asm__ __volatile__("xor %%eax,%%eax\n\t"
1842	"xor %%ecx,%%ecx\n\t"
1843	"xor %%edx,%%edx\n\t"
1844	"xchgl %%ebx, %3\n\t"
1845	"lock; cmpxchg8b (%4)\n\t"
1846	"movl %3, %%ebx\n\t"
1847	: "=A" (u64),
1848	# if RT_GNUC_PREREQ(4, 3)
1849	"+m" (*pu64),
1850	# else
1851	"=m" (*pu64),
1852	# endif
1853	"=c" (u32Spill)
1854	: "m" (u32EBX),
1855	"S" (pu64));
1856	# else /* !PIC */
1857	__asm__ __volatile__("lock; cmpxchg8b %1\n\t"
1858	: "=A" (u64),
1859	"+m" (*pu64)
1860	: "0" (0ULL),
1861	"b" (0),
1862	"c" (0));
1863	# endif
1864	# else
1865	Assert(!((uintptr_t)pu64 & 7));
1866	__asm
1867	{
1868	xor eax, eax
1869	xor edx, edx
1870	mov edi, pu64
1871	xor ecx, ecx
1872	xor ebx, ebx
1873	lock cmpxchg8b [edi]
1874	mov dword ptr [u64], eax
1875	mov dword ptr [u64 + 4], edx
1876	}
1877	# endif
1878	# endif /* !RT_ARCH_AMD64 */
1879	return u64;
1880	}
1881	#endif
1882
1883
1884	/**
1885	* Atomically reads a signed 64-bit value, ordered.
1886	*
1887	* @returns Current *pi64 value
1888	* @param pi64 Pointer to the 64-bit variable to read.
1889	* The memory pointed to must be writable.
1890	*
1891	* @remarks This may fault if the memory is read-only!
1892	* @remarks x86: Requires a Pentium or later.
1893	*/
1894	DECLINLINE(int64_t) ASMAtomicReadS64(volatile int64_t *pi64)
1895	{
1896	return (int64_t)ASMAtomicReadU64((volatile uint64_t *)pi64);
1897	}
1898
1899
1900	/**
1901	* Atomically reads a signed 64-bit value, unordered.
1902	*
1903	* @returns Current *pi64 value
1904	* @param pi64 Pointer to the 64-bit variable to read.
1905	* The memory pointed to must be writable.
1906	*
1907	* @remarks This will fault if the memory is read-only!
1908	* @remarks x86: Requires a Pentium or later.
1909	*/
1910	DECLINLINE(int64_t) ASMAtomicUoReadS64(volatile int64_t *pi64)
1911	{
1912	return (int64_t)ASMAtomicUoReadU64((volatile uint64_t *)pi64);
1913	}
1914
1915
1916	/**
1917	* Atomically reads a size_t value, ordered.
1918	*
1919	* @returns Current *pcb value
1920	* @param pcb Pointer to the size_t variable to read.
1921	*/
1922	DECLINLINE(size_t) ASMAtomicReadZ(size_t volatile *pcb)
1923	{
1924	#if ARCH_BITS == 64
1925	return ASMAtomicReadU64((uint64_t volatile *)pcb);
1926	#elif ARCH_BITS == 32
1927	return ASMAtomicReadU32((uint32_t volatile *)pcb);
1928	#elif ARCH_BITS == 16
1929	AssertCompileSize(size_t, 2);
1930	return ASMAtomicReadU16((uint16_t volatile *)pcb);
1931	#else
1932	# error "Unsupported ARCH_BITS value"
1933	#endif
1934	}
1935
1936
1937	/**
1938	* Atomically reads a size_t value, unordered.
1939	*
1940	* @returns Current *pcb value
1941	* @param pcb Pointer to the size_t variable to read.
1942	*/
1943	DECLINLINE(size_t) ASMAtomicUoReadZ(size_t volatile *pcb)
1944	{
1945	#if ARCH_BITS == 64 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
1946	return ASMAtomicUoReadU64((uint64_t volatile *)pcb);
1947	#elif ARCH_BITS == 32
1948	return ASMAtomicUoReadU32((uint32_t volatile *)pcb);
1949	#elif ARCH_BITS == 16
1950	AssertCompileSize(size_t, 2);
1951	return ASMAtomicUoReadU16((uint16_t volatile *)pcb);
1952	#else
1953	# error "Unsupported ARCH_BITS value"
1954	#endif
1955	}
1956
1957
1958	/**
1959	* Atomically reads a pointer value, ordered.
1960	*
1961	* @returns Current *pv value
1962	* @param ppv Pointer to the pointer variable to read.
1963	*
1964	* @remarks Please use ASMAtomicReadPtrT, it provides better type safety and
1965	* requires less typing (no casts).
1966	*/
1967	DECLINLINE(void ) ASMAtomicReadPtr(void volatile *ppv)
1968	{
1969	#if ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
1970	return (void )ASMAtomicReadU32((volatile uint32_t )(void *)ppv);
1971	#elif ARCH_BITS == 64
1972	return (void )ASMAtomicReadU64((volatile uint64_t )(void *)ppv);
1973	#else
1974	# error "ARCH_BITS is bogus"
1975	#endif
1976	}
1977
1978	/**
1979	* Convenience macro for avoiding the annoying casting with ASMAtomicReadPtr.
1980	*
1981	* @returns Current *pv value
1982	* @param ppv Pointer to the pointer variable to read.
1983	* @param Type The type of *ppv, sans volatile.
1984	*/
1985	#ifdef __GNUC__
1986	# define ASMAtomicReadPtrT(ppv, Type) \
1987	__extension__ \
1988	({\
1989	__typeof__((ppv)) volatile ppvTypeChecked = (ppv); \
1990	Type pvTypeChecked = (__typeof__((ppv))) ASMAtomicReadPtr((void volatile *)ppvTypeChecked); \
1991	pvTypeChecked; \
1992	})
1993	#else
1994	# define ASMAtomicReadPtrT(ppv, Type) \
1995	(Type)ASMAtomicReadPtr((void * volatile *)(ppv))
1996	#endif
1997
1998
1999	/**
2000	* Atomically reads a pointer value, unordered.
2001	*
2002	* @returns Current *pv value
2003	* @param ppv Pointer to the pointer variable to read.
2004	*
2005	* @remarks Please use ASMAtomicUoReadPtrT, it provides better type safety and
2006	* requires less typing (no casts).
2007	*/
2008	DECLINLINE(void ) ASMAtomicUoReadPtr(void volatile *ppv)
2009	{
2010	#if ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
2011	return (void )ASMAtomicUoReadU32((volatile uint32_t )(void *)ppv);
2012	#elif ARCH_BITS == 64
2013	return (void )ASMAtomicUoReadU64((volatile uint64_t )(void *)ppv);
2014	#else
2015	# error "ARCH_BITS is bogus"
2016	#endif
2017	}
2018
2019
2020	/**
2021	* Convenience macro for avoiding the annoying casting with ASMAtomicUoReadPtr.
2022	*
2023	* @returns Current *pv value
2024	* @param ppv Pointer to the pointer variable to read.
2025	* @param Type The type of *ppv, sans volatile.
2026	*/
2027	#ifdef __GNUC__
2028	# define ASMAtomicUoReadPtrT(ppv, Type) \
2029	__extension__ \
2030	({\
2031	__typeof__((ppv)) volatile const ppvTypeChecked = (ppv); \
2032	Type pvTypeChecked = (__typeof__((ppv))) ASMAtomicUoReadPtr((void volatile *)ppvTypeChecked); \
2033	pvTypeChecked; \
2034	})
2035	#else
2036	# define ASMAtomicUoReadPtrT(ppv, Type) \
2037	(Type)ASMAtomicUoReadPtr((void * volatile *)(ppv))
2038	#endif
2039
2040
2041	/**
2042	* Atomically reads a boolean value, ordered.
2043	*
2044	* @returns Current *pf value
2045	* @param pf Pointer to the boolean variable to read.
2046	*/
2047	DECLINLINE(bool) ASMAtomicReadBool(volatile bool *pf)
2048	{
2049	ASMMemoryFence();
2050	return pf; / byte reads are atomic on x86 */
2051	}
2052
2053
2054	/**
2055	* Atomically reads a boolean value, unordered.
2056	*
2057	* @returns Current *pf value
2058	* @param pf Pointer to the boolean variable to read.
2059	*/
2060	DECLINLINE(bool) ASMAtomicUoReadBool(volatile bool *pf)
2061	{
2062	return pf; / byte reads are atomic on x86 */
2063	}
2064
2065
2066	/**
2067	* Atomically read a typical IPRT handle value, ordered.
2068	*
2069	* @param ph Pointer to the handle variable to read.
2070	* @param phRes Where to store the result.
2071	*
2072	* @remarks This doesn't currently work for all handles (like RTFILE).
2073	*/
2074	#if HC_ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
2075	# define ASMAtomicReadHandle(ph, phRes) \
2076	do { \
2077	AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
2078	AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
2079	(uint32_t )(phRes) = ASMAtomicReadU32((uint32_t volatile *)(ph)); \
2080	} while (0)
2081	#elif HC_ARCH_BITS == 64
2082	# define ASMAtomicReadHandle(ph, phRes) \
2083	do { \
2084	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
2085	AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
2086	(uint64_t )(phRes) = ASMAtomicReadU64((uint64_t volatile *)(ph)); \
2087	} while (0)
2088	#else
2089	# error HC_ARCH_BITS
2090	#endif
2091
2092
2093	/**
2094	* Atomically read a typical IPRT handle value, unordered.
2095	*
2096	* @param ph Pointer to the handle variable to read.
2097	* @param phRes Where to store the result.
2098	*
2099	* @remarks This doesn't currently work for all handles (like RTFILE).
2100	*/
2101	#if HC_ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
2102	# define ASMAtomicUoReadHandle(ph, phRes) \
2103	do { \
2104	AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
2105	AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
2106	(uint32_t )(phRes) = ASMAtomicUoReadU32((uint32_t volatile *)(ph)); \
2107	} while (0)
2108	#elif HC_ARCH_BITS == 64
2109	# define ASMAtomicUoReadHandle(ph, phRes) \
2110	do { \
2111	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
2112	AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
2113	(uint64_t )(phRes) = ASMAtomicUoReadU64((uint64_t volatile *)(ph)); \
2114	} while (0)
2115	#else
2116	# error HC_ARCH_BITS
2117	#endif
2118
2119
2120	/**
2121	* Atomically read a value which size might differ
2122	* between platforms or compilers, ordered.
2123	*
2124	* @param pu Pointer to the variable to read.
2125	* @param puRes Where to store the result.
2126	*/
2127	#define ASMAtomicReadSize(pu, puRes) \
2128	do { \
2129	switch (sizeof(*(pu))) { \
2130	case 1: (uint8_t )(puRes) = ASMAtomicReadU8( (volatile uint8_t )(void )(pu)); break; \
2131	case 2: (uint16_t )(puRes) = ASMAtomicReadU16((volatile uint16_t )(void )(pu)); break; \
2132	case 4: (uint32_t )(puRes) = ASMAtomicReadU32((volatile uint32_t )(void )(pu)); break; \
2133	case 8: (uint64_t )(puRes) = ASMAtomicReadU64((volatile uint64_t )(void )(pu)); break; \
2134	default: AssertMsgFailed(("ASMAtomicReadSize: size %d is not supported\n", sizeof(*(pu)))); \
2135	} \
2136	} while (0)
2137
2138
2139	/**
2140	* Atomically read a value which size might differ
2141	* between platforms or compilers, unordered.
2142	*
2143	* @param pu Pointer to the variable to read.
2144	* @param puRes Where to store the result.
2145	*/
2146	#define ASMAtomicUoReadSize(pu, puRes) \
2147	do { \
2148	switch (sizeof(*(pu))) { \
2149	case 1: (uint8_t )(puRes) = ASMAtomicUoReadU8( (volatile uint8_t )(void )(pu)); break; \
2150	case 2: (uint16_t )(puRes) = ASMAtomicUoReadU16((volatile uint16_t )(void )(pu)); break; \
2151	case 4: (uint32_t )(puRes) = ASMAtomicUoReadU32((volatile uint32_t )(void )(pu)); break; \
2152	case 8: (uint64_t )(puRes) = ASMAtomicUoReadU64((volatile uint64_t )(void )(pu)); break; \
2153	default: AssertMsgFailed(("ASMAtomicReadSize: size %d is not supported\n", sizeof(*(pu)))); \
2154	} \
2155	} while (0)
2156
2157
2158	/**
2159	* Atomically writes an unsigned 8-bit value, ordered.
2160	*
2161	* @param pu8 Pointer to the 8-bit variable.
2162	* @param u8 The 8-bit value to assign to *pu8.
2163	*/
2164	DECLINLINE(void) ASMAtomicWriteU8(volatile uint8_t *pu8, uint8_t u8)
2165	{
2166	ASMAtomicXchgU8(pu8, u8);
2167	}
2168
2169
2170	/**
2171	* Atomically writes an unsigned 8-bit value, unordered.
2172	*
2173	* @param pu8 Pointer to the 8-bit variable.
2174	* @param u8 The 8-bit value to assign to *pu8.
2175	*/
2176	DECLINLINE(void) ASMAtomicUoWriteU8(volatile uint8_t *pu8, uint8_t u8)
2177	{
2178	pu8 = u8; / byte writes are atomic on x86 */
2179	}
2180
2181
2182	/**
2183	* Atomically writes a signed 8-bit value, ordered.
2184	*
2185	* @param pi8 Pointer to the 8-bit variable to read.
2186	* @param i8 The 8-bit value to assign to *pi8.
2187	*/
2188	DECLINLINE(void) ASMAtomicWriteS8(volatile int8_t *pi8, int8_t i8)
2189	{
2190	ASMAtomicXchgS8(pi8, i8);
2191	}
2192
2193
2194	/**
2195	* Atomically writes a signed 8-bit value, unordered.
2196	*
2197	* @param pi8 Pointer to the 8-bit variable to write.
2198	* @param i8 The 8-bit value to assign to *pi8.
2199	*/
2200	DECLINLINE(void) ASMAtomicUoWriteS8(volatile int8_t *pi8, int8_t i8)
2201	{
2202	pi8 = i8; / byte writes are atomic on x86 */
2203	}
2204
2205
2206	/**
2207	* Atomically writes an unsigned 16-bit value, ordered.
2208	*
2209	* @param pu16 Pointer to the 16-bit variable to write.
2210	* @param u16 The 16-bit value to assign to *pu16.
2211	*/
2212	DECLINLINE(void) ASMAtomicWriteU16(volatile uint16_t *pu16, uint16_t u16)
2213	{
2214	ASMAtomicXchgU16(pu16, u16);
2215	}
2216
2217
2218	/**
2219	* Atomically writes an unsigned 16-bit value, unordered.
2220	*
2221	* @param pu16 Pointer to the 16-bit variable to write.
2222	* @param u16 The 16-bit value to assign to *pu16.
2223	*/
2224	DECLINLINE(void) ASMAtomicUoWriteU16(volatile uint16_t *pu16, uint16_t u16)
2225	{
2226	Assert(!((uintptr_t)pu16 & 1));
2227	*pu16 = u16;
2228	}
2229
2230
2231	/**
2232	* Atomically writes a signed 16-bit value, ordered.
2233	*
2234	* @param pi16 Pointer to the 16-bit variable to write.
2235	* @param i16 The 16-bit value to assign to *pi16.
2236	*/
2237	DECLINLINE(void) ASMAtomicWriteS16(volatile int16_t *pi16, int16_t i16)
2238	{
2239	ASMAtomicXchgS16(pi16, i16);
2240	}
2241
2242
2243	/**
2244	* Atomically writes a signed 16-bit value, unordered.
2245	*
2246	* @param pi16 Pointer to the 16-bit variable to write.
2247	* @param i16 The 16-bit value to assign to *pi16.
2248	*/
2249	DECLINLINE(void) ASMAtomicUoWriteS16(volatile int16_t *pi16, int16_t i16)
2250	{
2251	Assert(!((uintptr_t)pi16 & 1));
2252	*pi16 = i16;
2253	}
2254
2255
2256	/**
2257	* Atomically writes an unsigned 32-bit value, ordered.
2258	*
2259	* @param pu32 Pointer to the 32-bit variable to write.
2260	* @param u32 The 32-bit value to assign to *pu32.
2261	*/
2262	DECLINLINE(void) ASMAtomicWriteU32(volatile uint32_t *pu32, uint32_t u32)
2263	{
2264	ASMAtomicXchgU32(pu32, u32);
2265	}
2266
2267
2268	/**
2269	* Atomically writes an unsigned 32-bit value, unordered.
2270	*
2271	* @param pu32 Pointer to the 32-bit variable to write.
2272	* @param u32 The 32-bit value to assign to *pu32.
2273	*/
2274	DECLINLINE(void) ASMAtomicUoWriteU32(volatile uint32_t *pu32, uint32_t u32)
2275	{
2276	Assert(!((uintptr_t)pu32 & 3));
2277	#if ARCH_BITS >= 32
2278	*pu32 = u32;
2279	#else
2280	ASMAtomicXchgU32(pu32, u32);
2281	#endif
2282	}
2283
2284
2285	/**
2286	* Atomically writes a signed 32-bit value, ordered.
2287	*
2288	* @param pi32 Pointer to the 32-bit variable to write.
2289	* @param i32 The 32-bit value to assign to *pi32.
2290	*/
2291	DECLINLINE(void) ASMAtomicWriteS32(volatile int32_t *pi32, int32_t i32)
2292	{
2293	ASMAtomicXchgS32(pi32, i32);
2294	}
2295
2296
2297	/**
2298	* Atomically writes a signed 32-bit value, unordered.
2299	*
2300	* @param pi32 Pointer to the 32-bit variable to write.
2301	* @param i32 The 32-bit value to assign to *pi32.
2302	*/
2303	DECLINLINE(void) ASMAtomicUoWriteS32(volatile int32_t *pi32, int32_t i32)
2304	{
2305	Assert(!((uintptr_t)pi32 & 3));
2306	#if ARCH_BITS >= 32
2307	*pi32 = i32;
2308	#else
2309	ASMAtomicXchgS32(pi32, i32);
2310	#endif
2311	}
2312
2313
2314	/**
2315	* Atomically writes an unsigned 64-bit value, ordered.
2316	*
2317	* @param pu64 Pointer to the 64-bit variable to write.
2318	* @param u64 The 64-bit value to assign to *pu64.
2319	*/
2320	DECLINLINE(void) ASMAtomicWriteU64(volatile uint64_t *pu64, uint64_t u64)
2321	{
2322	ASMAtomicXchgU64(pu64, u64);
2323	}
2324
2325
2326	/**
2327	* Atomically writes an unsigned 64-bit value, unordered.
2328	*
2329	* @param pu64 Pointer to the 64-bit variable to write.
2330	* @param u64 The 64-bit value to assign to *pu64.
2331	*/
2332	DECLINLINE(void) ASMAtomicUoWriteU64(volatile uint64_t *pu64, uint64_t u64)
2333	{
2334	Assert(!((uintptr_t)pu64 & 7));
2335	#if ARCH_BITS == 64
2336	*pu64 = u64;
2337	#else
2338	ASMAtomicXchgU64(pu64, u64);
2339	#endif
2340	}
2341
2342
2343	/**
2344	* Atomically writes a signed 64-bit value, ordered.
2345	*
2346	* @param pi64 Pointer to the 64-bit variable to write.
2347	* @param i64 The 64-bit value to assign to *pi64.
2348	*/
2349	DECLINLINE(void) ASMAtomicWriteS64(volatile int64_t *pi64, int64_t i64)
2350	{
2351	ASMAtomicXchgS64(pi64, i64);
2352	}
2353
2354
2355	/**
2356	* Atomically writes a signed 64-bit value, unordered.
2357	*
2358	* @param pi64 Pointer to the 64-bit variable to write.
2359	* @param i64 The 64-bit value to assign to *pi64.
2360	*/
2361	DECLINLINE(void) ASMAtomicUoWriteS64(volatile int64_t *pi64, int64_t i64)
2362	{
2363	Assert(!((uintptr_t)pi64 & 7));
2364	#if ARCH_BITS == 64
2365	*pi64 = i64;
2366	#else
2367	ASMAtomicXchgS64(pi64, i64);
2368	#endif
2369	}
2370
2371
2372	/**
2373	* Atomically writes a boolean value, unordered.
2374	*
2375	* @param pf Pointer to the boolean variable to write.
2376	* @param f The boolean value to assign to *pf.
2377	*/
2378	DECLINLINE(void) ASMAtomicWriteBool(volatile bool *pf, bool f)
2379	{
2380	ASMAtomicWriteU8((uint8_t volatile *)pf, f);
2381	}
2382
2383
2384	/**
2385	* Atomically writes a boolean value, unordered.
2386	*
2387	* @param pf Pointer to the boolean variable to write.
2388	* @param f The boolean value to assign to *pf.
2389	*/
2390	DECLINLINE(void) ASMAtomicUoWriteBool(volatile bool *pf, bool f)
2391	{
2392	pf = f; / byte writes are atomic on x86 */
2393	}
2394
2395
2396	/**
2397	* Atomically writes a pointer value, ordered.
2398	*
2399	* @param ppv Pointer to the pointer variable to write.
2400	* @param pv The pointer value to assign to *ppv.
2401	*/
2402	DECLINLINE(void) ASMAtomicWritePtrVoid(void * volatile ppv, const void pv)
2403	{
2404	#if ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
2405	ASMAtomicWriteU32((volatile uint32_t )(void )ppv, (uint32_t)pv);
2406	#elif ARCH_BITS == 64
2407	ASMAtomicWriteU64((volatile uint64_t )(void )ppv, (uint64_t)pv);
2408	#else
2409	# error "ARCH_BITS is bogus"
2410	#endif
2411	}
2412
2413
2414	/**
2415	* Atomically writes a pointer value, ordered.
2416	*
2417	* @param ppv Pointer to the pointer variable to write.
2418	* @param pv The pointer value to assign to *ppv. If NULL use
2419	* ASMAtomicWriteNullPtr or you'll land in trouble.
2420	*
2421	* @remarks This is relatively type safe on GCC platforms when @a pv isn't
2422	* NULL.
2423	*/
2424	#ifdef __GNUC__
2425	# define ASMAtomicWritePtr(ppv, pv) \
2426	do \
2427	{ \
2428	__typeof__((ppv)) volatile const ppvTypeChecked = (ppv); \
2429	__typeof__(*(ppv)) const pvTypeChecked = (pv); \
2430	\
2431	AssertCompile(sizeof(ppv) == sizeof(void )); \
2432	AssertCompile(sizeof(pv) == sizeof(void *)); \
2433	Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2434	\
2435	ASMAtomicWritePtrVoid((void * volatile )(ppvTypeChecked), (void )(pvTypeChecked)); \
2436	} while (0)
2437	#else
2438	# define ASMAtomicWritePtr(ppv, pv) \
2439	do \
2440	{ \
2441	AssertCompile(sizeof(ppv) == sizeof(void )); \
2442	AssertCompile(sizeof(pv) == sizeof(void *)); \
2443	Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2444	\
2445	ASMAtomicWritePtrVoid((void * volatile )(ppv), (void )(pv)); \
2446	} while (0)
2447	#endif
2448
2449
2450	/**
2451	* Atomically sets a pointer to NULL, ordered.
2452	*
2453	* @param ppv Pointer to the pointer variable that should be set to NULL.
2454	*
2455	* @remarks This is relatively type safe on GCC platforms.
2456	*/
2457	#ifdef __GNUC__
2458	# define ASMAtomicWriteNullPtr(ppv) \
2459	do \
2460	{ \
2461	__typeof__((ppv)) const ppvTypeChecked = (ppv); \
2462	AssertCompile(sizeof(ppv) == sizeof(void )); \
2463	Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2464	ASMAtomicWritePtrVoid((void * volatile *)(ppvTypeChecked), NULL); \
2465	} while (0)
2466	#else
2467	# define ASMAtomicWriteNullPtr(ppv) \
2468	do \
2469	{ \
2470	AssertCompile(sizeof(ppv) == sizeof(void )); \
2471	Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2472	ASMAtomicWritePtrVoid((void * volatile *)(ppv), NULL); \
2473	} while (0)
2474	#endif
2475
2476
2477	/**
2478	* Atomically writes a pointer value, unordered.
2479	*
2480	* @returns Current *pv value
2481	* @param ppv Pointer to the pointer variable.
2482	* @param pv The pointer value to assign to *ppv. If NULL use
2483	* ASMAtomicUoWriteNullPtr or you'll land in trouble.
2484	*
2485	* @remarks This is relatively type safe on GCC platforms when @a pv isn't
2486	* NULL.
2487	*/
2488	#ifdef __GNUC__
2489	# define ASMAtomicUoWritePtr(ppv, pv) \
2490	do \
2491	{ \
2492	__typeof__((ppv)) volatile const ppvTypeChecked = (ppv); \
2493	__typeof__(*(ppv)) const pvTypeChecked = (pv); \
2494	\
2495	AssertCompile(sizeof(ppv) == sizeof(void )); \
2496	AssertCompile(sizeof(pv) == sizeof(void *)); \
2497	Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2498	\
2499	*(ppvTypeChecked) = pvTypeChecked; \
2500	} while (0)
2501	#else
2502	# define ASMAtomicUoWritePtr(ppv, pv) \
2503	do \
2504	{ \
2505	AssertCompile(sizeof(ppv) == sizeof(void )); \
2506	AssertCompile(sizeof(pv) == sizeof(void *)); \
2507	Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2508	*(ppv) = pv; \
2509	} while (0)
2510	#endif
2511
2512
2513	/**
2514	* Atomically sets a pointer to NULL, unordered.
2515	*
2516	* @param ppv Pointer to the pointer variable that should be set to NULL.
2517	*
2518	* @remarks This is relatively type safe on GCC platforms.
2519	*/
2520	#ifdef __GNUC__
2521	# define ASMAtomicUoWriteNullPtr(ppv) \
2522	do \
2523	{ \
2524	__typeof__((ppv)) volatile const ppvTypeChecked = (ppv); \
2525	AssertCompile(sizeof(ppv) == sizeof(void )); \
2526	Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2527	*(ppvTypeChecked) = NULL; \
2528	} while (0)
2529	#else
2530	# define ASMAtomicUoWriteNullPtr(ppv) \
2531	do \
2532	{ \
2533	AssertCompile(sizeof(ppv) == sizeof(void )); \
2534	Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2535	*(ppv) = NULL; \
2536	} while (0)
2537	#endif
2538
2539
2540	/**
2541	* Atomically write a typical IPRT handle value, ordered.
2542	*
2543	* @param ph Pointer to the variable to update.
2544	* @param hNew The value to assign to *ph.
2545	*
2546	* @remarks This doesn't currently work for all handles (like RTFILE).
2547	*/
2548	#if HC_ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
2549	# define ASMAtomicWriteHandle(ph, hNew) \
2550	do { \
2551	AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
2552	ASMAtomicWriteU32((uint32_t volatile *)(ph), (const uint32_t)(hNew)); \
2553	} while (0)
2554	#elif HC_ARCH_BITS == 64
2555	# define ASMAtomicWriteHandle(ph, hNew) \
2556	do { \
2557	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
2558	ASMAtomicWriteU64((uint64_t volatile *)(ph), (const uint64_t)(hNew)); \
2559	} while (0)
2560	#else
2561	# error HC_ARCH_BITS
2562	#endif
2563
2564
2565	/**
2566	* Atomically write a typical IPRT handle value, unordered.
2567	*
2568	* @param ph Pointer to the variable to update.
2569	* @param hNew The value to assign to *ph.
2570	*
2571	* @remarks This doesn't currently work for all handles (like RTFILE).
2572	*/
2573	#if HC_ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
2574	# define ASMAtomicUoWriteHandle(ph, hNew) \
2575	do { \
2576	AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
2577	ASMAtomicUoWriteU32((uint32_t volatile *)(ph), (const uint32_t)hNew); \
2578	} while (0)
2579	#elif HC_ARCH_BITS == 64
2580	# define ASMAtomicUoWriteHandle(ph, hNew) \
2581	do { \
2582	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
2583	ASMAtomicUoWriteU64((uint64_t volatile *)(ph), (const uint64_t)hNew); \
2584	} while (0)
2585	#else
2586	# error HC_ARCH_BITS
2587	#endif
2588
2589
2590	/**
2591	* Atomically write a value which size might differ
2592	* between platforms or compilers, ordered.
2593	*
2594	* @param pu Pointer to the variable to update.
2595	* @param uNew The value to assign to *pu.
2596	*/
2597	#define ASMAtomicWriteSize(pu, uNew) \
2598	do { \
2599	switch (sizeof(*(pu))) { \
2600	case 1: ASMAtomicWriteU8( (volatile uint8_t )(void )(pu), (uint8_t )(uNew)); break; \
2601	case 2: ASMAtomicWriteU16((volatile uint16_t )(void )(pu), (uint16_t)(uNew)); break; \
2602	case 4: ASMAtomicWriteU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew)); break; \
2603	case 8: ASMAtomicWriteU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew)); break; \
2604	default: AssertMsgFailed(("ASMAtomicWriteSize: size %d is not supported\n", sizeof(*(pu)))); \
2605	} \
2606	} while (0)
2607
2608	/**
2609	* Atomically write a value which size might differ
2610	* between platforms or compilers, unordered.
2611	*
2612	* @param pu Pointer to the variable to update.
2613	* @param uNew The value to assign to *pu.
2614	*/
2615	#define ASMAtomicUoWriteSize(pu, uNew) \
2616	do { \
2617	switch (sizeof(*(pu))) { \
2618	case 1: ASMAtomicUoWriteU8( (volatile uint8_t )(void )(pu), (uint8_t )(uNew)); break; \
2619	case 2: ASMAtomicUoWriteU16((volatile uint16_t )(void )(pu), (uint16_t)(uNew)); break; \
2620	case 4: ASMAtomicUoWriteU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew)); break; \
2621	case 8: ASMAtomicUoWriteU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew)); break; \
2622	default: AssertMsgFailed(("ASMAtomicWriteSize: size %d is not supported\n", sizeof(*(pu)))); \
2623	} \
2624	} while (0)
2625
2626
2627
2628	/**
2629	* Atomically exchanges and adds to a 16-bit value, ordered.
2630	*
2631	* @returns The old value.
2632	* @param pu16 Pointer to the value.
2633	* @param u16 Number to add.
2634	*
2635	* @remarks Currently not implemented, just to make 16-bit code happy.
2636	* @remarks x86: Requires a 486 or later.
2637	*/
2638	DECLASM(uint16_t) ASMAtomicAddU16(uint16_t volatile *pu16, uint32_t u16);
2639
2640
2641	/**
2642	* Atomically exchanges and adds to a 32-bit value, ordered.
2643	*
2644	* @returns The old value.
2645	* @param pu32 Pointer to the value.
2646	* @param u32 Number to add.
2647	*
2648	* @remarks x86: Requires a 486 or later.
2649	*/
2650	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2651	DECLASM(uint32_t) ASMAtomicAddU32(uint32_t volatile *pu32, uint32_t u32);
2652	#else
2653	DECLINLINE(uint32_t) ASMAtomicAddU32(uint32_t volatile *pu32, uint32_t u32)
2654	{
2655	# if RT_INLINE_ASM_USES_INTRIN
2656	u32 = _InterlockedExchangeAdd((long *)pu32, u32);
2657	return u32;
2658
2659	# elif RT_INLINE_ASM_GNU_STYLE
2660	__asm__ __volatile__("lock; xaddl %0, %1\n\t"
2661	: "=r" (u32),
2662	"=m" (*pu32)
2663	: "0" (u32),
2664	"m" (*pu32)
2665	: "memory");
2666	return u32;
2667	# else
2668	__asm
2669	{
2670	mov eax, [u32]
2671	# ifdef RT_ARCH_AMD64
2672	mov rdx, [pu32]
2673	lock xadd [rdx], eax
2674	# else
2675	mov edx, [pu32]
2676	lock xadd [edx], eax
2677	# endif
2678	mov [u32], eax
2679	}
2680	return u32;
2681	# endif
2682	}
2683	#endif
2684
2685
2686	/**
2687	* Atomically exchanges and adds to a signed 32-bit value, ordered.
2688	*
2689	* @returns The old value.
2690	* @param pi32 Pointer to the value.
2691	* @param i32 Number to add.
2692	*
2693	* @remarks x86: Requires a 486 or later.
2694	*/
2695	DECLINLINE(int32_t) ASMAtomicAddS32(int32_t volatile *pi32, int32_t i32)
2696	{
2697	return (int32_t)ASMAtomicAddU32((uint32_t volatile *)pi32, (uint32_t)i32);
2698	}
2699
2700
2701	/**
2702	* Atomically exchanges and adds to a 64-bit value, ordered.
2703	*
2704	* @returns The old value.
2705	* @param pu64 Pointer to the value.
2706	* @param u64 Number to add.
2707	*
2708	* @remarks x86: Requires a Pentium or later.
2709	*/
2710	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2711	DECLASM(uint64_t) ASMAtomicAddU64(uint64_t volatile *pu64, uint64_t u64);
2712	#else
2713	DECLINLINE(uint64_t) ASMAtomicAddU64(uint64_t volatile *pu64, uint64_t u64)
2714	{
2715	# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
2716	u64 = _InterlockedExchangeAdd64((__int64 *)pu64, u64);
2717	return u64;
2718
2719	# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
2720	__asm__ __volatile__("lock; xaddq %0, %1\n\t"
2721	: "=r" (u64),
2722	"=m" (*pu64)
2723	: "0" (u64),
2724	"m" (*pu64)
2725	: "memory");
2726	return u64;
2727	# else
2728	uint64_t u64Old;
2729	for (;;)
2730	{
2731	uint64_t u64New;
2732	u64Old = ASMAtomicUoReadU64(pu64);
2733	u64New = u64Old + u64;
2734	if (ASMAtomicCmpXchgU64(pu64, u64New, u64Old))
2735	break;
2736	ASMNopPause();
2737	}
2738	return u64Old;
2739	# endif
2740	}
2741	#endif
2742
2743
2744	/**
2745	* Atomically exchanges and adds to a signed 64-bit value, ordered.
2746	*
2747	* @returns The old value.
2748	* @param pi64 Pointer to the value.
2749	* @param i64 Number to add.
2750	*
2751	* @remarks x86: Requires a Pentium or later.
2752	*/
2753	DECLINLINE(int64_t) ASMAtomicAddS64(int64_t volatile *pi64, int64_t i64)
2754	{
2755	return (int64_t)ASMAtomicAddU64((uint64_t volatile *)pi64, (uint64_t)i64);
2756	}
2757
2758
2759	/**
2760	* Atomically exchanges and adds to a size_t value, ordered.
2761	*
2762	* @returns The old value.
2763	* @param pcb Pointer to the size_t value.
2764	* @param cb Number to add.
2765	*/
2766	DECLINLINE(size_t) ASMAtomicAddZ(size_t volatile *pcb, size_t cb)
2767	{
2768	#if ARCH_BITS == 64
2769	AssertCompileSize(size_t, 8);
2770	return ASMAtomicAddU64((uint64_t volatile *)pcb, cb);
2771	#elif ARCH_BITS == 32
2772	AssertCompileSize(size_t, 4);
2773	return ASMAtomicAddU32((uint32_t volatile *)pcb, cb);
2774	#elif ARCH_BITS == 16
2775	AssertCompileSize(size_t, 2);
2776	return ASMAtomicAddU16((uint16_t volatile *)pcb, cb);
2777	#else
2778	# error "Unsupported ARCH_BITS value"
2779	#endif
2780	}
2781
2782
2783	/**
2784	* Atomically exchanges and adds a value which size might differ between
2785	* platforms or compilers, ordered.
2786	*
2787	* @param pu Pointer to the variable to update.
2788	* @param uNew The value to add to *pu.
2789	* @param puOld Where to store the old value.
2790	*/
2791	#define ASMAtomicAddSize(pu, uNew, puOld) \
2792	do { \
2793	switch (sizeof(*(pu))) { \
2794	case 4: (uint32_t )(puOld) = ASMAtomicAddU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew)); break; \
2795	case 8: (uint64_t )(puOld) = ASMAtomicAddU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew)); break; \
2796	default: AssertMsgFailed(("ASMAtomicAddSize: size %d is not supported\n", sizeof(*(pu)))); \
2797	} \
2798	} while (0)
2799
2800
2801
2802	/**
2803	* Atomically exchanges and subtracts to an unsigned 16-bit value, ordered.
2804	*
2805	* @returns The old value.
2806	* @param pu16 Pointer to the value.
2807	* @param u16 Number to subtract.
2808	*
2809	* @remarks x86: Requires a 486 or later.
2810	*/
2811	DECLINLINE(uint16_t) ASMAtomicSubU16(uint16_t volatile *pu16, uint32_t u16)
2812	{
2813	return ASMAtomicAddU16(pu16, (uint16_t)-(int16_t)u16);
2814	}
2815
2816
2817	/**
2818	* Atomically exchanges and subtracts to a signed 16-bit value, ordered.
2819	*
2820	* @returns The old value.
2821	* @param pi16 Pointer to the value.
2822	* @param i16 Number to subtract.
2823	*
2824	* @remarks x86: Requires a 486 or later.
2825	*/
2826	DECLINLINE(int16_t) ASMAtomicSubS16(int16_t volatile *pi16, int16_t i16)
2827	{
2828	return (int16_t)ASMAtomicAddU16((uint16_t volatile *)pi16, (uint16_t)-i16);
2829	}
2830
2831
2832	/**
2833	* Atomically exchanges and subtracts to an unsigned 32-bit value, ordered.
2834	*
2835	* @returns The old value.
2836	* @param pu32 Pointer to the value.
2837	* @param u32 Number to subtract.
2838	*
2839	* @remarks x86: Requires a 486 or later.
2840	*/
2841	DECLINLINE(uint32_t) ASMAtomicSubU32(uint32_t volatile *pu32, uint32_t u32)
2842	{
2843	return ASMAtomicAddU32(pu32, (uint32_t)-(int32_t)u32);
2844	}
2845
2846
2847	/**
2848	* Atomically exchanges and subtracts to a signed 32-bit value, ordered.
2849	*
2850	* @returns The old value.
2851	* @param pi32 Pointer to the value.
2852	* @param i32 Number to subtract.
2853	*
2854	* @remarks x86: Requires a 486 or later.
2855	*/
2856	DECLINLINE(int32_t) ASMAtomicSubS32(int32_t volatile *pi32, int32_t i32)
2857	{
2858	return (int32_t)ASMAtomicAddU32((uint32_t volatile *)pi32, (uint32_t)-i32);
2859	}
2860
2861
2862	/**
2863	* Atomically exchanges and subtracts to an unsigned 64-bit value, ordered.
2864	*
2865	* @returns The old value.
2866	* @param pu64 Pointer to the value.
2867	* @param u64 Number to subtract.
2868	*
2869	* @remarks x86: Requires a Pentium or later.
2870	*/
2871	DECLINLINE(uint64_t) ASMAtomicSubU64(uint64_t volatile *pu64, uint64_t u64)
2872	{
2873	return ASMAtomicAddU64(pu64, (uint64_t)-(int64_t)u64);
2874	}
2875
2876
2877	/**
2878	* Atomically exchanges and subtracts to a signed 64-bit value, ordered.
2879	*
2880	* @returns The old value.
2881	* @param pi64 Pointer to the value.
2882	* @param i64 Number to subtract.
2883	*
2884	* @remarks x86: Requires a Pentium or later.
2885	*/
2886	DECLINLINE(int64_t) ASMAtomicSubS64(int64_t volatile *pi64, int64_t i64)
2887	{
2888	return (int64_t)ASMAtomicAddU64((uint64_t volatile *)pi64, (uint64_t)-i64);
2889	}
2890
2891
2892	/**
2893	* Atomically exchanges and subtracts to a size_t value, ordered.
2894	*
2895	* @returns The old value.
2896	* @param pcb Pointer to the size_t value.
2897	* @param cb Number to subtract.
2898	*
2899	* @remarks x86: Requires a 486 or later.
2900	*/
2901	DECLINLINE(size_t) ASMAtomicSubZ(size_t volatile *pcb, size_t cb)
2902	{
2903	#if ARCH_BITS == 64
2904	return ASMAtomicSubU64((uint64_t volatile *)pcb, cb);
2905	#elif ARCH_BITS == 32
2906	return ASMAtomicSubU32((uint32_t volatile *)pcb, cb);
2907	#elif ARCH_BITS == 16
2908	AssertCompileSize(size_t, 2);
2909	return ASMAtomicSubU16((uint16_t volatile *)pcb, cb);
2910	#else
2911	# error "Unsupported ARCH_BITS value"
2912	#endif
2913	}
2914
2915
2916	/**
2917	* Atomically exchanges and subtracts a value which size might differ between
2918	* platforms or compilers, ordered.
2919	*
2920	* @param pu Pointer to the variable to update.
2921	* @param uNew The value to subtract to *pu.
2922	* @param puOld Where to store the old value.
2923	*
2924	* @remarks x86: Requires a 486 or later.
2925	*/
2926	#define ASMAtomicSubSize(pu, uNew, puOld) \
2927	do { \
2928	switch (sizeof(*(pu))) { \
2929	case 4: (uint32_t )(puOld) = ASMAtomicSubU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew)); break; \
2930	case 8: (uint64_t )(puOld) = ASMAtomicSubU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew)); break; \
2931	default: AssertMsgFailed(("ASMAtomicSubSize: size %d is not supported\n", sizeof(*(pu)))); \
2932	} \
2933	} while (0)
2934
2935
2936
2937	/**
2938	* Atomically increment a 16-bit value, ordered.
2939	*
2940	* @returns The new value.
2941	* @param pu16 Pointer to the value to increment.
2942	* @remarks Not implemented. Just to make 16-bit code happy.
2943	*
2944	* @remarks x86: Requires a 486 or later.
2945	*/
2946	DECLASM(uint16_t) ASMAtomicIncU16(uint16_t volatile *pu16);
2947
2948
2949	/**
2950	* Atomically increment a 32-bit value, ordered.
2951	*
2952	* @returns The new value.
2953	* @param pu32 Pointer to the value to increment.
2954	*
2955	* @remarks x86: Requires a 486 or later.
2956	*/
2957	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2958	DECLASM(uint32_t) ASMAtomicIncU32(uint32_t volatile *pu32);
2959	#else
2960	DECLINLINE(uint32_t) ASMAtomicIncU32(uint32_t volatile *pu32)
2961	{
2962	uint32_t u32;
2963	# if RT_INLINE_ASM_USES_INTRIN
2964	u32 = _InterlockedIncrement((long *)pu32);
2965	return u32;
2966
2967	# elif RT_INLINE_ASM_GNU_STYLE
2968	__asm__ __volatile__("lock; xaddl %0, %1\n\t"
2969	: "=r" (u32),
2970	"=m" (*pu32)
2971	: "0" (1),
2972	"m" (*pu32)
2973	: "memory");
2974	return u32+1;
2975	# else
2976	__asm
2977	{
2978	mov eax, 1
2979	# ifdef RT_ARCH_AMD64
2980	mov rdx, [pu32]
2981	lock xadd [rdx], eax
2982	# else
2983	mov edx, [pu32]
2984	lock xadd [edx], eax
2985	# endif
2986	mov u32, eax
2987	}
2988	return u32+1;
2989	# endif
2990	}
2991	#endif
2992
2993
2994	/**
2995	* Atomically increment a signed 32-bit value, ordered.
2996	*
2997	* @returns The new value.
2998	* @param pi32 Pointer to the value to increment.
2999	*
3000	* @remarks x86: Requires a 486 or later.
3001	*/
3002	DECLINLINE(int32_t) ASMAtomicIncS32(int32_t volatile *pi32)
3003	{
3004	return (int32_t)ASMAtomicIncU32((uint32_t volatile *)pi32);
3005	}
3006
3007
3008	/**
3009	* Atomically increment a 64-bit value, ordered.
3010	*
3011	* @returns The new value.
3012	* @param pu64 Pointer to the value to increment.
3013	*
3014	* @remarks x86: Requires a Pentium or later.
3015	*/
3016	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3017	DECLASM(uint64_t) ASMAtomicIncU64(uint64_t volatile *pu64);
3018	#else
3019	DECLINLINE(uint64_t) ASMAtomicIncU64(uint64_t volatile *pu64)
3020	{
3021	# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
3022	uint64_t u64;
3023	u64 = _InterlockedIncrement64((__int64 *)pu64);
3024	return u64;
3025
3026	# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
3027	uint64_t u64;
3028	__asm__ __volatile__("lock; xaddq %0, %1\n\t"
3029	: "=r" (u64),
3030	"=m" (*pu64)
3031	: "0" (1),
3032	"m" (*pu64)
3033	: "memory");
3034	return u64 + 1;
3035	# else
3036	return ASMAtomicAddU64(pu64, 1) + 1;
3037	# endif
3038	}
3039	#endif
3040
3041
3042	/**
3043	* Atomically increment a signed 64-bit value, ordered.
3044	*
3045	* @returns The new value.
3046	* @param pi64 Pointer to the value to increment.
3047	*
3048	* @remarks x86: Requires a Pentium or later.
3049	*/
3050	DECLINLINE(int64_t) ASMAtomicIncS64(int64_t volatile *pi64)
3051	{
3052	return (int64_t)ASMAtomicIncU64((uint64_t volatile *)pi64);
3053	}
3054
3055
3056	/**
3057	* Atomically increment a size_t value, ordered.
3058	*
3059	* @returns The new value.
3060	* @param pcb Pointer to the value to increment.
3061	*
3062	* @remarks x86: Requires a 486 or later.
3063	*/
3064	DECLINLINE(int64_t) ASMAtomicIncZ(size_t volatile *pcb)
3065	{
3066	#if ARCH_BITS == 64
3067	return ASMAtomicIncU64((uint64_t volatile *)pcb);
3068	#elif ARCH_BITS == 32
3069	return ASMAtomicIncU32((uint32_t volatile *)pcb);
3070	#elif ARCH_BITS == 16
3071	return ASMAtomicIncU16((uint16_t volatile *)pcb);
3072	#else
3073	# error "Unsupported ARCH_BITS value"
3074	#endif
3075	}
3076
3077
3078
3079	/**
3080	* Atomically decrement an unsigned 32-bit value, ordered.
3081	*
3082	* @returns The new value.
3083	* @param pu16 Pointer to the value to decrement.
3084	* @remarks Not implemented. Just to make 16-bit code happy.
3085	*
3086	* @remarks x86: Requires a 486 or later.
3087	*/
3088	DECLASM(uint32_t) ASMAtomicDecU16(uint16_t volatile *pu16);
3089
3090
3091	/**
3092	* Atomically decrement an unsigned 32-bit value, ordered.
3093	*
3094	* @returns The new value.
3095	* @param pu32 Pointer to the value to decrement.
3096	*
3097	* @remarks x86: Requires a 486 or later.
3098	*/
3099	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3100	DECLASM(uint32_t) ASMAtomicDecU32(uint32_t volatile *pu32);
3101	#else
3102	DECLINLINE(uint32_t) ASMAtomicDecU32(uint32_t volatile *pu32)
3103	{
3104	uint32_t u32;
3105	# if RT_INLINE_ASM_USES_INTRIN
3106	u32 = _InterlockedDecrement((long *)pu32);
3107	return u32;
3108
3109	# elif RT_INLINE_ASM_GNU_STYLE
3110	__asm__ __volatile__("lock; xaddl %0, %1\n\t"
3111	: "=r" (u32),
3112	"=m" (*pu32)
3113	: "0" (-1),
3114	"m" (*pu32)
3115	: "memory");
3116	return u32-1;
3117	# else
3118	__asm
3119	{
3120	mov eax, -1
3121	# ifdef RT_ARCH_AMD64
3122	mov rdx, [pu32]
3123	lock xadd [rdx], eax
3124	# else
3125	mov edx, [pu32]
3126	lock xadd [edx], eax
3127	# endif
3128	mov u32, eax
3129	}
3130	return u32-1;
3131	# endif
3132	}
3133	#endif
3134
3135
3136	/**
3137	* Atomically decrement a signed 32-bit value, ordered.
3138	*
3139	* @returns The new value.
3140	* @param pi32 Pointer to the value to decrement.
3141	*
3142	* @remarks x86: Requires a 486 or later.
3143	*/
3144	DECLINLINE(int32_t) ASMAtomicDecS32(int32_t volatile *pi32)
3145	{
3146	return (int32_t)ASMAtomicDecU32((uint32_t volatile *)pi32);
3147	}
3148
3149
3150	/**
3151	* Atomically decrement an unsigned 64-bit value, ordered.
3152	*
3153	* @returns The new value.
3154	* @param pu64 Pointer to the value to decrement.
3155	*
3156	* @remarks x86: Requires a Pentium or later.
3157	*/
3158	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3159	DECLASM(uint64_t) ASMAtomicDecU64(uint64_t volatile *pu64);
3160	#else
3161	DECLINLINE(uint64_t) ASMAtomicDecU64(uint64_t volatile *pu64)
3162	{
3163	# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
3164	uint64_t u64 = _InterlockedDecrement64((__int64 volatile *)pu64);
3165	return u64;
3166
3167	# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
3168	uint64_t u64;
3169	__asm__ __volatile__("lock; xaddq %q0, %1\n\t"
3170	: "=r" (u64),
3171	"=m" (*pu64)
3172	: "0" (~(uint64_t)0),
3173	"m" (*pu64)
3174	: "memory");
3175	return u64-1;
3176	# else
3177	return ASMAtomicAddU64(pu64, UINT64_MAX) - 1;
3178	# endif
3179	}
3180	#endif
3181
3182
3183	/**
3184	* Atomically decrement a signed 64-bit value, ordered.
3185	*
3186	* @returns The new value.
3187	* @param pi64 Pointer to the value to decrement.
3188	*
3189	* @remarks x86: Requires a Pentium or later.
3190	*/
3191	DECLINLINE(int64_t) ASMAtomicDecS64(int64_t volatile *pi64)
3192	{
3193	return (int64_t)ASMAtomicDecU64((uint64_t volatile *)pi64);
3194	}
3195
3196
3197	/**
3198	* Atomically decrement a size_t value, ordered.
3199	*
3200	* @returns The new value.
3201	* @param pcb Pointer to the value to decrement.
3202	*
3203	* @remarks x86: Requires a 486 or later.
3204	*/
3205	DECLINLINE(int64_t) ASMAtomicDecZ(size_t volatile *pcb)
3206	{
3207	#if ARCH_BITS == 64
3208	return ASMAtomicDecU64((uint64_t volatile *)pcb);
3209	#elif ARCH_BITS == 32
3210	return ASMAtomicDecU32((uint32_t volatile *)pcb);
3211	#elif ARCH_BITS == 16
3212	return ASMAtomicDecU16((uint16_t volatile *)pcb);
3213	#else
3214	# error "Unsupported ARCH_BITS value"
3215	#endif
3216	}
3217
3218
3219	/**
3220	* Atomically Or an unsigned 32-bit value, ordered.
3221	*
3222	* @param pu32 Pointer to the pointer variable to OR u32 with.
3223	* @param u32 The value to OR *pu32 with.
3224	*
3225	* @remarks x86: Requires a 386 or later.
3226	*/
3227	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3228	DECLASM(void) ASMAtomicOrU32(uint32_t volatile *pu32, uint32_t u32);
3229	#else
3230	DECLINLINE(void) ASMAtomicOrU32(uint32_t volatile *pu32, uint32_t u32)
3231	{
3232	# if RT_INLINE_ASM_USES_INTRIN
3233	_InterlockedOr((long volatile *)pu32, (long)u32);
3234
3235	# elif RT_INLINE_ASM_GNU_STYLE
3236	__asm__ __volatile__("lock; orl %1, %0\n\t"
3237	: "=m" (*pu32)
3238	: "ir" (u32),
3239	"m" (*pu32));
3240	# else
3241	__asm
3242	{
3243	mov eax, [u32]
3244	# ifdef RT_ARCH_AMD64
3245	mov rdx, [pu32]
3246	lock or [rdx], eax
3247	# else
3248	mov edx, [pu32]
3249	lock or [edx], eax
3250	# endif
3251	}
3252	# endif
3253	}
3254	#endif
3255
3256
3257	/**
3258	* Atomically Or a signed 32-bit value, ordered.
3259	*
3260	* @param pi32 Pointer to the pointer variable to OR u32 with.
3261	* @param i32 The value to OR *pu32 with.
3262	*
3263	* @remarks x86: Requires a 386 or later.
3264	*/
3265	DECLINLINE(void) ASMAtomicOrS32(int32_t volatile *pi32, int32_t i32)
3266	{
3267	ASMAtomicOrU32((uint32_t volatile *)pi32, i32);
3268	}
3269
3270
3271	/**
3272	* Atomically Or an unsigned 64-bit value, ordered.
3273	*
3274	* @param pu64 Pointer to the pointer variable to OR u64 with.
3275	* @param u64 The value to OR *pu64 with.
3276	*
3277	* @remarks x86: Requires a Pentium or later.
3278	*/
3279	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3280	DECLASM(void) ASMAtomicOrU64(uint64_t volatile *pu64, uint64_t u64);
3281	#else
3282	DECLINLINE(void) ASMAtomicOrU64(uint64_t volatile *pu64, uint64_t u64)
3283	{
3284	# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
3285	_InterlockedOr64((__int64 volatile *)pu64, (__int64)u64);
3286
3287	# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
3288	__asm__ __volatile__("lock; orq %1, %q0\n\t"
3289	: "=m" (*pu64)
3290	: "r" (u64),
3291	"m" (*pu64));
3292	# else
3293	for (;;)
3294	{
3295	uint64_t u64Old = ASMAtomicUoReadU64(pu64);
3296	uint64_t u64New = u64Old \| u64;
3297	if (ASMAtomicCmpXchgU64(pu64, u64New, u64Old))
3298	break;
3299	ASMNopPause();
3300	}
3301	# endif
3302	}
3303	#endif
3304
3305
3306	/**
3307	* Atomically Or a signed 64-bit value, ordered.
3308	*
3309	* @param pi64 Pointer to the pointer variable to OR u64 with.
3310	* @param i64 The value to OR *pu64 with.
3311	*
3312	* @remarks x86: Requires a Pentium or later.
3313	*/
3314	DECLINLINE(void) ASMAtomicOrS64(int64_t volatile *pi64, int64_t i64)
3315	{
3316	ASMAtomicOrU64((uint64_t volatile *)pi64, i64);
3317	}
3318
3319
3320	/**
3321	* Atomically And an unsigned 32-bit value, ordered.
3322	*
3323	* @param pu32 Pointer to the pointer variable to AND u32 with.
3324	* @param u32 The value to AND *pu32 with.
3325	*
3326	* @remarks x86: Requires a 386 or later.
3327	*/
3328	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3329	DECLASM(void) ASMAtomicAndU32(uint32_t volatile *pu32, uint32_t u32);
3330	#else
3331	DECLINLINE(void) ASMAtomicAndU32(uint32_t volatile *pu32, uint32_t u32)
3332	{
3333	# if RT_INLINE_ASM_USES_INTRIN
3334	_InterlockedAnd((long volatile *)pu32, u32);
3335
3336	# elif RT_INLINE_ASM_GNU_STYLE
3337	__asm__ __volatile__("lock; andl %1, %0\n\t"
3338	: "=m" (*pu32)
3339	: "ir" (u32),
3340	"m" (*pu32));
3341	# else
3342	__asm
3343	{
3344	mov eax, [u32]
3345	# ifdef RT_ARCH_AMD64
3346	mov rdx, [pu32]
3347	lock and [rdx], eax
3348	# else
3349	mov edx, [pu32]
3350	lock and [edx], eax
3351	# endif
3352	}
3353	# endif
3354	}
3355	#endif
3356
3357
3358	/**
3359	* Atomically And a signed 32-bit value, ordered.
3360	*
3361	* @param pi32 Pointer to the pointer variable to AND i32 with.
3362	* @param i32 The value to AND *pi32 with.
3363	*
3364	* @remarks x86: Requires a 386 or later.
3365	*/
3366	DECLINLINE(void) ASMAtomicAndS32(int32_t volatile *pi32, int32_t i32)
3367	{
3368	ASMAtomicAndU32((uint32_t volatile *)pi32, (uint32_t)i32);
3369	}
3370
3371
3372	/**
3373	* Atomically And an unsigned 64-bit value, ordered.
3374	*
3375	* @param pu64 Pointer to the pointer variable to AND u64 with.
3376	* @param u64 The value to AND *pu64 with.
3377	*
3378	* @remarks x86: Requires a Pentium or later.
3379	*/
3380	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3381	DECLASM(void) ASMAtomicAndU64(uint64_t volatile *pu64, uint64_t u64);
3382	#else
3383	DECLINLINE(void) ASMAtomicAndU64(uint64_t volatile *pu64, uint64_t u64)
3384	{
3385	# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
3386	_InterlockedAnd64((__int64 volatile *)pu64, u64);
3387
3388	# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
3389	__asm__ __volatile__("lock; andq %1, %0\n\t"
3390	: "=m" (*pu64)
3391	: "r" (u64),
3392	"m" (*pu64));
3393	# else
3394	for (;;)
3395	{
3396	uint64_t u64Old = ASMAtomicUoReadU64(pu64);
3397	uint64_t u64New = u64Old & u64;
3398	if (ASMAtomicCmpXchgU64(pu64, u64New, u64Old))
3399	break;
3400	ASMNopPause();
3401	}
3402	# endif
3403	}
3404	#endif
3405
3406
3407	/**
3408	* Atomically And a signed 64-bit value, ordered.
3409	*
3410	* @param pi64 Pointer to the pointer variable to AND i64 with.
3411	* @param i64 The value to AND *pi64 with.
3412	*
3413	* @remarks x86: Requires a Pentium or later.
3414	*/
3415	DECLINLINE(void) ASMAtomicAndS64(int64_t volatile *pi64, int64_t i64)
3416	{
3417	ASMAtomicAndU64((uint64_t volatile *)pi64, (uint64_t)i64);
3418	}
3419
3420
3421	/**
3422	* Atomically OR an unsigned 32-bit value, unordered but interrupt safe.
3423	*
3424	* @param pu32 Pointer to the pointer variable to OR u32 with.
3425	* @param u32 The value to OR *pu32 with.
3426	*
3427	* @remarks x86: Requires a 386 or later.
3428	*/
3429	#if RT_INLINE_ASM_EXTERNAL
3430	DECLASM(void) ASMAtomicUoOrU32(uint32_t volatile *pu32, uint32_t u32);
3431	#else
3432	DECLINLINE(void) ASMAtomicUoOrU32(uint32_t volatile *pu32, uint32_t u32)
3433	{
3434	# if RT_INLINE_ASM_GNU_STYLE
3435	__asm__ __volatile__("orl %1, %0\n\t"
3436	: "=m" (*pu32)
3437	: "ir" (u32),
3438	"m" (*pu32));
3439	# else
3440	__asm
3441	{
3442	mov eax, [u32]
3443	# ifdef RT_ARCH_AMD64
3444	mov rdx, [pu32]
3445	or [rdx], eax
3446	# else
3447	mov edx, [pu32]
3448	or [edx], eax
3449	# endif
3450	}
3451	# endif
3452	}
3453	#endif
3454
3455
3456	/**
3457	* Atomically OR a signed 32-bit value, unordered.
3458	*
3459	* @param pi32 Pointer to the pointer variable to OR u32 with.
3460	* @param i32 The value to OR *pu32 with.
3461	*
3462	* @remarks x86: Requires a 386 or later.
3463	*/
3464	DECLINLINE(void) ASMAtomicUoOrS32(int32_t volatile *pi32, int32_t i32)
3465	{
3466	ASMAtomicUoOrU32((uint32_t volatile *)pi32, i32);
3467	}
3468
3469
3470	/**
3471	* Atomically OR an unsigned 64-bit value, unordered.
3472	*
3473	* @param pu64 Pointer to the pointer variable to OR u64 with.
3474	* @param u64 The value to OR *pu64 with.
3475	*
3476	* @remarks x86: Requires a Pentium or later.
3477	*/
3478	#if RT_INLINE_ASM_EXTERNAL
3479	DECLASM(void) ASMAtomicUoOrU64(uint64_t volatile *pu64, uint64_t u64);
3480	#else
3481	DECLINLINE(void) ASMAtomicUoOrU64(uint64_t volatile *pu64, uint64_t u64)
3482	{
3483	# if RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
3484	__asm__ __volatile__("orq %1, %q0\n\t"
3485	: "=m" (*pu64)
3486	: "r" (u64),
3487	"m" (*pu64));
3488	# else
3489	for (;;)
3490	{
3491	uint64_t u64Old = ASMAtomicUoReadU64(pu64);
3492	uint64_t u64New = u64Old \| u64;
3493	if (ASMAtomicCmpXchgU64(pu64, u64New, u64Old))
3494	break;
3495	ASMNopPause();
3496	}
3497	# endif
3498	}
3499	#endif
3500
3501
3502	/**
3503	* Atomically Or a signed 64-bit value, unordered.
3504	*
3505	* @param pi64 Pointer to the pointer variable to OR u64 with.
3506	* @param i64 The value to OR *pu64 with.
3507	*
3508	* @remarks x86: Requires a Pentium or later.
3509	*/
3510	DECLINLINE(void) ASMAtomicUoOrS64(int64_t volatile *pi64, int64_t i64)
3511	{
3512	ASMAtomicUoOrU64((uint64_t volatile *)pi64, i64);
3513	}
3514
3515
3516	/**
3517	* Atomically And an unsigned 32-bit value, unordered.
3518	*
3519	* @param pu32 Pointer to the pointer variable to AND u32 with.
3520	* @param u32 The value to AND *pu32 with.
3521	*
3522	* @remarks x86: Requires a 386 or later.
3523	*/
3524	#if RT_INLINE_ASM_EXTERNAL
3525	DECLASM(void) ASMAtomicUoAndU32(uint32_t volatile *pu32, uint32_t u32);
3526	#else
3527	DECLINLINE(void) ASMAtomicUoAndU32(uint32_t volatile *pu32, uint32_t u32)
3528	{
3529	# if RT_INLINE_ASM_GNU_STYLE
3530	__asm__ __volatile__("andl %1, %0\n\t"
3531	: "=m" (*pu32)
3532	: "ir" (u32),
3533	"m" (*pu32));
3534	# else
3535	__asm
3536	{
3537	mov eax, [u32]
3538	# ifdef RT_ARCH_AMD64
3539	mov rdx, [pu32]
3540	and [rdx], eax
3541	# else
3542	mov edx, [pu32]
3543	and [edx], eax
3544	# endif
3545	}
3546	# endif
3547	}
3548	#endif
3549
3550
3551	/**
3552	* Atomically And a signed 32-bit value, unordered.
3553	*
3554	* @param pi32 Pointer to the pointer variable to AND i32 with.
3555	* @param i32 The value to AND *pi32 with.
3556	*
3557	* @remarks x86: Requires a 386 or later.
3558	*/
3559	DECLINLINE(void) ASMAtomicUoAndS32(int32_t volatile *pi32, int32_t i32)
3560	{
3561	ASMAtomicUoAndU32((uint32_t volatile *)pi32, (uint32_t)i32);
3562	}
3563
3564
3565	/**
3566	* Atomically And an unsigned 64-bit value, unordered.
3567	*
3568	* @param pu64 Pointer to the pointer variable to AND u64 with.
3569	* @param u64 The value to AND *pu64 with.
3570	*
3571	* @remarks x86: Requires a Pentium or later.
3572	*/
3573	#if RT_INLINE_ASM_EXTERNAL
3574	DECLASM(void) ASMAtomicUoAndU64(uint64_t volatile *pu64, uint64_t u64);
3575	#else
3576	DECLINLINE(void) ASMAtomicUoAndU64(uint64_t volatile *pu64, uint64_t u64)
3577	{
3578	# if RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
3579	__asm__ __volatile__("andq %1, %0\n\t"
3580	: "=m" (*pu64)
3581	: "r" (u64),
3582	"m" (*pu64));
3583	# else
3584	for (;;)
3585	{
3586	uint64_t u64Old = ASMAtomicUoReadU64(pu64);
3587	uint64_t u64New = u64Old & u64;
3588	if (ASMAtomicCmpXchgU64(pu64, u64New, u64Old))
3589	break;
3590	ASMNopPause();
3591	}
3592	# endif
3593	}
3594	#endif
3595
3596
3597	/**
3598	* Atomically And a signed 64-bit value, unordered.
3599	*
3600	* @param pi64 Pointer to the pointer variable to AND i64 with.
3601	* @param i64 The value to AND *pi64 with.
3602	*
3603	* @remarks x86: Requires a Pentium or later.
3604	*/
3605	DECLINLINE(void) ASMAtomicUoAndS64(int64_t volatile *pi64, int64_t i64)
3606	{
3607	ASMAtomicUoAndU64((uint64_t volatile *)pi64, (uint64_t)i64);
3608	}
3609
3610
3611	/**
3612	* Atomically increment an unsigned 32-bit value, unordered.
3613	*
3614	* @returns the new value.
3615	* @param pu32 Pointer to the variable to increment.
3616	*
3617	* @remarks x86: Requires a 486 or later.
3618	*/
3619	#if RT_INLINE_ASM_EXTERNAL
3620	DECLASM(uint32_t) ASMAtomicUoIncU32(uint32_t volatile *pu32);
3621	#else
3622	DECLINLINE(uint32_t) ASMAtomicUoIncU32(uint32_t volatile *pu32)
3623	{
3624	uint32_t u32;
3625	# if RT_INLINE_ASM_GNU_STYLE
3626	__asm__ __volatile__("xaddl %0, %1\n\t"
3627	: "=r" (u32),
3628	"=m" (*pu32)
3629	: "0" (1),
3630	"m" (*pu32)
3631	: "memory");
3632	return u32 + 1;
3633	# else
3634	__asm
3635	{
3636	mov eax, 1
3637	# ifdef RT_ARCH_AMD64
3638	mov rdx, [pu32]
3639	xadd [rdx], eax
3640	# else
3641	mov edx, [pu32]
3642	xadd [edx], eax
3643	# endif
3644	mov u32, eax
3645	}
3646	return u32 + 1;
3647	# endif
3648	}
3649	#endif
3650
3651
3652	/**
3653	* Atomically decrement an unsigned 32-bit value, unordered.
3654	*
3655	* @returns the new value.
3656	* @param pu32 Pointer to the variable to decrement.
3657	*
3658	* @remarks x86: Requires a 486 or later.
3659	*/
3660	#if RT_INLINE_ASM_EXTERNAL
3661	DECLASM(uint32_t) ASMAtomicUoDecU32(uint32_t volatile *pu32);
3662	#else
3663	DECLINLINE(uint32_t) ASMAtomicUoDecU32(uint32_t volatile *pu32)
3664	{
3665	uint32_t u32;
3666	# if RT_INLINE_ASM_GNU_STYLE
3667	__asm__ __volatile__("lock; xaddl %0, %1\n\t"
3668	: "=r" (u32),
3669	"=m" (*pu32)
3670	: "0" (-1),
3671	"m" (*pu32)
3672	: "memory");
3673	return u32 - 1;
3674	# else
3675	__asm
3676	{
3677	mov eax, -1
3678	# ifdef RT_ARCH_AMD64
3679	mov rdx, [pu32]
3680	xadd [rdx], eax
3681	# else
3682	mov edx, [pu32]
3683	xadd [edx], eax
3684	# endif
3685	mov u32, eax
3686	}
3687	return u32 - 1;
3688	# endif
3689	}
3690	#endif
3691
3692
3693	/** @def RT_ASM_PAGE_SIZE
3694	* We try avoid dragging in iprt/param.h here.
3695	* @internal
3696	*/
3697	#if defined(RT_ARCH_SPARC64)
3698	# define RT_ASM_PAGE_SIZE 0x2000
3699	# if defined(PAGE_SIZE) && !defined(NT_INCLUDED)
3700	# if PAGE_SIZE != 0x2000
3701	# error "PAGE_SIZE is not 0x2000!"
3702	# endif
3703	# endif
3704	#else
3705	# define RT_ASM_PAGE_SIZE 0x1000
3706	# if defined(PAGE_SIZE) && !defined(NT_INCLUDED)
3707	# if PAGE_SIZE != 0x1000
3708	# error "PAGE_SIZE is not 0x1000!"
3709	# endif
3710	# endif
3711	#endif
3712
3713	/**
3714	* Zeros a 4K memory page.
3715	*
3716	* @param pv Pointer to the memory block. This must be page aligned.
3717	*/
3718	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3719	DECLASM(void) ASMMemZeroPage(volatile void *pv);
3720	# else
3721	DECLINLINE(void) ASMMemZeroPage(volatile void *pv)
3722	{
3723	# if RT_INLINE_ASM_USES_INTRIN
3724	# ifdef RT_ARCH_AMD64
3725	__stosq((unsigned __int64 *)pv, 0, RT_ASM_PAGE_SIZE / 8);
3726	# else
3727	__stosd((unsigned long *)pv, 0, RT_ASM_PAGE_SIZE / 4);
3728	# endif
3729
3730	# elif RT_INLINE_ASM_GNU_STYLE
3731	RTCCUINTREG uDummy;
3732	# ifdef RT_ARCH_AMD64
3733	__asm__ __volatile__("rep stosq"
3734	: "=D" (pv),
3735	"=c" (uDummy)
3736	: "0" (pv),
3737	"c" (RT_ASM_PAGE_SIZE >> 3),
3738	"a" (0)
3739	: "memory");
3740	# else
3741	__asm__ __volatile__("rep stosl"
3742	: "=D" (pv),
3743	"=c" (uDummy)
3744	: "0" (pv),
3745	"c" (RT_ASM_PAGE_SIZE >> 2),
3746	"a" (0)
3747	: "memory");
3748	# endif
3749	# else
3750	__asm
3751	{
3752	# ifdef RT_ARCH_AMD64
3753	xor rax, rax
3754	mov ecx, 0200h
3755	mov rdi, [pv]
3756	rep stosq
3757	# else
3758	xor eax, eax
3759	mov ecx, 0400h
3760	mov edi, [pv]
3761	rep stosd
3762	# endif
3763	}
3764	# endif
3765	}
3766	# endif
3767
3768
3769	/**
3770	* Zeros a memory block with a 32-bit aligned size.
3771	*
3772	* @param pv Pointer to the memory block.
3773	* @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
3774	*/
3775	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3776	DECLASM(void) ASMMemZero32(volatile void *pv, size_t cb);
3777	#else
3778	DECLINLINE(void) ASMMemZero32(volatile void *pv, size_t cb)
3779	{
3780	# if RT_INLINE_ASM_USES_INTRIN
3781	# ifdef RT_ARCH_AMD64
3782	if (!(cb & 7))
3783	__stosq((unsigned __int64 *)pv, 0, cb / 8);
3784	else
3785	# endif
3786	__stosd((unsigned long *)pv, 0, cb / 4);
3787
3788	# elif RT_INLINE_ASM_GNU_STYLE
3789	__asm__ __volatile__("rep stosl"
3790	: "=D" (pv),
3791	"=c" (cb)
3792	: "0" (pv),
3793	"1" (cb >> 2),
3794	"a" (0)
3795	: "memory");
3796	# else
3797	__asm
3798	{
3799	xor eax, eax
3800	# ifdef RT_ARCH_AMD64
3801	mov rcx, [cb]
3802	shr rcx, 2
3803	mov rdi, [pv]
3804	# else
3805	mov ecx, [cb]
3806	shr ecx, 2
3807	mov edi, [pv]
3808	# endif
3809	rep stosd
3810	}
3811	# endif
3812	}
3813	#endif
3814
3815
3816	/**
3817	* Fills a memory block with a 32-bit aligned size.
3818	*
3819	* @param pv Pointer to the memory block.
3820	* @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
3821	* @param u32 The value to fill with.
3822	*/
3823	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3824	DECLASM(void) ASMMemFill32(volatile void *pv, size_t cb, uint32_t u32);
3825	#else
3826	DECLINLINE(void) ASMMemFill32(volatile void *pv, size_t cb, uint32_t u32)
3827	{
3828	# if RT_INLINE_ASM_USES_INTRIN
3829	# ifdef RT_ARCH_AMD64
3830	if (!(cb & 7))
3831	__stosq((unsigned __int64 *)pv, RT_MAKE_U64(u32, u32), cb / 8);
3832	else
3833	# endif
3834	__stosd((unsigned long *)pv, u32, cb / 4);
3835
3836	# elif RT_INLINE_ASM_GNU_STYLE
3837	__asm__ __volatile__("rep stosl"
3838	: "=D" (pv),
3839	"=c" (cb)
3840	: "0" (pv),
3841	"1" (cb >> 2),
3842	"a" (u32)
3843	: "memory");
3844	# else
3845	__asm
3846	{
3847	# ifdef RT_ARCH_AMD64
3848	mov rcx, [cb]
3849	shr rcx, 2
3850	mov rdi, [pv]
3851	# else
3852	mov ecx, [cb]
3853	shr ecx, 2
3854	mov edi, [pv]
3855	# endif
3856	mov eax, [u32]
3857	rep stosd
3858	}
3859	# endif
3860	}
3861	#endif
3862
3863
3864	/**
3865	* Checks if a memory block is all zeros.
3866	*
3867	* @returns Pointer to the first non-zero byte.
3868	* @returns NULL if all zero.
3869	*
3870	* @param pv Pointer to the memory block.
3871	* @param cb Number of bytes in the block.
3872	*
3873	* @todo Fix name, it is a predicate function but it's not returning boolean!
3874	*/
3875	#if !defined(RT_OS_LINUX) \|\| !defined(__KERNEL__)
3876	DECLASM(void ) ASMMemFirstNonZero(void const pv, size_t cb);
3877	#else
3878	DECLINLINE(void ) ASMMemFirstNonZero(void const pv, size_t cb)
3879	{
3880	uint8_t const pb = (uint8_t const )pv;
3881	for (; cb; cb--, pb++)
3882	if (RT_LIKELY(*pb == 0))
3883	{ /* likely */ }
3884	else
3885	return (void *)pb;
3886	return NULL;
3887	}
3888	#endif
3889
3890
3891	/**
3892	* Checks if a memory block is all zeros.
3893	*
3894	* @returns true if zero, false if not.
3895	*
3896	* @param pv Pointer to the memory block.
3897	* @param cb Number of bytes in the block.
3898	*
3899	* @sa ASMMemFirstNonZero
3900	*/
3901	DECLINLINE(bool) ASMMemIsZero(void const *pv, size_t cb)
3902	{
3903	return ASMMemFirstNonZero(pv, cb) == NULL;
3904	}
3905
3906
3907	/**
3908	* Checks if a memory page is all zeros.
3909	*
3910	* @returns true / false.
3911	*
3912	* @param pvPage Pointer to the page. Must be aligned on 16 byte
3913	* boundary
3914	*/
3915	DECLINLINE(bool) ASMMemIsZeroPage(void const *pvPage)
3916	{
3917	# if 0 /RT_INLINE_ASM_GNU_STYLE - this is actually slower... /
3918	union { RTCCUINTREG r; bool f; } uAX;
3919	RTCCUINTREG xCX, xDI;
3920	Assert(!((uintptr_t)pvPage & 15));
3921	__asm__ __volatile__("repe; "
3922	# ifdef RT_ARCH_AMD64
3923	"scasq\n\t"
3924	# else
3925	"scasl\n\t"
3926	# endif
3927	"setnc %%al\n\t"
3928	: "=&c" (xCX),
3929	"=&D" (xDI),
3930	"=&a" (uAX.r)
3931	: "mr" (pvPage),
3932	# ifdef RT_ARCH_AMD64
3933	"0" (RT_ASM_PAGE_SIZE/8),
3934	# else
3935	"0" (RT_ASM_PAGE_SIZE/4),
3936	# endif
3937	"1" (pvPage),
3938	"2" (0));
3939	return uAX.f;
3940	# else
3941	uintptr_t const puPtr = (uintptr_t const )pvPage;
3942	size_t cLeft = RT_ASM_PAGE_SIZE / sizeof(uintptr_t) / 8;
3943	Assert(!((uintptr_t)pvPage & 15));
3944	for (;;)
3945	{
3946	if (puPtr[0]) return false;
3947	if (puPtr[4]) return false;
3948
3949	if (puPtr[2]) return false;
3950	if (puPtr[6]) return false;
3951
3952	if (puPtr[1]) return false;
3953	if (puPtr[5]) return false;
3954
3955	if (puPtr[3]) return false;
3956	if (puPtr[7]) return false;
3957
3958	if (!--cLeft)
3959	return true;
3960	puPtr += 8;
3961	}
3962	# endif
3963	}
3964
3965
3966	/**
3967	* Checks if a memory block is filled with the specified byte, returning the
3968	* first mismatch.
3969	*
3970	* This is sort of an inverted memchr.
3971	*
3972	* @returns Pointer to the byte which doesn't equal u8.
3973	* @returns NULL if all equal to u8.
3974	*
3975	* @param pv Pointer to the memory block.
3976	* @param cb Number of bytes in the block.
3977	* @param u8 The value it's supposed to be filled with.
3978	*
3979	* @remarks No alignment requirements.
3980	*/
3981	#if (!defined(RT_OS_LINUX) \|\| !defined(__KERNEL__)) \
3982	&& (!defined(RT_OS_FREEBSD) \|\| !defined(_KERNEL))
3983	DECLASM(void ) ASMMemFirstMismatchingU8(void const pv, size_t cb, uint8_t u8);
3984	#else
3985	DECLINLINE(void ) ASMMemFirstMismatchingU8(void const pv, size_t cb, uint8_t u8)
3986	{
3987	uint8_t const pb = (uint8_t const )pv;
3988	for (; cb; cb--, pb++)
3989	if (RT_LIKELY(*pb == u8))
3990	{ /* likely */ }
3991	else
3992	return (void *)pb;
3993	return NULL;
3994	}
3995	#endif
3996
3997
3998	/**
3999	* Checks if a memory block is filled with the specified byte.
4000	*
4001	* @returns true if all matching, false if not.
4002	*
4003	* @param pv Pointer to the memory block.
4004	* @param cb Number of bytes in the block.
4005	* @param u8 The value it's supposed to be filled with.
4006	*
4007	* @remarks No alignment requirements.
4008	*/
4009	DECLINLINE(bool) ASMMemIsAllU8(void const *pv, size_t cb, uint8_t u8)
4010	{
4011	return ASMMemFirstMismatchingU8(pv, cb, u8) == NULL;
4012	}
4013
4014
4015	/**
4016	* Checks if a memory block is filled with the specified 32-bit value.
4017	*
4018	* This is a sort of inverted memchr.
4019	*
4020	* @returns Pointer to the first value which doesn't equal u32.
4021	* @returns NULL if all equal to u32.
4022	*
4023	* @param pv Pointer to the memory block.
4024	* @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
4025	* @param u32 The value it's supposed to be filled with.
4026	*/
4027	DECLINLINE(uint32_t ) ASMMemFirstMismatchingU32(void const pv, size_t cb, uint32_t u32)
4028	{
4029	/** @todo rewrite this in inline assembly? */
4030	uint32_t const pu32 = (uint32_t const )pv;
4031	for (; cb; cb -= 4, pu32++)
4032	if (RT_LIKELY(*pu32 == u32))
4033	{ /* likely */ }
4034	else
4035	return (uint32_t *)pu32;
4036	return NULL;
4037	}
4038
4039
4040	/**
4041	* Probes a byte pointer for read access.
4042	*
4043	* While the function will not fault if the byte is not read accessible,
4044	* the idea is to do this in a safe place like before acquiring locks
4045	* and such like.
4046	*
4047	* Also, this functions guarantees that an eager compiler is not going
4048	* to optimize the probing away.
4049	*
4050	* @param pvByte Pointer to the byte.
4051	*/
4052	#if RT_INLINE_ASM_EXTERNAL
4053	DECLASM(uint8_t) ASMProbeReadByte(const void *pvByte);
4054	#else
4055	DECLINLINE(uint8_t) ASMProbeReadByte(const void *pvByte)
4056	{
4057	/** @todo verify that the compiler actually doesn't optimize this away. (intel & gcc) */
4058	uint8_t u8;
4059	# if RT_INLINE_ASM_GNU_STYLE
4060	__asm__ __volatile__("movb (%1), %0\n\t"
4061	: "=r" (u8)
4062	: "r" (pvByte));
4063	# else
4064	__asm
4065	{
4066	# ifdef RT_ARCH_AMD64
4067	mov rax, [pvByte]
4068	mov al, [rax]
4069	# else
4070	mov eax, [pvByte]
4071	mov al, [eax]
4072	# endif
4073	mov [u8], al
4074	}
4075	# endif
4076	return u8;
4077	}
4078	#endif
4079
4080	/**
4081	* Probes a buffer for read access page by page.
4082	*
4083	* While the function will fault if the buffer is not fully read
4084	* accessible, the idea is to do this in a safe place like before
4085	* acquiring locks and such like.
4086	*
4087	* Also, this functions guarantees that an eager compiler is not going
4088	* to optimize the probing away.
4089	*
4090	* @param pvBuf Pointer to the buffer.
4091	* @param cbBuf The size of the buffer in bytes. Must be >= 1.
4092	*/
4093	DECLINLINE(void) ASMProbeReadBuffer(const void *pvBuf, size_t cbBuf)
4094	{
4095	/** @todo verify that the compiler actually doesn't optimize this away. (intel & gcc) */
4096	/* the first byte */
4097	const uint8_t pu8 = (const uint8_t )pvBuf;
4098	ASMProbeReadByte(pu8);
4099
4100	/* the pages in between pages. */
4101	while (cbBuf > RT_ASM_PAGE_SIZE)
4102	{
4103	ASMProbeReadByte(pu8);
4104	cbBuf -= RT_ASM_PAGE_SIZE;
4105	pu8 += RT_ASM_PAGE_SIZE;
4106	}
4107
4108	/* the last byte */
4109	ASMProbeReadByte(pu8 + cbBuf - 1);
4110	}
4111
4112
4113
4114	/** @defgroup grp_inline_bits Bit Operations
4115	* @{
4116	*/
4117
4118
4119	/**
4120	* Sets a bit in a bitmap.
4121	*
4122	* @param pvBitmap Pointer to the bitmap. This should be 32-bit aligned.
4123	* @param iBit The bit to set.
4124	*
4125	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
4126	* However, doing so will yield better performance as well as avoiding
4127	* traps accessing the last bits in the bitmap.
4128	*/
4129	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4130	DECLASM(void) ASMBitSet(volatile void *pvBitmap, int32_t iBit);
4131	#else
4132	DECLINLINE(void) ASMBitSet(volatile void *pvBitmap, int32_t iBit)
4133	{
4134	# if RT_INLINE_ASM_USES_INTRIN
4135	_bittestandset((long *)pvBitmap, iBit);
4136
4137	# elif RT_INLINE_ASM_GNU_STYLE
4138	__asm__ __volatile__("btsl %1, %0"
4139	: "=m" ((volatile long )pvBitmap)
4140	: "Ir" (iBit),
4141	"m" ((volatile long )pvBitmap)
4142	: "memory");
4143	# else
4144	__asm
4145	{
4146	# ifdef RT_ARCH_AMD64
4147	mov rax, [pvBitmap]
4148	mov edx, [iBit]
4149	bts [rax], edx
4150	# else
4151	mov eax, [pvBitmap]
4152	mov edx, [iBit]
4153	bts [eax], edx
4154	# endif
4155	}
4156	# endif
4157	}
4158	#endif
4159
4160
4161	/**
4162	* Atomically sets a bit in a bitmap, ordered.
4163	*
4164	* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
4165	* the memory access isn't atomic!
4166	* @param iBit The bit to set.
4167	*
4168	* @remarks x86: Requires a 386 or later.
4169	*/
4170	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4171	DECLASM(void) ASMAtomicBitSet(volatile void *pvBitmap, int32_t iBit);
4172	#else
4173	DECLINLINE(void) ASMAtomicBitSet(volatile void *pvBitmap, int32_t iBit)
4174	{
4175	AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
4176	# if RT_INLINE_ASM_USES_INTRIN
4177	_interlockedbittestandset((long *)pvBitmap, iBit);
4178	# elif RT_INLINE_ASM_GNU_STYLE
4179	__asm__ __volatile__("lock; btsl %1, %0"
4180	: "=m" ((volatile long )pvBitmap)
4181	: "Ir" (iBit),
4182	"m" ((volatile long )pvBitmap)
4183	: "memory");
4184	# else
4185	__asm
4186	{
4187	# ifdef RT_ARCH_AMD64
4188	mov rax, [pvBitmap]
4189	mov edx, [iBit]
4190	lock bts [rax], edx
4191	# else
4192	mov eax, [pvBitmap]
4193	mov edx, [iBit]
4194	lock bts [eax], edx
4195	# endif
4196	}
4197	# endif
4198	}
4199	#endif
4200
4201
4202	/**
4203	* Clears a bit in a bitmap.
4204	*
4205	* @param pvBitmap Pointer to the bitmap.
4206	* @param iBit The bit to clear.
4207	*
4208	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
4209	* However, doing so will yield better performance as well as avoiding
4210	* traps accessing the last bits in the bitmap.
4211	*/
4212	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4213	DECLASM(void) ASMBitClear(volatile void *pvBitmap, int32_t iBit);
4214	#else
4215	DECLINLINE(void) ASMBitClear(volatile void *pvBitmap, int32_t iBit)
4216	{
4217	# if RT_INLINE_ASM_USES_INTRIN
4218	_bittestandreset((long *)pvBitmap, iBit);
4219
4220	# elif RT_INLINE_ASM_GNU_STYLE
4221	__asm__ __volatile__("btrl %1, %0"
4222	: "=m" ((volatile long )pvBitmap)
4223	: "Ir" (iBit),
4224	"m" ((volatile long )pvBitmap)
4225	: "memory");
4226	# else
4227	__asm
4228	{
4229	# ifdef RT_ARCH_AMD64
4230	mov rax, [pvBitmap]
4231	mov edx, [iBit]
4232	btr [rax], edx
4233	# else
4234	mov eax, [pvBitmap]
4235	mov edx, [iBit]
4236	btr [eax], edx
4237	# endif
4238	}
4239	# endif
4240	}
4241	#endif
4242
4243
4244	/**
4245	* Atomically clears a bit in a bitmap, ordered.
4246	*
4247	* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
4248	* the memory access isn't atomic!
4249	* @param iBit The bit to toggle set.
4250	*
4251	* @remarks No memory barrier, take care on smp.
4252	* @remarks x86: Requires a 386 or later.
4253	*/
4254	#if RT_INLINE_ASM_EXTERNAL
4255	DECLASM(void) ASMAtomicBitClear(volatile void *pvBitmap, int32_t iBit);
4256	#else
4257	DECLINLINE(void) ASMAtomicBitClear(volatile void *pvBitmap, int32_t iBit)
4258	{
4259	AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
4260	# if RT_INLINE_ASM_GNU_STYLE
4261	__asm__ __volatile__("lock; btrl %1, %0"
4262	: "=m" ((volatile long )pvBitmap)
4263	: "Ir" (iBit),
4264	"m" ((volatile long )pvBitmap)
4265	: "memory");
4266	# else
4267	__asm
4268	{
4269	# ifdef RT_ARCH_AMD64
4270	mov rax, [pvBitmap]
4271	mov edx, [iBit]
4272	lock btr [rax], edx
4273	# else
4274	mov eax, [pvBitmap]
4275	mov edx, [iBit]
4276	lock btr [eax], edx
4277	# endif
4278	}
4279	# endif
4280	}
4281	#endif
4282
4283
4284	/**
4285	* Toggles a bit in a bitmap.
4286	*
4287	* @param pvBitmap Pointer to the bitmap.
4288	* @param iBit The bit to toggle.
4289	*
4290	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
4291	* However, doing so will yield better performance as well as avoiding
4292	* traps accessing the last bits in the bitmap.
4293	*/
4294	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4295	DECLASM(void) ASMBitToggle(volatile void *pvBitmap, int32_t iBit);
4296	#else
4297	DECLINLINE(void) ASMBitToggle(volatile void *pvBitmap, int32_t iBit)
4298	{
4299	# if RT_INLINE_ASM_USES_INTRIN
4300	_bittestandcomplement((long *)pvBitmap, iBit);
4301	# elif RT_INLINE_ASM_GNU_STYLE
4302	__asm__ __volatile__("btcl %1, %0"
4303	: "=m" ((volatile long )pvBitmap)
4304	: "Ir" (iBit),
4305	"m" ((volatile long )pvBitmap)
4306	: "memory");
4307	# else
4308	__asm
4309	{
4310	# ifdef RT_ARCH_AMD64
4311	mov rax, [pvBitmap]
4312	mov edx, [iBit]
4313	btc [rax], edx
4314	# else
4315	mov eax, [pvBitmap]
4316	mov edx, [iBit]
4317	btc [eax], edx
4318	# endif
4319	}
4320	# endif
4321	}
4322	#endif
4323
4324
4325	/**
4326	* Atomically toggles a bit in a bitmap, ordered.
4327	*
4328	* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
4329	* the memory access isn't atomic!
4330	* @param iBit The bit to test and set.
4331	*
4332	* @remarks x86: Requires a 386 or later.
4333	*/
4334	#if RT_INLINE_ASM_EXTERNAL
4335	DECLASM(void) ASMAtomicBitToggle(volatile void *pvBitmap, int32_t iBit);
4336	#else
4337	DECLINLINE(void) ASMAtomicBitToggle(volatile void *pvBitmap, int32_t iBit)
4338	{
4339	AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
4340	# if RT_INLINE_ASM_GNU_STYLE
4341	__asm__ __volatile__("lock; btcl %1, %0"
4342	: "=m" ((volatile long )pvBitmap)
4343	: "Ir" (iBit),
4344	"m" ((volatile long )pvBitmap)
4345	: "memory");
4346	# else
4347	__asm
4348	{
4349	# ifdef RT_ARCH_AMD64
4350	mov rax, [pvBitmap]
4351	mov edx, [iBit]
4352	lock btc [rax], edx
4353	# else
4354	mov eax, [pvBitmap]
4355	mov edx, [iBit]
4356	lock btc [eax], edx
4357	# endif
4358	}
4359	# endif
4360	}
4361	#endif
4362
4363
4364	/**
4365	* Tests and sets a bit in a bitmap.
4366	*
4367	* @returns true if the bit was set.
4368	* @returns false if the bit was clear.
4369	*
4370	* @param pvBitmap Pointer to the bitmap.
4371	* @param iBit The bit to test and set.
4372	*
4373	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
4374	* However, doing so will yield better performance as well as avoiding
4375	* traps accessing the last bits in the bitmap.
4376	*/
4377	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4378	DECLASM(bool) ASMBitTestAndSet(volatile void *pvBitmap, int32_t iBit);
4379	#else
4380	DECLINLINE(bool) ASMBitTestAndSet(volatile void *pvBitmap, int32_t iBit)
4381	{
4382	union { bool f; uint32_t u32; uint8_t u8; } rc;
4383	# if RT_INLINE_ASM_USES_INTRIN
4384	rc.u8 = _bittestandset((long *)pvBitmap, iBit);
4385
4386	# elif RT_INLINE_ASM_GNU_STYLE
4387	__asm__ __volatile__("btsl %2, %1\n\t"
4388	"setc %b0\n\t"
4389	"andl $1, %0\n\t"
4390	: "=q" (rc.u32),
4391	"=m" ((volatile long )pvBitmap)
4392	: "Ir" (iBit),
4393	"m" ((volatile long )pvBitmap)
4394	: "memory");
4395	# else
4396	__asm
4397	{
4398	mov edx, [iBit]
4399	# ifdef RT_ARCH_AMD64
4400	mov rax, [pvBitmap]
4401	bts [rax], edx
4402	# else
4403	mov eax, [pvBitmap]
4404	bts [eax], edx
4405	# endif
4406	setc al
4407	and eax, 1
4408	mov [rc.u32], eax
4409	}
4410	# endif
4411	return rc.f;
4412	}
4413	#endif
4414
4415
4416	/**
4417	* Atomically tests and sets a bit in a bitmap, ordered.
4418	*
4419	* @returns true if the bit was set.
4420	* @returns false if the bit was clear.
4421	*
4422	* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
4423	* the memory access isn't atomic!
4424	* @param iBit The bit to set.
4425	*
4426	* @remarks x86: Requires a 386 or later.
4427	*/
4428	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4429	DECLASM(bool) ASMAtomicBitTestAndSet(volatile void *pvBitmap, int32_t iBit);
4430	#else
4431	DECLINLINE(bool) ASMAtomicBitTestAndSet(volatile void *pvBitmap, int32_t iBit)
4432	{
4433	union { bool f; uint32_t u32; uint8_t u8; } rc;
4434	AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
4435	# if RT_INLINE_ASM_USES_INTRIN
4436	rc.u8 = _interlockedbittestandset((long *)pvBitmap, iBit);
4437	# elif RT_INLINE_ASM_GNU_STYLE
4438	__asm__ __volatile__("lock; btsl %2, %1\n\t"
4439	"setc %b0\n\t"
4440	"andl $1, %0\n\t"
4441	: "=q" (rc.u32),
4442	"=m" ((volatile long )pvBitmap)
4443	: "Ir" (iBit),
4444	"m" ((volatile long )pvBitmap)
4445	: "memory");
4446	# else
4447	__asm
4448	{
4449	mov edx, [iBit]
4450	# ifdef RT_ARCH_AMD64
4451	mov rax, [pvBitmap]
4452	lock bts [rax], edx
4453	# else
4454	mov eax, [pvBitmap]
4455	lock bts [eax], edx
4456	# endif
4457	setc al
4458	and eax, 1
4459	mov [rc.u32], eax
4460	}
4461	# endif
4462	return rc.f;
4463	}
4464	#endif
4465
4466
4467	/**
4468	* Tests and clears a bit in a bitmap.
4469	*
4470	* @returns true if the bit was set.
4471	* @returns false if the bit was clear.
4472	*
4473	* @param pvBitmap Pointer to the bitmap.
4474	* @param iBit The bit to test and clear.
4475	*
4476	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
4477	* However, doing so will yield better performance as well as avoiding
4478	* traps accessing the last bits in the bitmap.
4479	*/
4480	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4481	DECLASM(bool) ASMBitTestAndClear(volatile void *pvBitmap, int32_t iBit);
4482	#else
4483	DECLINLINE(bool) ASMBitTestAndClear(volatile void *pvBitmap, int32_t iBit)
4484	{
4485	union { bool f; uint32_t u32; uint8_t u8; } rc;
4486	# if RT_INLINE_ASM_USES_INTRIN
4487	rc.u8 = _bittestandreset((long *)pvBitmap, iBit);
4488
4489	# elif RT_INLINE_ASM_GNU_STYLE
4490	__asm__ __volatile__("btrl %2, %1\n\t"
4491	"setc %b0\n\t"
4492	"andl $1, %0\n\t"
4493	: "=q" (rc.u32),
4494	"=m" ((volatile long )pvBitmap)
4495	: "Ir" (iBit),
4496	"m" ((volatile long )pvBitmap)
4497	: "memory");
4498	# else
4499	__asm
4500	{
4501	mov edx, [iBit]
4502	# ifdef RT_ARCH_AMD64
4503	mov rax, [pvBitmap]
4504	btr [rax], edx
4505	# else
4506	mov eax, [pvBitmap]
4507	btr [eax], edx
4508	# endif
4509	setc al
4510	and eax, 1
4511	mov [rc.u32], eax
4512	}
4513	# endif
4514	return rc.f;
4515	}
4516	#endif
4517
4518
4519	/**
4520	* Atomically tests and clears a bit in a bitmap, ordered.
4521	*
4522	* @returns true if the bit was set.
4523	* @returns false if the bit was clear.
4524	*
4525	* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
4526	* the memory access isn't atomic!
4527	* @param iBit The bit to test and clear.
4528	*
4529	* @remarks No memory barrier, take care on smp.
4530	* @remarks x86: Requires a 386 or later.
4531	*/
4532	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4533	DECLASM(bool) ASMAtomicBitTestAndClear(volatile void *pvBitmap, int32_t iBit);
4534	#else
4535	DECLINLINE(bool) ASMAtomicBitTestAndClear(volatile void *pvBitmap, int32_t iBit)
4536	{
4537	union { bool f; uint32_t u32; uint8_t u8; } rc;
4538	AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
4539	# if RT_INLINE_ASM_USES_INTRIN
4540	rc.u8 = _interlockedbittestandreset((long *)pvBitmap, iBit);
4541
4542	# elif RT_INLINE_ASM_GNU_STYLE
4543	__asm__ __volatile__("lock; btrl %2, %1\n\t"
4544	"setc %b0\n\t"
4545	"andl $1, %0\n\t"
4546	: "=q" (rc.u32),
4547	"=m" ((volatile long )pvBitmap)
4548	: "Ir" (iBit),
4549	"m" ((volatile long )pvBitmap)
4550	: "memory");
4551	# else
4552	__asm
4553	{
4554	mov edx, [iBit]
4555	# ifdef RT_ARCH_AMD64
4556	mov rax, [pvBitmap]
4557	lock btr [rax], edx
4558	# else
4559	mov eax, [pvBitmap]
4560	lock btr [eax], edx
4561	# endif
4562	setc al
4563	and eax, 1
4564	mov [rc.u32], eax
4565	}
4566	# endif
4567	return rc.f;
4568	}
4569	#endif
4570
4571
4572	/**
4573	* Tests and toggles a bit in a bitmap.
4574	*
4575	* @returns true if the bit was set.
4576	* @returns false if the bit was clear.
4577	*
4578	* @param pvBitmap Pointer to the bitmap.
4579	* @param iBit The bit to test and toggle.
4580	*
4581	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
4582	* However, doing so will yield better performance as well as avoiding
4583	* traps accessing the last bits in the bitmap.
4584	*/
4585	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4586	DECLASM(bool) ASMBitTestAndToggle(volatile void *pvBitmap, int32_t iBit);
4587	#else
4588	DECLINLINE(bool) ASMBitTestAndToggle(volatile void *pvBitmap, int32_t iBit)
4589	{
4590	union { bool f; uint32_t u32; uint8_t u8; } rc;
4591	# if RT_INLINE_ASM_USES_INTRIN
4592	rc.u8 = _bittestandcomplement((long *)pvBitmap, iBit);
4593
4594	# elif RT_INLINE_ASM_GNU_STYLE
4595	__asm__ __volatile__("btcl %2, %1\n\t"
4596	"setc %b0\n\t"
4597	"andl $1, %0\n\t"
4598	: "=q" (rc.u32),
4599	"=m" ((volatile long )pvBitmap)
4600	: "Ir" (iBit),
4601	"m" ((volatile long )pvBitmap)
4602	: "memory");
4603	# else
4604	__asm
4605	{
4606	mov edx, [iBit]
4607	# ifdef RT_ARCH_AMD64
4608	mov rax, [pvBitmap]
4609	btc [rax], edx
4610	# else
4611	mov eax, [pvBitmap]
4612	btc [eax], edx
4613	# endif
4614	setc al
4615	and eax, 1
4616	mov [rc.u32], eax
4617	}
4618	# endif
4619	return rc.f;
4620	}
4621	#endif
4622
4623
4624	/**
4625	* Atomically tests and toggles a bit in a bitmap, ordered.
4626	*
4627	* @returns true if the bit was set.
4628	* @returns false if the bit was clear.
4629	*
4630	* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
4631	* the memory access isn't atomic!
4632	* @param iBit The bit to test and toggle.
4633	*
4634	* @remarks x86: Requires a 386 or later.
4635	*/
4636	#if RT_INLINE_ASM_EXTERNAL
4637	DECLASM(bool) ASMAtomicBitTestAndToggle(volatile void *pvBitmap, int32_t iBit);
4638	#else
4639	DECLINLINE(bool) ASMAtomicBitTestAndToggle(volatile void *pvBitmap, int32_t iBit)
4640	{
4641	union { bool f; uint32_t u32; uint8_t u8; } rc;
4642	AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
4643	# if RT_INLINE_ASM_GNU_STYLE
4644	__asm__ __volatile__("lock; btcl %2, %1\n\t"
4645	"setc %b0\n\t"
4646	"andl $1, %0\n\t"
4647	: "=q" (rc.u32),
4648	"=m" ((volatile long )pvBitmap)
4649	: "Ir" (iBit),
4650	"m" ((volatile long )pvBitmap)
4651	: "memory");
4652	# else
4653	__asm
4654	{
4655	mov edx, [iBit]
4656	# ifdef RT_ARCH_AMD64
4657	mov rax, [pvBitmap]
4658	lock btc [rax], edx
4659	# else
4660	mov eax, [pvBitmap]
4661	lock btc [eax], edx
4662	# endif
4663	setc al
4664	and eax, 1
4665	mov [rc.u32], eax
4666	}
4667	# endif
4668	return rc.f;
4669	}
4670	#endif
4671
4672
4673	/**
4674	* Tests if a bit in a bitmap is set.
4675	*
4676	* @returns true if the bit is set.
4677	* @returns false if the bit is clear.
4678	*
4679	* @param pvBitmap Pointer to the bitmap.
4680	* @param iBit The bit to test.
4681	*
4682	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
4683	* However, doing so will yield better performance as well as avoiding
4684	* traps accessing the last bits in the bitmap.
4685	*/
4686	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4687	DECLASM(bool) ASMBitTest(const volatile void *pvBitmap, int32_t iBit);
4688	#else
4689	DECLINLINE(bool) ASMBitTest(const volatile void *pvBitmap, int32_t iBit)
4690	{
4691	union { bool f; uint32_t u32; uint8_t u8; } rc;
4692	# if RT_INLINE_ASM_USES_INTRIN
4693	rc.u32 = _bittest((long *)pvBitmap, iBit);
4694	# elif RT_INLINE_ASM_GNU_STYLE
4695
4696	__asm__ __volatile__("btl %2, %1\n\t"
4697	"setc %b0\n\t"
4698	"andl $1, %0\n\t"
4699	: "=q" (rc.u32)
4700	: "m" ((const volatile long )pvBitmap),
4701	"Ir" (iBit)
4702	: "memory");
4703	# else
4704	__asm
4705	{
4706	mov edx, [iBit]
4707	# ifdef RT_ARCH_AMD64
4708	mov rax, [pvBitmap]
4709	bt [rax], edx
4710	# else
4711	mov eax, [pvBitmap]
4712	bt [eax], edx
4713	# endif
4714	setc al
4715	and eax, 1
4716	mov [rc.u32], eax
4717	}
4718	# endif
4719	return rc.f;
4720	}
4721	#endif
4722
4723
4724	/**
4725	* Clears a bit range within a bitmap.
4726	*
4727	* @param pvBitmap Pointer to the bitmap.
4728	* @param iBitStart The First bit to clear.
4729	* @param iBitEnd The first bit not to clear.
4730	*/
4731	DECLINLINE(void) ASMBitClearRange(volatile void *pvBitmap, int32_t iBitStart, int32_t iBitEnd)
4732	{
4733	if (iBitStart < iBitEnd)
4734	{
4735	volatile uint32_t pu32 = (volatile uint32_t )pvBitmap + (iBitStart >> 5);
4736	int32_t iStart = iBitStart & ~31;
4737	int32_t iEnd = iBitEnd & ~31;
4738	if (iStart == iEnd)
4739	*pu32 &= ((UINT32_C(1) << (iBitStart & 31)) - 1) \| ~((UINT32_C(1) << (iBitEnd & 31)) - 1);
4740	else
4741	{
4742	/* bits in first dword. */
4743	if (iBitStart & 31)
4744	{
4745	*pu32 &= (UINT32_C(1) << (iBitStart & 31)) - 1;
4746	pu32++;
4747	iBitStart = iStart + 32;
4748	}
4749
4750	/* whole dword. */
4751	if (iBitStart != iEnd)
4752	ASMMemZero32(pu32, (iEnd - iBitStart) >> 3);
4753
4754	/* bits in last dword. */
4755	if (iBitEnd & 31)
4756	{
4757	pu32 = (volatile uint32_t *)pvBitmap + (iBitEnd >> 5);
4758	*pu32 &= ~((UINT32_C(1) << (iBitEnd & 31)) - 1);
4759	}
4760	}
4761	}
4762	}
4763
4764
4765	/**
4766	* Sets a bit range within a bitmap.
4767	*
4768	* @param pvBitmap Pointer to the bitmap.
4769	* @param iBitStart The First bit to set.
4770	* @param iBitEnd The first bit not to set.
4771	*/
4772	DECLINLINE(void) ASMBitSetRange(volatile void *pvBitmap, int32_t iBitStart, int32_t iBitEnd)
4773	{
4774	if (iBitStart < iBitEnd)
4775	{
4776	volatile uint32_t pu32 = (volatile uint32_t )pvBitmap + (iBitStart >> 5);
4777	int32_t iStart = iBitStart & ~31;
4778	int32_t iEnd = iBitEnd & ~31;
4779	if (iStart == iEnd)
4780	*pu32 \|= ((UINT32_C(1) << (iBitEnd - iBitStart)) - 1) << (iBitStart & 31);
4781	else
4782	{
4783	/* bits in first dword. */
4784	if (iBitStart & 31)
4785	{
4786	*pu32 \|= ~((UINT32_C(1) << (iBitStart & 31)) - 1);
4787	pu32++;
4788	iBitStart = iStart + 32;
4789	}
4790
4791	/* whole dword. */
4792	if (iBitStart != iEnd)
4793	ASMMemFill32(pu32, (iEnd - iBitStart) >> 3, ~UINT32_C(0));
4794
4795	/* bits in last dword. */
4796	if (iBitEnd & 31)
4797	{
4798	pu32 = (volatile uint32_t *)pvBitmap + (iBitEnd >> 5);
4799	*pu32 \|= (UINT32_C(1) << (iBitEnd & 31)) - 1;
4800	}
4801	}
4802	}
4803	}
4804
4805
4806	/**
4807	* Finds the first clear bit in a bitmap.
4808	*
4809	* @returns Index of the first zero bit.
4810	* @returns -1 if no clear bit was found.
4811	* @param pvBitmap Pointer to the bitmap.
4812	* @param cBits The number of bits in the bitmap. Multiple of 32.
4813	*/
4814	#if RT_INLINE_ASM_EXTERNAL
4815	DECLASM(int32_t) ASMBitFirstClear(const volatile void *pvBitmap, uint32_t cBits);
4816	#else
4817	DECLINLINE(int32_t) ASMBitFirstClear(const volatile void *pvBitmap, uint32_t cBits)
4818	{
4819	if (cBits)
4820	{
4821	int32_t iBit;
4822	# if RT_INLINE_ASM_GNU_STYLE
4823	RTCCUINTREG uEAX, uECX, uEDI;
4824	cBits = RT_ALIGN_32(cBits, 32);
4825	__asm__ __volatile__("repe; scasl\n\t"
4826	"je 1f\n\t"
4827	# ifdef RT_ARCH_AMD64
4828	"lea -4(%%rdi), %%rdi\n\t"
4829	"xorl (%%rdi), %%eax\n\t"
4830	"subq %5, %%rdi\n\t"
4831	# else
4832	"lea -4(%%edi), %%edi\n\t"
4833	"xorl (%%edi), %%eax\n\t"
4834	"subl %5, %%edi\n\t"
4835	# endif
4836	"shll $3, %%edi\n\t"
4837	"bsfl %%eax, %%edx\n\t"
4838	"addl %%edi, %%edx\n\t"
4839	"1:\t\n"
4840	: "=d" (iBit),
4841	"=&c" (uECX),
4842	"=&D" (uEDI),
4843	"=&a" (uEAX)
4844	: "0" (0xffffffff),
4845	"mr" (pvBitmap),
4846	"1" (cBits >> 5),
4847	"2" (pvBitmap),
4848	"3" (0xffffffff));
4849	# else
4850	cBits = RT_ALIGN_32(cBits, 32);
4851	__asm
4852	{
4853	# ifdef RT_ARCH_AMD64
4854	mov rdi, [pvBitmap]
4855	mov rbx, rdi
4856	# else
4857	mov edi, [pvBitmap]
4858	mov ebx, edi
4859	# endif
4860	mov edx, 0ffffffffh
4861	mov eax, edx
4862	mov ecx, [cBits]
4863	shr ecx, 5
4864	repe scasd
4865	je done
4866
4867	# ifdef RT_ARCH_AMD64
4868	lea rdi, [rdi - 4]
4869	xor eax, [rdi]
4870	sub rdi, rbx
4871	# else
4872	lea edi, [edi - 4]
4873	xor eax, [edi]
4874	sub edi, ebx
4875	# endif
4876	shl edi, 3
4877	bsf edx, eax
4878	add edx, edi
4879	done:
4880	mov [iBit], edx
4881	}
4882	# endif
4883	return iBit;
4884	}
4885	return -1;
4886	}
4887	#endif
4888
4889
4890	/**
4891	* Finds the next clear bit in a bitmap.
4892	*
4893	* @returns Index of the first zero bit.
4894	* @returns -1 if no clear bit was found.
4895	* @param pvBitmap Pointer to the bitmap.
4896	* @param cBits The number of bits in the bitmap. Multiple of 32.
4897	* @param iBitPrev The bit returned from the last search.
4898	* The search will start at iBitPrev + 1.
4899	*/
4900	#if RT_INLINE_ASM_EXTERNAL
4901	DECLASM(int) ASMBitNextClear(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev);
4902	#else
4903	DECLINLINE(int) ASMBitNextClear(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev)
4904	{
4905	const volatile uint32_t pau32Bitmap = (const volatile uint32_t )pvBitmap;
4906	int iBit = ++iBitPrev & 31;
4907	if (iBit)
4908	{
4909	/*
4910	* Inspect the 32-bit word containing the unaligned bit.
4911	*/
4912	uint32_t u32 = ~pau32Bitmap[iBitPrev / 32] >> iBit;
4913
4914	# if RT_INLINE_ASM_USES_INTRIN
4915	unsigned long ulBit = 0;
4916	if (_BitScanForward(&ulBit, u32))
4917	return ulBit + iBitPrev;
4918	# else
4919	# if RT_INLINE_ASM_GNU_STYLE
4920	__asm__ __volatile__("bsf %1, %0\n\t"
4921	"jnz 1f\n\t"
4922	"movl $-1, %0\n\t"
4923	"1:\n\t"
4924	: "=r" (iBit)
4925	: "r" (u32));
4926	# else
4927	__asm
4928	{
4929	mov edx, [u32]
4930	bsf eax, edx
4931	jnz done
4932	mov eax, 0ffffffffh
4933	done:
4934	mov [iBit], eax
4935	}
4936	# endif
4937	if (iBit >= 0)
4938	return iBit + iBitPrev;
4939	# endif
4940
4941	/*
4942	* Skip ahead and see if there is anything left to search.
4943	*/
4944	iBitPrev \|= 31;
4945	iBitPrev++;
4946	if (cBits <= (uint32_t)iBitPrev)
4947	return -1;
4948	}
4949
4950	/*
4951	* 32-bit aligned search, let ASMBitFirstClear do the dirty work.
4952	*/
4953	iBit = ASMBitFirstClear(&pau32Bitmap[iBitPrev / 32], cBits - iBitPrev);
4954	if (iBit >= 0)
4955	iBit += iBitPrev;
4956	return iBit;
4957	}
4958	#endif
4959
4960
4961	/**
4962	* Finds the first set bit in a bitmap.
4963	*
4964	* @returns Index of the first set bit.
4965	* @returns -1 if no clear bit was found.
4966	* @param pvBitmap Pointer to the bitmap.
4967	* @param cBits The number of bits in the bitmap. Multiple of 32.
4968	*/
4969	#if RT_INLINE_ASM_EXTERNAL
4970	DECLASM(int32_t) ASMBitFirstSet(const volatile void *pvBitmap, uint32_t cBits);
4971	#else
4972	DECLINLINE(int32_t) ASMBitFirstSet(const volatile void *pvBitmap, uint32_t cBits)
4973	{
4974	if (cBits)
4975	{
4976	int32_t iBit;
4977	# if RT_INLINE_ASM_GNU_STYLE
4978	RTCCUINTREG uEAX, uECX, uEDI;
4979	cBits = RT_ALIGN_32(cBits, 32);
4980	__asm__ __volatile__("repe; scasl\n\t"
4981	"je 1f\n\t"
4982	# ifdef RT_ARCH_AMD64
4983	"lea -4(%%rdi), %%rdi\n\t"
4984	"movl (%%rdi), %%eax\n\t"
4985	"subq %5, %%rdi\n\t"
4986	# else
4987	"lea -4(%%edi), %%edi\n\t"
4988	"movl (%%edi), %%eax\n\t"
4989	"subl %5, %%edi\n\t"
4990	# endif
4991	"shll $3, %%edi\n\t"
4992	"bsfl %%eax, %%edx\n\t"
4993	"addl %%edi, %%edx\n\t"
4994	"1:\t\n"
4995	: "=d" (iBit),
4996	"=&c" (uECX),
4997	"=&D" (uEDI),
4998	"=&a" (uEAX)
4999	: "0" (0xffffffff),
5000	"mr" (pvBitmap),
5001	"1" (cBits >> 5),
5002	"2" (pvBitmap),
5003	"3" (0));
5004	# else
5005	cBits = RT_ALIGN_32(cBits, 32);
5006	__asm
5007	{
5008	# ifdef RT_ARCH_AMD64
5009	mov rdi, [pvBitmap]
5010	mov rbx, rdi
5011	# else
5012	mov edi, [pvBitmap]
5013	mov ebx, edi
5014	# endif
5015	mov edx, 0ffffffffh
5016	xor eax, eax
5017	mov ecx, [cBits]
5018	shr ecx, 5
5019	repe scasd
5020	je done
5021	# ifdef RT_ARCH_AMD64
5022	lea rdi, [rdi - 4]
5023	mov eax, [rdi]
5024	sub rdi, rbx
5025	# else
5026	lea edi, [edi - 4]
5027	mov eax, [edi]
5028	sub edi, ebx
5029	# endif
5030	shl edi, 3
5031	bsf edx, eax
5032	add edx, edi
5033	done:
5034	mov [iBit], edx
5035	}
5036	# endif
5037	return iBit;
5038	}
5039	return -1;
5040	}
5041	#endif
5042
5043
5044	/**
5045	* Finds the next set bit in a bitmap.
5046	*
5047	* @returns Index of the next set bit.
5048	* @returns -1 if no set bit was found.
5049	* @param pvBitmap Pointer to the bitmap.
5050	* @param cBits The number of bits in the bitmap. Multiple of 32.
5051	* @param iBitPrev The bit returned from the last search.
5052	* The search will start at iBitPrev + 1.
5053	*/
5054	#if RT_INLINE_ASM_EXTERNAL
5055	DECLASM(int) ASMBitNextSet(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev);
5056	#else
5057	DECLINLINE(int) ASMBitNextSet(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev)
5058	{
5059	const volatile uint32_t pau32Bitmap = (const volatile uint32_t )pvBitmap;
5060	int iBit = ++iBitPrev & 31;
5061	if (iBit)
5062	{
5063	/*
5064	* Inspect the 32-bit word containing the unaligned bit.
5065	*/
5066	uint32_t u32 = pau32Bitmap[iBitPrev / 32] >> iBit;
5067
5068	# if RT_INLINE_ASM_USES_INTRIN
5069	unsigned long ulBit = 0;
5070	if (_BitScanForward(&ulBit, u32))
5071	return ulBit + iBitPrev;
5072	# else
5073	# if RT_INLINE_ASM_GNU_STYLE
5074	__asm__ __volatile__("bsf %1, %0\n\t"
5075	"jnz 1f\n\t"
5076	"movl $-1, %0\n\t"
5077	"1:\n\t"
5078	: "=r" (iBit)
5079	: "r" (u32));
5080	# else
5081	__asm
5082	{
5083	mov edx, [u32]
5084	bsf eax, edx
5085	jnz done
5086	mov eax, 0ffffffffh
5087	done:
5088	mov [iBit], eax
5089	}
5090	# endif
5091	if (iBit >= 0)
5092	return iBit + iBitPrev;
5093	# endif
5094
5095	/*
5096	* Skip ahead and see if there is anything left to search.
5097	*/
5098	iBitPrev \|= 31;
5099	iBitPrev++;
5100	if (cBits <= (uint32_t)iBitPrev)
5101	return -1;
5102	}
5103
5104	/*
5105	* 32-bit aligned search, let ASMBitFirstClear do the dirty work.
5106	*/
5107	iBit = ASMBitFirstSet(&pau32Bitmap[iBitPrev / 32], cBits - iBitPrev);
5108	if (iBit >= 0)
5109	iBit += iBitPrev;
5110	return iBit;
5111	}
5112	#endif
5113
5114
5115	/**
5116	* Finds the first bit which is set in the given 32-bit integer.
5117	* Bits are numbered from 1 (least significant) to 32.
5118	*
5119	* @returns index [1..32] of the first set bit.
5120	* @returns 0 if all bits are cleared.
5121	* @param u32 Integer to search for set bits.
5122	* @remarks Similar to ffs() in BSD.
5123	*/
5124	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5125	DECLASM(unsigned) ASMBitFirstSetU32(uint32_t u32);
5126	#else
5127	DECLINLINE(unsigned) ASMBitFirstSetU32(uint32_t u32)
5128	{
5129	# if RT_INLINE_ASM_USES_INTRIN
5130	unsigned long iBit;
5131	if (_BitScanForward(&iBit, u32))
5132	iBit++;
5133	else
5134	iBit = 0;
5135	# elif RT_INLINE_ASM_GNU_STYLE
5136	uint32_t iBit;
5137	__asm__ __volatile__("bsf %1, %0\n\t"
5138	"jnz 1f\n\t"
5139	"xorl %0, %0\n\t"
5140	"jmp 2f\n"
5141	"1:\n\t"
5142	"incl %0\n"
5143	"2:\n\t"
5144	: "=r" (iBit)
5145	: "rm" (u32));
5146	# else
5147	uint32_t iBit;
5148	_asm
5149	{
5150	bsf eax, [u32]
5151	jnz found
5152	xor eax, eax
5153	jmp done
5154	found:
5155	inc eax
5156	done:
5157	mov [iBit], eax
5158	}
5159	# endif
5160	return iBit;
5161	}
5162	#endif
5163
5164
5165	/**
5166	* Finds the first bit which is set in the given 32-bit integer.
5167	* Bits are numbered from 1 (least significant) to 32.
5168	*
5169	* @returns index [1..32] of the first set bit.
5170	* @returns 0 if all bits are cleared.
5171	* @param i32 Integer to search for set bits.
5172	* @remark Similar to ffs() in BSD.
5173	*/
5174	DECLINLINE(unsigned) ASMBitFirstSetS32(int32_t i32)
5175	{
5176	return ASMBitFirstSetU32((uint32_t)i32);
5177	}
5178
5179
5180	/**
5181	* Finds the first bit which is set in the given 64-bit integer.
5182	*
5183	* Bits are numbered from 1 (least significant) to 64.
5184	*
5185	* @returns index [1..64] of the first set bit.
5186	* @returns 0 if all bits are cleared.
5187	* @param u64 Integer to search for set bits.
5188	* @remarks Similar to ffs() in BSD.
5189	*/
5190	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5191	DECLASM(unsigned) ASMBitFirstSetU64(uint64_t u64);
5192	#else
5193	DECLINLINE(unsigned) ASMBitFirstSetU64(uint64_t u64)
5194	{
5195	# if RT_INLINE_ASM_USES_INTRIN
5196	unsigned long iBit;
5197	# if ARCH_BITS == 64
5198	if (_BitScanForward64(&iBit, u64))
5199	iBit++;
5200	else
5201	iBit = 0;
5202	# else
5203	if (_BitScanForward(&iBit, (uint32_t)u64))
5204	iBit++;
5205	else if (_BitScanForward(&iBit, (uint32_t)(u64 >> 32)))
5206	iBit += 33;
5207	else
5208	iBit = 0;
5209	# endif
5210	# elif RT_INLINE_ASM_GNU_STYLE && ARCH_BITS == 64
5211	uint64_t iBit;
5212	__asm__ __volatile__("bsfq %1, %0\n\t"
5213	"jnz 1f\n\t"
5214	"xorl %k0, %k0\n\t"
5215	"jmp 2f\n"
5216	"1:\n\t"
5217	"incl %k0\n"
5218	"2:\n\t"
5219	: "=r" (iBit)
5220	: "rm" (u64));
5221	# else
5222	unsigned iBit = ASMBitFirstSetU32((uint32_t)u64);
5223	if (!iBit)
5224	{
5225	iBit = ASMBitFirstSetU32((uint32_t)(u64 >> 32));
5226	if (iBit)
5227	iBit += 32;
5228	}
5229	# endif
5230	return (unsigned)iBit;
5231	}
5232	#endif
5233
5234
5235	/**
5236	* Finds the first bit which is set in the given 16-bit integer.
5237	*
5238	* Bits are numbered from 1 (least significant) to 16.
5239	*
5240	* @returns index [1..16] of the first set bit.
5241	* @returns 0 if all bits are cleared.
5242	* @param u16 Integer to search for set bits.
5243	* @remarks For 16-bit bs3kit code.
5244	*/
5245	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5246	DECLASM(unsigned) ASMBitFirstSetU16(uint16_t u16);
5247	#else
5248	DECLINLINE(unsigned) ASMBitFirstSetU16(uint16_t u16)
5249	{
5250	return ASMBitFirstSetU32((uint32_t)u16);
5251	}
5252	#endif
5253
5254
5255	/**
5256	* Finds the last bit which is set in the given 32-bit integer.
5257	* Bits are numbered from 1 (least significant) to 32.
5258	*
5259	* @returns index [1..32] of the last set bit.
5260	* @returns 0 if all bits are cleared.
5261	* @param u32 Integer to search for set bits.
5262	* @remark Similar to fls() in BSD.
5263	*/
5264	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5265	DECLASM(unsigned) ASMBitLastSetU32(uint32_t u32);
5266	#else
5267	DECLINLINE(unsigned) ASMBitLastSetU32(uint32_t u32)
5268	{
5269	# if RT_INLINE_ASM_USES_INTRIN
5270	unsigned long iBit;
5271	if (_BitScanReverse(&iBit, u32))
5272	iBit++;
5273	else
5274	iBit = 0;
5275	# elif RT_INLINE_ASM_GNU_STYLE
5276	uint32_t iBit;
5277	__asm__ __volatile__("bsrl %1, %0\n\t"
5278	"jnz 1f\n\t"
5279	"xorl %0, %0\n\t"
5280	"jmp 2f\n"
5281	"1:\n\t"
5282	"incl %0\n"
5283	"2:\n\t"
5284	: "=r" (iBit)
5285	: "rm" (u32));
5286	# else
5287	uint32_t iBit;
5288	_asm
5289	{
5290	bsr eax, [u32]
5291	jnz found
5292	xor eax, eax
5293	jmp done
5294	found:
5295	inc eax
5296	done:
5297	mov [iBit], eax
5298	}
5299	# endif
5300	return iBit;
5301	}
5302	#endif
5303
5304
5305	/**
5306	* Finds the last bit which is set in the given 32-bit integer.
5307	* Bits are numbered from 1 (least significant) to 32.
5308	*
5309	* @returns index [1..32] of the last set bit.
5310	* @returns 0 if all bits are cleared.
5311	* @param i32 Integer to search for set bits.
5312	* @remark Similar to fls() in BSD.
5313	*/
5314	DECLINLINE(unsigned) ASMBitLastSetS32(int32_t i32)
5315	{
5316	return ASMBitLastSetU32((uint32_t)i32);
5317	}
5318
5319
5320	/**
5321	* Finds the last bit which is set in the given 64-bit integer.
5322	*
5323	* Bits are numbered from 1 (least significant) to 64.
5324	*
5325	* @returns index [1..64] of the last set bit.
5326	* @returns 0 if all bits are cleared.
5327	* @param u64 Integer to search for set bits.
5328	* @remark Similar to fls() in BSD.
5329	*/
5330	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5331	DECLASM(unsigned) ASMBitLastSetU64(uint64_t u64);
5332	#else
5333	DECLINLINE(unsigned) ASMBitLastSetU64(uint64_t u64)
5334	{
5335	# if RT_INLINE_ASM_USES_INTRIN
5336	unsigned long iBit;
5337	# if ARCH_BITS == 64
5338	if (_BitScanReverse64(&iBit, u64))
5339	iBit++;
5340	else
5341	iBit = 0;
5342	# else
5343	if (_BitScanReverse(&iBit, (uint32_t)(u64 >> 32)))
5344	iBit += 33;
5345	else if (_BitScanReverse(&iBit, (uint32_t)u64))
5346	iBit++;
5347	else
5348	iBit = 0;
5349	# endif
5350	# elif RT_INLINE_ASM_GNU_STYLE && ARCH_BITS == 64
5351	uint64_t iBit;
5352	__asm__ __volatile__("bsrq %1, %0\n\t"
5353	"jnz 1f\n\t"
5354	"xorl %k0, %k0\n\t"
5355	"jmp 2f\n"
5356	"1:\n\t"
5357	"incl %k0\n"
5358	"2:\n\t"
5359	: "=r" (iBit)
5360	: "rm" (u64));
5361	# else
5362	unsigned iBit = ASMBitLastSetU32((uint32_t)(u64 >> 32));
5363	if (iBit)
5364	iBit += 32;
5365	else
5366	iBit = ASMBitLastSetU32((uint32_t)u64);
5367	#endif
5368	return (unsigned)iBit;
5369	}
5370	#endif
5371
5372
5373	/**
5374	* Finds the last bit which is set in the given 16-bit integer.
5375	*
5376	* Bits are numbered from 1 (least significant) to 16.
5377	*
5378	* @returns index [1..16] of the last set bit.
5379	* @returns 0 if all bits are cleared.
5380	* @param u16 Integer to search for set bits.
5381	* @remarks For 16-bit bs3kit code.
5382	*/
5383	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5384	DECLASM(unsigned) ASMBitLastSetU16(uint16_t u16);
5385	#else
5386	DECLINLINE(unsigned) ASMBitLastSetU16(uint16_t u16)
5387	{
5388	return ASMBitLastSetU32((uint32_t)u16);
5389	}
5390	#endif
5391
5392
5393	/**
5394	* Reverse the byte order of the given 16-bit integer.
5395	*
5396	* @returns Revert
5397	* @param u16 16-bit integer value.
5398	*/
5399	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5400	DECLASM(uint16_t) ASMByteSwapU16(uint16_t u16);
5401	#else
5402	DECLINLINE(uint16_t) ASMByteSwapU16(uint16_t u16)
5403	{
5404	# if RT_INLINE_ASM_USES_INTRIN
5405	u16 = _byteswap_ushort(u16);
5406	# elif RT_INLINE_ASM_GNU_STYLE
5407	__asm__ ("rorw $8, %0" : "=r" (u16) : "0" (u16));
5408	# else
5409	_asm
5410	{
5411	mov ax, [u16]
5412	ror ax, 8
5413	mov [u16], ax
5414	}
5415	# endif
5416	return u16;
5417	}
5418	#endif
5419
5420
5421	/**
5422	* Reverse the byte order of the given 32-bit integer.
5423	*
5424	* @returns Revert
5425	* @param u32 32-bit integer value.
5426	*/
5427	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5428	DECLASM(uint32_t) ASMByteSwapU32(uint32_t u32);
5429	#else
5430	DECLINLINE(uint32_t) ASMByteSwapU32(uint32_t u32)
5431	{
5432	# if RT_INLINE_ASM_USES_INTRIN
5433	u32 = _byteswap_ulong(u32);
5434	# elif RT_INLINE_ASM_GNU_STYLE
5435	__asm__ ("bswapl %0" : "=r" (u32) : "0" (u32));
5436	# else
5437	_asm
5438	{
5439	mov eax, [u32]
5440	bswap eax
5441	mov [u32], eax
5442	}
5443	# endif
5444	return u32;
5445	}
5446	#endif
5447
5448
5449	/**
5450	* Reverse the byte order of the given 64-bit integer.
5451	*
5452	* @returns Revert
5453	* @param u64 64-bit integer value.
5454	*/
5455	DECLINLINE(uint64_t) ASMByteSwapU64(uint64_t u64)
5456	{
5457	#if defined(RT_ARCH_AMD64) && RT_INLINE_ASM_USES_INTRIN
5458	u64 = _byteswap_uint64(u64);
5459	#else
5460	u64 = (uint64_t)ASMByteSwapU32((uint32_t)u64) << 32
5461	\| (uint64_t)ASMByteSwapU32((uint32_t)(u64 >> 32));
5462	#endif
5463	return u64;
5464	}
5465
5466
5467	/**
5468	* Rotate 32-bit unsigned value to the left by @a cShift.
5469	*
5470	* @returns Rotated value.
5471	* @param u32 The value to rotate.
5472	* @param cShift How many bits to rotate by.
5473	*/
5474	#ifdef __WATCOMC__
5475	DECLASM(uint32_t) ASMRotateLeftU32(uint32_t u32, unsigned cShift);
5476	#else
5477	DECLINLINE(uint32_t) ASMRotateLeftU32(uint32_t u32, uint32_t cShift)
5478	{
5479	# if RT_INLINE_ASM_USES_INTRIN
5480	return _rotl(u32, cShift);
5481	# elif RT_INLINE_ASM_GNU_STYLE && (defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86))
5482	__asm__ __volatile__("roll %b1, %0" : "=g" (u32) : "Ic" (cShift), "0" (u32));
5483	return u32;
5484	# else
5485	cShift &= 31;
5486	return (u32 << cShift) \| (u32 >> (32 - cShift));
5487	# endif
5488	}
5489	#endif
5490
5491
5492	/**
5493	* Rotate 32-bit unsigned value to the right by @a cShift.
5494	*
5495	* @returns Rotated value.
5496	* @param u32 The value to rotate.
5497	* @param cShift How many bits to rotate by.
5498	*/
5499	#ifdef __WATCOMC__
5500	DECLASM(uint32_t) ASMRotateRightU32(uint32_t u32, unsigned cShift);
5501	#else
5502	DECLINLINE(uint32_t) ASMRotateRightU32(uint32_t u32, uint32_t cShift)
5503	{
5504	# if RT_INLINE_ASM_USES_INTRIN
5505	return _rotr(u32, cShift);
5506	# elif RT_INLINE_ASM_GNU_STYLE && (defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86))
5507	__asm__ __volatile__("rorl %b1, %0" : "=g" (u32) : "Ic" (cShift), "0" (u32));
5508	return u32;
5509	# else
5510	cShift &= 31;
5511	return (u32 >> cShift) \| (u32 << (32 - cShift));
5512	# endif
5513	}
5514	#endif
5515
5516
5517	/**
5518	* Rotate 64-bit unsigned value to the left by @a cShift.
5519	*
5520	* @returns Rotated value.
5521	* @param u64 The value to rotate.
5522	* @param cShift How many bits to rotate by.
5523	*/
5524	DECLINLINE(uint64_t) ASMRotateLeftU64(uint64_t u64, uint32_t cShift)
5525	{
5526	#if RT_INLINE_ASM_USES_INTRIN
5527	return _rotl64(u64, cShift);
5528	#elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
5529	__asm__ __volatile__("rolq %b1, %0" : "=g" (u64) : "Jc" (cShift), "0" (u64));
5530	return u64;
5531	#elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_X86)
5532	uint32_t uSpill;
5533	__asm__ __volatile__("testb $0x20, %%cl\n\t" /* if (cShift >= 0x20) { swap(u64.hi, u64lo); cShift -= 0x20; } */
5534	"jz 1f\n\t"
5535	"xchgl %%eax, %%edx\n\t"
5536	"1:\n\t"
5537	"andb $0x1f, %%cl\n\t" /* if (cShift & 0x1f) { */
5538	"jz 2f\n\t"
5539	"movl %%edx, %2\n\t" /* save the hi value in %3. */
5540	"shldl %%cl,%%eax,%%edx\n\t" /* shift the hi value left, feeding MSBits from the low value. */
5541	"shldl %%cl,%2,%%eax\n\t" /* shift the lo value left, feeding MSBits from the saved hi value. */
5542	"2:\n\t" /* } */
5543	: "=A" (u64), "=c" (cShift), "=r" (uSpill)
5544	: "0" (u64),
5545	"1" (cShift));
5546	return u64;
5547	#else
5548	cShift &= 63;
5549	return (u64 << cShift) \| (u64 >> (64 - cShift));
5550	#endif
5551	}
5552
5553
5554	/**
5555	* Rotate 64-bit unsigned value to the right by @a cShift.
5556	*
5557	* @returns Rotated value.
5558	* @param u64 The value to rotate.
5559	* @param cShift How many bits to rotate by.
5560	*/
5561	DECLINLINE(uint64_t) ASMRotateRightU64(uint64_t u64, uint32_t cShift)
5562	{
5563	#if RT_INLINE_ASM_USES_INTRIN
5564	return _rotr64(u64, cShift);
5565	#elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
5566	__asm__ __volatile__("rorq %b1, %0" : "=g" (u64) : "Jc" (cShift), "0" (u64));
5567	return u64;
5568	#elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_X86)
5569	uint32_t uSpill;
5570	__asm__ __volatile__("testb $0x20, %%cl\n\t" /* if (cShift >= 0x20) { swap(u64.hi, u64lo); cShift -= 0x20; } */
5571	"jz 1f\n\t"
5572	"xchgl %%eax, %%edx\n\t"
5573	"1:\n\t"
5574	"andb $0x1f, %%cl\n\t" /* if (cShift & 0x1f) { */
5575	"jz 2f\n\t"
5576	"movl %%edx, %2\n\t" /* save the hi value in %3. */
5577	"shrdl %%cl,%%eax,%%edx\n\t" /* shift the hi value right, feeding LSBits from the low value. */
5578	"shrdl %%cl,%2,%%eax\n\t" /* shift the lo value right, feeding LSBits from the saved hi value. */
5579	"2:\n\t" /* } */
5580	: "=A" (u64), "=c" (cShift), "=r" (uSpill)
5581	: "0" (u64),
5582	"1" (cShift));
5583	return u64;
5584	#else
5585	cShift &= 63;
5586	return (u64 >> cShift) \| (u64 << (64 - cShift));
5587	#endif
5588	}
5589
5590	/** @} */
5591
5592
5593	/** @} */
5594
5595	#endif
5596

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/include/iprt/asm.h@ 63688

Download in other formats: