asm.h@ 60547

Last change on this file since 60547 was 60228, checked in by vboxsync, 9 years ago
asm-amd64-x86.h,*: made ASMInvalidatePage take a flat unsigned pointer instead of a void pointer (simpler for 16-bit mode). Added some of the bits for 1GB pages.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 157.5 KB

Line
1	/** @file
2	* IPRT - Assembly Functions.
3	*/
4
5	/*
6	* Copyright (C) 2006-2015 Oracle Corporation
7	*
8	* This file is part of VirtualBox Open Source Edition (OSE), as
9	* available from http://www.virtualbox.org. This file is free software;
10	* you can redistribute it and/or modify it under the terms of the GNU
11	* General Public License (GPL) as published by the Free Software
12	* Foundation, in version 2 as it comes in the "COPYING" file of the
13	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
14	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
15	*
16	* The contents of this file may alternatively be used under the terms
17	* of the Common Development and Distribution License Version 1.0
18	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
19	* VirtualBox OSE distribution, in which case the provisions of the
20	* CDDL are applicable instead of those of the GPL.
21	*
22	* You may elect to license modified versions of this file under the
23	* terms and conditions of either the GPL or the CDDL or both.
24	*/
25
26	#ifndef ___iprt_asm_h
27	#define ___iprt_asm_h
28
29	#include <iprt/cdefs.h>
30	#include <iprt/types.h>
31	#include <iprt/assert.h>
32	/** @def RT_INLINE_ASM_USES_INTRIN
33	* Defined as 1 if we're using a _MSC_VER 1400.
34	* Otherwise defined as 0.
35	*/
36
37	/* Solaris 10 header ugliness */
38	#ifdef u
39	# undef u
40	#endif
41
42	#if defined(_MSC_VER) && RT_INLINE_ASM_USES_INTRIN
43	# include <intrin.h>
44	/* Emit the intrinsics at all optimization levels. */
45	# pragma intrinsic(_ReadWriteBarrier)
46	# pragma intrinsic(__cpuid)
47	# pragma intrinsic(__stosd)
48	# pragma intrinsic(__stosw)
49	# pragma intrinsic(__stosb)
50	# pragma intrinsic(_BitScanForward)
51	# pragma intrinsic(_BitScanReverse)
52	# pragma intrinsic(_bittest)
53	# pragma intrinsic(_bittestandset)
54	# pragma intrinsic(_bittestandreset)
55	# pragma intrinsic(_bittestandcomplement)
56	# pragma intrinsic(_byteswap_ushort)
57	# pragma intrinsic(_byteswap_ulong)
58	# pragma intrinsic(_interlockedbittestandset)
59	# pragma intrinsic(_interlockedbittestandreset)
60	# pragma intrinsic(_InterlockedAnd)
61	# pragma intrinsic(_InterlockedOr)
62	# pragma intrinsic(_InterlockedIncrement)
63	# pragma intrinsic(_InterlockedDecrement)
64	# pragma intrinsic(_InterlockedExchange)
65	# pragma intrinsic(_InterlockedExchangeAdd)
66	# pragma intrinsic(_InterlockedCompareExchange)
67	# pragma intrinsic(_InterlockedCompareExchange64)
68	# pragma intrinsic(_rotl)
69	# pragma intrinsic(_rotr)
70	# pragma intrinsic(_rotl64)
71	# pragma intrinsic(_rotr64)
72	# ifdef RT_ARCH_AMD64
73	# pragma intrinsic(__stosq)
74	# pragma intrinsic(_byteswap_uint64)
75	# pragma intrinsic(_InterlockedExchange64)
76	# pragma intrinsic(_InterlockedExchangeAdd64)
77	# pragma intrinsic(_InterlockedAnd64)
78	# pragma intrinsic(_InterlockedOr64)
79	# pragma intrinsic(_InterlockedIncrement64)
80	# pragma intrinsic(_InterlockedDecrement64)
81	# endif
82	#endif
83
84	/*
85	* Include #pragma aux definitions for Watcom C/C++.
86	*/
87	#if defined(__WATCOMC__) && ARCH_BITS == 16 && defined(RT_ARCH_X86)
88	# include "asm-watcom-x86-16.h"
89	#elif defined(__WATCOMC__) && ARCH_BITS == 32 && defined(RT_ARCH_X86)
90	# include "asm-watcom-x86-32.h"
91	#endif
92
93
94
95	/** @defgroup grp_rt_asm ASM - Assembly Routines
96	* @ingroup grp_rt
97	*
98	* @remarks The difference between ordered and unordered atomic operations are that
99	* the former will complete outstanding reads and writes before continuing
100	* while the latter doesn't make any promises about the order. Ordered
101	* operations doesn't, it seems, make any 100% promise wrt to whether
102	* the operation will complete before any subsequent memory access.
103	* (please, correct if wrong.)
104	*
105	* ASMAtomicSomething operations are all ordered, while ASMAtomicUoSomething
106	* are unordered (note the Uo).
107	*
108	* @remarks Some remarks about __volatile__: Without this keyword gcc is allowed to reorder
109	* or even optimize assembler instructions away. For instance, in the following code
110	* the second rdmsr instruction is optimized away because gcc treats that instruction
111	* as deterministic:
112	*
113	* @code
114	* static inline uint64_t rdmsr_low(int idx)
115	* {
116	* uint32_t low;
117	* __asm__ ("rdmsr" : "=a"(low) : "c"(idx) : "edx");
118	* }
119	* ...
120	* uint32_t msr1 = rdmsr_low(1);
121	* foo(msr1);
122	* msr1 = rdmsr_low(1);
123	* bar(msr1);
124	* @endcode
125	*
126	* The input parameter of rdmsr_low is the same for both calls and therefore gcc will
127	* use the result of the first call as input parameter for bar() as well. For rdmsr this
128	* is not acceptable as this instruction is _not_ deterministic. This applies to reading
129	* machine status information in general.
130	*
131	* @{
132	*/
133
134
135	/** @def RT_INLINE_ASM_GCC_4_3_X_X86
136	* Used to work around some 4.3.x register allocation issues in this version of
137	* the compiler. So far this workaround is still required for 4.4 and 4.5 but
138	* definitely not for 5.x */
139	#define RT_INLINE_ASM_GCC_4_3_X_X86 (RT_GNUC_PREREQ(4, 3) && !RT_GNUC_PREREQ(5, 0) && defined(__i386__))
140	#ifndef RT_INLINE_ASM_GCC_4_3_X_X86
141	# define RT_INLINE_ASM_GCC_4_3_X_X86 0
142	#endif
143
144	/** @def RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
145	* i686-apple-darwin9-gcc-4.0.1 (GCC) 4.0.1 (Apple Inc. build 5493) screws up
146	* RTSemRWRequestWrite semsemrw-lockless-generic.cpp in release builds. PIC
147	* mode, x86.
148	*
149	* Some gcc 4.3.x versions may have register allocation issues with cmpxchg8b
150	* when in PIC mode on x86.
151	*/
152	#ifndef RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
153	# if defined(DOXYGEN_RUNNING) \|\| defined(__WATCOMC__) /* Watcom has trouble with the expression below */
154	# define RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC 1
155	# else
156	# define RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC \
157	( (defined(PIC) \|\| defined(__PIC__)) \
158	&& defined(RT_ARCH_X86) \
159	&& ( RT_INLINE_ASM_GCC_4_3_X_X86 \
160	\|\| defined(RT_OS_DARWIN)) )
161	# endif
162	#endif
163
164
165	/** @def ASMReturnAddress
166	* Gets the return address of the current (or calling if you like) function or method.
167	*/
168	#ifdef _MSC_VER
169	# ifdef __cplusplus
170	extern "C"
171	# endif
172	void * _ReturnAddress(void);
173	# pragma intrinsic(_ReturnAddress)
174	# define ASMReturnAddress() _ReturnAddress()
175	#elif defined(__GNUC__) \|\| defined(DOXYGEN_RUNNING)
176	# define ASMReturnAddress() __builtin_return_address(0)
177	#elif defined(__WATCOMC__)
178	# define ASMReturnAddress() Watcom_does_not_appear_to_have_intrinsic_return_address_function()
179	#else
180	# error "Unsupported compiler."
181	#endif
182
183
184	/**
185	* Compiler memory barrier.
186	*
187	* Ensure that the compiler does not use any cached (register/tmp stack) memory
188	* values or any outstanding writes when returning from this function.
189	*
190	* This function must be used if non-volatile data is modified by a
191	* device or the VMM. Typical cases are port access, MMIO access,
192	* trapping instruction, etc.
193	*/
194	#if RT_INLINE_ASM_GNU_STYLE
195	# define ASMCompilerBarrier() do { __asm__ __volatile__("" : : : "memory"); } while (0)
196	#elif RT_INLINE_ASM_USES_INTRIN
197	# define ASMCompilerBarrier() do { _ReadWriteBarrier(); } while (0)
198	#elif defined(__WATCOMC__)
199	void ASMCompilerBarrier(void);
200	#else /* 2003 should have _ReadWriteBarrier() but I guess we're at 2002 level then... */
201	DECLINLINE(void) ASMCompilerBarrier(void)
202	{
203	__asm
204	{
205	}
206	}
207	#endif
208
209
210	/** @def ASMBreakpoint
211	* Debugger Breakpoint.
212	* @deprecated Use RT_BREAKPOINT instead.
213	* @internal
214	*/
215	#define ASMBreakpoint() RT_BREAKPOINT()
216
217
218	/**
219	* Spinloop hint for platforms that have these, empty function on the other
220	* platforms.
221	*
222	* x86 & AMD64: The PAUSE variant of NOP for helping hyperthreaded CPUs detecting
223	* spin locks.
224	*/
225	#if RT_INLINE_ASM_EXTERNAL && (defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86))
226	DECLASM(void) ASMNopPause(void);
227	#else
228	DECLINLINE(void) ASMNopPause(void)
229	{
230	# if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
231	# if RT_INLINE_ASM_GNU_STYLE
232	__asm__ __volatile__(".byte 0xf3,0x90\n\t");
233	# else
234	__asm {
235	_emit 0f3h
236	_emit 090h
237	}
238	# endif
239	# else
240	/* dummy */
241	# endif
242	}
243	#endif
244
245
246	/**
247	* Atomically Exchange an unsigned 8-bit value, ordered.
248	*
249	* @returns Current *pu8 value
250	* @param pu8 Pointer to the 8-bit variable to update.
251	* @param u8 The 8-bit value to assign to *pu8.
252	*/
253	#if RT_INLINE_ASM_EXTERNAL
254	DECLASM(uint8_t) ASMAtomicXchgU8(volatile uint8_t *pu8, uint8_t u8);
255	#else
256	DECLINLINE(uint8_t) ASMAtomicXchgU8(volatile uint8_t *pu8, uint8_t u8)
257	{
258	# if RT_INLINE_ASM_GNU_STYLE
259	__asm__ __volatile__("xchgb %0, %1\n\t"
260	: "=m" (*pu8),
261	"=q" (u8) /* =r - busted on g++ (GCC) 3.4.4 20050721 (Red Hat 3.4.4-2) */
262	: "1" (u8),
263	"m" (*pu8));
264	# else
265	__asm
266	{
267	# ifdef RT_ARCH_AMD64
268	mov rdx, [pu8]
269	mov al, [u8]
270	xchg [rdx], al
271	mov [u8], al
272	# else
273	mov edx, [pu8]
274	mov al, [u8]
275	xchg [edx], al
276	mov [u8], al
277	# endif
278	}
279	# endif
280	return u8;
281	}
282	#endif
283
284
285	/**
286	* Atomically Exchange a signed 8-bit value, ordered.
287	*
288	* @returns Current *pu8 value
289	* @param pi8 Pointer to the 8-bit variable to update.
290	* @param i8 The 8-bit value to assign to *pi8.
291	*/
292	DECLINLINE(int8_t) ASMAtomicXchgS8(volatile int8_t *pi8, int8_t i8)
293	{
294	return (int8_t)ASMAtomicXchgU8((volatile uint8_t *)pi8, (uint8_t)i8);
295	}
296
297
298	/**
299	* Atomically Exchange a bool value, ordered.
300	*
301	* @returns Current *pf value
302	* @param pf Pointer to the 8-bit variable to update.
303	* @param f The 8-bit value to assign to *pi8.
304	*/
305	DECLINLINE(bool) ASMAtomicXchgBool(volatile bool *pf, bool f)
306	{
307	#ifdef _MSC_VER
308	return !!ASMAtomicXchgU8((volatile uint8_t *)pf, (uint8_t)f);
309	#else
310	return (bool)ASMAtomicXchgU8((volatile uint8_t *)pf, (uint8_t)f);
311	#endif
312	}
313
314
315	/**
316	* Atomically Exchange an unsigned 16-bit value, ordered.
317	*
318	* @returns Current *pu16 value
319	* @param pu16 Pointer to the 16-bit variable to update.
320	* @param u16 The 16-bit value to assign to *pu16.
321	*/
322	#if RT_INLINE_ASM_EXTERNAL
323	DECLASM(uint16_t) ASMAtomicXchgU16(volatile uint16_t *pu16, uint16_t u16);
324	#else
325	DECLINLINE(uint16_t) ASMAtomicXchgU16(volatile uint16_t *pu16, uint16_t u16)
326	{
327	# if RT_INLINE_ASM_GNU_STYLE
328	__asm__ __volatile__("xchgw %0, %1\n\t"
329	: "=m" (*pu16),
330	"=r" (u16)
331	: "1" (u16),
332	"m" (*pu16));
333	# else
334	__asm
335	{
336	# ifdef RT_ARCH_AMD64
337	mov rdx, [pu16]
338	mov ax, [u16]
339	xchg [rdx], ax
340	mov [u16], ax
341	# else
342	mov edx, [pu16]
343	mov ax, [u16]
344	xchg [edx], ax
345	mov [u16], ax
346	# endif
347	}
348	# endif
349	return u16;
350	}
351	#endif
352
353
354	/**
355	* Atomically Exchange a signed 16-bit value, ordered.
356	*
357	* @returns Current *pu16 value
358	* @param pi16 Pointer to the 16-bit variable to update.
359	* @param i16 The 16-bit value to assign to *pi16.
360	*/
361	DECLINLINE(int16_t) ASMAtomicXchgS16(volatile int16_t *pi16, int16_t i16)
362	{
363	return (int16_t)ASMAtomicXchgU16((volatile uint16_t *)pi16, (uint16_t)i16);
364	}
365
366
367	/**
368	* Atomically Exchange an unsigned 32-bit value, ordered.
369	*
370	* @returns Current *pu32 value
371	* @param pu32 Pointer to the 32-bit variable to update.
372	* @param u32 The 32-bit value to assign to *pu32.
373	*
374	* @remarks Does not work on 286 and earlier.
375	*/
376	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
377	DECLASM(uint32_t) ASMAtomicXchgU32(volatile uint32_t *pu32, uint32_t u32);
378	#else
379	DECLINLINE(uint32_t) ASMAtomicXchgU32(volatile uint32_t *pu32, uint32_t u32)
380	{
381	# if RT_INLINE_ASM_GNU_STYLE
382	__asm__ __volatile__("xchgl %0, %1\n\t"
383	: "=m" (*pu32),
384	"=r" (u32)
385	: "1" (u32),
386	"m" (*pu32));
387
388	# elif RT_INLINE_ASM_USES_INTRIN
389	u32 = _InterlockedExchange((long *)pu32, u32);
390
391	# else
392	__asm
393	{
394	# ifdef RT_ARCH_AMD64
395	mov rdx, [pu32]
396	mov eax, u32
397	xchg [rdx], eax
398	mov [u32], eax
399	# else
400	mov edx, [pu32]
401	mov eax, u32
402	xchg [edx], eax
403	mov [u32], eax
404	# endif
405	}
406	# endif
407	return u32;
408	}
409	#endif
410
411
412	/**
413	* Atomically Exchange a signed 32-bit value, ordered.
414	*
415	* @returns Current *pu32 value
416	* @param pi32 Pointer to the 32-bit variable to update.
417	* @param i32 The 32-bit value to assign to *pi32.
418	*/
419	DECLINLINE(int32_t) ASMAtomicXchgS32(volatile int32_t *pi32, int32_t i32)
420	{
421	return (int32_t)ASMAtomicXchgU32((volatile uint32_t *)pi32, (uint32_t)i32);
422	}
423
424
425	/**
426	* Atomically Exchange an unsigned 64-bit value, ordered.
427	*
428	* @returns Current *pu64 value
429	* @param pu64 Pointer to the 64-bit variable to update.
430	* @param u64 The 64-bit value to assign to *pu64.
431	*
432	* @remarks Works on 32-bit x86 CPUs starting with Pentium.
433	*/
434	#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
435	\|\| RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
436	DECLASM(uint64_t) ASMAtomicXchgU64(volatile uint64_t *pu64, uint64_t u64);
437	#else
438	DECLINLINE(uint64_t) ASMAtomicXchgU64(volatile uint64_t *pu64, uint64_t u64)
439	{
440	# if defined(RT_ARCH_AMD64)
441	# if RT_INLINE_ASM_USES_INTRIN
442	u64 = _InterlockedExchange64((__int64 *)pu64, u64);
443
444	# elif RT_INLINE_ASM_GNU_STYLE
445	__asm__ __volatile__("xchgq %0, %1\n\t"
446	: "=m" (*pu64),
447	"=r" (u64)
448	: "1" (u64),
449	"m" (*pu64));
450	# else
451	__asm
452	{
453	mov rdx, [pu64]
454	mov rax, [u64]
455	xchg [rdx], rax
456	mov [u64], rax
457	}
458	# endif
459	# else /* !RT_ARCH_AMD64 */
460	# if RT_INLINE_ASM_GNU_STYLE
461	# if defined(PIC) \|\| defined(__PIC__)
462	uint32_t u32EBX = (uint32_t)u64;
463	__asm__ __volatile__(/"xchgl %%esi, %5\n\t"/
464	"xchgl %%ebx, %3\n\t"
465	"1:\n\t"
466	"lock; cmpxchg8b (%5)\n\t"
467	"jnz 1b\n\t"
468	"movl %3, %%ebx\n\t"
469	/"xchgl %%esi, %5\n\t"/
470	: "=A" (u64),
471	"=m" (*pu64)
472	: "0" (*pu64),
473	"m" ( u32EBX ),
474	"c" ( (uint32_t)(u64 >> 32) ),
475	"S" (pu64));
476	# else /* !PIC */
477	__asm__ __volatile__("1:\n\t"
478	"lock; cmpxchg8b %1\n\t"
479	"jnz 1b\n\t"
480	: "=A" (u64),
481	"=m" (*pu64)
482	: "0" (*pu64),
483	"b" ( (uint32_t)u64 ),
484	"c" ( (uint32_t)(u64 >> 32) ));
485	# endif
486	# else
487	__asm
488	{
489	mov ebx, dword ptr [u64]
490	mov ecx, dword ptr [u64 + 4]
491	mov edi, pu64
492	mov eax, dword ptr [edi]
493	mov edx, dword ptr [edi + 4]
494	retry:
495	lock cmpxchg8b [edi]
496	jnz retry
497	mov dword ptr [u64], eax
498	mov dword ptr [u64 + 4], edx
499	}
500	# endif
501	# endif /* !RT_ARCH_AMD64 */
502	return u64;
503	}
504	#endif
505
506
507	/**
508	* Atomically Exchange an signed 64-bit value, ordered.
509	*
510	* @returns Current *pi64 value
511	* @param pi64 Pointer to the 64-bit variable to update.
512	* @param i64 The 64-bit value to assign to *pi64.
513	*/
514	DECLINLINE(int64_t) ASMAtomicXchgS64(volatile int64_t *pi64, int64_t i64)
515	{
516	return (int64_t)ASMAtomicXchgU64((volatile uint64_t *)pi64, (uint64_t)i64);
517	}
518
519
520	/**
521	* Atomically Exchange a pointer value, ordered.
522	*
523	* @returns Current *ppv value
524	* @param ppv Pointer to the pointer variable to update.
525	* @param pv The pointer value to assign to *ppv.
526	*/
527	DECLINLINE(void ) ASMAtomicXchgPtr(void volatile ppv, const void pv)
528	{
529	#if ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
530	return (void )ASMAtomicXchgU32((volatile uint32_t )(void *)ppv, (uint32_t)pv);
531	#elif ARCH_BITS == 64
532	return (void )ASMAtomicXchgU64((volatile uint64_t )(void *)ppv, (uint64_t)pv);
533	#else
534	# error "ARCH_BITS is bogus"
535	#endif
536	}
537
538
539	/**
540	* Convenience macro for avoiding the annoying casting with ASMAtomicXchgPtr.
541	*
542	* @returns Current *pv value
543	* @param ppv Pointer to the pointer variable to update.
544	* @param pv The pointer value to assign to *ppv.
545	* @param Type The type of *ppv, sans volatile.
546	*/
547	#ifdef __GNUC__
548	# define ASMAtomicXchgPtrT(ppv, pv, Type) \
549	__extension__ \
550	({\
551	__typeof__((ppv)) volatile const ppvTypeChecked = (ppv); \
552	Type const pvTypeChecked = (pv); \
553	Type pvTypeCheckedRet = (__typeof__((ppv))) ASMAtomicXchgPtr((void volatile )ppvTypeChecked, (void )pvTypeChecked); \
554	pvTypeCheckedRet; \
555	})
556	#else
557	# define ASMAtomicXchgPtrT(ppv, pv, Type) \
558	(Type)ASMAtomicXchgPtr((void * volatile )(ppv), (void )(pv))
559	#endif
560
561
562	/**
563	* Atomically Exchange a raw-mode context pointer value, ordered.
564	*
565	* @returns Current *ppv value
566	* @param ppvRC Pointer to the pointer variable to update.
567	* @param pvRC The pointer value to assign to *ppv.
568	*/
569	DECLINLINE(RTRCPTR) ASMAtomicXchgRCPtr(RTRCPTR volatile *ppvRC, RTRCPTR pvRC)
570	{
571	return (RTRCPTR)ASMAtomicXchgU32((uint32_t volatile )(void )ppvRC, (uint32_t)pvRC);
572	}
573
574
575	/**
576	* Atomically Exchange a ring-0 pointer value, ordered.
577	*
578	* @returns Current *ppv value
579	* @param ppvR0 Pointer to the pointer variable to update.
580	* @param pvR0 The pointer value to assign to *ppv.
581	*/
582	DECLINLINE(RTR0PTR) ASMAtomicXchgR0Ptr(RTR0PTR volatile *ppvR0, RTR0PTR pvR0)
583	{
584	#if R0_ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
585	return (RTR0PTR)ASMAtomicXchgU32((volatile uint32_t )(void )ppvR0, (uint32_t)pvR0);
586	#elif R0_ARCH_BITS == 64
587	return (RTR0PTR)ASMAtomicXchgU64((volatile uint64_t )(void )ppvR0, (uint64_t)pvR0);
588	#else
589	# error "R0_ARCH_BITS is bogus"
590	#endif
591	}
592
593
594	/**
595	* Atomically Exchange a ring-3 pointer value, ordered.
596	*
597	* @returns Current *ppv value
598	* @param ppvR3 Pointer to the pointer variable to update.
599	* @param pvR3 The pointer value to assign to *ppv.
600	*/
601	DECLINLINE(RTR3PTR) ASMAtomicXchgR3Ptr(RTR3PTR volatile *ppvR3, RTR3PTR pvR3)
602	{
603	#if R3_ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
604	return (RTR3PTR)ASMAtomicXchgU32((volatile uint32_t )(void )ppvR3, (uint32_t)pvR3);
605	#elif R3_ARCH_BITS == 64
606	return (RTR3PTR)ASMAtomicXchgU64((volatile uint64_t )(void )ppvR3, (uint64_t)pvR3);
607	#else
608	# error "R3_ARCH_BITS is bogus"
609	#endif
610	}
611
612
613	/** @def ASMAtomicXchgHandle
614	* Atomically Exchange a typical IPRT handle value, ordered.
615	*
616	* @param ph Pointer to the value to update.
617	* @param hNew The new value to assigned to *pu.
618	* @param phRes Where to store the current *ph value.
619	*
620	* @remarks This doesn't currently work for all handles (like RTFILE).
621	*/
622	#if HC_ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
623	# define ASMAtomicXchgHandle(ph, hNew, phRes) \
624	do { \
625	AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
626	AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
627	(uint32_t )(phRes) = ASMAtomicXchgU32((uint32_t volatile *)(ph), (const uint32_t)(hNew)); \
628	} while (0)
629	#elif HC_ARCH_BITS == 64
630	# define ASMAtomicXchgHandle(ph, hNew, phRes) \
631	do { \
632	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
633	AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
634	(uint64_t )(phRes) = ASMAtomicXchgU64((uint64_t volatile *)(ph), (const uint64_t)(hNew)); \
635	} while (0)
636	#else
637	# error HC_ARCH_BITS
638	#endif
639
640
641	/**
642	* Atomically Exchange a value which size might differ
643	* between platforms or compilers, ordered.
644	*
645	* @param pu Pointer to the variable to update.
646	* @param uNew The value to assign to *pu.
647	* @todo This is busted as its missing the result argument.
648	*/
649	#define ASMAtomicXchgSize(pu, uNew) \
650	do { \
651	switch (sizeof(*(pu))) { \
652	case 1: ASMAtomicXchgU8((volatile uint8_t )(void )(pu), (uint8_t)(uNew)); break; \
653	case 2: ASMAtomicXchgU16((volatile uint16_t )(void )(pu), (uint16_t)(uNew)); break; \
654	case 4: ASMAtomicXchgU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew)); break; \
655	case 8: ASMAtomicXchgU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew)); break; \
656	default: AssertMsgFailed(("ASMAtomicXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
657	} \
658	} while (0)
659
660	/**
661	* Atomically Exchange a value which size might differ
662	* between platforms or compilers, ordered.
663	*
664	* @param pu Pointer to the variable to update.
665	* @param uNew The value to assign to *pu.
666	* @param puRes Where to store the current *pu value.
667	*/
668	#define ASMAtomicXchgSizeCorrect(pu, uNew, puRes) \
669	do { \
670	switch (sizeof(*(pu))) { \
671	case 1: (uint8_t )(puRes) = ASMAtomicXchgU8((volatile uint8_t )(void )(pu), (uint8_t)(uNew)); break; \
672	case 2: (uint16_t )(puRes) = ASMAtomicXchgU16((volatile uint16_t )(void )(pu), (uint16_t)(uNew)); break; \
673	case 4: (uint32_t )(puRes) = ASMAtomicXchgU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew)); break; \
674	case 8: (uint64_t )(puRes) = ASMAtomicXchgU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew)); break; \
675	default: AssertMsgFailed(("ASMAtomicXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
676	} \
677	} while (0)
678
679
680
681	/**
682	* Atomically Compare and Exchange an unsigned 8-bit value, ordered.
683	*
684	* @returns true if xchg was done.
685	* @returns false if xchg wasn't done.
686	*
687	* @param pu8 Pointer to the value to update.
688	* @param u8New The new value to assigned to *pu8.
689	* @param u8Old The old value to *pu8 compare with.
690	*
691	* @remarks x86: Requires a 486 or later.
692	*/
693	#if RT_INLINE_ASM_EXTERNAL \|\| !RT_INLINE_ASM_GNU_STYLE
694	DECLASM(bool) ASMAtomicCmpXchgU8(volatile uint8_t *pu8, const uint8_t u8New, const uint8_t u8Old);
695	#else
696	DECLINLINE(bool) ASMAtomicCmpXchgU8(volatile uint8_t *pu8, const uint8_t u8New, uint8_t u8Old)
697	{
698	uint8_t u8Ret;
699	__asm__ __volatile__("lock; cmpxchgb %3, %0\n\t"
700	"setz %1\n\t"
701	: "=m" (*pu8),
702	"=qm" (u8Ret),
703	"=a" (u8Old)
704	: "q" (u8New),
705	"2" (u8Old),
706	"m" (*pu8));
707	return (bool)u8Ret;
708	}
709	#endif
710
711
712	/**
713	* Atomically Compare and Exchange a signed 8-bit value, ordered.
714	*
715	* @returns true if xchg was done.
716	* @returns false if xchg wasn't done.
717	*
718	* @param pi8 Pointer to the value to update.
719	* @param i8New The new value to assigned to *pi8.
720	* @param i8Old The old value to *pi8 compare with.
721	*
722	* @remarks x86: Requires a 486 or later.
723	*/
724	DECLINLINE(bool) ASMAtomicCmpXchgS8(volatile int8_t *pi8, const int8_t i8New, const int8_t i8Old)
725	{
726	return ASMAtomicCmpXchgU8((volatile uint8_t *)pi8, (const uint8_t)i8New, (const uint8_t)i8Old);
727	}
728
729
730	/**
731	* Atomically Compare and Exchange a bool value, ordered.
732	*
733	* @returns true if xchg was done.
734	* @returns false if xchg wasn't done.
735	*
736	* @param pf Pointer to the value to update.
737	* @param fNew The new value to assigned to *pf.
738	* @param fOld The old value to *pf compare with.
739	*
740	* @remarks x86: Requires a 486 or later.
741	*/
742	DECLINLINE(bool) ASMAtomicCmpXchgBool(volatile bool *pf, const bool fNew, const bool fOld)
743	{
744	return ASMAtomicCmpXchgU8((volatile uint8_t *)pf, (const uint8_t)fNew, (const uint8_t)fOld);
745	}
746
747
748	/**
749	* Atomically Compare and Exchange an unsigned 32-bit value, ordered.
750	*
751	* @returns true if xchg was done.
752	* @returns false if xchg wasn't done.
753	*
754	* @param pu32 Pointer to the value to update.
755	* @param u32New The new value to assigned to *pu32.
756	* @param u32Old The old value to *pu32 compare with.
757	*
758	* @remarks x86: Requires a 486 or later.
759	*/
760	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
761	DECLASM(bool) ASMAtomicCmpXchgU32(volatile uint32_t *pu32, const uint32_t u32New, const uint32_t u32Old);
762	#else
763	DECLINLINE(bool) ASMAtomicCmpXchgU32(volatile uint32_t *pu32, const uint32_t u32New, uint32_t u32Old)
764	{
765	# if RT_INLINE_ASM_GNU_STYLE
766	uint8_t u8Ret;
767	__asm__ __volatile__("lock; cmpxchgl %3, %0\n\t"
768	"setz %1\n\t"
769	: "=m" (*pu32),
770	"=qm" (u8Ret),
771	"=a" (u32Old)
772	: "r" (u32New),
773	"2" (u32Old),
774	"m" (*pu32));
775	return (bool)u8Ret;
776
777	# elif RT_INLINE_ASM_USES_INTRIN
778	return (uint32_t)_InterlockedCompareExchange((long *)pu32, u32New, u32Old) == u32Old;
779
780	# else
781	uint32_t u32Ret;
782	__asm
783	{
784	# ifdef RT_ARCH_AMD64
785	mov rdx, [pu32]
786	# else
787	mov edx, [pu32]
788	# endif
789	mov eax, [u32Old]
790	mov ecx, [u32New]
791	# ifdef RT_ARCH_AMD64
792	lock cmpxchg [rdx], ecx
793	# else
794	lock cmpxchg [edx], ecx
795	# endif
796	setz al
797	movzx eax, al
798	mov [u32Ret], eax
799	}
800	return !!u32Ret;
801	# endif
802	}
803	#endif
804
805
806	/**
807	* Atomically Compare and Exchange a signed 32-bit value, ordered.
808	*
809	* @returns true if xchg was done.
810	* @returns false if xchg wasn't done.
811	*
812	* @param pi32 Pointer to the value to update.
813	* @param i32New The new value to assigned to *pi32.
814	* @param i32Old The old value to *pi32 compare with.
815	*
816	* @remarks x86: Requires a 486 or later.
817	*/
818	DECLINLINE(bool) ASMAtomicCmpXchgS32(volatile int32_t *pi32, const int32_t i32New, const int32_t i32Old)
819	{
820	return ASMAtomicCmpXchgU32((volatile uint32_t *)pi32, (uint32_t)i32New, (uint32_t)i32Old);
821	}
822
823
824	/**
825	* Atomically Compare and exchange an unsigned 64-bit value, ordered.
826	*
827	* @returns true if xchg was done.
828	* @returns false if xchg wasn't done.
829	*
830	* @param pu64 Pointer to the 64-bit variable to update.
831	* @param u64New The 64-bit value to assign to *pu64.
832	* @param u64Old The value to compare with.
833	*
834	* @remarks x86: Requires a Pentium or later.
835	*/
836	#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
837	\|\| RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
838	DECLASM(bool) ASMAtomicCmpXchgU64(volatile uint64_t *pu64, const uint64_t u64New, const uint64_t u64Old);
839	#else
840	DECLINLINE(bool) ASMAtomicCmpXchgU64(volatile uint64_t *pu64, uint64_t u64New, uint64_t u64Old)
841	{
842	# if RT_INLINE_ASM_USES_INTRIN
843	return (uint64_t)_InterlockedCompareExchange64((__int64 *)pu64, u64New, u64Old) == u64Old;
844
845	# elif defined(RT_ARCH_AMD64)
846	# if RT_INLINE_ASM_GNU_STYLE
847	uint8_t u8Ret;
848	__asm__ __volatile__("lock; cmpxchgq %3, %0\n\t"
849	"setz %1\n\t"
850	: "=m" (*pu64),
851	"=qm" (u8Ret),
852	"=a" (u64Old)
853	: "r" (u64New),
854	"2" (u64Old),
855	"m" (*pu64));
856	return (bool)u8Ret;
857	# else
858	bool fRet;
859	__asm
860	{
861	mov rdx, [pu32]
862	mov rax, [u64Old]
863	mov rcx, [u64New]
864	lock cmpxchg [rdx], rcx
865	setz al
866	mov [fRet], al
867	}
868	return fRet;
869	# endif
870	# else /* !RT_ARCH_AMD64 */
871	uint32_t u32Ret;
872	# if RT_INLINE_ASM_GNU_STYLE
873	# if defined(PIC) \|\| defined(__PIC__)
874	uint32_t u32EBX = (uint32_t)u64New;
875	uint32_t u32Spill;
876	__asm__ __volatile__("xchgl %%ebx, %4\n\t"
877	"lock; cmpxchg8b (%6)\n\t"
878	"setz %%al\n\t"
879	"movl %4, %%ebx\n\t"
880	"movzbl %%al, %%eax\n\t"
881	: "=a" (u32Ret),
882	"=d" (u32Spill),
883	# if RT_GNUC_PREREQ(4, 3)
884	"+m" (*pu64)
885	# else
886	"=m" (*pu64)
887	# endif
888	: "A" (u64Old),
889	"m" ( u32EBX ),
890	"c" ( (uint32_t)(u64New >> 32) ),
891	"S" (pu64));
892	# else /* !PIC */
893	uint32_t u32Spill;
894	__asm__ __volatile__("lock; cmpxchg8b %2\n\t"
895	"setz %%al\n\t"
896	"movzbl %%al, %%eax\n\t"
897	: "=a" (u32Ret),
898	"=d" (u32Spill),
899	"+m" (*pu64)
900	: "A" (u64Old),
901	"b" ( (uint32_t)u64New ),
902	"c" ( (uint32_t)(u64New >> 32) ));
903	# endif
904	return (bool)u32Ret;
905	# else
906	__asm
907	{
908	mov ebx, dword ptr [u64New]
909	mov ecx, dword ptr [u64New + 4]
910	mov edi, [pu64]
911	mov eax, dword ptr [u64Old]
912	mov edx, dword ptr [u64Old + 4]
913	lock cmpxchg8b [edi]
914	setz al
915	movzx eax, al
916	mov dword ptr [u32Ret], eax
917	}
918	return !!u32Ret;
919	# endif
920	# endif /* !RT_ARCH_AMD64 */
921	}
922	#endif
923
924
925	/**
926	* Atomically Compare and exchange a signed 64-bit value, ordered.
927	*
928	* @returns true if xchg was done.
929	* @returns false if xchg wasn't done.
930	*
931	* @param pi64 Pointer to the 64-bit variable to update.
932	* @param i64 The 64-bit value to assign to *pu64.
933	* @param i64Old The value to compare with.
934	*
935	* @remarks x86: Requires a Pentium or later.
936	*/
937	DECLINLINE(bool) ASMAtomicCmpXchgS64(volatile int64_t *pi64, const int64_t i64, const int64_t i64Old)
938	{
939	return ASMAtomicCmpXchgU64((volatile uint64_t *)pi64, (uint64_t)i64, (uint64_t)i64Old);
940	}
941
942
943	/**
944	* Atomically Compare and Exchange a pointer value, ordered.
945	*
946	* @returns true if xchg was done.
947	* @returns false if xchg wasn't done.
948	*
949	* @param ppv Pointer to the value to update.
950	* @param pvNew The new value to assigned to *ppv.
951	* @param pvOld The old value to *ppv compare with.
952	*
953	* @remarks x86: Requires a 486 or later.
954	*/
955	DECLINLINE(bool) ASMAtomicCmpXchgPtrVoid(void * volatile ppv, const void pvNew, const void *pvOld)
956	{
957	#if ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
958	return ASMAtomicCmpXchgU32((volatile uint32_t )(void )ppv, (uint32_t)pvNew, (uint32_t)pvOld);
959	#elif ARCH_BITS == 64
960	return ASMAtomicCmpXchgU64((volatile uint64_t )(void )ppv, (uint64_t)pvNew, (uint64_t)pvOld);
961	#else
962	# error "ARCH_BITS is bogus"
963	#endif
964	}
965
966
967	/**
968	* Atomically Compare and Exchange a pointer value, ordered.
969	*
970	* @returns true if xchg was done.
971	* @returns false if xchg wasn't done.
972	*
973	* @param ppv Pointer to the value to update.
974	* @param pvNew The new value to assigned to *ppv.
975	* @param pvOld The old value to *ppv compare with.
976	*
977	* @remarks This is relatively type safe on GCC platforms.
978	* @remarks x86: Requires a 486 or later.
979	*/
980	#ifdef __GNUC__
981	# define ASMAtomicCmpXchgPtr(ppv, pvNew, pvOld) \
982	__extension__ \
983	({\
984	__typeof__((ppv)) volatile const ppvTypeChecked = (ppv); \
985	__typeof__(*(ppv)) const pvNewTypeChecked = (pvNew); \
986	__typeof__(*(ppv)) const pvOldTypeChecked = (pvOld); \
987	bool fMacroRet = ASMAtomicCmpXchgPtrVoid((void * volatile *)ppvTypeChecked, \
988	(void )pvNewTypeChecked, (void )pvOldTypeChecked); \
989	fMacroRet; \
990	})
991	#else
992	# define ASMAtomicCmpXchgPtr(ppv, pvNew, pvOld) \
993	ASMAtomicCmpXchgPtrVoid((void * volatile )(ppv), (void )(pvNew), (void *)(pvOld))
994	#endif
995
996
997	/** @def ASMAtomicCmpXchgHandle
998	* Atomically Compare and Exchange a typical IPRT handle value, ordered.
999	*
1000	* @param ph Pointer to the value to update.
1001	* @param hNew The new value to assigned to *pu.
1002	* @param hOld The old value to *pu compare with.
1003	* @param fRc Where to store the result.
1004	*
1005	* @remarks This doesn't currently work for all handles (like RTFILE).
1006	* @remarks x86: Requires a 486 or later.
1007	*/
1008	#if HC_ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
1009	# define ASMAtomicCmpXchgHandle(ph, hNew, hOld, fRc) \
1010	do { \
1011	AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
1012	(fRc) = ASMAtomicCmpXchgU32((uint32_t volatile *)(ph), (const uint32_t)(hNew), (const uint32_t)(hOld)); \
1013	} while (0)
1014	#elif HC_ARCH_BITS == 64
1015	# define ASMAtomicCmpXchgHandle(ph, hNew, hOld, fRc) \
1016	do { \
1017	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
1018	(fRc) = ASMAtomicCmpXchgU64((uint64_t volatile *)(ph), (const uint64_t)(hNew), (const uint64_t)(hOld)); \
1019	} while (0)
1020	#else
1021	# error HC_ARCH_BITS
1022	#endif
1023
1024
1025	/** @def ASMAtomicCmpXchgSize
1026	* Atomically Compare and Exchange a value which size might differ
1027	* between platforms or compilers, ordered.
1028	*
1029	* @param pu Pointer to the value to update.
1030	* @param uNew The new value to assigned to *pu.
1031	* @param uOld The old value to *pu compare with.
1032	* @param fRc Where to store the result.
1033	*
1034	* @remarks x86: Requires a 486 or later.
1035	*/
1036	#define ASMAtomicCmpXchgSize(pu, uNew, uOld, fRc) \
1037	do { \
1038	switch (sizeof(*(pu))) { \
1039	case 4: (fRc) = ASMAtomicCmpXchgU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew), (uint32_t)(uOld)); \
1040	break; \
1041	case 8: (fRc) = ASMAtomicCmpXchgU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew), (uint64_t)(uOld)); \
1042	break; \
1043	default: AssertMsgFailed(("ASMAtomicCmpXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
1044	(fRc) = false; \
1045	break; \
1046	} \
1047	} while (0)
1048
1049
1050	/**
1051	* Atomically Compare and Exchange an unsigned 32-bit value, additionally
1052	* passes back old value, ordered.
1053	*
1054	* @returns true if xchg was done.
1055	* @returns false if xchg wasn't done.
1056	*
1057	* @param pu32 Pointer to the value to update.
1058	* @param u32New The new value to assigned to *pu32.
1059	* @param u32Old The old value to *pu32 compare with.
1060	* @param pu32Old Pointer store the old value at.
1061	*
1062	* @remarks x86: Requires a 486 or later.
1063	*/
1064	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1065	DECLASM(bool) ASMAtomicCmpXchgExU32(volatile uint32_t pu32, const uint32_t u32New, const uint32_t u32Old, uint32_t pu32Old);
1066	#else
1067	DECLINLINE(bool) ASMAtomicCmpXchgExU32(volatile uint32_t pu32, const uint32_t u32New, const uint32_t u32Old, uint32_t pu32Old)
1068	{
1069	# if RT_INLINE_ASM_GNU_STYLE
1070	uint8_t u8Ret;
1071	__asm__ __volatile__("lock; cmpxchgl %3, %0\n\t"
1072	"setz %1\n\t"
1073	: "=m" (*pu32),
1074	"=qm" (u8Ret),
1075	"=a" (*pu32Old)
1076	: "r" (u32New),
1077	"a" (u32Old),
1078	"m" (*pu32));
1079	return (bool)u8Ret;
1080
1081	# elif RT_INLINE_ASM_USES_INTRIN
1082	return (pu32Old =_InterlockedCompareExchange((long )pu32, u32New, u32Old)) == u32Old;
1083
1084	# else
1085	uint32_t u32Ret;
1086	__asm
1087	{
1088	# ifdef RT_ARCH_AMD64
1089	mov rdx, [pu32]
1090	# else
1091	mov edx, [pu32]
1092	# endif
1093	mov eax, [u32Old]
1094	mov ecx, [u32New]
1095	# ifdef RT_ARCH_AMD64
1096	lock cmpxchg [rdx], ecx
1097	mov rdx, [pu32Old]
1098	mov [rdx], eax
1099	# else
1100	lock cmpxchg [edx], ecx
1101	mov edx, [pu32Old]
1102	mov [edx], eax
1103	# endif
1104	setz al
1105	movzx eax, al
1106	mov [u32Ret], eax
1107	}
1108	return !!u32Ret;
1109	# endif
1110	}
1111	#endif
1112
1113
1114	/**
1115	* Atomically Compare and Exchange a signed 32-bit value, additionally
1116	* passes back old value, ordered.
1117	*
1118	* @returns true if xchg was done.
1119	* @returns false if xchg wasn't done.
1120	*
1121	* @param pi32 Pointer to the value to update.
1122	* @param i32New The new value to assigned to *pi32.
1123	* @param i32Old The old value to *pi32 compare with.
1124	* @param pi32Old Pointer store the old value at.
1125	*
1126	* @remarks x86: Requires a 486 or later.
1127	*/
1128	DECLINLINE(bool) ASMAtomicCmpXchgExS32(volatile int32_t pi32, const int32_t i32New, const int32_t i32Old, int32_t pi32Old)
1129	{
1130	return ASMAtomicCmpXchgExU32((volatile uint32_t )pi32, (uint32_t)i32New, (uint32_t)i32Old, (uint32_t )pi32Old);
1131	}
1132
1133
1134	/**
1135	* Atomically Compare and exchange an unsigned 64-bit value, additionally
1136	* passing back old value, ordered.
1137	*
1138	* @returns true if xchg was done.
1139	* @returns false if xchg wasn't done.
1140	*
1141	* @param pu64 Pointer to the 64-bit variable to update.
1142	* @param u64New The 64-bit value to assign to *pu64.
1143	* @param u64Old The value to compare with.
1144	* @param pu64Old Pointer store the old value at.
1145	*
1146	* @remarks x86: Requires a Pentium or later.
1147	*/
1148	#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
1149	\|\| RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
1150	DECLASM(bool) ASMAtomicCmpXchgExU64(volatile uint64_t pu64, const uint64_t u64New, const uint64_t u64Old, uint64_t pu64Old);
1151	#else
1152	DECLINLINE(bool) ASMAtomicCmpXchgExU64(volatile uint64_t pu64, const uint64_t u64New, const uint64_t u64Old, uint64_t pu64Old)
1153	{
1154	# if RT_INLINE_ASM_USES_INTRIN
1155	return (pu64Old =_InterlockedCompareExchange64((__int64 )pu64, u64New, u64Old)) == u64Old;
1156
1157	# elif defined(RT_ARCH_AMD64)
1158	# if RT_INLINE_ASM_GNU_STYLE
1159	uint8_t u8Ret;
1160	__asm__ __volatile__("lock; cmpxchgq %3, %0\n\t"
1161	"setz %1\n\t"
1162	: "=m" (*pu64),
1163	"=qm" (u8Ret),
1164	"=a" (*pu64Old)
1165	: "r" (u64New),
1166	"a" (u64Old),
1167	"m" (*pu64));
1168	return (bool)u8Ret;
1169	# else
1170	bool fRet;
1171	__asm
1172	{
1173	mov rdx, [pu32]
1174	mov rax, [u64Old]
1175	mov rcx, [u64New]
1176	lock cmpxchg [rdx], rcx
1177	mov rdx, [pu64Old]
1178	mov [rdx], rax
1179	setz al
1180	mov [fRet], al
1181	}
1182	return fRet;
1183	# endif
1184	# else /* !RT_ARCH_AMD64 */
1185	# if RT_INLINE_ASM_GNU_STYLE
1186	uint64_t u64Ret;
1187	# if defined(PIC) \|\| defined(__PIC__)
1188	/* NB: this code uses a memory clobber description, because the clean
1189	* solution with an output value for *pu64 makes gcc run out of registers.
1190	* This will cause suboptimal code, and anyone with a better solution is
1191	* welcome to improve this. */
1192	__asm__ __volatile__("xchgl %%ebx, %1\n\t"
1193	"lock; cmpxchg8b %3\n\t"
1194	"xchgl %%ebx, %1\n\t"
1195	: "=A" (u64Ret)
1196	: "DS" ((uint32_t)u64New),
1197	"c" ((uint32_t)(u64New >> 32)),
1198	"m" (*pu64),
1199	"0" (u64Old)
1200	: "memory" );
1201	# else /* !PIC */
1202	__asm__ __volatile__("lock; cmpxchg8b %4\n\t"
1203	: "=A" (u64Ret),
1204	"=m" (*pu64)
1205	: "b" ((uint32_t)u64New),
1206	"c" ((uint32_t)(u64New >> 32)),
1207	"m" (*pu64),
1208	"0" (u64Old));
1209	# endif
1210	*pu64Old = u64Ret;
1211	return u64Ret == u64Old;
1212	# else
1213	uint32_t u32Ret;
1214	__asm
1215	{
1216	mov ebx, dword ptr [u64New]
1217	mov ecx, dword ptr [u64New + 4]
1218	mov edi, [pu64]
1219	mov eax, dword ptr [u64Old]
1220	mov edx, dword ptr [u64Old + 4]
1221	lock cmpxchg8b [edi]
1222	mov ebx, [pu64Old]
1223	mov [ebx], eax
1224	setz al
1225	movzx eax, al
1226	add ebx, 4
1227	mov [ebx], edx
1228	mov dword ptr [u32Ret], eax
1229	}
1230	return !!u32Ret;
1231	# endif
1232	# endif /* !RT_ARCH_AMD64 */
1233	}
1234	#endif
1235
1236
1237	/**
1238	* Atomically Compare and exchange a signed 64-bit value, additionally
1239	* passing back old value, ordered.
1240	*
1241	* @returns true if xchg was done.
1242	* @returns false if xchg wasn't done.
1243	*
1244	* @param pi64 Pointer to the 64-bit variable to update.
1245	* @param i64 The 64-bit value to assign to *pu64.
1246	* @param i64Old The value to compare with.
1247	* @param pi64Old Pointer store the old value at.
1248	*
1249	* @remarks x86: Requires a Pentium or later.
1250	*/
1251	DECLINLINE(bool) ASMAtomicCmpXchgExS64(volatile int64_t pi64, const int64_t i64, const int64_t i64Old, int64_t pi64Old)
1252	{
1253	return ASMAtomicCmpXchgExU64((volatile uint64_t )pi64, (uint64_t)i64, (uint64_t)i64Old, (uint64_t )pi64Old);
1254	}
1255
1256	/** @def ASMAtomicCmpXchgExHandle
1257	* Atomically Compare and Exchange a typical IPRT handle value, ordered.
1258	*
1259	* @param ph Pointer to the value to update.
1260	* @param hNew The new value to assigned to *pu.
1261	* @param hOld The old value to *pu compare with.
1262	* @param fRc Where to store the result.
1263	* @param phOldVal Pointer to where to store the old value.
1264	*
1265	* @remarks This doesn't currently work for all handles (like RTFILE).
1266	*/
1267	#if HC_ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
1268	# define ASMAtomicCmpXchgExHandle(ph, hNew, hOld, fRc, phOldVal) \
1269	do { \
1270	AssertCompile(sizeof(*ph) == sizeof(uint32_t)); \
1271	AssertCompile(sizeof(*phOldVal) == sizeof(uint32_t)); \
1272	(fRc) = ASMAtomicCmpXchgExU32((volatile uint32_t )(pu), (uint32_t)(uNew), (uint32_t)(uOld), (uint32_t )(puOldVal)); \
1273	} while (0)
1274	#elif HC_ARCH_BITS == 64
1275	# define ASMAtomicCmpXchgExHandle(ph, hNew, hOld, fRc, phOldVal) \
1276	do { \
1277	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
1278	AssertCompile(sizeof(*(phOldVal)) == sizeof(uint64_t)); \
1279	(fRc) = ASMAtomicCmpXchgExU64((volatile uint64_t )(pu), (uint64_t)(uNew), (uint64_t)(uOld), (uint64_t )(puOldVal)); \
1280	} while (0)
1281	#else
1282	# error HC_ARCH_BITS
1283	#endif
1284
1285
1286	/** @def ASMAtomicCmpXchgExSize
1287	* Atomically Compare and Exchange a value which size might differ
1288	* between platforms or compilers. Additionally passes back old value.
1289	*
1290	* @param pu Pointer to the value to update.
1291	* @param uNew The new value to assigned to *pu.
1292	* @param uOld The old value to *pu compare with.
1293	* @param fRc Where to store the result.
1294	* @param puOldVal Pointer to where to store the old value.
1295	*
1296	* @remarks x86: Requires a 486 or later.
1297	*/
1298	#define ASMAtomicCmpXchgExSize(pu, uNew, uOld, fRc, puOldVal) \
1299	do { \
1300	switch (sizeof(*(pu))) { \
1301	case 4: (fRc) = ASMAtomicCmpXchgExU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew), (uint32_t)(uOld), (uint32_t *)(uOldVal)); \
1302	break; \
1303	case 8: (fRc) = ASMAtomicCmpXchgExU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew), (uint64_t)(uOld), (uint64_t *)(uOldVal)); \
1304	break; \
1305	default: AssertMsgFailed(("ASMAtomicCmpXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
1306	(fRc) = false; \
1307	(uOldVal) = 0; \
1308	break; \
1309	} \
1310	} while (0)
1311
1312
1313	/**
1314	* Atomically Compare and Exchange a pointer value, additionally
1315	* passing back old value, ordered.
1316	*
1317	* @returns true if xchg was done.
1318	* @returns false if xchg wasn't done.
1319	*
1320	* @param ppv Pointer to the value to update.
1321	* @param pvNew The new value to assigned to *ppv.
1322	* @param pvOld The old value to *ppv compare with.
1323	* @param ppvOld Pointer store the old value at.
1324	*
1325	* @remarks x86: Requires a 486 or later.
1326	*/
1327	DECLINLINE(bool) ASMAtomicCmpXchgExPtrVoid(void * volatile ppv, const void pvNew, const void pvOld, void *ppvOld)
1328	{
1329	#if ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
1330	return ASMAtomicCmpXchgExU32((volatile uint32_t )(void )ppv, (uint32_t)pvNew, (uint32_t)pvOld, (uint32_t *)ppvOld);
1331	#elif ARCH_BITS == 64
1332	return ASMAtomicCmpXchgExU64((volatile uint64_t )(void )ppv, (uint64_t)pvNew, (uint64_t)pvOld, (uint64_t *)ppvOld);
1333	#else
1334	# error "ARCH_BITS is bogus"
1335	#endif
1336	}
1337
1338
1339	/**
1340	* Atomically Compare and Exchange a pointer value, additionally
1341	* passing back old value, ordered.
1342	*
1343	* @returns true if xchg was done.
1344	* @returns false if xchg wasn't done.
1345	*
1346	* @param ppv Pointer to the value to update.
1347	* @param pvNew The new value to assigned to *ppv.
1348	* @param pvOld The old value to *ppv compare with.
1349	* @param ppvOld Pointer store the old value at.
1350	*
1351	* @remarks This is relatively type safe on GCC platforms.
1352	* @remarks x86: Requires a 486 or later.
1353	*/
1354	#ifdef __GNUC__
1355	# define ASMAtomicCmpXchgExPtr(ppv, pvNew, pvOld, ppvOld) \
1356	__extension__ \
1357	({\
1358	__typeof__((ppv)) volatile const ppvTypeChecked = (ppv); \
1359	__typeof__(*(ppv)) const pvNewTypeChecked = (pvNew); \
1360	__typeof__(*(ppv)) const pvOldTypeChecked = (pvOld); \
1361	__typeof__((ppv)) const ppvOldTypeChecked = (ppvOld); \
1362	bool fMacroRet = ASMAtomicCmpXchgExPtrVoid((void * volatile *)ppvTypeChecked, \
1363	(void )pvNewTypeChecked, (void )pvOldTypeChecked, \
1364	(void **)ppvOldTypeChecked); \
1365	fMacroRet; \
1366	})
1367	#else
1368	# define ASMAtomicCmpXchgExPtr(ppv, pvNew, pvOld, ppvOld) \
1369	ASMAtomicCmpXchgExPtrVoid((void * volatile )(ppv), (void )(pvNew), (void )(pvOld), (void *)(ppvOld))
1370	#endif
1371
1372
1373	/**
1374	* Virtualization unfriendly serializing instruction, always exits.
1375	*/
1376	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1377	DECLASM(void) ASMSerializeInstructionCpuId(void);
1378	#else
1379	DECLINLINE(void) ASMSerializeInstructionCpuId(void)
1380	{
1381	# if RT_INLINE_ASM_GNU_STYLE
1382	RTCCUINTREG xAX = 0;
1383	# ifdef RT_ARCH_AMD64
1384	__asm__ __volatile__ ("cpuid"
1385	: "=a" (xAX)
1386	: "0" (xAX)
1387	: "rbx", "rcx", "rdx", "memory");
1388	# elif (defined(PIC) \|\| defined(__PIC__)) && defined(__i386__)
1389	__asm__ __volatile__ ("push %%ebx\n\t"
1390	"cpuid\n\t"
1391	"pop %%ebx\n\t"
1392	: "=a" (xAX)
1393	: "0" (xAX)
1394	: "ecx", "edx", "memory");
1395	# else
1396	__asm__ __volatile__ ("cpuid"
1397	: "=a" (xAX)
1398	: "0" (xAX)
1399	: "ebx", "ecx", "edx", "memory");
1400	# endif
1401
1402	# elif RT_INLINE_ASM_USES_INTRIN
1403	int aInfo[4];
1404	_ReadWriteBarrier();
1405	__cpuid(aInfo, 0);
1406
1407	# else
1408	__asm
1409	{
1410	push ebx
1411	xor eax, eax
1412	cpuid
1413	pop ebx
1414	}
1415	# endif
1416	}
1417	#endif
1418
1419	/**
1420	* Virtualization friendly serializing instruction, though more expensive.
1421	*/
1422	#if RT_INLINE_ASM_EXTERNAL
1423	DECLASM(void) ASMSerializeInstructionIRet(void);
1424	#else
1425	DECLINLINE(void) ASMSerializeInstructionIRet(void)
1426	{
1427	# if RT_INLINE_ASM_GNU_STYLE
1428	# ifdef RT_ARCH_AMD64
1429	__asm__ __volatile__ ("movq %%rsp,%%r10\n\t"
1430	"subq $128, %%rsp\n\t" /redzone/
1431	"mov %%ss, %%eax\n\t"
1432	"pushq %%rax\n\t"
1433	"pushq %%r10\n\t"
1434	"pushfq\n\t"
1435	"movl %%cs, %%eax\n\t"
1436	"pushq %%rax\n\t"
1437	"leaq 1f(%%rip), %%rax\n\t"
1438	"pushq %%rax\n\t"
1439	"iretq\n\t"
1440	"1:\n\t"
1441	::: "rax", "r10", "memory");
1442	# else
1443	__asm__ __volatile__ ("pushfl\n\t"
1444	"pushl %%cs\n\t"
1445	"pushl $1f\n\t"
1446	"iretl\n\t"
1447	"1:\n\t"
1448	::: "memory");
1449	# endif
1450
1451	# else
1452	__asm
1453	{
1454	pushfd
1455	push cs
1456	push la_ret
1457	iretd
1458	la_ret:
1459	}
1460	# endif
1461	}
1462	#endif
1463
1464	/**
1465	* Virtualization friendlier serializing instruction, may still cause exits.
1466	*/
1467	#if RT_INLINE_ASM_EXTERNAL && RT_INLINE_ASM_USES_INTRIN < 15
1468	DECLASM(void) ASMSerializeInstructionRdTscp(void);
1469	#else
1470	DECLINLINE(void) ASMSerializeInstructionRdTscp(void)
1471	{
1472	# if RT_INLINE_ASM_GNU_STYLE
1473	/* rdtscp is not supported by ancient linux build VM of course :-( */
1474	# ifdef RT_ARCH_AMD64
1475	/__asm__ __volatile__("rdtscp\n\t" ::: "rax", "rdx, "rcx"); /
1476	__asm__ __volatile__(".byte 0x0f,0x01,0xf9\n\t" ::: "rax", "rdx", "rcx", "memory");
1477	# else
1478	/__asm__ __volatile__("rdtscp\n\t" ::: "eax", "edx, "ecx"); /
1479	__asm__ __volatile__(".byte 0x0f,0x01,0xf9\n\t" ::: "eax", "edx", "ecx", "memory");
1480	# endif
1481	# else
1482	# if RT_INLINE_ASM_USES_INTRIN >= 15
1483	uint32_t uIgnore;
1484	_ReadWriteBarrier();
1485	(void)__rdtscp(&uIgnore);
1486	(void)uIgnore;
1487	# else
1488	__asm
1489	{
1490	rdtscp
1491	}
1492	# endif
1493	# endif
1494	}
1495	#endif
1496
1497
1498	/**
1499	* Serialize Instruction.
1500	*/
1501	#if (defined(RT_ARCH_X86) && ARCH_BITS == 16) \|\| defined(IN_GUEST)
1502	# define ASMSerializeInstruction() ASMSerializeInstructionIRet()
1503	#else
1504	# define ASMSerializeInstruction() ASMSerializeInstructionCpuId()
1505	#endif
1506
1507
1508	/**
1509	* Memory fence, waits for any pending writes and reads to complete.
1510	*/
1511	DECLINLINE(void) ASMMemoryFence(void)
1512	{
1513	/** @todo use mfence? check if all cpus we care for support it. */
1514	#if ARCH_BITS == 16
1515	uint16_t volatile u16;
1516	ASMAtomicXchgU16(&u16, 0);
1517	#else
1518	uint32_t volatile u32;
1519	ASMAtomicXchgU32(&u32, 0);
1520	#endif
1521	}
1522
1523
1524	/**
1525	* Write fence, waits for any pending writes to complete.
1526	*/
1527	DECLINLINE(void) ASMWriteFence(void)
1528	{
1529	/** @todo use sfence? check if all cpus we care for support it. */
1530	ASMMemoryFence();
1531	}
1532
1533
1534	/**
1535	* Read fence, waits for any pending reads to complete.
1536	*/
1537	DECLINLINE(void) ASMReadFence(void)
1538	{
1539	/** @todo use lfence? check if all cpus we care for support it. */
1540	ASMMemoryFence();
1541	}
1542
1543
1544	/**
1545	* Atomically reads an unsigned 8-bit value, ordered.
1546	*
1547	* @returns Current *pu8 value
1548	* @param pu8 Pointer to the 8-bit variable to read.
1549	*/
1550	DECLINLINE(uint8_t) ASMAtomicReadU8(volatile uint8_t *pu8)
1551	{
1552	ASMMemoryFence();
1553	return pu8; / byte reads are atomic on x86 */
1554	}
1555
1556
1557	/**
1558	* Atomically reads an unsigned 8-bit value, unordered.
1559	*
1560	* @returns Current *pu8 value
1561	* @param pu8 Pointer to the 8-bit variable to read.
1562	*/
1563	DECLINLINE(uint8_t) ASMAtomicUoReadU8(volatile uint8_t *pu8)
1564	{
1565	return pu8; / byte reads are atomic on x86 */
1566	}
1567
1568
1569	/**
1570	* Atomically reads a signed 8-bit value, ordered.
1571	*
1572	* @returns Current *pi8 value
1573	* @param pi8 Pointer to the 8-bit variable to read.
1574	*/
1575	DECLINLINE(int8_t) ASMAtomicReadS8(volatile int8_t *pi8)
1576	{
1577	ASMMemoryFence();
1578	return pi8; / byte reads are atomic on x86 */
1579	}
1580
1581
1582	/**
1583	* Atomically reads a signed 8-bit value, unordered.
1584	*
1585	* @returns Current *pi8 value
1586	* @param pi8 Pointer to the 8-bit variable to read.
1587	*/
1588	DECLINLINE(int8_t) ASMAtomicUoReadS8(volatile int8_t *pi8)
1589	{
1590	return pi8; / byte reads are atomic on x86 */
1591	}
1592
1593
1594	/**
1595	* Atomically reads an unsigned 16-bit value, ordered.
1596	*
1597	* @returns Current *pu16 value
1598	* @param pu16 Pointer to the 16-bit variable to read.
1599	*/
1600	DECLINLINE(uint16_t) ASMAtomicReadU16(volatile uint16_t *pu16)
1601	{
1602	ASMMemoryFence();
1603	Assert(!((uintptr_t)pu16 & 1));
1604	return *pu16;
1605	}
1606
1607
1608	/**
1609	* Atomically reads an unsigned 16-bit value, unordered.
1610	*
1611	* @returns Current *pu16 value
1612	* @param pu16 Pointer to the 16-bit variable to read.
1613	*/
1614	DECLINLINE(uint16_t) ASMAtomicUoReadU16(volatile uint16_t *pu16)
1615	{
1616	Assert(!((uintptr_t)pu16 & 1));
1617	return *pu16;
1618	}
1619
1620
1621	/**
1622	* Atomically reads a signed 16-bit value, ordered.
1623	*
1624	* @returns Current *pi16 value
1625	* @param pi16 Pointer to the 16-bit variable to read.
1626	*/
1627	DECLINLINE(int16_t) ASMAtomicReadS16(volatile int16_t *pi16)
1628	{
1629	ASMMemoryFence();
1630	Assert(!((uintptr_t)pi16 & 1));
1631	return *pi16;
1632	}
1633
1634
1635	/**
1636	* Atomically reads a signed 16-bit value, unordered.
1637	*
1638	* @returns Current *pi16 value
1639	* @param pi16 Pointer to the 16-bit variable to read.
1640	*/
1641	DECLINLINE(int16_t) ASMAtomicUoReadS16(volatile int16_t *pi16)
1642	{
1643	Assert(!((uintptr_t)pi16 & 1));
1644	return *pi16;
1645	}
1646
1647
1648	/**
1649	* Atomically reads an unsigned 32-bit value, ordered.
1650	*
1651	* @returns Current *pu32 value
1652	* @param pu32 Pointer to the 32-bit variable to read.
1653	*/
1654	DECLINLINE(uint32_t) ASMAtomicReadU32(volatile uint32_t *pu32)
1655	{
1656	ASMMemoryFence();
1657	Assert(!((uintptr_t)pu32 & 3));
1658	return *pu32;
1659	}
1660
1661
1662	/**
1663	* Atomically reads an unsigned 32-bit value, unordered.
1664	*
1665	* @returns Current *pu32 value
1666	* @param pu32 Pointer to the 32-bit variable to read.
1667	*/
1668	DECLINLINE(uint32_t) ASMAtomicUoReadU32(volatile uint32_t *pu32)
1669	{
1670	Assert(!((uintptr_t)pu32 & 3));
1671	return *pu32;
1672	}
1673
1674
1675	/**
1676	* Atomically reads a signed 32-bit value, ordered.
1677	*
1678	* @returns Current *pi32 value
1679	* @param pi32 Pointer to the 32-bit variable to read.
1680	*/
1681	DECLINLINE(int32_t) ASMAtomicReadS32(volatile int32_t *pi32)
1682	{
1683	ASMMemoryFence();
1684	Assert(!((uintptr_t)pi32 & 3));
1685	return *pi32;
1686	}
1687
1688
1689	/**
1690	* Atomically reads a signed 32-bit value, unordered.
1691	*
1692	* @returns Current *pi32 value
1693	* @param pi32 Pointer to the 32-bit variable to read.
1694	*/
1695	DECLINLINE(int32_t) ASMAtomicUoReadS32(volatile int32_t *pi32)
1696	{
1697	Assert(!((uintptr_t)pi32 & 3));
1698	return *pi32;
1699	}
1700
1701
1702	/**
1703	* Atomically reads an unsigned 64-bit value, ordered.
1704	*
1705	* @returns Current *pu64 value
1706	* @param pu64 Pointer to the 64-bit variable to read.
1707	* The memory pointed to must be writable.
1708	*
1709	* @remarks This may fault if the memory is read-only!
1710	* @remarks x86: Requires a Pentium or later.
1711	*/
1712	#if (RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)) \
1713	\|\| RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
1714	DECLASM(uint64_t) ASMAtomicReadU64(volatile uint64_t *pu64);
1715	#else
1716	DECLINLINE(uint64_t) ASMAtomicReadU64(volatile uint64_t *pu64)
1717	{
1718	uint64_t u64;
1719	# ifdef RT_ARCH_AMD64
1720	Assert(!((uintptr_t)pu64 & 7));
1721	/*# if RT_INLINE_ASM_GNU_STYLE
1722	__asm__ __volatile__( "mfence\n\t"
1723	"movq %1, %0\n\t"
1724	: "=r" (u64)
1725	: "m" (*pu64));
1726	# else
1727	__asm
1728	{
1729	mfence
1730	mov rdx, [pu64]
1731	mov rax, [rdx]
1732	mov [u64], rax
1733	}
1734	# endif*/
1735	ASMMemoryFence();
1736	u64 = *pu64;
1737	# else /* !RT_ARCH_AMD64 */
1738	# if RT_INLINE_ASM_GNU_STYLE
1739	# if defined(PIC) \|\| defined(__PIC__)
1740	uint32_t u32EBX = 0;
1741	Assert(!((uintptr_t)pu64 & 7));
1742	__asm__ __volatile__("xchgl %%ebx, %3\n\t"
1743	"lock; cmpxchg8b (%5)\n\t"
1744	"movl %3, %%ebx\n\t"
1745	: "=A" (u64),
1746	# if RT_GNUC_PREREQ(4, 3)
1747	"+m" (*pu64)
1748	# else
1749	"=m" (*pu64)
1750	# endif
1751	: "0" (0ULL),
1752	"m" (u32EBX),
1753	"c" (0),
1754	"S" (pu64));
1755	# else /* !PIC */
1756	__asm__ __volatile__("lock; cmpxchg8b %1\n\t"
1757	: "=A" (u64),
1758	"+m" (*pu64)
1759	: "0" (0ULL),
1760	"b" (0),
1761	"c" (0));
1762	# endif
1763	# else
1764	Assert(!((uintptr_t)pu64 & 7));
1765	__asm
1766	{
1767	xor eax, eax
1768	xor edx, edx
1769	mov edi, pu64
1770	xor ecx, ecx
1771	xor ebx, ebx
1772	lock cmpxchg8b [edi]
1773	mov dword ptr [u64], eax
1774	mov dword ptr [u64 + 4], edx
1775	}
1776	# endif
1777	# endif /* !RT_ARCH_AMD64 */
1778	return u64;
1779	}
1780	#endif
1781
1782
1783	/**
1784	* Atomically reads an unsigned 64-bit value, unordered.
1785	*
1786	* @returns Current *pu64 value
1787	* @param pu64 Pointer to the 64-bit variable to read.
1788	* The memory pointed to must be writable.
1789	*
1790	* @remarks This may fault if the memory is read-only!
1791	* @remarks x86: Requires a Pentium or later.
1792	*/
1793	#if !defined(RT_ARCH_AMD64) \
1794	&& ( (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
1795	\|\| RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC)
1796	DECLASM(uint64_t) ASMAtomicUoReadU64(volatile uint64_t *pu64);
1797	#else
1798	DECLINLINE(uint64_t) ASMAtomicUoReadU64(volatile uint64_t *pu64)
1799	{
1800	uint64_t u64;
1801	# ifdef RT_ARCH_AMD64
1802	Assert(!((uintptr_t)pu64 & 7));
1803	/*# if RT_INLINE_ASM_GNU_STYLE
1804	Assert(!((uintptr_t)pu64 & 7));
1805	__asm__ __volatile__("movq %1, %0\n\t"
1806	: "=r" (u64)
1807	: "m" (*pu64));
1808	# else
1809	__asm
1810	{
1811	mov rdx, [pu64]
1812	mov rax, [rdx]
1813	mov [u64], rax
1814	}
1815	# endif */
1816	u64 = *pu64;
1817	# else /* !RT_ARCH_AMD64 */
1818	# if RT_INLINE_ASM_GNU_STYLE
1819	# if defined(PIC) \|\| defined(__PIC__)
1820	uint32_t u32EBX = 0;
1821	uint32_t u32Spill;
1822	Assert(!((uintptr_t)pu64 & 7));
1823	__asm__ __volatile__("xor %%eax,%%eax\n\t"
1824	"xor %%ecx,%%ecx\n\t"
1825	"xor %%edx,%%edx\n\t"
1826	"xchgl %%ebx, %3\n\t"
1827	"lock; cmpxchg8b (%4)\n\t"
1828	"movl %3, %%ebx\n\t"
1829	: "=A" (u64),
1830	# if RT_GNUC_PREREQ(4, 3)
1831	"+m" (*pu64),
1832	# else
1833	"=m" (*pu64),
1834	# endif
1835	"=c" (u32Spill)
1836	: "m" (u32EBX),
1837	"S" (pu64));
1838	# else /* !PIC */
1839	__asm__ __volatile__("lock; cmpxchg8b %1\n\t"
1840	: "=A" (u64),
1841	"+m" (*pu64)
1842	: "0" (0ULL),
1843	"b" (0),
1844	"c" (0));
1845	# endif
1846	# else
1847	Assert(!((uintptr_t)pu64 & 7));
1848	__asm
1849	{
1850	xor eax, eax
1851	xor edx, edx
1852	mov edi, pu64
1853	xor ecx, ecx
1854	xor ebx, ebx
1855	lock cmpxchg8b [edi]
1856	mov dword ptr [u64], eax
1857	mov dword ptr [u64 + 4], edx
1858	}
1859	# endif
1860	# endif /* !RT_ARCH_AMD64 */
1861	return u64;
1862	}
1863	#endif
1864
1865
1866	/**
1867	* Atomically reads a signed 64-bit value, ordered.
1868	*
1869	* @returns Current *pi64 value
1870	* @param pi64 Pointer to the 64-bit variable to read.
1871	* The memory pointed to must be writable.
1872	*
1873	* @remarks This may fault if the memory is read-only!
1874	* @remarks x86: Requires a Pentium or later.
1875	*/
1876	DECLINLINE(int64_t) ASMAtomicReadS64(volatile int64_t *pi64)
1877	{
1878	return (int64_t)ASMAtomicReadU64((volatile uint64_t *)pi64);
1879	}
1880
1881
1882	/**
1883	* Atomically reads a signed 64-bit value, unordered.
1884	*
1885	* @returns Current *pi64 value
1886	* @param pi64 Pointer to the 64-bit variable to read.
1887	* The memory pointed to must be writable.
1888	*
1889	* @remarks This will fault if the memory is read-only!
1890	* @remarks x86: Requires a Pentium or later.
1891	*/
1892	DECLINLINE(int64_t) ASMAtomicUoReadS64(volatile int64_t *pi64)
1893	{
1894	return (int64_t)ASMAtomicUoReadU64((volatile uint64_t *)pi64);
1895	}
1896
1897
1898	/**
1899	* Atomically reads a size_t value, ordered.
1900	*
1901	* @returns Current *pcb value
1902	* @param pcb Pointer to the size_t variable to read.
1903	*/
1904	DECLINLINE(size_t) ASMAtomicReadZ(size_t volatile *pcb)
1905	{
1906	#if ARCH_BITS == 64
1907	return ASMAtomicReadU64((uint64_t volatile *)pcb);
1908	#elif ARCH_BITS == 32
1909	return ASMAtomicReadU32((uint32_t volatile *)pcb);
1910	#elif ARCH_BITS == 16
1911	AssertCompileSize(size_t, 2);
1912	return ASMAtomicReadU16((uint16_t volatile *)pcb);
1913	#else
1914	# error "Unsupported ARCH_BITS value"
1915	#endif
1916	}
1917
1918
1919	/**
1920	* Atomically reads a size_t value, unordered.
1921	*
1922	* @returns Current *pcb value
1923	* @param pcb Pointer to the size_t variable to read.
1924	*/
1925	DECLINLINE(size_t) ASMAtomicUoReadZ(size_t volatile *pcb)
1926	{
1927	#if ARCH_BITS == 64 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
1928	return ASMAtomicUoReadU64((uint64_t volatile *)pcb);
1929	#elif ARCH_BITS == 32
1930	return ASMAtomicUoReadU32((uint32_t volatile *)pcb);
1931	#elif ARCH_BITS == 16
1932	AssertCompileSize(size_t, 2);
1933	return ASMAtomicUoReadU16((uint16_t volatile *)pcb);
1934	#else
1935	# error "Unsupported ARCH_BITS value"
1936	#endif
1937	}
1938
1939
1940	/**
1941	* Atomically reads a pointer value, ordered.
1942	*
1943	* @returns Current *pv value
1944	* @param ppv Pointer to the pointer variable to read.
1945	*
1946	* @remarks Please use ASMAtomicReadPtrT, it provides better type safety and
1947	* requires less typing (no casts).
1948	*/
1949	DECLINLINE(void ) ASMAtomicReadPtr(void volatile *ppv)
1950	{
1951	#if ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
1952	return (void )ASMAtomicReadU32((volatile uint32_t )(void *)ppv);
1953	#elif ARCH_BITS == 64
1954	return (void )ASMAtomicReadU64((volatile uint64_t )(void *)ppv);
1955	#else
1956	# error "ARCH_BITS is bogus"
1957	#endif
1958	}
1959
1960	/**
1961	* Convenience macro for avoiding the annoying casting with ASMAtomicReadPtr.
1962	*
1963	* @returns Current *pv value
1964	* @param ppv Pointer to the pointer variable to read.
1965	* @param Type The type of *ppv, sans volatile.
1966	*/
1967	#ifdef __GNUC__
1968	# define ASMAtomicReadPtrT(ppv, Type) \
1969	__extension__ \
1970	({\
1971	__typeof__((ppv)) volatile ppvTypeChecked = (ppv); \
1972	Type pvTypeChecked = (__typeof__((ppv))) ASMAtomicReadPtr((void volatile *)ppvTypeChecked); \
1973	pvTypeChecked; \
1974	})
1975	#else
1976	# define ASMAtomicReadPtrT(ppv, Type) \
1977	(Type)ASMAtomicReadPtr((void * volatile *)(ppv))
1978	#endif
1979
1980
1981	/**
1982	* Atomically reads a pointer value, unordered.
1983	*
1984	* @returns Current *pv value
1985	* @param ppv Pointer to the pointer variable to read.
1986	*
1987	* @remarks Please use ASMAtomicUoReadPtrT, it provides better type safety and
1988	* requires less typing (no casts).
1989	*/
1990	DECLINLINE(void ) ASMAtomicUoReadPtr(void volatile *ppv)
1991	{
1992	#if ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
1993	return (void )ASMAtomicUoReadU32((volatile uint32_t )(void *)ppv);
1994	#elif ARCH_BITS == 64
1995	return (void )ASMAtomicUoReadU64((volatile uint64_t )(void *)ppv);
1996	#else
1997	# error "ARCH_BITS is bogus"
1998	#endif
1999	}
2000
2001
2002	/**
2003	* Convenience macro for avoiding the annoying casting with ASMAtomicUoReadPtr.
2004	*
2005	* @returns Current *pv value
2006	* @param ppv Pointer to the pointer variable to read.
2007	* @param Type The type of *ppv, sans volatile.
2008	*/
2009	#ifdef __GNUC__
2010	# define ASMAtomicUoReadPtrT(ppv, Type) \
2011	__extension__ \
2012	({\
2013	__typeof__((ppv)) volatile const ppvTypeChecked = (ppv); \
2014	Type pvTypeChecked = (__typeof__((ppv))) ASMAtomicUoReadPtr((void volatile *)ppvTypeChecked); \
2015	pvTypeChecked; \
2016	})
2017	#else
2018	# define ASMAtomicUoReadPtrT(ppv, Type) \
2019	(Type)ASMAtomicUoReadPtr((void * volatile *)(ppv))
2020	#endif
2021
2022
2023	/**
2024	* Atomically reads a boolean value, ordered.
2025	*
2026	* @returns Current *pf value
2027	* @param pf Pointer to the boolean variable to read.
2028	*/
2029	DECLINLINE(bool) ASMAtomicReadBool(volatile bool *pf)
2030	{
2031	ASMMemoryFence();
2032	return pf; / byte reads are atomic on x86 */
2033	}
2034
2035
2036	/**
2037	* Atomically reads a boolean value, unordered.
2038	*
2039	* @returns Current *pf value
2040	* @param pf Pointer to the boolean variable to read.
2041	*/
2042	DECLINLINE(bool) ASMAtomicUoReadBool(volatile bool *pf)
2043	{
2044	return pf; / byte reads are atomic on x86 */
2045	}
2046
2047
2048	/**
2049	* Atomically read a typical IPRT handle value, ordered.
2050	*
2051	* @param ph Pointer to the handle variable to read.
2052	* @param phRes Where to store the result.
2053	*
2054	* @remarks This doesn't currently work for all handles (like RTFILE).
2055	*/
2056	#if HC_ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
2057	# define ASMAtomicReadHandle(ph, phRes) \
2058	do { \
2059	AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
2060	AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
2061	(uint32_t )(phRes) = ASMAtomicReadU32((uint32_t volatile *)(ph)); \
2062	} while (0)
2063	#elif HC_ARCH_BITS == 64
2064	# define ASMAtomicReadHandle(ph, phRes) \
2065	do { \
2066	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
2067	AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
2068	(uint64_t )(phRes) = ASMAtomicReadU64((uint64_t volatile *)(ph)); \
2069	} while (0)
2070	#else
2071	# error HC_ARCH_BITS
2072	#endif
2073
2074
2075	/**
2076	* Atomically read a typical IPRT handle value, unordered.
2077	*
2078	* @param ph Pointer to the handle variable to read.
2079	* @param phRes Where to store the result.
2080	*
2081	* @remarks This doesn't currently work for all handles (like RTFILE).
2082	*/
2083	#if HC_ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
2084	# define ASMAtomicUoReadHandle(ph, phRes) \
2085	do { \
2086	AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
2087	AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
2088	(uint32_t )(phRes) = ASMAtomicUoReadU32((uint32_t volatile *)(ph)); \
2089	} while (0)
2090	#elif HC_ARCH_BITS == 64
2091	# define ASMAtomicUoReadHandle(ph, phRes) \
2092	do { \
2093	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
2094	AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
2095	(uint64_t )(phRes) = ASMAtomicUoReadU64((uint64_t volatile *)(ph)); \
2096	} while (0)
2097	#else
2098	# error HC_ARCH_BITS
2099	#endif
2100
2101
2102	/**
2103	* Atomically read a value which size might differ
2104	* between platforms or compilers, ordered.
2105	*
2106	* @param pu Pointer to the variable to read.
2107	* @param puRes Where to store the result.
2108	*/
2109	#define ASMAtomicReadSize(pu, puRes) \
2110	do { \
2111	switch (sizeof(*(pu))) { \
2112	case 1: (uint8_t )(puRes) = ASMAtomicReadU8( (volatile uint8_t )(void )(pu)); break; \
2113	case 2: (uint16_t )(puRes) = ASMAtomicReadU16((volatile uint16_t )(void )(pu)); break; \
2114	case 4: (uint32_t )(puRes) = ASMAtomicReadU32((volatile uint32_t )(void )(pu)); break; \
2115	case 8: (uint64_t )(puRes) = ASMAtomicReadU64((volatile uint64_t )(void )(pu)); break; \
2116	default: AssertMsgFailed(("ASMAtomicReadSize: size %d is not supported\n", sizeof(*(pu)))); \
2117	} \
2118	} while (0)
2119
2120
2121	/**
2122	* Atomically read a value which size might differ
2123	* between platforms or compilers, unordered.
2124	*
2125	* @param pu Pointer to the variable to read.
2126	* @param puRes Where to store the result.
2127	*/
2128	#define ASMAtomicUoReadSize(pu, puRes) \
2129	do { \
2130	switch (sizeof(*(pu))) { \
2131	case 1: (uint8_t )(puRes) = ASMAtomicUoReadU8( (volatile uint8_t )(void )(pu)); break; \
2132	case 2: (uint16_t )(puRes) = ASMAtomicUoReadU16((volatile uint16_t )(void )(pu)); break; \
2133	case 4: (uint32_t )(puRes) = ASMAtomicUoReadU32((volatile uint32_t )(void )(pu)); break; \
2134	case 8: (uint64_t )(puRes) = ASMAtomicUoReadU64((volatile uint64_t )(void )(pu)); break; \
2135	default: AssertMsgFailed(("ASMAtomicReadSize: size %d is not supported\n", sizeof(*(pu)))); \
2136	} \
2137	} while (0)
2138
2139
2140	/**
2141	* Atomically writes an unsigned 8-bit value, ordered.
2142	*
2143	* @param pu8 Pointer to the 8-bit variable.
2144	* @param u8 The 8-bit value to assign to *pu8.
2145	*/
2146	DECLINLINE(void) ASMAtomicWriteU8(volatile uint8_t *pu8, uint8_t u8)
2147	{
2148	ASMAtomicXchgU8(pu8, u8);
2149	}
2150
2151
2152	/**
2153	* Atomically writes an unsigned 8-bit value, unordered.
2154	*
2155	* @param pu8 Pointer to the 8-bit variable.
2156	* @param u8 The 8-bit value to assign to *pu8.
2157	*/
2158	DECLINLINE(void) ASMAtomicUoWriteU8(volatile uint8_t *pu8, uint8_t u8)
2159	{
2160	pu8 = u8; / byte writes are atomic on x86 */
2161	}
2162
2163
2164	/**
2165	* Atomically writes a signed 8-bit value, ordered.
2166	*
2167	* @param pi8 Pointer to the 8-bit variable to read.
2168	* @param i8 The 8-bit value to assign to *pi8.
2169	*/
2170	DECLINLINE(void) ASMAtomicWriteS8(volatile int8_t *pi8, int8_t i8)
2171	{
2172	ASMAtomicXchgS8(pi8, i8);
2173	}
2174
2175
2176	/**
2177	* Atomically writes a signed 8-bit value, unordered.
2178	*
2179	* @param pi8 Pointer to the 8-bit variable to write.
2180	* @param i8 The 8-bit value to assign to *pi8.
2181	*/
2182	DECLINLINE(void) ASMAtomicUoWriteS8(volatile int8_t *pi8, int8_t i8)
2183	{
2184	pi8 = i8; / byte writes are atomic on x86 */
2185	}
2186
2187
2188	/**
2189	* Atomically writes an unsigned 16-bit value, ordered.
2190	*
2191	* @param pu16 Pointer to the 16-bit variable to write.
2192	* @param u16 The 16-bit value to assign to *pu16.
2193	*/
2194	DECLINLINE(void) ASMAtomicWriteU16(volatile uint16_t *pu16, uint16_t u16)
2195	{
2196	ASMAtomicXchgU16(pu16, u16);
2197	}
2198
2199
2200	/**
2201	* Atomically writes an unsigned 16-bit value, unordered.
2202	*
2203	* @param pu16 Pointer to the 16-bit variable to write.
2204	* @param u16 The 16-bit value to assign to *pu16.
2205	*/
2206	DECLINLINE(void) ASMAtomicUoWriteU16(volatile uint16_t *pu16, uint16_t u16)
2207	{
2208	Assert(!((uintptr_t)pu16 & 1));
2209	*pu16 = u16;
2210	}
2211
2212
2213	/**
2214	* Atomically writes a signed 16-bit value, ordered.
2215	*
2216	* @param pi16 Pointer to the 16-bit variable to write.
2217	* @param i16 The 16-bit value to assign to *pi16.
2218	*/
2219	DECLINLINE(void) ASMAtomicWriteS16(volatile int16_t *pi16, int16_t i16)
2220	{
2221	ASMAtomicXchgS16(pi16, i16);
2222	}
2223
2224
2225	/**
2226	* Atomically writes a signed 16-bit value, unordered.
2227	*
2228	* @param pi16 Pointer to the 16-bit variable to write.
2229	* @param i16 The 16-bit value to assign to *pi16.
2230	*/
2231	DECLINLINE(void) ASMAtomicUoWriteS16(volatile int16_t *pi16, int16_t i16)
2232	{
2233	Assert(!((uintptr_t)pi16 & 1));
2234	*pi16 = i16;
2235	}
2236
2237
2238	/**
2239	* Atomically writes an unsigned 32-bit value, ordered.
2240	*
2241	* @param pu32 Pointer to the 32-bit variable to write.
2242	* @param u32 The 32-bit value to assign to *pu32.
2243	*/
2244	DECLINLINE(void) ASMAtomicWriteU32(volatile uint32_t *pu32, uint32_t u32)
2245	{
2246	ASMAtomicXchgU32(pu32, u32);
2247	}
2248
2249
2250	/**
2251	* Atomically writes an unsigned 32-bit value, unordered.
2252	*
2253	* @param pu32 Pointer to the 32-bit variable to write.
2254	* @param u32 The 32-bit value to assign to *pu32.
2255	*/
2256	DECLINLINE(void) ASMAtomicUoWriteU32(volatile uint32_t *pu32, uint32_t u32)
2257	{
2258	Assert(!((uintptr_t)pu32 & 3));
2259	*pu32 = u32;
2260	}
2261
2262
2263	/**
2264	* Atomically writes a signed 32-bit value, ordered.
2265	*
2266	* @param pi32 Pointer to the 32-bit variable to write.
2267	* @param i32 The 32-bit value to assign to *pi32.
2268	*/
2269	DECLINLINE(void) ASMAtomicWriteS32(volatile int32_t *pi32, int32_t i32)
2270	{
2271	ASMAtomicXchgS32(pi32, i32);
2272	}
2273
2274
2275	/**
2276	* Atomically writes a signed 32-bit value, unordered.
2277	*
2278	* @param pi32 Pointer to the 32-bit variable to write.
2279	* @param i32 The 32-bit value to assign to *pi32.
2280	*/
2281	DECLINLINE(void) ASMAtomicUoWriteS32(volatile int32_t *pi32, int32_t i32)
2282	{
2283	Assert(!((uintptr_t)pi32 & 3));
2284	*pi32 = i32;
2285	}
2286
2287
2288	/**
2289	* Atomically writes an unsigned 64-bit value, ordered.
2290	*
2291	* @param pu64 Pointer to the 64-bit variable to write.
2292	* @param u64 The 64-bit value to assign to *pu64.
2293	*/
2294	DECLINLINE(void) ASMAtomicWriteU64(volatile uint64_t *pu64, uint64_t u64)
2295	{
2296	ASMAtomicXchgU64(pu64, u64);
2297	}
2298
2299
2300	/**
2301	* Atomically writes an unsigned 64-bit value, unordered.
2302	*
2303	* @param pu64 Pointer to the 64-bit variable to write.
2304	* @param u64 The 64-bit value to assign to *pu64.
2305	*/
2306	DECLINLINE(void) ASMAtomicUoWriteU64(volatile uint64_t *pu64, uint64_t u64)
2307	{
2308	Assert(!((uintptr_t)pu64 & 7));
2309	#if ARCH_BITS == 64
2310	*pu64 = u64;
2311	#else
2312	ASMAtomicXchgU64(pu64, u64);
2313	#endif
2314	}
2315
2316
2317	/**
2318	* Atomically writes a signed 64-bit value, ordered.
2319	*
2320	* @param pi64 Pointer to the 64-bit variable to write.
2321	* @param i64 The 64-bit value to assign to *pi64.
2322	*/
2323	DECLINLINE(void) ASMAtomicWriteS64(volatile int64_t *pi64, int64_t i64)
2324	{
2325	ASMAtomicXchgS64(pi64, i64);
2326	}
2327
2328
2329	/**
2330	* Atomically writes a signed 64-bit value, unordered.
2331	*
2332	* @param pi64 Pointer to the 64-bit variable to write.
2333	* @param i64 The 64-bit value to assign to *pi64.
2334	*/
2335	DECLINLINE(void) ASMAtomicUoWriteS64(volatile int64_t *pi64, int64_t i64)
2336	{
2337	Assert(!((uintptr_t)pi64 & 7));
2338	#if ARCH_BITS == 64
2339	*pi64 = i64;
2340	#else
2341	ASMAtomicXchgS64(pi64, i64);
2342	#endif
2343	}
2344
2345
2346	/**
2347	* Atomically writes a boolean value, unordered.
2348	*
2349	* @param pf Pointer to the boolean variable to write.
2350	* @param f The boolean value to assign to *pf.
2351	*/
2352	DECLINLINE(void) ASMAtomicWriteBool(volatile bool *pf, bool f)
2353	{
2354	ASMAtomicWriteU8((uint8_t volatile *)pf, f);
2355	}
2356
2357
2358	/**
2359	* Atomically writes a boolean value, unordered.
2360	*
2361	* @param pf Pointer to the boolean variable to write.
2362	* @param f The boolean value to assign to *pf.
2363	*/
2364	DECLINLINE(void) ASMAtomicUoWriteBool(volatile bool *pf, bool f)
2365	{
2366	pf = f; / byte writes are atomic on x86 */
2367	}
2368
2369
2370	/**
2371	* Atomically writes a pointer value, ordered.
2372	*
2373	* @param ppv Pointer to the pointer variable to write.
2374	* @param pv The pointer value to assign to *ppv.
2375	*/
2376	DECLINLINE(void) ASMAtomicWritePtrVoid(void * volatile ppv, const void pv)
2377	{
2378	#if ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
2379	ASMAtomicWriteU32((volatile uint32_t )(void )ppv, (uint32_t)pv);
2380	#elif ARCH_BITS == 64
2381	ASMAtomicWriteU64((volatile uint64_t )(void )ppv, (uint64_t)pv);
2382	#else
2383	# error "ARCH_BITS is bogus"
2384	#endif
2385	}
2386
2387
2388	/**
2389	* Atomically writes a pointer value, ordered.
2390	*
2391	* @param ppv Pointer to the pointer variable to write.
2392	* @param pv The pointer value to assign to *ppv. If NULL use
2393	* ASMAtomicWriteNullPtr or you'll land in trouble.
2394	*
2395	* @remarks This is relatively type safe on GCC platforms when @a pv isn't
2396	* NULL.
2397	*/
2398	#ifdef __GNUC__
2399	# define ASMAtomicWritePtr(ppv, pv) \
2400	do \
2401	{ \
2402	__typeof__((ppv)) volatile const ppvTypeChecked = (ppv); \
2403	__typeof__(*(ppv)) const pvTypeChecked = (pv); \
2404	\
2405	AssertCompile(sizeof(ppv) == sizeof(void )); \
2406	AssertCompile(sizeof(pv) == sizeof(void *)); \
2407	Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2408	\
2409	ASMAtomicWritePtrVoid((void * volatile )(ppvTypeChecked), (void )(pvTypeChecked)); \
2410	} while (0)
2411	#else
2412	# define ASMAtomicWritePtr(ppv, pv) \
2413	do \
2414	{ \
2415	AssertCompile(sizeof(ppv) == sizeof(void )); \
2416	AssertCompile(sizeof(pv) == sizeof(void *)); \
2417	Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2418	\
2419	ASMAtomicWritePtrVoid((void * volatile )(ppv), (void )(pv)); \
2420	} while (0)
2421	#endif
2422
2423
2424	/**
2425	* Atomically sets a pointer to NULL, ordered.
2426	*
2427	* @param ppv Pointer to the pointer variable that should be set to NULL.
2428	*
2429	* @remarks This is relatively type safe on GCC platforms.
2430	*/
2431	#ifdef __GNUC__
2432	# define ASMAtomicWriteNullPtr(ppv) \
2433	do \
2434	{ \
2435	__typeof__((ppv)) const ppvTypeChecked = (ppv); \
2436	AssertCompile(sizeof(ppv) == sizeof(void )); \
2437	Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2438	ASMAtomicWritePtrVoid((void * volatile *)(ppvTypeChecked), NULL); \
2439	} while (0)
2440	#else
2441	# define ASMAtomicWriteNullPtr(ppv) \
2442	do \
2443	{ \
2444	AssertCompile(sizeof(ppv) == sizeof(void )); \
2445	Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2446	ASMAtomicWritePtrVoid((void * volatile *)(ppv), NULL); \
2447	} while (0)
2448	#endif
2449
2450
2451	/**
2452	* Atomically writes a pointer value, unordered.
2453	*
2454	* @returns Current *pv value
2455	* @param ppv Pointer to the pointer variable.
2456	* @param pv The pointer value to assign to *ppv. If NULL use
2457	* ASMAtomicUoWriteNullPtr or you'll land in trouble.
2458	*
2459	* @remarks This is relatively type safe on GCC platforms when @a pv isn't
2460	* NULL.
2461	*/
2462	#ifdef __GNUC__
2463	# define ASMAtomicUoWritePtr(ppv, pv) \
2464	do \
2465	{ \
2466	__typeof__((ppv)) volatile const ppvTypeChecked = (ppv); \
2467	__typeof__(*(ppv)) const pvTypeChecked = (pv); \
2468	\
2469	AssertCompile(sizeof(ppv) == sizeof(void )); \
2470	AssertCompile(sizeof(pv) == sizeof(void *)); \
2471	Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2472	\
2473	*(ppvTypeChecked) = pvTypeChecked; \
2474	} while (0)
2475	#else
2476	# define ASMAtomicUoWritePtr(ppv, pv) \
2477	do \
2478	{ \
2479	AssertCompile(sizeof(ppv) == sizeof(void )); \
2480	AssertCompile(sizeof(pv) == sizeof(void *)); \
2481	Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2482	*(ppv) = pv; \
2483	} while (0)
2484	#endif
2485
2486
2487	/**
2488	* Atomically sets a pointer to NULL, unordered.
2489	*
2490	* @param ppv Pointer to the pointer variable that should be set to NULL.
2491	*
2492	* @remarks This is relatively type safe on GCC platforms.
2493	*/
2494	#ifdef __GNUC__
2495	# define ASMAtomicUoWriteNullPtr(ppv) \
2496	do \
2497	{ \
2498	__typeof__((ppv)) volatile const ppvTypeChecked = (ppv); \
2499	AssertCompile(sizeof(ppv) == sizeof(void )); \
2500	Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2501	*(ppvTypeChecked) = NULL; \
2502	} while (0)
2503	#else
2504	# define ASMAtomicUoWriteNullPtr(ppv) \
2505	do \
2506	{ \
2507	AssertCompile(sizeof(ppv) == sizeof(void )); \
2508	Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2509	*(ppv) = NULL; \
2510	} while (0)
2511	#endif
2512
2513
2514	/**
2515	* Atomically write a typical IPRT handle value, ordered.
2516	*
2517	* @param ph Pointer to the variable to update.
2518	* @param hNew The value to assign to *ph.
2519	*
2520	* @remarks This doesn't currently work for all handles (like RTFILE).
2521	*/
2522	#if HC_ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
2523	# define ASMAtomicWriteHandle(ph, hNew) \
2524	do { \
2525	AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
2526	ASMAtomicWriteU32((uint32_t volatile *)(ph), (const uint32_t)(hNew)); \
2527	} while (0)
2528	#elif HC_ARCH_BITS == 64
2529	# define ASMAtomicWriteHandle(ph, hNew) \
2530	do { \
2531	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
2532	ASMAtomicWriteU64((uint64_t volatile *)(ph), (const uint64_t)(hNew)); \
2533	} while (0)
2534	#else
2535	# error HC_ARCH_BITS
2536	#endif
2537
2538
2539	/**
2540	* Atomically write a typical IPRT handle value, unordered.
2541	*
2542	* @param ph Pointer to the variable to update.
2543	* @param hNew The value to assign to *ph.
2544	*
2545	* @remarks This doesn't currently work for all handles (like RTFILE).
2546	*/
2547	#if HC_ARCH_BITS == 32 \|\| (ARCH_BITS == 16 && RT_FAR_DATA)
2548	# define ASMAtomicUoWriteHandle(ph, hNew) \
2549	do { \
2550	AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
2551	ASMAtomicUoWriteU32((uint32_t volatile *)(ph), (const uint32_t)hNew); \
2552	} while (0)
2553	#elif HC_ARCH_BITS == 64
2554	# define ASMAtomicUoWriteHandle(ph, hNew) \
2555	do { \
2556	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
2557	ASMAtomicUoWriteU64((uint64_t volatile *)(ph), (const uint64_t)hNew); \
2558	} while (0)
2559	#else
2560	# error HC_ARCH_BITS
2561	#endif
2562
2563
2564	/**
2565	* Atomically write a value which size might differ
2566	* between platforms or compilers, ordered.
2567	*
2568	* @param pu Pointer to the variable to update.
2569	* @param uNew The value to assign to *pu.
2570	*/
2571	#define ASMAtomicWriteSize(pu, uNew) \
2572	do { \
2573	switch (sizeof(*(pu))) { \
2574	case 1: ASMAtomicWriteU8( (volatile uint8_t )(void )(pu), (uint8_t )(uNew)); break; \
2575	case 2: ASMAtomicWriteU16((volatile uint16_t )(void )(pu), (uint16_t)(uNew)); break; \
2576	case 4: ASMAtomicWriteU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew)); break; \
2577	case 8: ASMAtomicWriteU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew)); break; \
2578	default: AssertMsgFailed(("ASMAtomicWriteSize: size %d is not supported\n", sizeof(*(pu)))); \
2579	} \
2580	} while (0)
2581
2582	/**
2583	* Atomically write a value which size might differ
2584	* between platforms or compilers, unordered.
2585	*
2586	* @param pu Pointer to the variable to update.
2587	* @param uNew The value to assign to *pu.
2588	*/
2589	#define ASMAtomicUoWriteSize(pu, uNew) \
2590	do { \
2591	switch (sizeof(*(pu))) { \
2592	case 1: ASMAtomicUoWriteU8( (volatile uint8_t )(void )(pu), (uint8_t )(uNew)); break; \
2593	case 2: ASMAtomicUoWriteU16((volatile uint16_t )(void )(pu), (uint16_t)(uNew)); break; \
2594	case 4: ASMAtomicUoWriteU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew)); break; \
2595	case 8: ASMAtomicUoWriteU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew)); break; \
2596	default: AssertMsgFailed(("ASMAtomicWriteSize: size %d is not supported\n", sizeof(*(pu)))); \
2597	} \
2598	} while (0)
2599
2600
2601
2602	/**
2603	* Atomically exchanges and adds to a 16-bit value, ordered.
2604	*
2605	* @returns The old value.
2606	* @param pu16 Pointer to the value.
2607	* @param u16 Number to add.
2608	*
2609	* @remarks Currently not implemented, just to make 16-bit code happy.
2610	* @remarks x86: Requires a 486 or later.
2611	*/
2612	DECLASM(uint16_t) ASMAtomicAddU16(uint16_t volatile *pu16, uint32_t u16);
2613
2614
2615	/**
2616	* Atomically exchanges and adds to a 32-bit value, ordered.
2617	*
2618	* @returns The old value.
2619	* @param pu32 Pointer to the value.
2620	* @param u32 Number to add.
2621	*
2622	* @remarks x86: Requires a 486 or later.
2623	*/
2624	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2625	DECLASM(uint32_t) ASMAtomicAddU32(uint32_t volatile *pu32, uint32_t u32);
2626	#else
2627	DECLINLINE(uint32_t) ASMAtomicAddU32(uint32_t volatile *pu32, uint32_t u32)
2628	{
2629	# if RT_INLINE_ASM_USES_INTRIN
2630	u32 = _InterlockedExchangeAdd((long *)pu32, u32);
2631	return u32;
2632
2633	# elif RT_INLINE_ASM_GNU_STYLE
2634	__asm__ __volatile__("lock; xaddl %0, %1\n\t"
2635	: "=r" (u32),
2636	"=m" (*pu32)
2637	: "0" (u32),
2638	"m" (*pu32)
2639	: "memory");
2640	return u32;
2641	# else
2642	__asm
2643	{
2644	mov eax, [u32]
2645	# ifdef RT_ARCH_AMD64
2646	mov rdx, [pu32]
2647	lock xadd [rdx], eax
2648	# else
2649	mov edx, [pu32]
2650	lock xadd [edx], eax
2651	# endif
2652	mov [u32], eax
2653	}
2654	return u32;
2655	# endif
2656	}
2657	#endif
2658
2659
2660	/**
2661	* Atomically exchanges and adds to a signed 32-bit value, ordered.
2662	*
2663	* @returns The old value.
2664	* @param pi32 Pointer to the value.
2665	* @param i32 Number to add.
2666	*
2667	* @remarks x86: Requires a 486 or later.
2668	*/
2669	DECLINLINE(int32_t) ASMAtomicAddS32(int32_t volatile *pi32, int32_t i32)
2670	{
2671	return (int32_t)ASMAtomicAddU32((uint32_t volatile *)pi32, (uint32_t)i32);
2672	}
2673
2674
2675	/**
2676	* Atomically exchanges and adds to a 64-bit value, ordered.
2677	*
2678	* @returns The old value.
2679	* @param pu64 Pointer to the value.
2680	* @param u64 Number to add.
2681	*
2682	* @remarks x86: Requires a Pentium or later.
2683	*/
2684	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2685	DECLASM(uint64_t) ASMAtomicAddU64(uint64_t volatile *pu64, uint64_t u64);
2686	#else
2687	DECLINLINE(uint64_t) ASMAtomicAddU64(uint64_t volatile *pu64, uint64_t u64)
2688	{
2689	# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
2690	u64 = _InterlockedExchangeAdd64((__int64 *)pu64, u64);
2691	return u64;
2692
2693	# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
2694	__asm__ __volatile__("lock; xaddq %0, %1\n\t"
2695	: "=r" (u64),
2696	"=m" (*pu64)
2697	: "0" (u64),
2698	"m" (*pu64)
2699	: "memory");
2700	return u64;
2701	# else
2702	uint64_t u64Old;
2703	for (;;)
2704	{
2705	uint64_t u64New;
2706	u64Old = ASMAtomicUoReadU64(pu64);
2707	u64New = u64Old + u64;
2708	if (ASMAtomicCmpXchgU64(pu64, u64New, u64Old))
2709	break;
2710	ASMNopPause();
2711	}
2712	return u64Old;
2713	# endif
2714	}
2715	#endif
2716
2717
2718	/**
2719	* Atomically exchanges and adds to a signed 64-bit value, ordered.
2720	*
2721	* @returns The old value.
2722	* @param pi64 Pointer to the value.
2723	* @param i64 Number to add.
2724	*
2725	* @remarks x86: Requires a Pentium or later.
2726	*/
2727	DECLINLINE(int64_t) ASMAtomicAddS64(int64_t volatile *pi64, int64_t i64)
2728	{
2729	return (int64_t)ASMAtomicAddU64((uint64_t volatile *)pi64, (uint64_t)i64);
2730	}
2731
2732
2733	/**
2734	* Atomically exchanges and adds to a size_t value, ordered.
2735	*
2736	* @returns The old value.
2737	* @param pcb Pointer to the size_t value.
2738	* @param cb Number to add.
2739	*/
2740	DECLINLINE(size_t) ASMAtomicAddZ(size_t volatile *pcb, size_t cb)
2741	{
2742	#if ARCH_BITS == 64
2743	AssertCompileSize(size_t, 8);
2744	return ASMAtomicAddU64((uint64_t volatile *)pcb, cb);
2745	#elif ARCH_BITS == 32
2746	AssertCompileSize(size_t, 4);
2747	return ASMAtomicAddU32((uint32_t volatile *)pcb, cb);
2748	#elif ARCH_BITS == 16
2749	AssertCompileSize(size_t, 2);
2750	return ASMAtomicAddU16((uint16_t volatile *)pcb, cb);
2751	#else
2752	# error "Unsupported ARCH_BITS value"
2753	#endif
2754	}
2755
2756
2757	/**
2758	* Atomically exchanges and adds a value which size might differ between
2759	* platforms or compilers, ordered.
2760	*
2761	* @param pu Pointer to the variable to update.
2762	* @param uNew The value to add to *pu.
2763	* @param puOld Where to store the old value.
2764	*/
2765	#define ASMAtomicAddSize(pu, uNew, puOld) \
2766	do { \
2767	switch (sizeof(*(pu))) { \
2768	case 4: (uint32_t )(puOld) = ASMAtomicAddU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew)); break; \
2769	case 8: (uint64_t )(puOld) = ASMAtomicAddU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew)); break; \
2770	default: AssertMsgFailed(("ASMAtomicAddSize: size %d is not supported\n", sizeof(*(pu)))); \
2771	} \
2772	} while (0)
2773
2774
2775
2776	/**
2777	* Atomically exchanges and subtracts to an unsigned 16-bit value, ordered.
2778	*
2779	* @returns The old value.
2780	* @param pu16 Pointer to the value.
2781	* @param u16 Number to subtract.
2782	*
2783	* @remarks x86: Requires a 486 or later.
2784	*/
2785	DECLINLINE(uint16_t) ASMAtomicSubU16(uint16_t volatile *pu16, uint32_t u16)
2786	{
2787	return ASMAtomicAddU16(pu16, (uint16_t)-(int16_t)u16);
2788	}
2789
2790
2791	/**
2792	* Atomically exchanges and subtracts to a signed 16-bit value, ordered.
2793	*
2794	* @returns The old value.
2795	* @param pi16 Pointer to the value.
2796	* @param i16 Number to subtract.
2797	*
2798	* @remarks x86: Requires a 486 or later.
2799	*/
2800	DECLINLINE(int16_t) ASMAtomicSubS16(int16_t volatile *pi16, int16_t i16)
2801	{
2802	return (int16_t)ASMAtomicAddU16((uint16_t volatile *)pi16, (uint16_t)-i16);
2803	}
2804
2805
2806	/**
2807	* Atomically exchanges and subtracts to an unsigned 32-bit value, ordered.
2808	*
2809	* @returns The old value.
2810	* @param pu32 Pointer to the value.
2811	* @param u32 Number to subtract.
2812	*
2813	* @remarks x86: Requires a 486 or later.
2814	*/
2815	DECLINLINE(uint32_t) ASMAtomicSubU32(uint32_t volatile *pu32, uint32_t u32)
2816	{
2817	return ASMAtomicAddU32(pu32, (uint32_t)-(int32_t)u32);
2818	}
2819
2820
2821	/**
2822	* Atomically exchanges and subtracts to a signed 32-bit value, ordered.
2823	*
2824	* @returns The old value.
2825	* @param pi32 Pointer to the value.
2826	* @param i32 Number to subtract.
2827	*
2828	* @remarks x86: Requires a 486 or later.
2829	*/
2830	DECLINLINE(int32_t) ASMAtomicSubS32(int32_t volatile *pi32, int32_t i32)
2831	{
2832	return (int32_t)ASMAtomicAddU32((uint32_t volatile *)pi32, (uint32_t)-i32);
2833	}
2834
2835
2836	/**
2837	* Atomically exchanges and subtracts to an unsigned 64-bit value, ordered.
2838	*
2839	* @returns The old value.
2840	* @param pu64 Pointer to the value.
2841	* @param u64 Number to subtract.
2842	*
2843	* @remarks x86: Requires a Pentium or later.
2844	*/
2845	DECLINLINE(uint64_t) ASMAtomicSubU64(uint64_t volatile *pu64, uint64_t u64)
2846	{
2847	return ASMAtomicAddU64(pu64, (uint64_t)-(int64_t)u64);
2848	}
2849
2850
2851	/**
2852	* Atomically exchanges and subtracts to a signed 64-bit value, ordered.
2853	*
2854	* @returns The old value.
2855	* @param pi64 Pointer to the value.
2856	* @param i64 Number to subtract.
2857	*
2858	* @remarks x86: Requires a Pentium or later.
2859	*/
2860	DECLINLINE(int64_t) ASMAtomicSubS64(int64_t volatile *pi64, int64_t i64)
2861	{
2862	return (int64_t)ASMAtomicAddU64((uint64_t volatile *)pi64, (uint64_t)-i64);
2863	}
2864
2865
2866	/**
2867	* Atomically exchanges and subtracts to a size_t value, ordered.
2868	*
2869	* @returns The old value.
2870	* @param pcb Pointer to the size_t value.
2871	* @param cb Number to subtract.
2872	*
2873	* @remarks x86: Requires a 486 or later.
2874	*/
2875	DECLINLINE(size_t) ASMAtomicSubZ(size_t volatile *pcb, size_t cb)
2876	{
2877	#if ARCH_BITS == 64
2878	return ASMAtomicSubU64((uint64_t volatile *)pcb, cb);
2879	#elif ARCH_BITS == 32
2880	return ASMAtomicSubU32((uint32_t volatile *)pcb, cb);
2881	#elif ARCH_BITS == 16
2882	AssertCompileSize(size_t, 2);
2883	return ASMAtomicSubU16((uint16_t volatile *)pcb, cb);
2884	#else
2885	# error "Unsupported ARCH_BITS value"
2886	#endif
2887	}
2888
2889
2890	/**
2891	* Atomically exchanges and subtracts a value which size might differ between
2892	* platforms or compilers, ordered.
2893	*
2894	* @param pu Pointer to the variable to update.
2895	* @param uNew The value to subtract to *pu.
2896	* @param puOld Where to store the old value.
2897	*
2898	* @remarks x86: Requires a 486 or later.
2899	*/
2900	#define ASMAtomicSubSize(pu, uNew, puOld) \
2901	do { \
2902	switch (sizeof(*(pu))) { \
2903	case 4: (uint32_t )(puOld) = ASMAtomicSubU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew)); break; \
2904	case 8: (uint64_t )(puOld) = ASMAtomicSubU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew)); break; \
2905	default: AssertMsgFailed(("ASMAtomicSubSize: size %d is not supported\n", sizeof(*(pu)))); \
2906	} \
2907	} while (0)
2908
2909
2910
2911	/**
2912	* Atomically increment a 16-bit value, ordered.
2913	*
2914	* @returns The new value.
2915	* @param pu16 Pointer to the value to increment.
2916	* @remarks Not implemented. Just to make 16-bit code happy.
2917	*
2918	* @remarks x86: Requires a 486 or later.
2919	*/
2920	DECLASM(uint16_t) ASMAtomicIncU16(uint16_t volatile *pu16);
2921
2922
2923	/**
2924	* Atomically increment a 32-bit value, ordered.
2925	*
2926	* @returns The new value.
2927	* @param pu32 Pointer to the value to increment.
2928	*
2929	* @remarks x86: Requires a 486 or later.
2930	*/
2931	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2932	DECLASM(uint32_t) ASMAtomicIncU32(uint32_t volatile *pu32);
2933	#else
2934	DECLINLINE(uint32_t) ASMAtomicIncU32(uint32_t volatile *pu32)
2935	{
2936	uint32_t u32;
2937	# if RT_INLINE_ASM_USES_INTRIN
2938	u32 = _InterlockedIncrement((long *)pu32);
2939	return u32;
2940
2941	# elif RT_INLINE_ASM_GNU_STYLE
2942	__asm__ __volatile__("lock; xaddl %0, %1\n\t"
2943	: "=r" (u32),
2944	"=m" (*pu32)
2945	: "0" (1),
2946	"m" (*pu32)
2947	: "memory");
2948	return u32+1;
2949	# else
2950	__asm
2951	{
2952	mov eax, 1
2953	# ifdef RT_ARCH_AMD64
2954	mov rdx, [pu32]
2955	lock xadd [rdx], eax
2956	# else
2957	mov edx, [pu32]
2958	lock xadd [edx], eax
2959	# endif
2960	mov u32, eax
2961	}
2962	return u32+1;
2963	# endif
2964	}
2965	#endif
2966
2967
2968	/**
2969	* Atomically increment a signed 32-bit value, ordered.
2970	*
2971	* @returns The new value.
2972	* @param pi32 Pointer to the value to increment.
2973	*
2974	* @remarks x86: Requires a 486 or later.
2975	*/
2976	DECLINLINE(int32_t) ASMAtomicIncS32(int32_t volatile *pi32)
2977	{
2978	return (int32_t)ASMAtomicIncU32((uint32_t volatile *)pi32);
2979	}
2980
2981
2982	/**
2983	* Atomically increment a 64-bit value, ordered.
2984	*
2985	* @returns The new value.
2986	* @param pu64 Pointer to the value to increment.
2987	*
2988	* @remarks x86: Requires a Pentium or later.
2989	*/
2990	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2991	DECLASM(uint64_t) ASMAtomicIncU64(uint64_t volatile *pu64);
2992	#else
2993	DECLINLINE(uint64_t) ASMAtomicIncU64(uint64_t volatile *pu64)
2994	{
2995	# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
2996	uint64_t u64;
2997	u64 = _InterlockedIncrement64((__int64 *)pu64);
2998	return u64;
2999
3000	# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
3001	uint64_t u64;
3002	__asm__ __volatile__("lock; xaddq %0, %1\n\t"
3003	: "=r" (u64),
3004	"=m" (*pu64)
3005	: "0" (1),
3006	"m" (*pu64)
3007	: "memory");
3008	return u64 + 1;
3009	# else
3010	return ASMAtomicAddU64(pu64, 1) + 1;
3011	# endif
3012	}
3013	#endif
3014
3015
3016	/**
3017	* Atomically increment a signed 64-bit value, ordered.
3018	*
3019	* @returns The new value.
3020	* @param pi64 Pointer to the value to increment.
3021	*
3022	* @remarks x86: Requires a Pentium or later.
3023	*/
3024	DECLINLINE(int64_t) ASMAtomicIncS64(int64_t volatile *pi64)
3025	{
3026	return (int64_t)ASMAtomicIncU64((uint64_t volatile *)pi64);
3027	}
3028
3029
3030	/**
3031	* Atomically increment a size_t value, ordered.
3032	*
3033	* @returns The new value.
3034	* @param pcb Pointer to the value to increment.
3035	*
3036	* @remarks x86: Requires a 486 or later.
3037	*/
3038	DECLINLINE(int64_t) ASMAtomicIncZ(size_t volatile *pcb)
3039	{
3040	#if ARCH_BITS == 64
3041	return ASMAtomicIncU64((uint64_t volatile *)pcb);
3042	#elif ARCH_BITS == 32
3043	return ASMAtomicIncU32((uint32_t volatile *)pcb);
3044	#elif ARCH_BITS == 16
3045	return ASMAtomicIncU16((uint16_t volatile *)pcb);
3046	#else
3047	# error "Unsupported ARCH_BITS value"
3048	#endif
3049	}
3050
3051
3052
3053	/**
3054	* Atomically decrement an unsigned 32-bit value, ordered.
3055	*
3056	* @returns The new value.
3057	* @param pu16 Pointer to the value to decrement.
3058	* @remarks Not implemented. Just to make 16-bit code happy.
3059	*
3060	* @remarks x86: Requires a 486 or later.
3061	*/
3062	DECLASM(uint32_t) ASMAtomicDecU16(uint16_t volatile *pu16);
3063
3064
3065	/**
3066	* Atomically decrement an unsigned 32-bit value, ordered.
3067	*
3068	* @returns The new value.
3069	* @param pu32 Pointer to the value to decrement.
3070	*
3071	* @remarks x86: Requires a 486 or later.
3072	*/
3073	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3074	DECLASM(uint32_t) ASMAtomicDecU32(uint32_t volatile *pu32);
3075	#else
3076	DECLINLINE(uint32_t) ASMAtomicDecU32(uint32_t volatile *pu32)
3077	{
3078	uint32_t u32;
3079	# if RT_INLINE_ASM_USES_INTRIN
3080	u32 = _InterlockedDecrement((long *)pu32);
3081	return u32;
3082
3083	# elif RT_INLINE_ASM_GNU_STYLE
3084	__asm__ __volatile__("lock; xaddl %0, %1\n\t"
3085	: "=r" (u32),
3086	"=m" (*pu32)
3087	: "0" (-1),
3088	"m" (*pu32)
3089	: "memory");
3090	return u32-1;
3091	# else
3092	__asm
3093	{
3094	mov eax, -1
3095	# ifdef RT_ARCH_AMD64
3096	mov rdx, [pu32]
3097	lock xadd [rdx], eax
3098	# else
3099	mov edx, [pu32]
3100	lock xadd [edx], eax
3101	# endif
3102	mov u32, eax
3103	}
3104	return u32-1;
3105	# endif
3106	}
3107	#endif
3108
3109
3110	/**
3111	* Atomically decrement a signed 32-bit value, ordered.
3112	*
3113	* @returns The new value.
3114	* @param pi32 Pointer to the value to decrement.
3115	*
3116	* @remarks x86: Requires a 486 or later.
3117	*/
3118	DECLINLINE(int32_t) ASMAtomicDecS32(int32_t volatile *pi32)
3119	{
3120	return (int32_t)ASMAtomicDecU32((uint32_t volatile *)pi32);
3121	}
3122
3123
3124	/**
3125	* Atomically decrement an unsigned 64-bit value, ordered.
3126	*
3127	* @returns The new value.
3128	* @param pu64 Pointer to the value to decrement.
3129	*
3130	* @remarks x86: Requires a Pentium or later.
3131	*/
3132	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3133	DECLASM(uint64_t) ASMAtomicDecU64(uint64_t volatile *pu64);
3134	#else
3135	DECLINLINE(uint64_t) ASMAtomicDecU64(uint64_t volatile *pu64)
3136	{
3137	# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
3138	uint64_t u64 = _InterlockedDecrement64((__int64 volatile *)pu64);
3139	return u64;
3140
3141	# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
3142	uint64_t u64;
3143	__asm__ __volatile__("lock; xaddq %q0, %1\n\t"
3144	: "=r" (u64),
3145	"=m" (*pu64)
3146	: "0" (~(uint64_t)0),
3147	"m" (*pu64)
3148	: "memory");
3149	return u64-1;
3150	# else
3151	return ASMAtomicAddU64(pu64, UINT64_MAX) - 1;
3152	# endif
3153	}
3154	#endif
3155
3156
3157	/**
3158	* Atomically decrement a signed 64-bit value, ordered.
3159	*
3160	* @returns The new value.
3161	* @param pi64 Pointer to the value to decrement.
3162	*
3163	* @remarks x86: Requires a Pentium or later.
3164	*/
3165	DECLINLINE(int64_t) ASMAtomicDecS64(int64_t volatile *pi64)
3166	{
3167	return (int64_t)ASMAtomicDecU64((uint64_t volatile *)pi64);
3168	}
3169
3170
3171	/**
3172	* Atomically decrement a size_t value, ordered.
3173	*
3174	* @returns The new value.
3175	* @param pcb Pointer to the value to decrement.
3176	*
3177	* @remarks x86: Requires a 486 or later.
3178	*/
3179	DECLINLINE(int64_t) ASMAtomicDecZ(size_t volatile *pcb)
3180	{
3181	#if ARCH_BITS == 64
3182	return ASMAtomicDecU64((uint64_t volatile *)pcb);
3183	#elif ARCH_BITS == 32
3184	return ASMAtomicDecU32((uint32_t volatile *)pcb);
3185	#elif ARCH_BITS == 16
3186	return ASMAtomicDecU16((uint16_t volatile *)pcb);
3187	#else
3188	# error "Unsupported ARCH_BITS value"
3189	#endif
3190	}
3191
3192
3193	/**
3194	* Atomically Or an unsigned 32-bit value, ordered.
3195	*
3196	* @param pu32 Pointer to the pointer variable to OR u32 with.
3197	* @param u32 The value to OR *pu32 with.
3198	*
3199	* @remarks x86: Requires a 386 or later.
3200	*/
3201	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3202	DECLASM(void) ASMAtomicOrU32(uint32_t volatile *pu32, uint32_t u32);
3203	#else
3204	DECLINLINE(void) ASMAtomicOrU32(uint32_t volatile *pu32, uint32_t u32)
3205	{
3206	# if RT_INLINE_ASM_USES_INTRIN
3207	_InterlockedOr((long volatile *)pu32, (long)u32);
3208
3209	# elif RT_INLINE_ASM_GNU_STYLE
3210	__asm__ __volatile__("lock; orl %1, %0\n\t"
3211	: "=m" (*pu32)
3212	: "ir" (u32),
3213	"m" (*pu32));
3214	# else
3215	__asm
3216	{
3217	mov eax, [u32]
3218	# ifdef RT_ARCH_AMD64
3219	mov rdx, [pu32]
3220	lock or [rdx], eax
3221	# else
3222	mov edx, [pu32]
3223	lock or [edx], eax
3224	# endif
3225	}
3226	# endif
3227	}
3228	#endif
3229
3230
3231	/**
3232	* Atomically Or a signed 32-bit value, ordered.
3233	*
3234	* @param pi32 Pointer to the pointer variable to OR u32 with.
3235	* @param i32 The value to OR *pu32 with.
3236	*
3237	* @remarks x86: Requires a 386 or later.
3238	*/
3239	DECLINLINE(void) ASMAtomicOrS32(int32_t volatile *pi32, int32_t i32)
3240	{
3241	ASMAtomicOrU32((uint32_t volatile *)pi32, i32);
3242	}
3243
3244
3245	/**
3246	* Atomically Or an unsigned 64-bit value, ordered.
3247	*
3248	* @param pu64 Pointer to the pointer variable to OR u64 with.
3249	* @param u64 The value to OR *pu64 with.
3250	*
3251	* @remarks x86: Requires a Pentium or later.
3252	*/
3253	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3254	DECLASM(void) ASMAtomicOrU64(uint64_t volatile *pu64, uint64_t u64);
3255	#else
3256	DECLINLINE(void) ASMAtomicOrU64(uint64_t volatile *pu64, uint64_t u64)
3257	{
3258	# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
3259	_InterlockedOr64((__int64 volatile *)pu64, (__int64)u64);
3260
3261	# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
3262	__asm__ __volatile__("lock; orq %1, %q0\n\t"
3263	: "=m" (*pu64)
3264	: "r" (u64),
3265	"m" (*pu64));
3266	# else
3267	for (;;)
3268	{
3269	uint64_t u64Old = ASMAtomicUoReadU64(pu64);
3270	uint64_t u64New = u64Old \| u64;
3271	if (ASMAtomicCmpXchgU64(pu64, u64New, u64Old))
3272	break;
3273	ASMNopPause();
3274	}
3275	# endif
3276	}
3277	#endif
3278
3279
3280	/**
3281	* Atomically Or a signed 64-bit value, ordered.
3282	*
3283	* @param pi64 Pointer to the pointer variable to OR u64 with.
3284	* @param i64 The value to OR *pu64 with.
3285	*
3286	* @remarks x86: Requires a Pentium or later.
3287	*/
3288	DECLINLINE(void) ASMAtomicOrS64(int64_t volatile *pi64, int64_t i64)
3289	{
3290	ASMAtomicOrU64((uint64_t volatile *)pi64, i64);
3291	}
3292
3293
3294	/**
3295	* Atomically And an unsigned 32-bit value, ordered.
3296	*
3297	* @param pu32 Pointer to the pointer variable to AND u32 with.
3298	* @param u32 The value to AND *pu32 with.
3299	*
3300	* @remarks x86: Requires a 386 or later.
3301	*/
3302	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3303	DECLASM(void) ASMAtomicAndU32(uint32_t volatile *pu32, uint32_t u32);
3304	#else
3305	DECLINLINE(void) ASMAtomicAndU32(uint32_t volatile *pu32, uint32_t u32)
3306	{
3307	# if RT_INLINE_ASM_USES_INTRIN
3308	_InterlockedAnd((long volatile *)pu32, u32);
3309
3310	# elif RT_INLINE_ASM_GNU_STYLE
3311	__asm__ __volatile__("lock; andl %1, %0\n\t"
3312	: "=m" (*pu32)
3313	: "ir" (u32),
3314	"m" (*pu32));
3315	# else
3316	__asm
3317	{
3318	mov eax, [u32]
3319	# ifdef RT_ARCH_AMD64
3320	mov rdx, [pu32]
3321	lock and [rdx], eax
3322	# else
3323	mov edx, [pu32]
3324	lock and [edx], eax
3325	# endif
3326	}
3327	# endif
3328	}
3329	#endif
3330
3331
3332	/**
3333	* Atomically And a signed 32-bit value, ordered.
3334	*
3335	* @param pi32 Pointer to the pointer variable to AND i32 with.
3336	* @param i32 The value to AND *pi32 with.
3337	*
3338	* @remarks x86: Requires a 386 or later.
3339	*/
3340	DECLINLINE(void) ASMAtomicAndS32(int32_t volatile *pi32, int32_t i32)
3341	{
3342	ASMAtomicAndU32((uint32_t volatile *)pi32, (uint32_t)i32);
3343	}
3344
3345
3346	/**
3347	* Atomically And an unsigned 64-bit value, ordered.
3348	*
3349	* @param pu64 Pointer to the pointer variable to AND u64 with.
3350	* @param u64 The value to AND *pu64 with.
3351	*
3352	* @remarks x86: Requires a Pentium or later.
3353	*/
3354	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3355	DECLASM(void) ASMAtomicAndU64(uint64_t volatile *pu64, uint64_t u64);
3356	#else
3357	DECLINLINE(void) ASMAtomicAndU64(uint64_t volatile *pu64, uint64_t u64)
3358	{
3359	# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
3360	_InterlockedAnd64((__int64 volatile *)pu64, u64);
3361
3362	# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
3363	__asm__ __volatile__("lock; andq %1, %0\n\t"
3364	: "=m" (*pu64)
3365	: "r" (u64),
3366	"m" (*pu64));
3367	# else
3368	for (;;)
3369	{
3370	uint64_t u64Old = ASMAtomicUoReadU64(pu64);
3371	uint64_t u64New = u64Old & u64;
3372	if (ASMAtomicCmpXchgU64(pu64, u64New, u64Old))
3373	break;
3374	ASMNopPause();
3375	}
3376	# endif
3377	}
3378	#endif
3379
3380
3381	/**
3382	* Atomically And a signed 64-bit value, ordered.
3383	*
3384	* @param pi64 Pointer to the pointer variable to AND i64 with.
3385	* @param i64 The value to AND *pi64 with.
3386	*
3387	* @remarks x86: Requires a Pentium or later.
3388	*/
3389	DECLINLINE(void) ASMAtomicAndS64(int64_t volatile *pi64, int64_t i64)
3390	{
3391	ASMAtomicAndU64((uint64_t volatile *)pi64, (uint64_t)i64);
3392	}
3393
3394
3395	/**
3396	* Atomically OR an unsigned 32-bit value, unordered but interrupt safe.
3397	*
3398	* @param pu32 Pointer to the pointer variable to OR u32 with.
3399	* @param u32 The value to OR *pu32 with.
3400	*
3401	* @remarks x86: Requires a 386 or later.
3402	*/
3403	#if RT_INLINE_ASM_EXTERNAL
3404	DECLASM(void) ASMAtomicUoOrU32(uint32_t volatile *pu32, uint32_t u32);
3405	#else
3406	DECLINLINE(void) ASMAtomicUoOrU32(uint32_t volatile *pu32, uint32_t u32)
3407	{
3408	# if RT_INLINE_ASM_GNU_STYLE
3409	__asm__ __volatile__("orl %1, %0\n\t"
3410	: "=m" (*pu32)
3411	: "ir" (u32),
3412	"m" (*pu32));
3413	# else
3414	__asm
3415	{
3416	mov eax, [u32]
3417	# ifdef RT_ARCH_AMD64
3418	mov rdx, [pu32]
3419	or [rdx], eax
3420	# else
3421	mov edx, [pu32]
3422	or [edx], eax
3423	# endif
3424	}
3425	# endif
3426	}
3427	#endif
3428
3429
3430	/**
3431	* Atomically OR a signed 32-bit value, unordered.
3432	*
3433	* @param pi32 Pointer to the pointer variable to OR u32 with.
3434	* @param i32 The value to OR *pu32 with.
3435	*
3436	* @remarks x86: Requires a 386 or later.
3437	*/
3438	DECLINLINE(void) ASMAtomicUoOrS32(int32_t volatile *pi32, int32_t i32)
3439	{
3440	ASMAtomicUoOrU32((uint32_t volatile *)pi32, i32);
3441	}
3442
3443
3444	/**
3445	* Atomically OR an unsigned 64-bit value, unordered.
3446	*
3447	* @param pu64 Pointer to the pointer variable to OR u64 with.
3448	* @param u64 The value to OR *pu64 with.
3449	*
3450	* @remarks x86: Requires a Pentium or later.
3451	*/
3452	#if RT_INLINE_ASM_EXTERNAL
3453	DECLASM(void) ASMAtomicUoOrU64(uint64_t volatile *pu64, uint64_t u64);
3454	#else
3455	DECLINLINE(void) ASMAtomicUoOrU64(uint64_t volatile *pu64, uint64_t u64)
3456	{
3457	# if RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
3458	__asm__ __volatile__("orq %1, %q0\n\t"
3459	: "=m" (*pu64)
3460	: "r" (u64),
3461	"m" (*pu64));
3462	# else
3463	for (;;)
3464	{
3465	uint64_t u64Old = ASMAtomicUoReadU64(pu64);
3466	uint64_t u64New = u64Old \| u64;
3467	if (ASMAtomicCmpXchgU64(pu64, u64New, u64Old))
3468	break;
3469	ASMNopPause();
3470	}
3471	# endif
3472	}
3473	#endif
3474
3475
3476	/**
3477	* Atomically Or a signed 64-bit value, unordered.
3478	*
3479	* @param pi64 Pointer to the pointer variable to OR u64 with.
3480	* @param i64 The value to OR *pu64 with.
3481	*
3482	* @remarks x86: Requires a Pentium or later.
3483	*/
3484	DECLINLINE(void) ASMAtomicUoOrS64(int64_t volatile *pi64, int64_t i64)
3485	{
3486	ASMAtomicUoOrU64((uint64_t volatile *)pi64, i64);
3487	}
3488
3489
3490	/**
3491	* Atomically And an unsigned 32-bit value, unordered.
3492	*
3493	* @param pu32 Pointer to the pointer variable to AND u32 with.
3494	* @param u32 The value to AND *pu32 with.
3495	*
3496	* @remarks x86: Requires a 386 or later.
3497	*/
3498	#if RT_INLINE_ASM_EXTERNAL
3499	DECLASM(void) ASMAtomicUoAndU32(uint32_t volatile *pu32, uint32_t u32);
3500	#else
3501	DECLINLINE(void) ASMAtomicUoAndU32(uint32_t volatile *pu32, uint32_t u32)
3502	{
3503	# if RT_INLINE_ASM_GNU_STYLE
3504	__asm__ __volatile__("andl %1, %0\n\t"
3505	: "=m" (*pu32)
3506	: "ir" (u32),
3507	"m" (*pu32));
3508	# else
3509	__asm
3510	{
3511	mov eax, [u32]
3512	# ifdef RT_ARCH_AMD64
3513	mov rdx, [pu32]
3514	and [rdx], eax
3515	# else
3516	mov edx, [pu32]
3517	and [edx], eax
3518	# endif
3519	}
3520	# endif
3521	}
3522	#endif
3523
3524
3525	/**
3526	* Atomically And a signed 32-bit value, unordered.
3527	*
3528	* @param pi32 Pointer to the pointer variable to AND i32 with.
3529	* @param i32 The value to AND *pi32 with.
3530	*
3531	* @remarks x86: Requires a 386 or later.
3532	*/
3533	DECLINLINE(void) ASMAtomicUoAndS32(int32_t volatile *pi32, int32_t i32)
3534	{
3535	ASMAtomicUoAndU32((uint32_t volatile *)pi32, (uint32_t)i32);
3536	}
3537
3538
3539	/**
3540	* Atomically And an unsigned 64-bit value, unordered.
3541	*
3542	* @param pu64 Pointer to the pointer variable to AND u64 with.
3543	* @param u64 The value to AND *pu64 with.
3544	*
3545	* @remarks x86: Requires a Pentium or later.
3546	*/
3547	#if RT_INLINE_ASM_EXTERNAL
3548	DECLASM(void) ASMAtomicUoAndU64(uint64_t volatile *pu64, uint64_t u64);
3549	#else
3550	DECLINLINE(void) ASMAtomicUoAndU64(uint64_t volatile *pu64, uint64_t u64)
3551	{
3552	# if RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
3553	__asm__ __volatile__("andq %1, %0\n\t"
3554	: "=m" (*pu64)
3555	: "r" (u64),
3556	"m" (*pu64));
3557	# else
3558	for (;;)
3559	{
3560	uint64_t u64Old = ASMAtomicUoReadU64(pu64);
3561	uint64_t u64New = u64Old & u64;
3562	if (ASMAtomicCmpXchgU64(pu64, u64New, u64Old))
3563	break;
3564	ASMNopPause();
3565	}
3566	# endif
3567	}
3568	#endif
3569
3570
3571	/**
3572	* Atomically And a signed 64-bit value, unordered.
3573	*
3574	* @param pi64 Pointer to the pointer variable to AND i64 with.
3575	* @param i64 The value to AND *pi64 with.
3576	*
3577	* @remarks x86: Requires a Pentium or later.
3578	*/
3579	DECLINLINE(void) ASMAtomicUoAndS64(int64_t volatile *pi64, int64_t i64)
3580	{
3581	ASMAtomicUoAndU64((uint64_t volatile *)pi64, (uint64_t)i64);
3582	}
3583
3584
3585	/**
3586	* Atomically increment an unsigned 32-bit value, unordered.
3587	*
3588	* @returns the new value.
3589	* @param pu32 Pointer to the variable to increment.
3590	*
3591	* @remarks x86: Requires a 486 or later.
3592	*/
3593	#if RT_INLINE_ASM_EXTERNAL
3594	DECLASM(uint32_t) ASMAtomicUoIncU32(uint32_t volatile *pu32);
3595	#else
3596	DECLINLINE(uint32_t) ASMAtomicUoIncU32(uint32_t volatile *pu32)
3597	{
3598	uint32_t u32;
3599	# if RT_INLINE_ASM_GNU_STYLE
3600	__asm__ __volatile__("xaddl %0, %1\n\t"
3601	: "=r" (u32),
3602	"=m" (*pu32)
3603	: "0" (1),
3604	"m" (*pu32)
3605	: "memory");
3606	return u32 + 1;
3607	# else
3608	__asm
3609	{
3610	mov eax, 1
3611	# ifdef RT_ARCH_AMD64
3612	mov rdx, [pu32]
3613	xadd [rdx], eax
3614	# else
3615	mov edx, [pu32]
3616	xadd [edx], eax
3617	# endif
3618	mov u32, eax
3619	}
3620	return u32 + 1;
3621	# endif
3622	}
3623	#endif
3624
3625
3626	/**
3627	* Atomically decrement an unsigned 32-bit value, unordered.
3628	*
3629	* @returns the new value.
3630	* @param pu32 Pointer to the variable to decrement.
3631	*
3632	* @remarks x86: Requires a 486 or later.
3633	*/
3634	#if RT_INLINE_ASM_EXTERNAL
3635	DECLASM(uint32_t) ASMAtomicUoDecU32(uint32_t volatile *pu32);
3636	#else
3637	DECLINLINE(uint32_t) ASMAtomicUoDecU32(uint32_t volatile *pu32)
3638	{
3639	uint32_t u32;
3640	# if RT_INLINE_ASM_GNU_STYLE
3641	__asm__ __volatile__("lock; xaddl %0, %1\n\t"
3642	: "=r" (u32),
3643	"=m" (*pu32)
3644	: "0" (-1),
3645	"m" (*pu32)
3646	: "memory");
3647	return u32 - 1;
3648	# else
3649	__asm
3650	{
3651	mov eax, -1
3652	# ifdef RT_ARCH_AMD64
3653	mov rdx, [pu32]
3654	xadd [rdx], eax
3655	# else
3656	mov edx, [pu32]
3657	xadd [edx], eax
3658	# endif
3659	mov u32, eax
3660	}
3661	return u32 - 1;
3662	# endif
3663	}
3664	#endif
3665
3666
3667	/** @def RT_ASM_PAGE_SIZE
3668	* We try avoid dragging in iprt/param.h here.
3669	* @internal
3670	*/
3671	#if defined(RT_ARCH_SPARC64)
3672	# define RT_ASM_PAGE_SIZE 0x2000
3673	# if defined(PAGE_SIZE) && !defined(NT_INCLUDED)
3674	# if PAGE_SIZE != 0x2000
3675	# error "PAGE_SIZE is not 0x2000!"
3676	# endif
3677	# endif
3678	#else
3679	# define RT_ASM_PAGE_SIZE 0x1000
3680	# if defined(PAGE_SIZE) && !defined(NT_INCLUDED)
3681	# if PAGE_SIZE != 0x1000
3682	# error "PAGE_SIZE is not 0x1000!"
3683	# endif
3684	# endif
3685	#endif
3686
3687	/**
3688	* Zeros a 4K memory page.
3689	*
3690	* @param pv Pointer to the memory block. This must be page aligned.
3691	*/
3692	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3693	DECLASM(void) ASMMemZeroPage(volatile void *pv);
3694	# else
3695	DECLINLINE(void) ASMMemZeroPage(volatile void *pv)
3696	{
3697	# if RT_INLINE_ASM_USES_INTRIN
3698	# ifdef RT_ARCH_AMD64
3699	__stosq((unsigned __int64 *)pv, 0, RT_ASM_PAGE_SIZE / 8);
3700	# else
3701	__stosd((unsigned long *)pv, 0, RT_ASM_PAGE_SIZE / 4);
3702	# endif
3703
3704	# elif RT_INLINE_ASM_GNU_STYLE
3705	RTCCUINTREG uDummy;
3706	# ifdef RT_ARCH_AMD64
3707	__asm__ __volatile__("rep stosq"
3708	: "=D" (pv),
3709	"=c" (uDummy)
3710	: "0" (pv),
3711	"c" (RT_ASM_PAGE_SIZE >> 3),
3712	"a" (0)
3713	: "memory");
3714	# else
3715	__asm__ __volatile__("rep stosl"
3716	: "=D" (pv),
3717	"=c" (uDummy)
3718	: "0" (pv),
3719	"c" (RT_ASM_PAGE_SIZE >> 2),
3720	"a" (0)
3721	: "memory");
3722	# endif
3723	# else
3724	__asm
3725	{
3726	# ifdef RT_ARCH_AMD64
3727	xor rax, rax
3728	mov ecx, 0200h
3729	mov rdi, [pv]
3730	rep stosq
3731	# else
3732	xor eax, eax
3733	mov ecx, 0400h
3734	mov edi, [pv]
3735	rep stosd
3736	# endif
3737	}
3738	# endif
3739	}
3740	# endif
3741
3742
3743	/**
3744	* Zeros a memory block with a 32-bit aligned size.
3745	*
3746	* @param pv Pointer to the memory block.
3747	* @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
3748	*/
3749	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3750	DECLASM(void) ASMMemZero32(volatile void *pv, size_t cb);
3751	#else
3752	DECLINLINE(void) ASMMemZero32(volatile void *pv, size_t cb)
3753	{
3754	# if RT_INLINE_ASM_USES_INTRIN
3755	# ifdef RT_ARCH_AMD64
3756	if (!(cb & 7))
3757	__stosq((unsigned __int64 *)pv, 0, cb / 8);
3758	else
3759	# endif
3760	__stosd((unsigned long *)pv, 0, cb / 4);
3761
3762	# elif RT_INLINE_ASM_GNU_STYLE
3763	__asm__ __volatile__("rep stosl"
3764	: "=D" (pv),
3765	"=c" (cb)
3766	: "0" (pv),
3767	"1" (cb >> 2),
3768	"a" (0)
3769	: "memory");
3770	# else
3771	__asm
3772	{
3773	xor eax, eax
3774	# ifdef RT_ARCH_AMD64
3775	mov rcx, [cb]
3776	shr rcx, 2
3777	mov rdi, [pv]
3778	# else
3779	mov ecx, [cb]
3780	shr ecx, 2
3781	mov edi, [pv]
3782	# endif
3783	rep stosd
3784	}
3785	# endif
3786	}
3787	#endif
3788
3789
3790	/**
3791	* Fills a memory block with a 32-bit aligned size.
3792	*
3793	* @param pv Pointer to the memory block.
3794	* @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
3795	* @param u32 The value to fill with.
3796	*/
3797	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3798	DECLASM(void) ASMMemFill32(volatile void *pv, size_t cb, uint32_t u32);
3799	#else
3800	DECLINLINE(void) ASMMemFill32(volatile void *pv, size_t cb, uint32_t u32)
3801	{
3802	# if RT_INLINE_ASM_USES_INTRIN
3803	# ifdef RT_ARCH_AMD64
3804	if (!(cb & 7))
3805	__stosq((unsigned __int64 *)pv, RT_MAKE_U64(u32, u32), cb / 8);
3806	else
3807	# endif
3808	__stosd((unsigned long *)pv, u32, cb / 4);
3809
3810	# elif RT_INLINE_ASM_GNU_STYLE
3811	__asm__ __volatile__("rep stosl"
3812	: "=D" (pv),
3813	"=c" (cb)
3814	: "0" (pv),
3815	"1" (cb >> 2),
3816	"a" (u32)
3817	: "memory");
3818	# else
3819	__asm
3820	{
3821	# ifdef RT_ARCH_AMD64
3822	mov rcx, [cb]
3823	shr rcx, 2
3824	mov rdi, [pv]
3825	# else
3826	mov ecx, [cb]
3827	shr ecx, 2
3828	mov edi, [pv]
3829	# endif
3830	mov eax, [u32]
3831	rep stosd
3832	}
3833	# endif
3834	}
3835	#endif
3836
3837
3838	/**
3839	* Checks if a memory block is all zeros.
3840	*
3841	* @returns Pointer to the first non-zero byte.
3842	* @returns NULL if all zero.
3843	*
3844	* @param pv Pointer to the memory block.
3845	* @param cb Number of bytes in the block.
3846	*
3847	* @todo Fix name, it is a predicate function but it's not returning boolean!
3848	*/
3849	#if !defined(RT_OS_LINUX) \|\| !defined(__KERNEL__)
3850	DECLASM(void ) ASMMemFirstNonZero(void const pv, size_t cb);
3851	#else
3852	DECLINLINE(void ) ASMMemFirstNonZero(void const pv, size_t cb)
3853	{
3854	uint8_t const pb = (uint8_t const )pv;
3855	for (; cb; cb--, pb++)
3856	if (RT_LIKELY(*pb == 0))
3857	{ /* likely */ }
3858	else
3859	return (void *)pb;
3860	return NULL;
3861	}
3862	#endif
3863
3864
3865	/**
3866	* Checks if a memory block is all zeros.
3867	*
3868	* @returns true if zero, false if not.
3869	*
3870	* @param pv Pointer to the memory block.
3871	* @param cb Number of bytes in the block.
3872	*
3873	* @sa ASMMemFirstNonZero
3874	*/
3875	DECLINLINE(bool) ASMMemIsZero(void const *pv, size_t cb)
3876	{
3877	return ASMMemFirstNonZero(pv, cb) == NULL;
3878	}
3879
3880
3881	/**
3882	* Checks if a memory page is all zeros.
3883	*
3884	* @returns true / false.
3885	*
3886	* @param pvPage Pointer to the page. Must be aligned on 16 byte
3887	* boundary
3888	*/
3889	DECLINLINE(bool) ASMMemIsZeroPage(void const *pvPage)
3890	{
3891	# if 0 /RT_INLINE_ASM_GNU_STYLE - this is actually slower... /
3892	union { RTCCUINTREG r; bool f; } uAX;
3893	RTCCUINTREG xCX, xDI;
3894	Assert(!((uintptr_t)pvPage & 15));
3895	__asm__ __volatile__("repe; "
3896	# ifdef RT_ARCH_AMD64
3897	"scasq\n\t"
3898	# else
3899	"scasl\n\t"
3900	# endif
3901	"setnc %%al\n\t"
3902	: "=&c" (xCX),
3903	"=&D" (xDI),
3904	"=&a" (uAX.r)
3905	: "mr" (pvPage),
3906	# ifdef RT_ARCH_AMD64
3907	"0" (RT_ASM_PAGE_SIZE/8),
3908	# else
3909	"0" (RT_ASM_PAGE_SIZE/4),
3910	# endif
3911	"1" (pvPage),
3912	"2" (0));
3913	return uAX.f;
3914	# else
3915	uintptr_t const puPtr = (uintptr_t const )pvPage;
3916	int cLeft = RT_ASM_PAGE_SIZE / sizeof(uintptr_t) / 8;
3917	Assert(!((uintptr_t)pvPage & 15));
3918	for (;;)
3919	{
3920	if (puPtr[0]) return false;
3921	if (puPtr[4]) return false;
3922
3923	if (puPtr[2]) return false;
3924	if (puPtr[6]) return false;
3925
3926	if (puPtr[1]) return false;
3927	if (puPtr[5]) return false;
3928
3929	if (puPtr[3]) return false;
3930	if (puPtr[7]) return false;
3931
3932	if (!--cLeft)
3933	return true;
3934	puPtr += 8;
3935	}
3936	return true;
3937	# endif
3938	}
3939
3940
3941	/**
3942	* Checks if a memory block is filled with the specified byte, returning the
3943	* first mismatch.
3944	*
3945	* This is sort of an inverted memchr.
3946	*
3947	* @returns Pointer to the byte which doesn't equal u8.
3948	* @returns NULL if all equal to u8.
3949	*
3950	* @param pv Pointer to the memory block.
3951	* @param cb Number of bytes in the block.
3952	* @param u8 The value it's supposed to be filled with.
3953	*
3954	* @remarks No alignment requirements.
3955	*/
3956	#if !defined(RT_OS_LINUX) \|\| !defined(__KERNEL__)
3957	DECLASM(void ) ASMMemFirstMismatchingU8(void const pv, size_t cb, uint8_t u8);
3958	#else
3959	DECLINLINE(void ) ASMMemFirstMismatchingU8(void const pv, size_t cb, uint8_t u8)
3960	{
3961	uint8_t const pb = (uint8_t const )pv;
3962	for (; cb; cb--, pb++)
3963	if (RT_LIKELY(*pb == u8))
3964	{ /* likely */ }
3965	else
3966	return (void *)pb;
3967	return NULL;
3968	}
3969	#endif
3970
3971
3972	/**
3973	* Checks if a memory block is filled with the specified byte.
3974	*
3975	* @returns true if all matching, false if not.
3976	*
3977	* @param pv Pointer to the memory block.
3978	* @param cb Number of bytes in the block.
3979	* @param u8 The value it's supposed to be filled with.
3980	*
3981	* @remarks No alignment requirements.
3982	*/
3983	DECLINLINE(bool) ASMMemIsAllU8(void const *pv, size_t cb, uint8_t u8)
3984	{
3985	return ASMMemFirstMismatchingU8(pv, cb, u8) == NULL;
3986	}
3987
3988
3989	/**
3990	* Checks if a memory block is filled with the specified 32-bit value.
3991	*
3992	* This is a sort of inverted memchr.
3993	*
3994	* @returns Pointer to the first value which doesn't equal u32.
3995	* @returns NULL if all equal to u32.
3996	*
3997	* @param pv Pointer to the memory block.
3998	* @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
3999	* @param u32 The value it's supposed to be filled with.
4000	*/
4001	DECLINLINE(uint32_t ) ASMMemFirstMismatchingU32(void const pv, size_t cb, uint32_t u32)
4002	{
4003	/** @todo rewrite this in inline assembly? */
4004	uint32_t const pu32 = (uint32_t const )pv;
4005	for (; cb; cb -= 4, pu32++)
4006	if (RT_LIKELY(*pu32 == u32))
4007	{ /* likely */ }
4008	else
4009	return (uint32_t *)pu32;
4010	return NULL;
4011	}
4012
4013
4014	/**
4015	* Probes a byte pointer for read access.
4016	*
4017	* While the function will not fault if the byte is not read accessible,
4018	* the idea is to do this in a safe place like before acquiring locks
4019	* and such like.
4020	*
4021	* Also, this functions guarantees that an eager compiler is not going
4022	* to optimize the probing away.
4023	*
4024	* @param pvByte Pointer to the byte.
4025	*/
4026	#if RT_INLINE_ASM_EXTERNAL
4027	DECLASM(uint8_t) ASMProbeReadByte(const void *pvByte);
4028	#else
4029	DECLINLINE(uint8_t) ASMProbeReadByte(const void *pvByte)
4030	{
4031	/** @todo verify that the compiler actually doesn't optimize this away. (intel & gcc) */
4032	uint8_t u8;
4033	# if RT_INLINE_ASM_GNU_STYLE
4034	__asm__ __volatile__("movb (%1), %0\n\t"
4035	: "=r" (u8)
4036	: "r" (pvByte));
4037	# else
4038	__asm
4039	{
4040	# ifdef RT_ARCH_AMD64
4041	mov rax, [pvByte]
4042	mov al, [rax]
4043	# else
4044	mov eax, [pvByte]
4045	mov al, [eax]
4046	# endif
4047	mov [u8], al
4048	}
4049	# endif
4050	return u8;
4051	}
4052	#endif
4053
4054	/**
4055	* Probes a buffer for read access page by page.
4056	*
4057	* While the function will fault if the buffer is not fully read
4058	* accessible, the idea is to do this in a safe place like before
4059	* acquiring locks and such like.
4060	*
4061	* Also, this functions guarantees that an eager compiler is not going
4062	* to optimize the probing away.
4063	*
4064	* @param pvBuf Pointer to the buffer.
4065	* @param cbBuf The size of the buffer in bytes. Must be >= 1.
4066	*/
4067	DECLINLINE(void) ASMProbeReadBuffer(const void *pvBuf, size_t cbBuf)
4068	{
4069	/** @todo verify that the compiler actually doesn't optimize this away. (intel & gcc) */
4070	/* the first byte */
4071	const uint8_t pu8 = (const uint8_t )pvBuf;
4072	ASMProbeReadByte(pu8);
4073
4074	/* the pages in between pages. */
4075	while (cbBuf > RT_ASM_PAGE_SIZE)
4076	{
4077	ASMProbeReadByte(pu8);
4078	cbBuf -= RT_ASM_PAGE_SIZE;
4079	pu8 += RT_ASM_PAGE_SIZE;
4080	}
4081
4082	/* the last byte */
4083	ASMProbeReadByte(pu8 + cbBuf - 1);
4084	}
4085
4086
4087
4088	/** @defgroup grp_inline_bits Bit Operations
4089	* @{
4090	*/
4091
4092
4093	/**
4094	* Sets a bit in a bitmap.
4095	*
4096	* @param pvBitmap Pointer to the bitmap. This should be 32-bit aligned.
4097	* @param iBit The bit to set.
4098	*
4099	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
4100	* However, doing so will yield better performance as well as avoiding
4101	* traps accessing the last bits in the bitmap.
4102	*/
4103	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4104	DECLASM(void) ASMBitSet(volatile void *pvBitmap, int32_t iBit);
4105	#else
4106	DECLINLINE(void) ASMBitSet(volatile void *pvBitmap, int32_t iBit)
4107	{
4108	# if RT_INLINE_ASM_USES_INTRIN
4109	_bittestandset((long *)pvBitmap, iBit);
4110
4111	# elif RT_INLINE_ASM_GNU_STYLE
4112	__asm__ __volatile__("btsl %1, %0"
4113	: "=m" ((volatile long )pvBitmap)
4114	: "Ir" (iBit),
4115	"m" ((volatile long )pvBitmap)
4116	: "memory");
4117	# else
4118	__asm
4119	{
4120	# ifdef RT_ARCH_AMD64
4121	mov rax, [pvBitmap]
4122	mov edx, [iBit]
4123	bts [rax], edx
4124	# else
4125	mov eax, [pvBitmap]
4126	mov edx, [iBit]
4127	bts [eax], edx
4128	# endif
4129	}
4130	# endif
4131	}
4132	#endif
4133
4134
4135	/**
4136	* Atomically sets a bit in a bitmap, ordered.
4137	*
4138	* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
4139	* the memory access isn't atomic!
4140	* @param iBit The bit to set.
4141	*
4142	* @remarks x86: Requires a 386 or later.
4143	*/
4144	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4145	DECLASM(void) ASMAtomicBitSet(volatile void *pvBitmap, int32_t iBit);
4146	#else
4147	DECLINLINE(void) ASMAtomicBitSet(volatile void *pvBitmap, int32_t iBit)
4148	{
4149	AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
4150	# if RT_INLINE_ASM_USES_INTRIN
4151	_interlockedbittestandset((long *)pvBitmap, iBit);
4152	# elif RT_INLINE_ASM_GNU_STYLE
4153	__asm__ __volatile__("lock; btsl %1, %0"
4154	: "=m" ((volatile long )pvBitmap)
4155	: "Ir" (iBit),
4156	"m" ((volatile long )pvBitmap)
4157	: "memory");
4158	# else
4159	__asm
4160	{
4161	# ifdef RT_ARCH_AMD64
4162	mov rax, [pvBitmap]
4163	mov edx, [iBit]
4164	lock bts [rax], edx
4165	# else
4166	mov eax, [pvBitmap]
4167	mov edx, [iBit]
4168	lock bts [eax], edx
4169	# endif
4170	}
4171	# endif
4172	}
4173	#endif
4174
4175
4176	/**
4177	* Clears a bit in a bitmap.
4178	*
4179	* @param pvBitmap Pointer to the bitmap.
4180	* @param iBit The bit to clear.
4181	*
4182	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
4183	* However, doing so will yield better performance as well as avoiding
4184	* traps accessing the last bits in the bitmap.
4185	*/
4186	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4187	DECLASM(void) ASMBitClear(volatile void *pvBitmap, int32_t iBit);
4188	#else
4189	DECLINLINE(void) ASMBitClear(volatile void *pvBitmap, int32_t iBit)
4190	{
4191	# if RT_INLINE_ASM_USES_INTRIN
4192	_bittestandreset((long *)pvBitmap, iBit);
4193
4194	# elif RT_INLINE_ASM_GNU_STYLE
4195	__asm__ __volatile__("btrl %1, %0"
4196	: "=m" ((volatile long )pvBitmap)
4197	: "Ir" (iBit),
4198	"m" ((volatile long )pvBitmap)
4199	: "memory");
4200	# else
4201	__asm
4202	{
4203	# ifdef RT_ARCH_AMD64
4204	mov rax, [pvBitmap]
4205	mov edx, [iBit]
4206	btr [rax], edx
4207	# else
4208	mov eax, [pvBitmap]
4209	mov edx, [iBit]
4210	btr [eax], edx
4211	# endif
4212	}
4213	# endif
4214	}
4215	#endif
4216
4217
4218	/**
4219	* Atomically clears a bit in a bitmap, ordered.
4220	*
4221	* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
4222	* the memory access isn't atomic!
4223	* @param iBit The bit to toggle set.
4224	*
4225	* @remarks No memory barrier, take care on smp.
4226	* @remarks x86: Requires a 386 or later.
4227	*/
4228	#if RT_INLINE_ASM_EXTERNAL
4229	DECLASM(void) ASMAtomicBitClear(volatile void *pvBitmap, int32_t iBit);
4230	#else
4231	DECLINLINE(void) ASMAtomicBitClear(volatile void *pvBitmap, int32_t iBit)
4232	{
4233	AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
4234	# if RT_INLINE_ASM_GNU_STYLE
4235	__asm__ __volatile__("lock; btrl %1, %0"
4236	: "=m" ((volatile long )pvBitmap)
4237	: "Ir" (iBit),
4238	"m" ((volatile long )pvBitmap)
4239	: "memory");
4240	# else
4241	__asm
4242	{
4243	# ifdef RT_ARCH_AMD64
4244	mov rax, [pvBitmap]
4245	mov edx, [iBit]
4246	lock btr [rax], edx
4247	# else
4248	mov eax, [pvBitmap]
4249	mov edx, [iBit]
4250	lock btr [eax], edx
4251	# endif
4252	}
4253	# endif
4254	}
4255	#endif
4256
4257
4258	/**
4259	* Toggles a bit in a bitmap.
4260	*
4261	* @param pvBitmap Pointer to the bitmap.
4262	* @param iBit The bit to toggle.
4263	*
4264	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
4265	* However, doing so will yield better performance as well as avoiding
4266	* traps accessing the last bits in the bitmap.
4267	*/
4268	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4269	DECLASM(void) ASMBitToggle(volatile void *pvBitmap, int32_t iBit);
4270	#else
4271	DECLINLINE(void) ASMBitToggle(volatile void *pvBitmap, int32_t iBit)
4272	{
4273	# if RT_INLINE_ASM_USES_INTRIN
4274	_bittestandcomplement((long *)pvBitmap, iBit);
4275	# elif RT_INLINE_ASM_GNU_STYLE
4276	__asm__ __volatile__("btcl %1, %0"
4277	: "=m" ((volatile long )pvBitmap)
4278	: "Ir" (iBit),
4279	"m" ((volatile long )pvBitmap)
4280	: "memory");
4281	# else
4282	__asm
4283	{
4284	# ifdef RT_ARCH_AMD64
4285	mov rax, [pvBitmap]
4286	mov edx, [iBit]
4287	btc [rax], edx
4288	# else
4289	mov eax, [pvBitmap]
4290	mov edx, [iBit]
4291	btc [eax], edx
4292	# endif
4293	}
4294	# endif
4295	}
4296	#endif
4297
4298
4299	/**
4300	* Atomically toggles a bit in a bitmap, ordered.
4301	*
4302	* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
4303	* the memory access isn't atomic!
4304	* @param iBit The bit to test and set.
4305	*
4306	* @remarks x86: Requires a 386 or later.
4307	*/
4308	#if RT_INLINE_ASM_EXTERNAL
4309	DECLASM(void) ASMAtomicBitToggle(volatile void *pvBitmap, int32_t iBit);
4310	#else
4311	DECLINLINE(void) ASMAtomicBitToggle(volatile void *pvBitmap, int32_t iBit)
4312	{
4313	AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
4314	# if RT_INLINE_ASM_GNU_STYLE
4315	__asm__ __volatile__("lock; btcl %1, %0"
4316	: "=m" ((volatile long )pvBitmap)
4317	: "Ir" (iBit),
4318	"m" ((volatile long )pvBitmap)
4319	: "memory");
4320	# else
4321	__asm
4322	{
4323	# ifdef RT_ARCH_AMD64
4324	mov rax, [pvBitmap]
4325	mov edx, [iBit]
4326	lock btc [rax], edx
4327	# else
4328	mov eax, [pvBitmap]
4329	mov edx, [iBit]
4330	lock btc [eax], edx
4331	# endif
4332	}
4333	# endif
4334	}
4335	#endif
4336
4337
4338	/**
4339	* Tests and sets a bit in a bitmap.
4340	*
4341	* @returns true if the bit was set.
4342	* @returns false if the bit was clear.
4343	*
4344	* @param pvBitmap Pointer to the bitmap.
4345	* @param iBit The bit to test and set.
4346	*
4347	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
4348	* However, doing so will yield better performance as well as avoiding
4349	* traps accessing the last bits in the bitmap.
4350	*/
4351	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4352	DECLASM(bool) ASMBitTestAndSet(volatile void *pvBitmap, int32_t iBit);
4353	#else
4354	DECLINLINE(bool) ASMBitTestAndSet(volatile void *pvBitmap, int32_t iBit)
4355	{
4356	union { bool f; uint32_t u32; uint8_t u8; } rc;
4357	# if RT_INLINE_ASM_USES_INTRIN
4358	rc.u8 = _bittestandset((long *)pvBitmap, iBit);
4359
4360	# elif RT_INLINE_ASM_GNU_STYLE
4361	__asm__ __volatile__("btsl %2, %1\n\t"
4362	"setc %b0\n\t"
4363	"andl $1, %0\n\t"
4364	: "=q" (rc.u32),
4365	"=m" ((volatile long )pvBitmap)
4366	: "Ir" (iBit),
4367	"m" ((volatile long )pvBitmap)
4368	: "memory");
4369	# else
4370	__asm
4371	{
4372	mov edx, [iBit]
4373	# ifdef RT_ARCH_AMD64
4374	mov rax, [pvBitmap]
4375	bts [rax], edx
4376	# else
4377	mov eax, [pvBitmap]
4378	bts [eax], edx
4379	# endif
4380	setc al
4381	and eax, 1
4382	mov [rc.u32], eax
4383	}
4384	# endif
4385	return rc.f;
4386	}
4387	#endif
4388
4389
4390	/**
4391	* Atomically tests and sets a bit in a bitmap, ordered.
4392	*
4393	* @returns true if the bit was set.
4394	* @returns false if the bit was clear.
4395	*
4396	* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
4397	* the memory access isn't atomic!
4398	* @param iBit The bit to set.
4399	*
4400	* @remarks x86: Requires a 386 or later.
4401	*/
4402	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4403	DECLASM(bool) ASMAtomicBitTestAndSet(volatile void *pvBitmap, int32_t iBit);
4404	#else
4405	DECLINLINE(bool) ASMAtomicBitTestAndSet(volatile void *pvBitmap, int32_t iBit)
4406	{
4407	union { bool f; uint32_t u32; uint8_t u8; } rc;
4408	AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
4409	# if RT_INLINE_ASM_USES_INTRIN
4410	rc.u8 = _interlockedbittestandset((long *)pvBitmap, iBit);
4411	# elif RT_INLINE_ASM_GNU_STYLE
4412	__asm__ __volatile__("lock; btsl %2, %1\n\t"
4413	"setc %b0\n\t"
4414	"andl $1, %0\n\t"
4415	: "=q" (rc.u32),
4416	"=m" ((volatile long )pvBitmap)
4417	: "Ir" (iBit),
4418	"m" ((volatile long )pvBitmap)
4419	: "memory");
4420	# else
4421	__asm
4422	{
4423	mov edx, [iBit]
4424	# ifdef RT_ARCH_AMD64
4425	mov rax, [pvBitmap]
4426	lock bts [rax], edx
4427	# else
4428	mov eax, [pvBitmap]
4429	lock bts [eax], edx
4430	# endif
4431	setc al
4432	and eax, 1
4433	mov [rc.u32], eax
4434	}
4435	# endif
4436	return rc.f;
4437	}
4438	#endif
4439
4440
4441	/**
4442	* Tests and clears a bit in a bitmap.
4443	*
4444	* @returns true if the bit was set.
4445	* @returns false if the bit was clear.
4446	*
4447	* @param pvBitmap Pointer to the bitmap.
4448	* @param iBit The bit to test and clear.
4449	*
4450	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
4451	* However, doing so will yield better performance as well as avoiding
4452	* traps accessing the last bits in the bitmap.
4453	*/
4454	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4455	DECLASM(bool) ASMBitTestAndClear(volatile void *pvBitmap, int32_t iBit);
4456	#else
4457	DECLINLINE(bool) ASMBitTestAndClear(volatile void *pvBitmap, int32_t iBit)
4458	{
4459	union { bool f; uint32_t u32; uint8_t u8; } rc;
4460	# if RT_INLINE_ASM_USES_INTRIN
4461	rc.u8 = _bittestandreset((long *)pvBitmap, iBit);
4462
4463	# elif RT_INLINE_ASM_GNU_STYLE
4464	__asm__ __volatile__("btrl %2, %1\n\t"
4465	"setc %b0\n\t"
4466	"andl $1, %0\n\t"
4467	: "=q" (rc.u32),
4468	"=m" ((volatile long )pvBitmap)
4469	: "Ir" (iBit),
4470	"m" ((volatile long )pvBitmap)
4471	: "memory");
4472	# else
4473	__asm
4474	{
4475	mov edx, [iBit]
4476	# ifdef RT_ARCH_AMD64
4477	mov rax, [pvBitmap]
4478	btr [rax], edx
4479	# else
4480	mov eax, [pvBitmap]
4481	btr [eax], edx
4482	# endif
4483	setc al
4484	and eax, 1
4485	mov [rc.u32], eax
4486	}
4487	# endif
4488	return rc.f;
4489	}
4490	#endif
4491
4492
4493	/**
4494	* Atomically tests and clears a bit in a bitmap, ordered.
4495	*
4496	* @returns true if the bit was set.
4497	* @returns false if the bit was clear.
4498	*
4499	* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
4500	* the memory access isn't atomic!
4501	* @param iBit The bit to test and clear.
4502	*
4503	* @remarks No memory barrier, take care on smp.
4504	* @remarks x86: Requires a 386 or later.
4505	*/
4506	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4507	DECLASM(bool) ASMAtomicBitTestAndClear(volatile void *pvBitmap, int32_t iBit);
4508	#else
4509	DECLINLINE(bool) ASMAtomicBitTestAndClear(volatile void *pvBitmap, int32_t iBit)
4510	{
4511	union { bool f; uint32_t u32; uint8_t u8; } rc;
4512	AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
4513	# if RT_INLINE_ASM_USES_INTRIN
4514	rc.u8 = _interlockedbittestandreset((long *)pvBitmap, iBit);
4515
4516	# elif RT_INLINE_ASM_GNU_STYLE
4517	__asm__ __volatile__("lock; btrl %2, %1\n\t"
4518	"setc %b0\n\t"
4519	"andl $1, %0\n\t"
4520	: "=q" (rc.u32),
4521	"=m" ((volatile long )pvBitmap)
4522	: "Ir" (iBit),
4523	"m" ((volatile long )pvBitmap)
4524	: "memory");
4525	# else
4526	__asm
4527	{
4528	mov edx, [iBit]
4529	# ifdef RT_ARCH_AMD64
4530	mov rax, [pvBitmap]
4531	lock btr [rax], edx
4532	# else
4533	mov eax, [pvBitmap]
4534	lock btr [eax], edx
4535	# endif
4536	setc al
4537	and eax, 1
4538	mov [rc.u32], eax
4539	}
4540	# endif
4541	return rc.f;
4542	}
4543	#endif
4544
4545
4546	/**
4547	* Tests and toggles a bit in a bitmap.
4548	*
4549	* @returns true if the bit was set.
4550	* @returns false if the bit was clear.
4551	*
4552	* @param pvBitmap Pointer to the bitmap.
4553	* @param iBit The bit to test and toggle.
4554	*
4555	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
4556	* However, doing so will yield better performance as well as avoiding
4557	* traps accessing the last bits in the bitmap.
4558	*/
4559	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4560	DECLASM(bool) ASMBitTestAndToggle(volatile void *pvBitmap, int32_t iBit);
4561	#else
4562	DECLINLINE(bool) ASMBitTestAndToggle(volatile void *pvBitmap, int32_t iBit)
4563	{
4564	union { bool f; uint32_t u32; uint8_t u8; } rc;
4565	# if RT_INLINE_ASM_USES_INTRIN
4566	rc.u8 = _bittestandcomplement((long *)pvBitmap, iBit);
4567
4568	# elif RT_INLINE_ASM_GNU_STYLE
4569	__asm__ __volatile__("btcl %2, %1\n\t"
4570	"setc %b0\n\t"
4571	"andl $1, %0\n\t"
4572	: "=q" (rc.u32),
4573	"=m" ((volatile long )pvBitmap)
4574	: "Ir" (iBit),
4575	"m" ((volatile long )pvBitmap)
4576	: "memory");
4577	# else
4578	__asm
4579	{
4580	mov edx, [iBit]
4581	# ifdef RT_ARCH_AMD64
4582	mov rax, [pvBitmap]
4583	btc [rax], edx
4584	# else
4585	mov eax, [pvBitmap]
4586	btc [eax], edx
4587	# endif
4588	setc al
4589	and eax, 1
4590	mov [rc.u32], eax
4591	}
4592	# endif
4593	return rc.f;
4594	}
4595	#endif
4596
4597
4598	/**
4599	* Atomically tests and toggles a bit in a bitmap, ordered.
4600	*
4601	* @returns true if the bit was set.
4602	* @returns false if the bit was clear.
4603	*
4604	* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
4605	* the memory access isn't atomic!
4606	* @param iBit The bit to test and toggle.
4607	*
4608	* @remarks x86: Requires a 386 or later.
4609	*/
4610	#if RT_INLINE_ASM_EXTERNAL
4611	DECLASM(bool) ASMAtomicBitTestAndToggle(volatile void *pvBitmap, int32_t iBit);
4612	#else
4613	DECLINLINE(bool) ASMAtomicBitTestAndToggle(volatile void *pvBitmap, int32_t iBit)
4614	{
4615	union { bool f; uint32_t u32; uint8_t u8; } rc;
4616	AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
4617	# if RT_INLINE_ASM_GNU_STYLE
4618	__asm__ __volatile__("lock; btcl %2, %1\n\t"
4619	"setc %b0\n\t"
4620	"andl $1, %0\n\t"
4621	: "=q" (rc.u32),
4622	"=m" ((volatile long )pvBitmap)
4623	: "Ir" (iBit),
4624	"m" ((volatile long )pvBitmap)
4625	: "memory");
4626	# else
4627	__asm
4628	{
4629	mov edx, [iBit]
4630	# ifdef RT_ARCH_AMD64
4631	mov rax, [pvBitmap]
4632	lock btc [rax], edx
4633	# else
4634	mov eax, [pvBitmap]
4635	lock btc [eax], edx
4636	# endif
4637	setc al
4638	and eax, 1
4639	mov [rc.u32], eax
4640	}
4641	# endif
4642	return rc.f;
4643	}
4644	#endif
4645
4646
4647	/**
4648	* Tests if a bit in a bitmap is set.
4649	*
4650	* @returns true if the bit is set.
4651	* @returns false if the bit is clear.
4652	*
4653	* @param pvBitmap Pointer to the bitmap.
4654	* @param iBit The bit to test.
4655	*
4656	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
4657	* However, doing so will yield better performance as well as avoiding
4658	* traps accessing the last bits in the bitmap.
4659	*/
4660	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4661	DECLASM(bool) ASMBitTest(const volatile void *pvBitmap, int32_t iBit);
4662	#else
4663	DECLINLINE(bool) ASMBitTest(const volatile void *pvBitmap, int32_t iBit)
4664	{
4665	union { bool f; uint32_t u32; uint8_t u8; } rc;
4666	# if RT_INLINE_ASM_USES_INTRIN
4667	rc.u32 = _bittest((long *)pvBitmap, iBit);
4668	# elif RT_INLINE_ASM_GNU_STYLE
4669
4670	__asm__ __volatile__("btl %2, %1\n\t"
4671	"setc %b0\n\t"
4672	"andl $1, %0\n\t"
4673	: "=q" (rc.u32)
4674	: "m" ((const volatile long )pvBitmap),
4675	"Ir" (iBit)
4676	: "memory");
4677	# else
4678	__asm
4679	{
4680	mov edx, [iBit]
4681	# ifdef RT_ARCH_AMD64
4682	mov rax, [pvBitmap]
4683	bt [rax], edx
4684	# else
4685	mov eax, [pvBitmap]
4686	bt [eax], edx
4687	# endif
4688	setc al
4689	and eax, 1
4690	mov [rc.u32], eax
4691	}
4692	# endif
4693	return rc.f;
4694	}
4695	#endif
4696
4697
4698	/**
4699	* Clears a bit range within a bitmap.
4700	*
4701	* @param pvBitmap Pointer to the bitmap.
4702	* @param iBitStart The First bit to clear.
4703	* @param iBitEnd The first bit not to clear.
4704	*/
4705	DECLINLINE(void) ASMBitClearRange(volatile void *pvBitmap, int32_t iBitStart, int32_t iBitEnd)
4706	{
4707	if (iBitStart < iBitEnd)
4708	{
4709	volatile uint32_t pu32 = (volatile uint32_t )pvBitmap + (iBitStart >> 5);
4710	int32_t iStart = iBitStart & ~31;
4711	int32_t iEnd = iBitEnd & ~31;
4712	if (iStart == iEnd)
4713	*pu32 &= ((UINT32_C(1) << (iBitStart & 31)) - 1) \| ~((UINT32_C(1) << (iBitEnd & 31)) - 1);
4714	else
4715	{
4716	/* bits in first dword. */
4717	if (iBitStart & 31)
4718	{
4719	*pu32 &= (UINT32_C(1) << (iBitStart & 31)) - 1;
4720	pu32++;
4721	iBitStart = iStart + 32;
4722	}
4723
4724	/* whole dword. */
4725	if (iBitStart != iEnd)
4726	ASMMemZero32(pu32, (iEnd - iBitStart) >> 3);
4727
4728	/* bits in last dword. */
4729	if (iBitEnd & 31)
4730	{
4731	pu32 = (volatile uint32_t *)pvBitmap + (iBitEnd >> 5);
4732	*pu32 &= ~((UINT32_C(1) << (iBitEnd & 31)) - 1);
4733	}
4734	}
4735	}
4736	}
4737
4738
4739	/**
4740	* Sets a bit range within a bitmap.
4741	*
4742	* @param pvBitmap Pointer to the bitmap.
4743	* @param iBitStart The First bit to set.
4744	* @param iBitEnd The first bit not to set.
4745	*/
4746	DECLINLINE(void) ASMBitSetRange(volatile void *pvBitmap, int32_t iBitStart, int32_t iBitEnd)
4747	{
4748	if (iBitStart < iBitEnd)
4749	{
4750	volatile uint32_t pu32 = (volatile uint32_t )pvBitmap + (iBitStart >> 5);
4751	int32_t iStart = iBitStart & ~31;
4752	int32_t iEnd = iBitEnd & ~31;
4753	if (iStart == iEnd)
4754	*pu32 \|= ((UINT32_C(1) << (iBitEnd - iBitStart)) - 1) << (iBitStart & 31);
4755	else
4756	{
4757	/* bits in first dword. */
4758	if (iBitStart & 31)
4759	{
4760	*pu32 \|= ~((UINT32_C(1) << (iBitStart & 31)) - 1);
4761	pu32++;
4762	iBitStart = iStart + 32;
4763	}
4764
4765	/* whole dword. */
4766	if (iBitStart != iEnd)
4767	ASMMemFill32(pu32, (iEnd - iBitStart) >> 3, ~UINT32_C(0));
4768
4769	/* bits in last dword. */
4770	if (iBitEnd & 31)
4771	{
4772	pu32 = (volatile uint32_t *)pvBitmap + (iBitEnd >> 5);
4773	*pu32 \|= (UINT32_C(1) << (iBitEnd & 31)) - 1;
4774	}
4775	}
4776	}
4777	}
4778
4779
4780	/**
4781	* Finds the first clear bit in a bitmap.
4782	*
4783	* @returns Index of the first zero bit.
4784	* @returns -1 if no clear bit was found.
4785	* @param pvBitmap Pointer to the bitmap.
4786	* @param cBits The number of bits in the bitmap. Multiple of 32.
4787	*/
4788	#if RT_INLINE_ASM_EXTERNAL
4789	DECLASM(int32_t) ASMBitFirstClear(const volatile void *pvBitmap, uint32_t cBits);
4790	#else
4791	DECLINLINE(int32_t) ASMBitFirstClear(const volatile void *pvBitmap, uint32_t cBits)
4792	{
4793	if (cBits)
4794	{
4795	int32_t iBit;
4796	# if RT_INLINE_ASM_GNU_STYLE
4797	RTCCUINTREG uEAX, uECX, uEDI;
4798	cBits = RT_ALIGN_32(cBits, 32);
4799	__asm__ __volatile__("repe; scasl\n\t"
4800	"je 1f\n\t"
4801	# ifdef RT_ARCH_AMD64
4802	"lea -4(%%rdi), %%rdi\n\t"
4803	"xorl (%%rdi), %%eax\n\t"
4804	"subq %5, %%rdi\n\t"
4805	# else
4806	"lea -4(%%edi), %%edi\n\t"
4807	"xorl (%%edi), %%eax\n\t"
4808	"subl %5, %%edi\n\t"
4809	# endif
4810	"shll $3, %%edi\n\t"
4811	"bsfl %%eax, %%edx\n\t"
4812	"addl %%edi, %%edx\n\t"
4813	"1:\t\n"
4814	: "=d" (iBit),
4815	"=&c" (uECX),
4816	"=&D" (uEDI),
4817	"=&a" (uEAX)
4818	: "0" (0xffffffff),
4819	"mr" (pvBitmap),
4820	"1" (cBits >> 5),
4821	"2" (pvBitmap),
4822	"3" (0xffffffff));
4823	# else
4824	cBits = RT_ALIGN_32(cBits, 32);
4825	__asm
4826	{
4827	# ifdef RT_ARCH_AMD64
4828	mov rdi, [pvBitmap]
4829	mov rbx, rdi
4830	# else
4831	mov edi, [pvBitmap]
4832	mov ebx, edi
4833	# endif
4834	mov edx, 0ffffffffh
4835	mov eax, edx
4836	mov ecx, [cBits]
4837	shr ecx, 5
4838	repe scasd
4839	je done
4840
4841	# ifdef RT_ARCH_AMD64
4842	lea rdi, [rdi - 4]
4843	xor eax, [rdi]
4844	sub rdi, rbx
4845	# else
4846	lea edi, [edi - 4]
4847	xor eax, [edi]
4848	sub edi, ebx
4849	# endif
4850	shl edi, 3
4851	bsf edx, eax
4852	add edx, edi
4853	done:
4854	mov [iBit], edx
4855	}
4856	# endif
4857	return iBit;
4858	}
4859	return -1;
4860	}
4861	#endif
4862
4863
4864	/**
4865	* Finds the next clear bit in a bitmap.
4866	*
4867	* @returns Index of the first zero bit.
4868	* @returns -1 if no clear bit was found.
4869	* @param pvBitmap Pointer to the bitmap.
4870	* @param cBits The number of bits in the bitmap. Multiple of 32.
4871	* @param iBitPrev The bit returned from the last search.
4872	* The search will start at iBitPrev + 1.
4873	*/
4874	#if RT_INLINE_ASM_EXTERNAL
4875	DECLASM(int) ASMBitNextClear(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev);
4876	#else
4877	DECLINLINE(int) ASMBitNextClear(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev)
4878	{
4879	const volatile uint32_t pau32Bitmap = (const volatile uint32_t )pvBitmap;
4880	int iBit = ++iBitPrev & 31;
4881	if (iBit)
4882	{
4883	/*
4884	* Inspect the 32-bit word containing the unaligned bit.
4885	*/
4886	uint32_t u32 = ~pau32Bitmap[iBitPrev / 32] >> iBit;
4887
4888	# if RT_INLINE_ASM_USES_INTRIN
4889	unsigned long ulBit = 0;
4890	if (_BitScanForward(&ulBit, u32))
4891	return ulBit + iBitPrev;
4892	# else
4893	# if RT_INLINE_ASM_GNU_STYLE
4894	__asm__ __volatile__("bsf %1, %0\n\t"
4895	"jnz 1f\n\t"
4896	"movl $-1, %0\n\t"
4897	"1:\n\t"
4898	: "=r" (iBit)
4899	: "r" (u32));
4900	# else
4901	__asm
4902	{
4903	mov edx, [u32]
4904	bsf eax, edx
4905	jnz done
4906	mov eax, 0ffffffffh
4907	done:
4908	mov [iBit], eax
4909	}
4910	# endif
4911	if (iBit >= 0)
4912	return iBit + iBitPrev;
4913	# endif
4914
4915	/*
4916	* Skip ahead and see if there is anything left to search.
4917	*/
4918	iBitPrev \|= 31;
4919	iBitPrev++;
4920	if (cBits <= (uint32_t)iBitPrev)
4921	return -1;
4922	}
4923
4924	/*
4925	* 32-bit aligned search, let ASMBitFirstClear do the dirty work.
4926	*/
4927	iBit = ASMBitFirstClear(&pau32Bitmap[iBitPrev / 32], cBits - iBitPrev);
4928	if (iBit >= 0)
4929	iBit += iBitPrev;
4930	return iBit;
4931	}
4932	#endif
4933
4934
4935	/**
4936	* Finds the first set bit in a bitmap.
4937	*
4938	* @returns Index of the first set bit.
4939	* @returns -1 if no clear bit was found.
4940	* @param pvBitmap Pointer to the bitmap.
4941	* @param cBits The number of bits in the bitmap. Multiple of 32.
4942	*/
4943	#if RT_INLINE_ASM_EXTERNAL
4944	DECLASM(int32_t) ASMBitFirstSet(const volatile void *pvBitmap, uint32_t cBits);
4945	#else
4946	DECLINLINE(int32_t) ASMBitFirstSet(const volatile void *pvBitmap, uint32_t cBits)
4947	{
4948	if (cBits)
4949	{
4950	int32_t iBit;
4951	# if RT_INLINE_ASM_GNU_STYLE
4952	RTCCUINTREG uEAX, uECX, uEDI;
4953	cBits = RT_ALIGN_32(cBits, 32);
4954	__asm__ __volatile__("repe; scasl\n\t"
4955	"je 1f\n\t"
4956	# ifdef RT_ARCH_AMD64
4957	"lea -4(%%rdi), %%rdi\n\t"
4958	"movl (%%rdi), %%eax\n\t"
4959	"subq %5, %%rdi\n\t"
4960	# else
4961	"lea -4(%%edi), %%edi\n\t"
4962	"movl (%%edi), %%eax\n\t"
4963	"subl %5, %%edi\n\t"
4964	# endif
4965	"shll $3, %%edi\n\t"
4966	"bsfl %%eax, %%edx\n\t"
4967	"addl %%edi, %%edx\n\t"
4968	"1:\t\n"
4969	: "=d" (iBit),
4970	"=&c" (uECX),
4971	"=&D" (uEDI),
4972	"=&a" (uEAX)
4973	: "0" (0xffffffff),
4974	"mr" (pvBitmap),
4975	"1" (cBits >> 5),
4976	"2" (pvBitmap),
4977	"3" (0));
4978	# else
4979	cBits = RT_ALIGN_32(cBits, 32);
4980	__asm
4981	{
4982	# ifdef RT_ARCH_AMD64
4983	mov rdi, [pvBitmap]
4984	mov rbx, rdi
4985	# else
4986	mov edi, [pvBitmap]
4987	mov ebx, edi
4988	# endif
4989	mov edx, 0ffffffffh
4990	xor eax, eax
4991	mov ecx, [cBits]
4992	shr ecx, 5
4993	repe scasd
4994	je done
4995	# ifdef RT_ARCH_AMD64
4996	lea rdi, [rdi - 4]
4997	mov eax, [rdi]
4998	sub rdi, rbx
4999	# else
5000	lea edi, [edi - 4]
5001	mov eax, [edi]
5002	sub edi, ebx
5003	# endif
5004	shl edi, 3
5005	bsf edx, eax
5006	add edx, edi
5007	done:
5008	mov [iBit], edx
5009	}
5010	# endif
5011	return iBit;
5012	}
5013	return -1;
5014	}
5015	#endif
5016
5017
5018	/**
5019	* Finds the next set bit in a bitmap.
5020	*
5021	* @returns Index of the next set bit.
5022	* @returns -1 if no set bit was found.
5023	* @param pvBitmap Pointer to the bitmap.
5024	* @param cBits The number of bits in the bitmap. Multiple of 32.
5025	* @param iBitPrev The bit returned from the last search.
5026	* The search will start at iBitPrev + 1.
5027	*/
5028	#if RT_INLINE_ASM_EXTERNAL
5029	DECLASM(int) ASMBitNextSet(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev);
5030	#else
5031	DECLINLINE(int) ASMBitNextSet(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev)
5032	{
5033	const volatile uint32_t pau32Bitmap = (const volatile uint32_t )pvBitmap;
5034	int iBit = ++iBitPrev & 31;
5035	if (iBit)
5036	{
5037	/*
5038	* Inspect the 32-bit word containing the unaligned bit.
5039	*/
5040	uint32_t u32 = pau32Bitmap[iBitPrev / 32] >> iBit;
5041
5042	# if RT_INLINE_ASM_USES_INTRIN
5043	unsigned long ulBit = 0;
5044	if (_BitScanForward(&ulBit, u32))
5045	return ulBit + iBitPrev;
5046	# else
5047	# if RT_INLINE_ASM_GNU_STYLE
5048	__asm__ __volatile__("bsf %1, %0\n\t"
5049	"jnz 1f\n\t"
5050	"movl $-1, %0\n\t"
5051	"1:\n\t"
5052	: "=r" (iBit)
5053	: "r" (u32));
5054	# else
5055	__asm
5056	{
5057	mov edx, [u32]
5058	bsf eax, edx
5059	jnz done
5060	mov eax, 0ffffffffh
5061	done:
5062	mov [iBit], eax
5063	}
5064	# endif
5065	if (iBit >= 0)
5066	return iBit + iBitPrev;
5067	# endif
5068
5069	/*
5070	* Skip ahead and see if there is anything left to search.
5071	*/
5072	iBitPrev \|= 31;
5073	iBitPrev++;
5074	if (cBits <= (uint32_t)iBitPrev)
5075	return -1;
5076	}
5077
5078	/*
5079	* 32-bit aligned search, let ASMBitFirstClear do the dirty work.
5080	*/
5081	iBit = ASMBitFirstSet(&pau32Bitmap[iBitPrev / 32], cBits - iBitPrev);
5082	if (iBit >= 0)
5083	iBit += iBitPrev;
5084	return iBit;
5085	}
5086	#endif
5087
5088
5089	/**
5090	* Finds the first bit which is set in the given 32-bit integer.
5091	* Bits are numbered from 1 (least significant) to 32.
5092	*
5093	* @returns index [1..32] of the first set bit.
5094	* @returns 0 if all bits are cleared.
5095	* @param u32 Integer to search for set bits.
5096	* @remarks Similar to ffs() in BSD.
5097	*/
5098	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5099	DECLASM(unsigned) ASMBitFirstSetU32(uint32_t u32);
5100	#else
5101	DECLINLINE(unsigned) ASMBitFirstSetU32(uint32_t u32)
5102	{
5103	# if RT_INLINE_ASM_USES_INTRIN
5104	unsigned long iBit;
5105	if (_BitScanForward(&iBit, u32))
5106	iBit++;
5107	else
5108	iBit = 0;
5109	# elif RT_INLINE_ASM_GNU_STYLE
5110	uint32_t iBit;
5111	__asm__ __volatile__("bsf %1, %0\n\t"
5112	"jnz 1f\n\t"
5113	"xorl %0, %0\n\t"
5114	"jmp 2f\n"
5115	"1:\n\t"
5116	"incl %0\n"
5117	"2:\n\t"
5118	: "=r" (iBit)
5119	: "rm" (u32));
5120	# else
5121	uint32_t iBit;
5122	_asm
5123	{
5124	bsf eax, [u32]
5125	jnz found
5126	xor eax, eax
5127	jmp done
5128	found:
5129	inc eax
5130	done:
5131	mov [iBit], eax
5132	}
5133	# endif
5134	return iBit;
5135	}
5136	#endif
5137
5138
5139	/**
5140	* Finds the first bit which is set in the given 32-bit integer.
5141	* Bits are numbered from 1 (least significant) to 32.
5142	*
5143	* @returns index [1..32] of the first set bit.
5144	* @returns 0 if all bits are cleared.
5145	* @param i32 Integer to search for set bits.
5146	* @remark Similar to ffs() in BSD.
5147	*/
5148	DECLINLINE(unsigned) ASMBitFirstSetS32(int32_t i32)
5149	{
5150	return ASMBitFirstSetU32((uint32_t)i32);
5151	}
5152
5153
5154	/**
5155	* Finds the first bit which is set in the given 64-bit integer.
5156	*
5157	* Bits are numbered from 1 (least significant) to 64.
5158	*
5159	* @returns index [1..64] of the first set bit.
5160	* @returns 0 if all bits are cleared.
5161	* @param u64 Integer to search for set bits.
5162	* @remarks Similar to ffs() in BSD.
5163	*/
5164	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5165	DECLASM(unsigned) ASMBitFirstSetU64(uint64_t u64);
5166	#else
5167	DECLINLINE(unsigned) ASMBitFirstSetU64(uint64_t u64)
5168	{
5169	# if RT_INLINE_ASM_USES_INTRIN
5170	unsigned long iBit;
5171	# if ARCH_BITS == 64
5172	if (_BitScanForward64(&iBit, u64))
5173	iBit++;
5174	else
5175	iBit = 0;
5176	# else
5177	if (_BitScanForward(&iBit, (uint32_t)u64))
5178	iBit++;
5179	else if (_BitScanForward(&iBit, (uint32_t)(u64 >> 32)))
5180	iBit += 33;
5181	else
5182	iBit = 0;
5183	# endif
5184	# elif RT_INLINE_ASM_GNU_STYLE && ARCH_BITS == 64
5185	uint64_t iBit;
5186	__asm__ __volatile__("bsfq %1, %0\n\t"
5187	"jnz 1f\n\t"
5188	"xorl %0, %0\n\t"
5189	"jmp 2f\n"
5190	"1:\n\t"
5191	"incl %0\n"
5192	"2:\n\t"
5193	: "=r" (iBit)
5194	: "rm" (u64));
5195	# else
5196	unsigned iBit = ASMBitFirstSetU32((uint32_t)u64);
5197	if (!iBit)
5198	{
5199	iBit = ASMBitFirstSetU32((uint32_t)(u64 >> 32));
5200	if (iBit)
5201	iBit += 32;
5202	}
5203	# endif
5204	return (unsigned)iBit;
5205	}
5206	#endif
5207
5208
5209	/**
5210	* Finds the first bit which is set in the given 16-bit integer.
5211	*
5212	* Bits are numbered from 1 (least significant) to 16.
5213	*
5214	* @returns index [1..16] of the first set bit.
5215	* @returns 0 if all bits are cleared.
5216	* @param u16 Integer to search for set bits.
5217	* @remarks For 16-bit bs3kit code.
5218	*/
5219	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5220	DECLASM(unsigned) ASMBitFirstSetU16(uint16_t u16);
5221	#else
5222	DECLINLINE(unsigned) ASMBitFirstSetU16(uint16_t u16)
5223	{
5224	return ASMBitFirstSetU32((uint32_t)u16);
5225	}
5226	#endif
5227
5228
5229	/**
5230	* Finds the last bit which is set in the given 32-bit integer.
5231	* Bits are numbered from 1 (least significant) to 32.
5232	*
5233	* @returns index [1..32] of the last set bit.
5234	* @returns 0 if all bits are cleared.
5235	* @param u32 Integer to search for set bits.
5236	* @remark Similar to fls() in BSD.
5237	*/
5238	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5239	DECLASM(unsigned) ASMBitLastSetU32(uint32_t u32);
5240	#else
5241	DECLINLINE(unsigned) ASMBitLastSetU32(uint32_t u32)
5242	{
5243	# if RT_INLINE_ASM_USES_INTRIN
5244	unsigned long iBit;
5245	if (_BitScanReverse(&iBit, u32))
5246	iBit++;
5247	else
5248	iBit = 0;
5249	# elif RT_INLINE_ASM_GNU_STYLE
5250	uint32_t iBit;
5251	__asm__ __volatile__("bsrl %1, %0\n\t"
5252	"jnz 1f\n\t"
5253	"xorl %0, %0\n\t"
5254	"jmp 2f\n"
5255	"1:\n\t"
5256	"incl %0\n"
5257	"2:\n\t"
5258	: "=r" (iBit)
5259	: "rm" (u32));
5260	# else
5261	uint32_t iBit;
5262	_asm
5263	{
5264	bsr eax, [u32]
5265	jnz found
5266	xor eax, eax
5267	jmp done
5268	found:
5269	inc eax
5270	done:
5271	mov [iBit], eax
5272	}
5273	# endif
5274	return iBit;
5275	}
5276	#endif
5277
5278
5279	/**
5280	* Finds the last bit which is set in the given 32-bit integer.
5281	* Bits are numbered from 1 (least significant) to 32.
5282	*
5283	* @returns index [1..32] of the last set bit.
5284	* @returns 0 if all bits are cleared.
5285	* @param i32 Integer to search for set bits.
5286	* @remark Similar to fls() in BSD.
5287	*/
5288	DECLINLINE(unsigned) ASMBitLastSetS32(int32_t i32)
5289	{
5290	return ASMBitLastSetU32((uint32_t)i32);
5291	}
5292
5293
5294	/**
5295	* Finds the last bit which is set in the given 64-bit integer.
5296	*
5297	* Bits are numbered from 1 (least significant) to 64.
5298	*
5299	* @returns index [1..64] of the last set bit.
5300	* @returns 0 if all bits are cleared.
5301	* @param u64 Integer to search for set bits.
5302	* @remark Similar to fls() in BSD.
5303	*/
5304	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5305	DECLASM(unsigned) ASMBitLastSetU64(uint64_t u64);
5306	#else
5307	DECLINLINE(unsigned) ASMBitLastSetU64(uint64_t u64)
5308	{
5309	# if RT_INLINE_ASM_USES_INTRIN
5310	unsigned long iBit;
5311	# if ARCH_BITS == 64
5312	if (_BitScanReverse64(&iBit, u64))
5313	iBit++;
5314	else
5315	iBit = 0;
5316	# else
5317	if (_BitScanReverse(&iBit, (uint32_t)(u64 >> 32)))
5318	iBit += 33;
5319	else if (_BitScanReverse(&iBit, (uint32_t)u64))
5320	iBit++;
5321	else
5322	iBit = 0;
5323	# endif
5324	# elif RT_INLINE_ASM_GNU_STYLE && ARCH_BITS == 64
5325	uint64_t iBit;
5326	__asm__ __volatile__("bsrq %1, %0\n\t"
5327	"jnz 1f\n\t"
5328	"xorl %0, %0\n\t"
5329	"jmp 2f\n"
5330	"1:\n\t"
5331	"incl %0\n"
5332	"2:\n\t"
5333	: "=r" (iBit)
5334	: "rm" (u64));
5335	# else
5336	unsigned iBit = ASMBitLastSetU32((uint32_t)(u64 >> 32));
5337	if (iBit)
5338	iBit += 32;
5339	else
5340	iBit = ASMBitLastSetU32((uint32_t)u64);
5341	#endif
5342	return (unsigned)iBit;
5343	}
5344	#endif
5345
5346
5347	/**
5348	* Finds the last bit which is set in the given 16-bit integer.
5349	*
5350	* Bits are numbered from 1 (least significant) to 16.
5351	*
5352	* @returns index [1..16] of the last set bit.
5353	* @returns 0 if all bits are cleared.
5354	* @param u16 Integer to search for set bits.
5355	* @remarks For 16-bit bs3kit code.
5356	*/
5357	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5358	DECLASM(unsigned) ASMBitLastSetU16(uint16_t u16);
5359	#else
5360	DECLINLINE(unsigned) ASMBitLastSetU16(uint16_t u16)
5361	{
5362	return ASMBitLastSetU32((uint32_t)u16);
5363	}
5364	#endif
5365
5366
5367	/**
5368	* Reverse the byte order of the given 16-bit integer.
5369	*
5370	* @returns Revert
5371	* @param u16 16-bit integer value.
5372	*/
5373	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5374	DECLASM(uint16_t) ASMByteSwapU16(uint16_t u16);
5375	#else
5376	DECLINLINE(uint16_t) ASMByteSwapU16(uint16_t u16)
5377	{
5378	# if RT_INLINE_ASM_USES_INTRIN
5379	u16 = _byteswap_ushort(u16);
5380	# elif RT_INLINE_ASM_GNU_STYLE
5381	__asm__ ("rorw $8, %0" : "=r" (u16) : "0" (u16));
5382	# else
5383	_asm
5384	{
5385	mov ax, [u16]
5386	ror ax, 8
5387	mov [u16], ax
5388	}
5389	# endif
5390	return u16;
5391	}
5392	#endif
5393
5394
5395	/**
5396	* Reverse the byte order of the given 32-bit integer.
5397	*
5398	* @returns Revert
5399	* @param u32 32-bit integer value.
5400	*/
5401	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5402	DECLASM(uint32_t) ASMByteSwapU32(uint32_t u32);
5403	#else
5404	DECLINLINE(uint32_t) ASMByteSwapU32(uint32_t u32)
5405	{
5406	# if RT_INLINE_ASM_USES_INTRIN
5407	u32 = _byteswap_ulong(u32);
5408	# elif RT_INLINE_ASM_GNU_STYLE
5409	__asm__ ("bswapl %0" : "=r" (u32) : "0" (u32));
5410	# else
5411	_asm
5412	{
5413	mov eax, [u32]
5414	bswap eax
5415	mov [u32], eax
5416	}
5417	# endif
5418	return u32;
5419	}
5420	#endif
5421
5422
5423	/**
5424	* Reverse the byte order of the given 64-bit integer.
5425	*
5426	* @returns Revert
5427	* @param u64 64-bit integer value.
5428	*/
5429	DECLINLINE(uint64_t) ASMByteSwapU64(uint64_t u64)
5430	{
5431	#if defined(RT_ARCH_AMD64) && RT_INLINE_ASM_USES_INTRIN
5432	u64 = _byteswap_uint64(u64);
5433	#else
5434	u64 = (uint64_t)ASMByteSwapU32((uint32_t)u64) << 32
5435	\| (uint64_t)ASMByteSwapU32((uint32_t)(u64 >> 32));
5436	#endif
5437	return u64;
5438	}
5439
5440
5441	/**
5442	* Rotate 32-bit unsigned value to the left by @a cShift.
5443	*
5444	* @returns Rotated value.
5445	* @param u32 The value to rotate.
5446	* @param cShift How many bits to rotate by.
5447	*/
5448	#ifdef __WATCOMC__
5449	DECLASM(uint32_t) ASMRotateLeftU32(uint32_t u32, unsigned cShift);
5450	#else
5451	DECLINLINE(uint32_t) ASMRotateLeftU32(uint32_t u32, uint32_t cShift)
5452	{
5453	# if RT_INLINE_ASM_USES_INTRIN
5454	return _rotl(u32, cShift);
5455	# elif RT_INLINE_ASM_GNU_STYLE && (defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86))
5456	__asm__ __volatile__("roll %b1, %0" : "=g" (u32) : "Ic" (cShift), "0" (u32));
5457	return u32;
5458	# else
5459	cShift &= 31;
5460	return (u32 << cShift) \| (u32 >> (32 - cShift));
5461	# endif
5462	}
5463	#endif
5464
5465
5466	/**
5467	* Rotate 32-bit unsigned value to the right by @a cShift.
5468	*
5469	* @returns Rotated value.
5470	* @param u32 The value to rotate.
5471	* @param cShift How many bits to rotate by.
5472	*/
5473	#ifdef __WATCOMC__
5474	DECLASM(uint32_t) ASMRotateRightU32(uint32_t u32, unsigned cShift);
5475	#else
5476	DECLINLINE(uint32_t) ASMRotateRightU32(uint32_t u32, uint32_t cShift)
5477	{
5478	# if RT_INLINE_ASM_USES_INTRIN
5479	return _rotr(u32, cShift);
5480	# elif RT_INLINE_ASM_GNU_STYLE && (defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86))
5481	__asm__ __volatile__("rorl %b1, %0" : "=g" (u32) : "Ic" (cShift), "0" (u32));
5482	return u32;
5483	# else
5484	cShift &= 31;
5485	return (u32 >> cShift) \| (u32 << (32 - cShift));
5486	# endif
5487	}
5488	#endif
5489
5490
5491	/**
5492	* Rotate 64-bit unsigned value to the left by @a cShift.
5493	*
5494	* @returns Rotated value.
5495	* @param u64 The value to rotate.
5496	* @param cShift How many bits to rotate by.
5497	*/
5498	DECLINLINE(uint64_t) ASMRotateLeftU64(uint64_t u64, uint32_t cShift)
5499	{
5500	#if RT_INLINE_ASM_USES_INTRIN
5501	return _rotl64(u64, cShift);
5502	#elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
5503	__asm__ __volatile__("rolq %b1, %0" : "=g" (u64) : "Jc" (cShift), "0" (u64));
5504	return u64;
5505	#elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_X86)
5506	uint32_t uSpill;
5507	__asm__ __volatile__("testb $0x20, %%cl\n\t" /* if (cShift >= 0x20) { swap(u64.hi, u64lo); cShift -= 0x20; } */
5508	"jz 1f\n\t"
5509	"xchgl %%eax, %%edx\n\t"
5510	"1:\n\t"
5511	"andb $0x1f, %%cl\n\t" /* if (cShift & 0x1f) { */
5512	"jz 2f\n\t"
5513	"movl %%edx, %2\n\t" /* save the hi value in %3. */
5514	"shldl %%cl,%%eax,%%edx\n\t" /* shift the hi value left, feeding MSBits from the low value. */
5515	"shldl %%cl,%2,%%eax\n\t" /* shift the lo value left, feeding MSBits from the saved hi value. */
5516	"2:\n\t" /* } */
5517	: "=A" (u64), "=c" (cShift), "=r" (uSpill)
5518	: "0" (u64),
5519	"1" (cShift));
5520	return u64;
5521	#else
5522	cShift &= 63;
5523	return (u64 << cShift) \| (u64 >> (64 - cShift));
5524	#endif
5525	}
5526
5527
5528	/**
5529	* Rotate 64-bit unsigned value to the right by @a cShift.
5530	*
5531	* @returns Rotated value.
5532	* @param u64 The value to rotate.
5533	* @param cShift How many bits to rotate by.
5534	*/
5535	DECLINLINE(uint64_t) ASMRotateRightU64(uint64_t u64, uint32_t cShift)
5536	{
5537	#if RT_INLINE_ASM_USES_INTRIN
5538	return _rotr64(u64, cShift);
5539	#elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
5540	__asm__ __volatile__("rorq %b1, %0" : "=g" (u64) : "Jc" (cShift), "0" (u64));
5541	return u64;
5542	#elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_X86)
5543	uint32_t uSpill;
5544	__asm__ __volatile__("testb $0x20, %%cl\n\t" /* if (cShift >= 0x20) { swap(u64.hi, u64lo); cShift -= 0x20; } */
5545	"jz 1f\n\t"
5546	"xchgl %%eax, %%edx\n\t"
5547	"1:\n\t"
5548	"andb $0x1f, %%cl\n\t" /* if (cShift & 0x1f) { */
5549	"jz 2f\n\t"
5550	"movl %%edx, %2\n\t" /* save the hi value in %3. */
5551	"shrdl %%cl,%%eax,%%edx\n\t" /* shift the hi value right, feeding LSBits from the low value. */
5552	"shrdl %%cl,%2,%%eax\n\t" /* shift the lo value right, feeding LSBits from the saved hi value. */
5553	"2:\n\t" /* } */
5554	: "=A" (u64), "=c" (cShift), "=r" (uSpill)
5555	: "0" (u64),
5556	"1" (cShift));
5557	return u64;
5558	#else
5559	cShift &= 63;
5560	return (u64 >> cShift) \| (u64 << (64 - cShift));
5561	#endif
5562	}
5563
5564	/** @} */
5565
5566
5567	/** @} */
5568
5569	#endif
5570

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/include/iprt/asm.h@ 60547

Download in other formats: