asm.h@ 22279

Last change on this file since 22279 was 22279, checked in by vboxsync, 15 years ago
ASMGetFlags() / gcc: don't use '=g' (includes '=m') as output constraint because this will not work without -fomit-frame-pointer
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 171.1 KB

Line
1	/** @file
2	* IPRT - Assembly Functions.
3	*/
4
5	/*
6	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
7	*
8	* This file is part of VirtualBox Open Source Edition (OSE), as
9	* available from http://www.virtualbox.org. This file is free software;
10	* you can redistribute it and/or modify it under the terms of the GNU
11	* General Public License (GPL) as published by the Free Software
12	* Foundation, in version 2 as it comes in the "COPYING" file of the
13	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
14	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
15	*
16	* The contents of this file may alternatively be used under the terms
17	* of the Common Development and Distribution License Version 1.0
18	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
19	* VirtualBox OSE distribution, in which case the provisions of the
20	* CDDL are applicable instead of those of the GPL.
21	*
22	* You may elect to license modified versions of this file under the
23	* terms and conditions of either the GPL or the CDDL or both.
24	*
25	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
26	* Clara, CA 95054 USA or visit http://www.sun.com if you need
27	* additional information or have any questions.
28	*/
29
30	#ifndef ___iprt_asm_h
31	#define ___iprt_asm_h
32
33	#include <iprt/cdefs.h>
34	#include <iprt/types.h>
35	#include <iprt/assert.h>
36	/** @todo #include <iprt/param.h> for PAGE_SIZE. */
37	/** @def RT_INLINE_ASM_USES_INTRIN
38	* Defined as 1 if we're using a _MSC_VER 1400.
39	* Otherwise defined as 0.
40	*/
41
42	/* Solaris 10 header ugliness */
43	#ifdef u
44	#undef u
45	#endif
46
47	#ifdef _MSC_VER
48	# if _MSC_VER >= 1400
49	# define RT_INLINE_ASM_USES_INTRIN 1
50	# include <intrin.h>
51	/* Emit the intrinsics at all optimization levels. */
52	# pragma intrinsic(_ReadWriteBarrier)
53	# pragma intrinsic(__cpuid)
54	# pragma intrinsic(_enable)
55	# pragma intrinsic(_disable)
56	# pragma intrinsic(__rdtsc)
57	# pragma intrinsic(__readmsr)
58	# pragma intrinsic(__writemsr)
59	# pragma intrinsic(__outbyte)
60	# pragma intrinsic(__outbytestring)
61	# pragma intrinsic(__outword)
62	# pragma intrinsic(__outwordstring)
63	# pragma intrinsic(__outdword)
64	# pragma intrinsic(__outdwordstring)
65	# pragma intrinsic(__inbyte)
66	# pragma intrinsic(__inbytestring)
67	# pragma intrinsic(__inword)
68	# pragma intrinsic(__inwordstring)
69	# pragma intrinsic(__indword)
70	# pragma intrinsic(__indwordstring)
71	# pragma intrinsic(__invlpg)
72	# pragma intrinsic(__wbinvd)
73	# pragma intrinsic(__stosd)
74	# pragma intrinsic(__stosw)
75	# pragma intrinsic(__stosb)
76	# pragma intrinsic(__readcr0)
77	# pragma intrinsic(__readcr2)
78	# pragma intrinsic(__readcr3)
79	# pragma intrinsic(__readcr4)
80	# pragma intrinsic(__writecr0)
81	# pragma intrinsic(__writecr3)
82	# pragma intrinsic(__writecr4)
83	# pragma intrinsic(__readdr)
84	# pragma intrinsic(__writedr)
85	# pragma intrinsic(_BitScanForward)
86	# pragma intrinsic(_BitScanReverse)
87	# pragma intrinsic(_bittest)
88	# pragma intrinsic(_bittestandset)
89	# pragma intrinsic(_bittestandreset)
90	# pragma intrinsic(_bittestandcomplement)
91	# pragma intrinsic(_byteswap_ushort)
92	# pragma intrinsic(_byteswap_ulong)
93	# pragma intrinsic(_interlockedbittestandset)
94	# pragma intrinsic(_interlockedbittestandreset)
95	# pragma intrinsic(_InterlockedAnd)
96	# pragma intrinsic(_InterlockedOr)
97	# pragma intrinsic(_InterlockedIncrement)
98	# pragma intrinsic(_InterlockedDecrement)
99	# pragma intrinsic(_InterlockedExchange)
100	# pragma intrinsic(_InterlockedExchangeAdd)
101	# pragma intrinsic(_InterlockedCompareExchange)
102	# pragma intrinsic(_InterlockedCompareExchange64)
103	# ifdef RT_ARCH_AMD64
104	# pragma intrinsic(_mm_mfence)
105	# pragma intrinsic(_mm_sfence)
106	# pragma intrinsic(_mm_lfence)
107	# pragma intrinsic(__stosq)
108	# pragma intrinsic(__readcr8)
109	# pragma intrinsic(__writecr8)
110	# pragma intrinsic(_byteswap_uint64)
111	# pragma intrinsic(_InterlockedExchange64)
112	# endif
113	# endif
114	#endif
115	#ifndef RT_INLINE_ASM_USES_INTRIN
116	# define RT_INLINE_ASM_USES_INTRIN 0
117	#endif
118
119	/** @def RT_INLINE_ASM_GCC_4_3_X_X86
120	* Used to work around some 4.3.x register allocation issues in this version of
121	* the compiler. */
122	#ifdef __GNUC__
123	# define RT_INLINE_ASM_GCC_4_3_X_X86 (__GNUC__ == 4 && __GNUC_MINOR__ == 3 && defined(__i386__))
124	#endif
125	#ifndef RT_INLINE_ASM_GCC_4_3_X_X86
126	# define RT_INLINE_ASM_GCC_4_3_X_X86 0
127	#endif
128
129
130
131	/** @defgroup grp_asm ASM - Assembly Routines
132	* @ingroup grp_rt
133	*
134	* @remarks The difference between ordered and unordered atomic operations are that
135	* the former will complete outstanding reads and writes before continuing
136	* while the latter doesn't make any promisses about the order. Ordered
137	* operations doesn't, it seems, make any 100% promise wrt to whether
138	* the operation will complete before any subsequent memory access.
139	* (please, correct if wrong.)
140	*
141	* ASMAtomicSomething operations are all ordered, while ASMAtomicUoSomething
142	* are unordered (note the Uo).
143	*
144	* @remarks Some remarks about __volatile__: Without this keyword gcc is allowed to reorder
145	* or even optimize assembler instructions away. For instance, in the following code
146	* the second rdmsr instruction is optimized away because gcc treats that instruction
147	* as deterministic:
148	*
149	* @code
150	* static inline uint64_t rdmsr_low(int idx)
151	* {
152	* uint32_t low;
153	* __asm__ ("rdmsr" : "=a"(low) : "c"(idx) : "edx");
154	* }
155	* ...
156	* uint32_t msr1 = rdmsr_low(1);
157	* foo(msr1);
158	* msr1 = rdmsr_low(1);
159	* bar(msr1);
160	* @endcode
161	*
162	* The input parameter of rdmsr_low is the same for both calls and therefore gcc will
163	* use the result of the first call as input parameter for bar() as well. For rdmsr this
164	* is not acceptable as this instruction is _not_ deterministic. This applies to reading
165	* machine status information in general.
166	*
167	* @{
168	*/
169
170	/** @def RT_INLINE_ASM_EXTERNAL
171	* Defined as 1 if the compiler does not support inline assembly.
172	* The ASM* functions will then be implemented in an external .asm file.
173	*
174	* @remark At the present time it's unconfirmed whether or not Microsoft skipped
175	* inline assembly in their AMD64 compiler.
176	*/
177	#if defined(_MSC_VER) && defined(RT_ARCH_AMD64)
178	# define RT_INLINE_ASM_EXTERNAL 1
179	#else
180	# define RT_INLINE_ASM_EXTERNAL 0
181	#endif
182
183	/** @def RT_INLINE_ASM_GNU_STYLE
184	* Defined as 1 if the compiler understands GNU style inline assembly.
185	*/
186	#if defined(_MSC_VER)
187	# define RT_INLINE_ASM_GNU_STYLE 0
188	#else
189	# define RT_INLINE_ASM_GNU_STYLE 1
190	#endif
191
192
193	/** @todo find a more proper place for this structure? */
194	#pragma pack(1)
195	/** IDTR */
196	typedef struct RTIDTR
197	{
198	/** Size of the IDT. */
199	uint16_t cbIdt;
200	/** Address of the IDT. */
201	uintptr_t pIdt;
202	} RTIDTR, *PRTIDTR;
203	#pragma pack()
204
205	#pragma pack(1)
206	/** GDTR */
207	typedef struct RTGDTR
208	{
209	/** Size of the GDT. */
210	uint16_t cbGdt;
211	/** Address of the GDT. */
212	uintptr_t pGdt;
213	} RTGDTR, *PRTGDTR;
214	#pragma pack()
215
216
217	/** @def ASMReturnAddress
218	* Gets the return address of the current (or calling if you like) function or method.
219	*/
220	#ifdef _MSC_VER
221	# ifdef __cplusplus
222	extern "C"
223	# endif
224	void * _ReturnAddress(void);
225	# pragma intrinsic(_ReturnAddress)
226	# define ASMReturnAddress() _ReturnAddress()
227	#elif defined(__GNUC__) \|\| defined(DOXYGEN_RUNNING)
228	# define ASMReturnAddress() __builtin_return_address(0)
229	#else
230	# error "Unsupported compiler."
231	#endif
232
233
234	/**
235	* Gets the content of the IDTR CPU register.
236	* @param pIdtr Where to store the IDTR contents.
237	*/
238	#if RT_INLINE_ASM_EXTERNAL
239	DECLASM(void) ASMGetIDTR(PRTIDTR pIdtr);
240	#else
241	DECLINLINE(void) ASMGetIDTR(PRTIDTR pIdtr)
242	{
243	# if RT_INLINE_ASM_GNU_STYLE
244	__asm__ __volatile__("sidt %0" : "=m" (*pIdtr));
245	# else
246	__asm
247	{
248	# ifdef RT_ARCH_AMD64
249	mov rax, [pIdtr]
250	sidt [rax]
251	# else
252	mov eax, [pIdtr]
253	sidt [eax]
254	# endif
255	}
256	# endif
257	}
258	#endif
259
260
261	/**
262	* Sets the content of the IDTR CPU register.
263	* @param pIdtr Where to load the IDTR contents from
264	*/
265	#if RT_INLINE_ASM_EXTERNAL
266	DECLASM(void) ASMSetIDTR(const RTIDTR *pIdtr);
267	#else
268	DECLINLINE(void) ASMSetIDTR(const RTIDTR *pIdtr)
269	{
270	# if RT_INLINE_ASM_GNU_STYLE
271	__asm__ __volatile__("lidt %0" : : "m" (*pIdtr));
272	# else
273	__asm
274	{
275	# ifdef RT_ARCH_AMD64
276	mov rax, [pIdtr]
277	lidt [rax]
278	# else
279	mov eax, [pIdtr]
280	lidt [eax]
281	# endif
282	}
283	# endif
284	}
285	#endif
286
287
288	/**
289	* Gets the content of the GDTR CPU register.
290	* @param pGdtr Where to store the GDTR contents.
291	*/
292	#if RT_INLINE_ASM_EXTERNAL
293	DECLASM(void) ASMGetGDTR(PRTGDTR pGdtr);
294	#else
295	DECLINLINE(void) ASMGetGDTR(PRTGDTR pGdtr)
296	{
297	# if RT_INLINE_ASM_GNU_STYLE
298	__asm__ __volatile__("sgdt %0" : "=m" (*pGdtr));
299	# else
300	__asm
301	{
302	# ifdef RT_ARCH_AMD64
303	mov rax, [pGdtr]
304	sgdt [rax]
305	# else
306	mov eax, [pGdtr]
307	sgdt [eax]
308	# endif
309	}
310	# endif
311	}
312	#endif
313
314	/**
315	* Get the cs register.
316	* @returns cs.
317	*/
318	#if RT_INLINE_ASM_EXTERNAL
319	DECLASM(RTSEL) ASMGetCS(void);
320	#else
321	DECLINLINE(RTSEL) ASMGetCS(void)
322	{
323	RTSEL SelCS;
324	# if RT_INLINE_ASM_GNU_STYLE
325	__asm__ __volatile__("movw %%cs, %0\n\t" : "=r" (SelCS));
326	# else
327	__asm
328	{
329	mov ax, cs
330	mov [SelCS], ax
331	}
332	# endif
333	return SelCS;
334	}
335	#endif
336
337
338	/**
339	* Get the DS register.
340	* @returns DS.
341	*/
342	#if RT_INLINE_ASM_EXTERNAL
343	DECLASM(RTSEL) ASMGetDS(void);
344	#else
345	DECLINLINE(RTSEL) ASMGetDS(void)
346	{
347	RTSEL SelDS;
348	# if RT_INLINE_ASM_GNU_STYLE
349	__asm__ __volatile__("movw %%ds, %0\n\t" : "=r" (SelDS));
350	# else
351	__asm
352	{
353	mov ax, ds
354	mov [SelDS], ax
355	}
356	# endif
357	return SelDS;
358	}
359	#endif
360
361
362	/**
363	* Get the ES register.
364	* @returns ES.
365	*/
366	#if RT_INLINE_ASM_EXTERNAL
367	DECLASM(RTSEL) ASMGetES(void);
368	#else
369	DECLINLINE(RTSEL) ASMGetES(void)
370	{
371	RTSEL SelES;
372	# if RT_INLINE_ASM_GNU_STYLE
373	__asm__ __volatile__("movw %%es, %0\n\t" : "=r" (SelES));
374	# else
375	__asm
376	{
377	mov ax, es
378	mov [SelES], ax
379	}
380	# endif
381	return SelES;
382	}
383	#endif
384
385
386	/**
387	* Get the FS register.
388	* @returns FS.
389	*/
390	#if RT_INLINE_ASM_EXTERNAL
391	DECLASM(RTSEL) ASMGetFS(void);
392	#else
393	DECLINLINE(RTSEL) ASMGetFS(void)
394	{
395	RTSEL SelFS;
396	# if RT_INLINE_ASM_GNU_STYLE
397	__asm__ __volatile__("movw %%fs, %0\n\t" : "=r" (SelFS));
398	# else
399	__asm
400	{
401	mov ax, fs
402	mov [SelFS], ax
403	}
404	# endif
405	return SelFS;
406	}
407	# endif
408
409
410	/**
411	* Get the GS register.
412	* @returns GS.
413	*/
414	#if RT_INLINE_ASM_EXTERNAL
415	DECLASM(RTSEL) ASMGetGS(void);
416	#else
417	DECLINLINE(RTSEL) ASMGetGS(void)
418	{
419	RTSEL SelGS;
420	# if RT_INLINE_ASM_GNU_STYLE
421	__asm__ __volatile__("movw %%gs, %0\n\t" : "=r" (SelGS));
422	# else
423	__asm
424	{
425	mov ax, gs
426	mov [SelGS], ax
427	}
428	# endif
429	return SelGS;
430	}
431	#endif
432
433
434	/**
435	* Get the SS register.
436	* @returns SS.
437	*/
438	#if RT_INLINE_ASM_EXTERNAL
439	DECLASM(RTSEL) ASMGetSS(void);
440	#else
441	DECLINLINE(RTSEL) ASMGetSS(void)
442	{
443	RTSEL SelSS;
444	# if RT_INLINE_ASM_GNU_STYLE
445	__asm__ __volatile__("movw %%ss, %0\n\t" : "=r" (SelSS));
446	# else
447	__asm
448	{
449	mov ax, ss
450	mov [SelSS], ax
451	}
452	# endif
453	return SelSS;
454	}
455	#endif
456
457
458	/**
459	* Get the TR register.
460	* @returns TR.
461	*/
462	#if RT_INLINE_ASM_EXTERNAL
463	DECLASM(RTSEL) ASMGetTR(void);
464	#else
465	DECLINLINE(RTSEL) ASMGetTR(void)
466	{
467	RTSEL SelTR;
468	# if RT_INLINE_ASM_GNU_STYLE
469	__asm__ __volatile__("str %w0\n\t" : "=r" (SelTR));
470	# else
471	__asm
472	{
473	str ax
474	mov [SelTR], ax
475	}
476	# endif
477	return SelTR;
478	}
479	#endif
480
481
482	/**
483	* Get the [RE]FLAGS register.
484	* @returns [RE]FLAGS.
485	*/
486	#if RT_INLINE_ASM_EXTERNAL
487	DECLASM(RTCCUINTREG) ASMGetFlags(void);
488	#else
489	DECLINLINE(RTCCUINTREG) ASMGetFlags(void)
490	{
491	RTCCUINTREG uFlags;
492	# if RT_INLINE_ASM_GNU_STYLE
493	# ifdef RT_ARCH_AMD64
494	__asm__ __volatile__("pushfq\n\t"
495	"popq %0\n\t"
496	: "=r" (uFlags));
497	# else
498	__asm__ __volatile__("pushfl\n\t"
499	"popl %0\n\t"
500	: "=r" (uFlags));
501	# endif
502	# else
503	__asm
504	{
505	# ifdef RT_ARCH_AMD64
506	pushfq
507	pop [uFlags]
508	# else
509	pushfd
510	pop [uFlags]
511	# endif
512	}
513	# endif
514	return uFlags;
515	}
516	#endif
517
518
519	/**
520	* Set the [RE]FLAGS register.
521	* @param uFlags The new [RE]FLAGS value.
522	*/
523	#if RT_INLINE_ASM_EXTERNAL
524	DECLASM(void) ASMSetFlags(RTCCUINTREG uFlags);
525	#else
526	DECLINLINE(void) ASMSetFlags(RTCCUINTREG uFlags)
527	{
528	# if RT_INLINE_ASM_GNU_STYLE
529	# ifdef RT_ARCH_AMD64
530	__asm__ __volatile__("pushq %0\n\t"
531	"popfq\n\t"
532	: : "g" (uFlags));
533	# else
534	__asm__ __volatile__("pushl %0\n\t"
535	"popfl\n\t"
536	: : "g" (uFlags));
537	# endif
538	# else
539	__asm
540	{
541	# ifdef RT_ARCH_AMD64
542	push [uFlags]
543	popfq
544	# else
545	push [uFlags]
546	popfd
547	# endif
548	}
549	# endif
550	}
551	#endif
552
553
554	/**
555	* Gets the content of the CPU timestamp counter register.
556	*
557	* @returns TSC.
558	*/
559	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
560	DECLASM(uint64_t) ASMReadTSC(void);
561	#else
562	DECLINLINE(uint64_t) ASMReadTSC(void)
563	{
564	RTUINT64U u;
565	# if RT_INLINE_ASM_GNU_STYLE
566	__asm__ __volatile__("rdtsc\n\t" : "=a" (u.s.Lo), "=d" (u.s.Hi));
567	# else
568	# if RT_INLINE_ASM_USES_INTRIN
569	u.u = __rdtsc();
570	# else
571	__asm
572	{
573	rdtsc
574	mov [u.s.Lo], eax
575	mov [u.s.Hi], edx
576	}
577	# endif
578	# endif
579	return u.u;
580	}
581	#endif
582
583
584	/**
585	* Performs the cpuid instruction returning all registers.
586	*
587	* @param uOperator CPUID operation (eax).
588	* @param pvEAX Where to store eax.
589	* @param pvEBX Where to store ebx.
590	* @param pvECX Where to store ecx.
591	* @param pvEDX Where to store edx.
592	* @remark We're using void pointers to ease the use of special bitfield structures and such.
593	*/
594	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
595	DECLASM(void) ASMCpuId(uint32_t uOperator, void pvEAX, void pvEBX, void pvECX, void pvEDX);
596	#else
597	DECLINLINE(void) ASMCpuId(uint32_t uOperator, void pvEAX, void pvEBX, void pvECX, void pvEDX)
598	{
599	# if RT_INLINE_ASM_GNU_STYLE
600	# ifdef RT_ARCH_AMD64
601	RTCCUINTREG uRAX, uRBX, uRCX, uRDX;
602	__asm__ ("cpuid\n\t"
603	: "=a" (uRAX),
604	"=b" (uRBX),
605	"=c" (uRCX),
606	"=d" (uRDX)
607	: "0" (uOperator));
608	(uint32_t )pvEAX = (uint32_t)uRAX;
609	(uint32_t )pvEBX = (uint32_t)uRBX;
610	(uint32_t )pvECX = (uint32_t)uRCX;
611	(uint32_t )pvEDX = (uint32_t)uRDX;
612	# else
613	__asm__ ("xchgl %%ebx, %1\n\t"
614	"cpuid\n\t"
615	"xchgl %%ebx, %1\n\t"
616	: "=a" ((uint32_t )pvEAX),
617	"=r" ((uint32_t )pvEBX),
618	"=c" ((uint32_t )pvECX),
619	"=d" ((uint32_t )pvEDX)
620	: "0" (uOperator));
621	# endif
622
623	# elif RT_INLINE_ASM_USES_INTRIN
624	int aInfo[4];
625	__cpuid(aInfo, uOperator);
626	(uint32_t )pvEAX = aInfo[0];
627	(uint32_t )pvEBX = aInfo[1];
628	(uint32_t )pvECX = aInfo[2];
629	(uint32_t )pvEDX = aInfo[3];
630
631	# else
632	uint32_t uEAX;
633	uint32_t uEBX;
634	uint32_t uECX;
635	uint32_t uEDX;
636	__asm
637	{
638	push ebx
639	mov eax, [uOperator]
640	cpuid
641	mov [uEAX], eax
642	mov [uEBX], ebx
643	mov [uECX], ecx
644	mov [uEDX], edx
645	pop ebx
646	}
647	(uint32_t )pvEAX = uEAX;
648	(uint32_t )pvEBX = uEBX;
649	(uint32_t )pvECX = uECX;
650	(uint32_t )pvEDX = uEDX;
651	# endif
652	}
653	#endif
654
655
656	/**
657	* Performs the cpuid instruction returning all registers.
658	* Some subfunctions of cpuid take ECX as additional parameter (currently known for EAX=4)
659	*
660	* @param uOperator CPUID operation (eax).
661	* @param uIdxECX ecx index
662	* @param pvEAX Where to store eax.
663	* @param pvEBX Where to store ebx.
664	* @param pvECX Where to store ecx.
665	* @param pvEDX Where to store edx.
666	* @remark We're using void pointers to ease the use of special bitfield structures and such.
667	*/
668	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
669	DECLASM(void) ASMCpuId_Idx_ECX(uint32_t uOperator, uint32_t uIdxECX, void pvEAX, void pvEBX, void pvECX, void pvEDX);
670	#else
671	DECLINLINE(void) ASMCpuId_Idx_ECX(uint32_t uOperator, uint32_t uIdxECX, void pvEAX, void pvEBX, void pvECX, void pvEDX)
672	{
673	# if RT_INLINE_ASM_GNU_STYLE
674	# ifdef RT_ARCH_AMD64
675	RTCCUINTREG uRAX, uRBX, uRCX, uRDX;
676	__asm__ ("cpuid\n\t"
677	: "=a" (uRAX),
678	"=b" (uRBX),
679	"=c" (uRCX),
680	"=d" (uRDX)
681	: "0" (uOperator),
682	"2" (uIdxECX));
683	(uint32_t )pvEAX = (uint32_t)uRAX;
684	(uint32_t )pvEBX = (uint32_t)uRBX;
685	(uint32_t )pvECX = (uint32_t)uRCX;
686	(uint32_t )pvEDX = (uint32_t)uRDX;
687	# else
688	__asm__ ("xchgl %%ebx, %1\n\t"
689	"cpuid\n\t"
690	"xchgl %%ebx, %1\n\t"
691	: "=a" ((uint32_t )pvEAX),
692	"=r" ((uint32_t )pvEBX),
693	"=c" ((uint32_t )pvECX),
694	"=d" ((uint32_t )pvEDX)
695	: "0" (uOperator),
696	"2" (uIdxECX));
697	# endif
698
699	# elif RT_INLINE_ASM_USES_INTRIN
700	int aInfo[4];
701	/* ??? another intrinsic ??? */
702	__cpuid(aInfo, uOperator);
703	(uint32_t )pvEAX = aInfo[0];
704	(uint32_t )pvEBX = aInfo[1];
705	(uint32_t )pvECX = aInfo[2];
706	(uint32_t )pvEDX = aInfo[3];
707
708	# else
709	uint32_t uEAX;
710	uint32_t uEBX;
711	uint32_t uECX;
712	uint32_t uEDX;
713	__asm
714	{
715	push ebx
716	mov eax, [uOperator]
717	mov ecx, [uIdxECX]
718	cpuid
719	mov [uEAX], eax
720	mov [uEBX], ebx
721	mov [uECX], ecx
722	mov [uEDX], edx
723	pop ebx
724	}
725	(uint32_t )pvEAX = uEAX;
726	(uint32_t )pvEBX = uEBX;
727	(uint32_t )pvECX = uECX;
728	(uint32_t )pvEDX = uEDX;
729	# endif
730	}
731	#endif
732
733
734	/**
735	* Performs the cpuid instruction returning ecx and edx.
736	*
737	* @param uOperator CPUID operation (eax).
738	* @param pvECX Where to store ecx.
739	* @param pvEDX Where to store edx.
740	* @remark We're using void pointers to ease the use of special bitfield structures and such.
741	*/
742	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
743	DECLASM(void) ASMCpuId_ECX_EDX(uint32_t uOperator, void pvECX, void pvEDX);
744	#else
745	DECLINLINE(void) ASMCpuId_ECX_EDX(uint32_t uOperator, void pvECX, void pvEDX)
746	{
747	uint32_t uEBX;
748	ASMCpuId(uOperator, &uOperator, &uEBX, pvECX, pvEDX);
749	}
750	#endif
751
752
753	/**
754	* Performs the cpuid instruction returning edx.
755	*
756	* @param uOperator CPUID operation (eax).
757	* @returns EDX after cpuid operation.
758	*/
759	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
760	DECLASM(uint32_t) ASMCpuId_EDX(uint32_t uOperator);
761	#else
762	DECLINLINE(uint32_t) ASMCpuId_EDX(uint32_t uOperator)
763	{
764	RTCCUINTREG xDX;
765	# if RT_INLINE_ASM_GNU_STYLE
766	# ifdef RT_ARCH_AMD64
767	RTCCUINTREG uSpill;
768	__asm__ ("cpuid"
769	: "=a" (uSpill),
770	"=d" (xDX)
771	: "0" (uOperator)
772	: "rbx", "rcx");
773	# elif (defined(PIC) \|\| defined(__PIC__)) && defined(__i386__)
774	__asm__ ("push %%ebx\n\t"
775	"cpuid\n\t"
776	"pop %%ebx\n\t"
777	: "=a" (uOperator),
778	"=d" (xDX)
779	: "0" (uOperator)
780	: "ecx");
781	# else
782	__asm__ ("cpuid"
783	: "=a" (uOperator),
784	"=d" (xDX)
785	: "0" (uOperator)
786	: "ebx", "ecx");
787	# endif
788
789	# elif RT_INLINE_ASM_USES_INTRIN
790	int aInfo[4];
791	__cpuid(aInfo, uOperator);
792	xDX = aInfo[3];
793
794	# else
795	__asm
796	{
797	push ebx
798	mov eax, [uOperator]
799	cpuid
800	mov [xDX], edx
801	pop ebx
802	}
803	# endif
804	return (uint32_t)xDX;
805	}
806	#endif
807
808
809	/**
810	* Performs the cpuid instruction returning ecx.
811	*
812	* @param uOperator CPUID operation (eax).
813	* @returns ECX after cpuid operation.
814	*/
815	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
816	DECLASM(uint32_t) ASMCpuId_ECX(uint32_t uOperator);
817	#else
818	DECLINLINE(uint32_t) ASMCpuId_ECX(uint32_t uOperator)
819	{
820	RTCCUINTREG xCX;
821	# if RT_INLINE_ASM_GNU_STYLE
822	# ifdef RT_ARCH_AMD64
823	RTCCUINTREG uSpill;
824	__asm__ ("cpuid"
825	: "=a" (uSpill),
826	"=c" (xCX)
827	: "0" (uOperator)
828	: "rbx", "rdx");
829	# elif (defined(PIC) \|\| defined(__PIC__)) && defined(__i386__)
830	__asm__ ("push %%ebx\n\t"
831	"cpuid\n\t"
832	"pop %%ebx\n\t"
833	: "=a" (uOperator),
834	"=c" (xCX)
835	: "0" (uOperator)
836	: "edx");
837	# else
838	__asm__ ("cpuid"
839	: "=a" (uOperator),
840	"=c" (xCX)
841	: "0" (uOperator)
842	: "ebx", "edx");
843
844	# endif
845
846	# elif RT_INLINE_ASM_USES_INTRIN
847	int aInfo[4];
848	__cpuid(aInfo, uOperator);
849	xCX = aInfo[2];
850
851	# else
852	__asm
853	{
854	push ebx
855	mov eax, [uOperator]
856	cpuid
857	mov [xCX], ecx
858	pop ebx
859	}
860	# endif
861	return (uint32_t)xCX;
862	}
863	#endif
864
865
866	/**
867	* Checks if the current CPU supports CPUID.
868	*
869	* @returns true if CPUID is supported.
870	*/
871	DECLINLINE(bool) ASMHasCpuId(void)
872	{
873	#ifdef RT_ARCH_AMD64
874	return true; /* ASSUME that all amd64 compatible CPUs have cpuid. */
875	#else /* !RT_ARCH_AMD64 */
876	bool fRet = false;
877	# if RT_INLINE_ASM_GNU_STYLE
878	uint32_t u1;
879	uint32_t u2;
880	__asm__ ("pushf\n\t"
881	"pop %1\n\t"
882	"mov %1, %2\n\t"
883	"xorl $0x200000, %1\n\t"
884	"push %1\n\t"
885	"popf\n\t"
886	"pushf\n\t"
887	"pop %1\n\t"
888	"cmpl %1, %2\n\t"
889	"setne %0\n\t"
890	"push %2\n\t"
891	"popf\n\t"
892	: "=m" (fRet), "=r" (u1), "=r" (u2));
893	# else
894	__asm
895	{
896	pushfd
897	pop eax
898	mov ebx, eax
899	xor eax, 0200000h
900	push eax
901	popfd
902	pushfd
903	pop eax
904	cmp eax, ebx
905	setne fRet
906	push ebx
907	popfd
908	}
909	# endif
910	return fRet;
911	#endif /* !RT_ARCH_AMD64 */
912	}
913
914
915	/**
916	* Gets the APIC ID of the current CPU.
917	*
918	* @returns the APIC ID.
919	*/
920	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
921	DECLASM(uint8_t) ASMGetApicId(void);
922	#else
923	DECLINLINE(uint8_t) ASMGetApicId(void)
924	{
925	RTCCUINTREG xBX;
926	# if RT_INLINE_ASM_GNU_STYLE
927	# ifdef RT_ARCH_AMD64
928	RTCCUINTREG uSpill;
929	__asm__ ("cpuid"
930	: "=a" (uSpill),
931	"=b" (xBX)
932	: "0" (1)
933	: "rcx", "rdx");
934	# elif (defined(PIC) \|\| defined(__PIC__)) && defined(__i386__)
935	RTCCUINTREG uSpill;
936	__asm__ ("mov %%ebx,%1\n\t"
937	"cpuid\n\t"
938	"xchgl %%ebx,%1\n\t"
939	: "=a" (uSpill),
940	"=r" (xBX)
941	: "0" (1)
942	: "ecx", "edx");
943	# else
944	RTCCUINTREG uSpill;
945	__asm__ ("cpuid"
946	: "=a" (uSpill),
947	"=b" (xBX)
948	: "0" (1)
949	: "ecx", "edx");
950	# endif
951
952	# elif RT_INLINE_ASM_USES_INTRIN
953	int aInfo[4];
954	__cpuid(aInfo, 1);
955	xBX = aInfo[1];
956
957	# else
958	__asm
959	{
960	push ebx
961	mov eax, 1
962	cpuid
963	mov [xBX], ebx
964	pop ebx
965	}
966	# endif
967	return (uint8_t)(xBX >> 24);
968	}
969	#endif
970
971
972	/**
973	* Tests if it a genuine Intel CPU based on the ASMCpuId(0) output.
974	*
975	* @returns true/false.
976	* @param uEBX EBX return from ASMCpuId(0)
977	* @param uECX ECX return from ASMCpuId(0)
978	* @param uEDX EDX return from ASMCpuId(0)
979	*/
980	DECLINLINE(bool) ASMIsIntelCpuEx(uint32_t uEBX, uint32_t uECX, uint32_t uEDX)
981	{
982	return uEBX == 0x756e6547
983	&& uECX == 0x6c65746e
984	&& uEDX == 0x49656e69;
985	}
986
987
988	/**
989	* Tests if this is a genuine Intel CPU.
990	*
991	* @returns true/false.
992	* @remarks ASSUMES that cpuid is supported by the CPU.
993	*/
994	DECLINLINE(bool) ASMIsIntelCpu(void)
995	{
996	uint32_t uEAX, uEBX, uECX, uEDX;
997	ASMCpuId(0, &uEAX, &uEBX, &uECX, &uEDX);
998	return ASMIsIntelCpuEx(uEBX, uECX, uEDX);
999	}
1000
1001
1002	/**
1003	* Extracts the CPU family from ASMCpuId(1) or ASMCpuId(0x80000001)
1004	*
1005	* @returns Family.
1006	* @param uEAX EAX return from ASMCpuId(1) or ASMCpuId(0x80000001).
1007	*/
1008	DECLINLINE(uint32_t) ASMGetCpuFamily(uint32_t uEAX)
1009	{
1010	return ((uEAX >> 8) & 0xf) == 0xf
1011	? ((uEAX >> 20) & 0x7f) + 0xf
1012	: ((uEAX >> 8) & 0xf);
1013	}
1014
1015
1016	/**
1017	* Extracts the CPU model from ASMCpuId(1) or ASMCpuId(0x80000001), Intel variant.
1018	*
1019	* @returns Model.
1020	* @param uEAX EAX from ASMCpuId(1) or ASMCpuId(0x80000001).
1021	* @param fIntel Whether it's an intel CPU.
1022	*/
1023	DECLINLINE(uint32_t) ASMGetCpuModelIntel(uint32_t uEAX)
1024	{
1025	return ((uEAX >> 8) & 0xf) == 0xf \|\| (((uEAX >> 8) & 0xf) == 0x6) /* family! */
1026	? ((uEAX >> 4) & 0xf) \| ((uEAX >> 12) & 0xf0)
1027	: ((uEAX >> 4) & 0xf);
1028	}
1029
1030
1031	/**
1032	* Extracts the CPU model from ASMCpuId(1) or ASMCpuId(0x80000001), AMD variant.
1033	*
1034	* @returns Model.
1035	* @param uEAX EAX from ASMCpuId(1) or ASMCpuId(0x80000001).
1036	* @param fIntel Whether it's an intel CPU.
1037	*/
1038	DECLINLINE(uint32_t) ASMGetCpuModelAMD(uint32_t uEAX)
1039	{
1040	return ((uEAX >> 8) & 0xf) == 0xf
1041	? ((uEAX >> 4) & 0xf) \| ((uEAX >> 12) & 0xf0)
1042	: ((uEAX >> 4) & 0xf);
1043	}
1044
1045
1046	/**
1047	* Extracts the CPU model from ASMCpuId(1) or ASMCpuId(0x80000001)
1048	*
1049	* @returns Model.
1050	* @param uEAX EAX from ASMCpuId(1) or ASMCpuId(0x80000001).
1051	* @param fIntel Whether it's an intel CPU. Use ASMIsIntelCpuEx() or ASMIsIntelCpu().
1052	*/
1053	DECLINLINE(uint32_t) ASMGetCpuModel(uint32_t uEAX, bool fIntel)
1054	{
1055	return ((uEAX >> 8) & 0xf) == 0xf \|\| (((uEAX >> 8) & 0xf) == 0x6 && fIntel) /* family! */
1056	? ((uEAX >> 4) & 0xf) \| ((uEAX >> 12) & 0xf0)
1057	: ((uEAX >> 4) & 0xf);
1058	}
1059
1060
1061	/**
1062	* Extracts the CPU stepping from ASMCpuId(1) or ASMCpuId(0x80000001)
1063	*
1064	* @returns Model.
1065	* @param uEAX EAX from ASMCpuId(1) or ASMCpuId(0x80000001).
1066	*/
1067	DECLINLINE(uint32_t) ASMGetCpuStepping(uint32_t uEAX)
1068	{
1069	return uEAX & 0xf;
1070	}
1071
1072
1073	/**
1074	* Get cr0.
1075	* @returns cr0.
1076	*/
1077	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1078	DECLASM(RTCCUINTREG) ASMGetCR0(void);
1079	#else
1080	DECLINLINE(RTCCUINTREG) ASMGetCR0(void)
1081	{
1082	RTCCUINTREG uCR0;
1083	# if RT_INLINE_ASM_USES_INTRIN
1084	uCR0 = __readcr0();
1085
1086	# elif RT_INLINE_ASM_GNU_STYLE
1087	# ifdef RT_ARCH_AMD64
1088	__asm__ __volatile__("movq %%cr0, %0\t\n" : "=r" (uCR0));
1089	# else
1090	__asm__ __volatile__("movl %%cr0, %0\t\n" : "=r" (uCR0));
1091	# endif
1092	# else
1093	__asm
1094	{
1095	# ifdef RT_ARCH_AMD64
1096	mov rax, cr0
1097	mov [uCR0], rax
1098	# else
1099	mov eax, cr0
1100	mov [uCR0], eax
1101	# endif
1102	}
1103	# endif
1104	return uCR0;
1105	}
1106	#endif
1107
1108
1109	/**
1110	* Sets the CR0 register.
1111	* @param uCR0 The new CR0 value.
1112	*/
1113	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1114	DECLASM(void) ASMSetCR0(RTCCUINTREG uCR0);
1115	#else
1116	DECLINLINE(void) ASMSetCR0(RTCCUINTREG uCR0)
1117	{
1118	# if RT_INLINE_ASM_USES_INTRIN
1119	__writecr0(uCR0);
1120
1121	# elif RT_INLINE_ASM_GNU_STYLE
1122	# ifdef RT_ARCH_AMD64
1123	__asm__ __volatile__("movq %0, %%cr0\n\t" :: "r" (uCR0));
1124	# else
1125	__asm__ __volatile__("movl %0, %%cr0\n\t" :: "r" (uCR0));
1126	# endif
1127	# else
1128	__asm
1129	{
1130	# ifdef RT_ARCH_AMD64
1131	mov rax, [uCR0]
1132	mov cr0, rax
1133	# else
1134	mov eax, [uCR0]
1135	mov cr0, eax
1136	# endif
1137	}
1138	# endif
1139	}
1140	#endif
1141
1142
1143	/**
1144	* Get cr2.
1145	* @returns cr2.
1146	*/
1147	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1148	DECLASM(RTCCUINTREG) ASMGetCR2(void);
1149	#else
1150	DECLINLINE(RTCCUINTREG) ASMGetCR2(void)
1151	{
1152	RTCCUINTREG uCR2;
1153	# if RT_INLINE_ASM_USES_INTRIN
1154	uCR2 = __readcr2();
1155
1156	# elif RT_INLINE_ASM_GNU_STYLE
1157	# ifdef RT_ARCH_AMD64
1158	__asm__ __volatile__("movq %%cr2, %0\t\n" : "=r" (uCR2));
1159	# else
1160	__asm__ __volatile__("movl %%cr2, %0\t\n" : "=r" (uCR2));
1161	# endif
1162	# else
1163	__asm
1164	{
1165	# ifdef RT_ARCH_AMD64
1166	mov rax, cr2
1167	mov [uCR2], rax
1168	# else
1169	mov eax, cr2
1170	mov [uCR2], eax
1171	# endif
1172	}
1173	# endif
1174	return uCR2;
1175	}
1176	#endif
1177
1178
1179	/**
1180	* Sets the CR2 register.
1181	* @param uCR2 The new CR0 value.
1182	*/
1183	#if RT_INLINE_ASM_EXTERNAL
1184	DECLASM(void) ASMSetCR2(RTCCUINTREG uCR2);
1185	#else
1186	DECLINLINE(void) ASMSetCR2(RTCCUINTREG uCR2)
1187	{
1188	# if RT_INLINE_ASM_GNU_STYLE
1189	# ifdef RT_ARCH_AMD64
1190	__asm__ __volatile__("movq %0, %%cr2\n\t" :: "r" (uCR2));
1191	# else
1192	__asm__ __volatile__("movl %0, %%cr2\n\t" :: "r" (uCR2));
1193	# endif
1194	# else
1195	__asm
1196	{
1197	# ifdef RT_ARCH_AMD64
1198	mov rax, [uCR2]
1199	mov cr2, rax
1200	# else
1201	mov eax, [uCR2]
1202	mov cr2, eax
1203	# endif
1204	}
1205	# endif
1206	}
1207	#endif
1208
1209
1210	/**
1211	* Get cr3.
1212	* @returns cr3.
1213	*/
1214	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1215	DECLASM(RTCCUINTREG) ASMGetCR3(void);
1216	#else
1217	DECLINLINE(RTCCUINTREG) ASMGetCR3(void)
1218	{
1219	RTCCUINTREG uCR3;
1220	# if RT_INLINE_ASM_USES_INTRIN
1221	uCR3 = __readcr3();
1222
1223	# elif RT_INLINE_ASM_GNU_STYLE
1224	# ifdef RT_ARCH_AMD64
1225	__asm__ __volatile__("movq %%cr3, %0\t\n" : "=r" (uCR3));
1226	# else
1227	__asm__ __volatile__("movl %%cr3, %0\t\n" : "=r" (uCR3));
1228	# endif
1229	# else
1230	__asm
1231	{
1232	# ifdef RT_ARCH_AMD64
1233	mov rax, cr3
1234	mov [uCR3], rax
1235	# else
1236	mov eax, cr3
1237	mov [uCR3], eax
1238	# endif
1239	}
1240	# endif
1241	return uCR3;
1242	}
1243	#endif
1244
1245
1246	/**
1247	* Sets the CR3 register.
1248	*
1249	* @param uCR3 New CR3 value.
1250	*/
1251	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1252	DECLASM(void) ASMSetCR3(RTCCUINTREG uCR3);
1253	#else
1254	DECLINLINE(void) ASMSetCR3(RTCCUINTREG uCR3)
1255	{
1256	# if RT_INLINE_ASM_USES_INTRIN
1257	__writecr3(uCR3);
1258
1259	# elif RT_INLINE_ASM_GNU_STYLE
1260	# ifdef RT_ARCH_AMD64
1261	__asm__ __volatile__("movq %0, %%cr3\n\t" : : "r" (uCR3));
1262	# else
1263	__asm__ __volatile__("movl %0, %%cr3\n\t" : : "r" (uCR3));
1264	# endif
1265	# else
1266	__asm
1267	{
1268	# ifdef RT_ARCH_AMD64
1269	mov rax, [uCR3]
1270	mov cr3, rax
1271	# else
1272	mov eax, [uCR3]
1273	mov cr3, eax
1274	# endif
1275	}
1276	# endif
1277	}
1278	#endif
1279
1280
1281	/**
1282	* Reloads the CR3 register.
1283	*/
1284	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1285	DECLASM(void) ASMReloadCR3(void);
1286	#else
1287	DECLINLINE(void) ASMReloadCR3(void)
1288	{
1289	# if RT_INLINE_ASM_USES_INTRIN
1290	__writecr3(__readcr3());
1291
1292	# elif RT_INLINE_ASM_GNU_STYLE
1293	RTCCUINTREG u;
1294	# ifdef RT_ARCH_AMD64
1295	__asm__ __volatile__("movq %%cr3, %0\n\t"
1296	"movq %0, %%cr3\n\t"
1297	: "=r" (u));
1298	# else
1299	__asm__ __volatile__("movl %%cr3, %0\n\t"
1300	"movl %0, %%cr3\n\t"
1301	: "=r" (u));
1302	# endif
1303	# else
1304	__asm
1305	{
1306	# ifdef RT_ARCH_AMD64
1307	mov rax, cr3
1308	mov cr3, rax
1309	# else
1310	mov eax, cr3
1311	mov cr3, eax
1312	# endif
1313	}
1314	# endif
1315	}
1316	#endif
1317
1318
1319	/**
1320	* Get cr4.
1321	* @returns cr4.
1322	*/
1323	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1324	DECLASM(RTCCUINTREG) ASMGetCR4(void);
1325	#else
1326	DECLINLINE(RTCCUINTREG) ASMGetCR4(void)
1327	{
1328	RTCCUINTREG uCR4;
1329	# if RT_INLINE_ASM_USES_INTRIN
1330	uCR4 = __readcr4();
1331
1332	# elif RT_INLINE_ASM_GNU_STYLE
1333	# ifdef RT_ARCH_AMD64
1334	__asm__ __volatile__("movq %%cr4, %0\t\n" : "=r" (uCR4));
1335	# else
1336	__asm__ __volatile__("movl %%cr4, %0\t\n" : "=r" (uCR4));
1337	# endif
1338	# else
1339	__asm
1340	{
1341	# ifdef RT_ARCH_AMD64
1342	mov rax, cr4
1343	mov [uCR4], rax
1344	# else
1345	push eax /* just in case */
1346	/mov eax, cr4/
1347	_emit 0x0f
1348	_emit 0x20
1349	_emit 0xe0
1350	mov [uCR4], eax
1351	pop eax
1352	# endif
1353	}
1354	# endif
1355	return uCR4;
1356	}
1357	#endif
1358
1359
1360	/**
1361	* Sets the CR4 register.
1362	*
1363	* @param uCR4 New CR4 value.
1364	*/
1365	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1366	DECLASM(void) ASMSetCR4(RTCCUINTREG uCR4);
1367	#else
1368	DECLINLINE(void) ASMSetCR4(RTCCUINTREG uCR4)
1369	{
1370	# if RT_INLINE_ASM_USES_INTRIN
1371	__writecr4(uCR4);
1372
1373	# elif RT_INLINE_ASM_GNU_STYLE
1374	# ifdef RT_ARCH_AMD64
1375	__asm__ __volatile__("movq %0, %%cr4\n\t" : : "r" (uCR4));
1376	# else
1377	__asm__ __volatile__("movl %0, %%cr4\n\t" : : "r" (uCR4));
1378	# endif
1379	# else
1380	__asm
1381	{
1382	# ifdef RT_ARCH_AMD64
1383	mov rax, [uCR4]
1384	mov cr4, rax
1385	# else
1386	mov eax, [uCR4]
1387	_emit 0x0F
1388	_emit 0x22
1389	_emit 0xE0 /* mov cr4, eax */
1390	# endif
1391	}
1392	# endif
1393	}
1394	#endif
1395
1396
1397	/**
1398	* Get cr8.
1399	* @returns cr8.
1400	* @remark The lock prefix hack for access from non-64-bit modes is NOT used and 0 is returned.
1401	*/
1402	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1403	DECLASM(RTCCUINTREG) ASMGetCR8(void);
1404	#else
1405	DECLINLINE(RTCCUINTREG) ASMGetCR8(void)
1406	{
1407	# ifdef RT_ARCH_AMD64
1408	RTCCUINTREG uCR8;
1409	# if RT_INLINE_ASM_USES_INTRIN
1410	uCR8 = __readcr8();
1411
1412	# elif RT_INLINE_ASM_GNU_STYLE
1413	__asm__ __volatile__("movq %%cr8, %0\t\n" : "=r" (uCR8));
1414	# else
1415	__asm
1416	{
1417	mov rax, cr8
1418	mov [uCR8], rax
1419	}
1420	# endif
1421	return uCR8;
1422	# else /* !RT_ARCH_AMD64 */
1423	return 0;
1424	# endif /* !RT_ARCH_AMD64 */
1425	}
1426	#endif
1427
1428
1429	/**
1430	* Enables interrupts (EFLAGS.IF).
1431	*/
1432	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1433	DECLASM(void) ASMIntEnable(void);
1434	#else
1435	DECLINLINE(void) ASMIntEnable(void)
1436	{
1437	# if RT_INLINE_ASM_GNU_STYLE
1438	__asm("sti\n");
1439	# elif RT_INLINE_ASM_USES_INTRIN
1440	_enable();
1441	# else
1442	__asm sti
1443	# endif
1444	}
1445	#endif
1446
1447
1448	/**
1449	* Disables interrupts (!EFLAGS.IF).
1450	*/
1451	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1452	DECLASM(void) ASMIntDisable(void);
1453	#else
1454	DECLINLINE(void) ASMIntDisable(void)
1455	{
1456	# if RT_INLINE_ASM_GNU_STYLE
1457	__asm("cli\n");
1458	# elif RT_INLINE_ASM_USES_INTRIN
1459	_disable();
1460	# else
1461	__asm cli
1462	# endif
1463	}
1464	#endif
1465
1466
1467	/**
1468	* Disables interrupts and returns previous xFLAGS.
1469	*/
1470	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1471	DECLASM(RTCCUINTREG) ASMIntDisableFlags(void);
1472	#else
1473	DECLINLINE(RTCCUINTREG) ASMIntDisableFlags(void)
1474	{
1475	RTCCUINTREG xFlags;
1476	# if RT_INLINE_ASM_GNU_STYLE
1477	# ifdef RT_ARCH_AMD64
1478	__asm__ __volatile__("pushfq\n\t"
1479	"cli\n\t"
1480	"popq %0\n\t"
1481	: "=rm" (xFlags));
1482	# else
1483	__asm__ __volatile__("pushfl\n\t"
1484	"cli\n\t"
1485	"popl %0\n\t"
1486	: "=rm" (xFlags));
1487	# endif
1488	# elif RT_INLINE_ASM_USES_INTRIN && !defined(RT_ARCH_X86)
1489	xFlags = ASMGetFlags();
1490	_disable();
1491	# else
1492	__asm {
1493	pushfd
1494	cli
1495	pop [xFlags]
1496	}
1497	# endif
1498	return xFlags;
1499	}
1500	#endif
1501
1502
1503	/**
1504	* Are interrupts enabled?
1505	*
1506	* @returns true / false.
1507	*/
1508	DECLINLINE(RTCCUINTREG) ASMIntAreEnabled(void)
1509	{
1510	RTCCUINTREG uFlags = ASMGetFlags();
1511	return uFlags & 0x200 /* X86_EFL_IF */ ? true : false;
1512	}
1513
1514
1515	/**
1516	* Halts the CPU until interrupted.
1517	*/
1518	#if RT_INLINE_ASM_EXTERNAL
1519	DECLASM(void) ASMHalt(void);
1520	#else
1521	DECLINLINE(void) ASMHalt(void)
1522	{
1523	# if RT_INLINE_ASM_GNU_STYLE
1524	__asm__ __volatile__("hlt\n\t");
1525	# else
1526	__asm {
1527	hlt
1528	}
1529	# endif
1530	}
1531	#endif
1532
1533
1534	/**
1535	* The PAUSE variant of NOP for helping hyperthreaded CPUs detecing spin locks.
1536	*/
1537	#if RT_INLINE_ASM_EXTERNAL
1538	DECLASM(void) ASMNopPause(void);
1539	#else
1540	DECLINLINE(void) ASMNopPause(void)
1541	{
1542	# if RT_INLINE_ASM_GNU_STYLE
1543	__asm__ __volatile__(".byte 0xf3,0x90\n\t");
1544	# else
1545	__asm {
1546	_emit 0f3h
1547	_emit 090h
1548	}
1549	# endif
1550	}
1551	#endif
1552
1553
1554	/**
1555	* Reads a machine specific register.
1556	*
1557	* @returns Register content.
1558	* @param uRegister Register to read.
1559	*/
1560	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1561	DECLASM(uint64_t) ASMRdMsr(uint32_t uRegister);
1562	#else
1563	DECLINLINE(uint64_t) ASMRdMsr(uint32_t uRegister)
1564	{
1565	RTUINT64U u;
1566	# if RT_INLINE_ASM_GNU_STYLE
1567	__asm__ __volatile__("rdmsr\n\t"
1568	: "=a" (u.s.Lo),
1569	"=d" (u.s.Hi)
1570	: "c" (uRegister));
1571
1572	# elif RT_INLINE_ASM_USES_INTRIN
1573	u.u = __readmsr(uRegister);
1574
1575	# else
1576	__asm
1577	{
1578	mov ecx, [uRegister]
1579	rdmsr
1580	mov [u.s.Lo], eax
1581	mov [u.s.Hi], edx
1582	}
1583	# endif
1584
1585	return u.u;
1586	}
1587	#endif
1588
1589
1590	/**
1591	* Writes a machine specific register.
1592	*
1593	* @returns Register content.
1594	* @param uRegister Register to write to.
1595	* @param u64Val Value to write.
1596	*/
1597	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1598	DECLASM(void) ASMWrMsr(uint32_t uRegister, uint64_t u64Val);
1599	#else
1600	DECLINLINE(void) ASMWrMsr(uint32_t uRegister, uint64_t u64Val)
1601	{
1602	RTUINT64U u;
1603
1604	u.u = u64Val;
1605	# if RT_INLINE_ASM_GNU_STYLE
1606	__asm__ __volatile__("wrmsr\n\t"
1607	::"a" (u.s.Lo),
1608	"d" (u.s.Hi),
1609	"c" (uRegister));
1610
1611	# elif RT_INLINE_ASM_USES_INTRIN
1612	__writemsr(uRegister, u.u);
1613
1614	# else
1615	__asm
1616	{
1617	mov ecx, [uRegister]
1618	mov edx, [u.s.Hi]
1619	mov eax, [u.s.Lo]
1620	wrmsr
1621	}
1622	# endif
1623	}
1624	#endif
1625
1626
1627	/**
1628	* Reads low part of a machine specific register.
1629	*
1630	* @returns Register content.
1631	* @param uRegister Register to read.
1632	*/
1633	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1634	DECLASM(uint32_t) ASMRdMsr_Low(uint32_t uRegister);
1635	#else
1636	DECLINLINE(uint32_t) ASMRdMsr_Low(uint32_t uRegister)
1637	{
1638	uint32_t u32;
1639	# if RT_INLINE_ASM_GNU_STYLE
1640	__asm__ __volatile__("rdmsr\n\t"
1641	: "=a" (u32)
1642	: "c" (uRegister)
1643	: "edx");
1644
1645	# elif RT_INLINE_ASM_USES_INTRIN
1646	u32 = (uint32_t)__readmsr(uRegister);
1647
1648	#else
1649	__asm
1650	{
1651	mov ecx, [uRegister]
1652	rdmsr
1653	mov [u32], eax
1654	}
1655	# endif
1656
1657	return u32;
1658	}
1659	#endif
1660
1661
1662	/**
1663	* Reads high part of a machine specific register.
1664	*
1665	* @returns Register content.
1666	* @param uRegister Register to read.
1667	*/
1668	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1669	DECLASM(uint32_t) ASMRdMsr_High(uint32_t uRegister);
1670	#else
1671	DECLINLINE(uint32_t) ASMRdMsr_High(uint32_t uRegister)
1672	{
1673	uint32_t u32;
1674	# if RT_INLINE_ASM_GNU_STYLE
1675	__asm__ __volatile__("rdmsr\n\t"
1676	: "=d" (u32)
1677	: "c" (uRegister)
1678	: "eax");
1679
1680	# elif RT_INLINE_ASM_USES_INTRIN
1681	u32 = (uint32_t)(__readmsr(uRegister) >> 32);
1682
1683	# else
1684	__asm
1685	{
1686	mov ecx, [uRegister]
1687	rdmsr
1688	mov [u32], edx
1689	}
1690	# endif
1691
1692	return u32;
1693	}
1694	#endif
1695
1696
1697	/**
1698	* Gets dr0.
1699	*
1700	* @returns dr0.
1701	*/
1702	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1703	DECLASM(RTCCUINTREG) ASMGetDR0(void);
1704	#else
1705	DECLINLINE(RTCCUINTREG) ASMGetDR0(void)
1706	{
1707	RTCCUINTREG uDR0;
1708	# if RT_INLINE_ASM_USES_INTRIN
1709	uDR0 = __readdr(0);
1710	# elif RT_INLINE_ASM_GNU_STYLE
1711	# ifdef RT_ARCH_AMD64
1712	__asm__ __volatile__("movq %%dr0, %0\n\t" : "=r" (uDR0));
1713	# else
1714	__asm__ __volatile__("movl %%dr0, %0\n\t" : "=r" (uDR0));
1715	# endif
1716	# else
1717	__asm
1718	{
1719	# ifdef RT_ARCH_AMD64
1720	mov rax, dr0
1721	mov [uDR0], rax
1722	# else
1723	mov eax, dr0
1724	mov [uDR0], eax
1725	# endif
1726	}
1727	# endif
1728	return uDR0;
1729	}
1730	#endif
1731
1732
1733	/**
1734	* Gets dr1.
1735	*
1736	* @returns dr1.
1737	*/
1738	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1739	DECLASM(RTCCUINTREG) ASMGetDR1(void);
1740	#else
1741	DECLINLINE(RTCCUINTREG) ASMGetDR1(void)
1742	{
1743	RTCCUINTREG uDR1;
1744	# if RT_INLINE_ASM_USES_INTRIN
1745	uDR1 = __readdr(1);
1746	# elif RT_INLINE_ASM_GNU_STYLE
1747	# ifdef RT_ARCH_AMD64
1748	__asm__ __volatile__("movq %%dr1, %0\n\t" : "=r" (uDR1));
1749	# else
1750	__asm__ __volatile__("movl %%dr1, %0\n\t" : "=r" (uDR1));
1751	# endif
1752	# else
1753	__asm
1754	{
1755	# ifdef RT_ARCH_AMD64
1756	mov rax, dr1
1757	mov [uDR1], rax
1758	# else
1759	mov eax, dr1
1760	mov [uDR1], eax
1761	# endif
1762	}
1763	# endif
1764	return uDR1;
1765	}
1766	#endif
1767
1768
1769	/**
1770	* Gets dr2.
1771	*
1772	* @returns dr2.
1773	*/
1774	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1775	DECLASM(RTCCUINTREG) ASMGetDR2(void);
1776	#else
1777	DECLINLINE(RTCCUINTREG) ASMGetDR2(void)
1778	{
1779	RTCCUINTREG uDR2;
1780	# if RT_INLINE_ASM_USES_INTRIN
1781	uDR2 = __readdr(2);
1782	# elif RT_INLINE_ASM_GNU_STYLE
1783	# ifdef RT_ARCH_AMD64
1784	__asm__ __volatile__("movq %%dr2, %0\n\t" : "=r" (uDR2));
1785	# else
1786	__asm__ __volatile__("movl %%dr2, %0\n\t" : "=r" (uDR2));
1787	# endif
1788	# else
1789	__asm
1790	{
1791	# ifdef RT_ARCH_AMD64
1792	mov rax, dr2
1793	mov [uDR2], rax
1794	# else
1795	mov eax, dr2
1796	mov [uDR2], eax
1797	# endif
1798	}
1799	# endif
1800	return uDR2;
1801	}
1802	#endif
1803
1804
1805	/**
1806	* Gets dr3.
1807	*
1808	* @returns dr3.
1809	*/
1810	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1811	DECLASM(RTCCUINTREG) ASMGetDR3(void);
1812	#else
1813	DECLINLINE(RTCCUINTREG) ASMGetDR3(void)
1814	{
1815	RTCCUINTREG uDR3;
1816	# if RT_INLINE_ASM_USES_INTRIN
1817	uDR3 = __readdr(3);
1818	# elif RT_INLINE_ASM_GNU_STYLE
1819	# ifdef RT_ARCH_AMD64
1820	__asm__ __volatile__("movq %%dr3, %0\n\t" : "=r" (uDR3));
1821	# else
1822	__asm__ __volatile__("movl %%dr3, %0\n\t" : "=r" (uDR3));
1823	# endif
1824	# else
1825	__asm
1826	{
1827	# ifdef RT_ARCH_AMD64
1828	mov rax, dr3
1829	mov [uDR3], rax
1830	# else
1831	mov eax, dr3
1832	mov [uDR3], eax
1833	# endif
1834	}
1835	# endif
1836	return uDR3;
1837	}
1838	#endif
1839
1840
1841	/**
1842	* Gets dr6.
1843	*
1844	* @returns dr6.
1845	*/
1846	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1847	DECLASM(RTCCUINTREG) ASMGetDR6(void);
1848	#else
1849	DECLINLINE(RTCCUINTREG) ASMGetDR6(void)
1850	{
1851	RTCCUINTREG uDR6;
1852	# if RT_INLINE_ASM_USES_INTRIN
1853	uDR6 = __readdr(6);
1854	# elif RT_INLINE_ASM_GNU_STYLE
1855	# ifdef RT_ARCH_AMD64
1856	__asm__ __volatile__("movq %%dr6, %0\n\t" : "=r" (uDR6));
1857	# else
1858	__asm__ __volatile__("movl %%dr6, %0\n\t" : "=r" (uDR6));
1859	# endif
1860	# else
1861	__asm
1862	{
1863	# ifdef RT_ARCH_AMD64
1864	mov rax, dr6
1865	mov [uDR6], rax
1866	# else
1867	mov eax, dr6
1868	mov [uDR6], eax
1869	# endif
1870	}
1871	# endif
1872	return uDR6;
1873	}
1874	#endif
1875
1876
1877	/**
1878	* Reads and clears DR6.
1879	*
1880	* @returns DR6.
1881	*/
1882	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1883	DECLASM(RTCCUINTREG) ASMGetAndClearDR6(void);
1884	#else
1885	DECLINLINE(RTCCUINTREG) ASMGetAndClearDR6(void)
1886	{
1887	RTCCUINTREG uDR6;
1888	# if RT_INLINE_ASM_USES_INTRIN
1889	uDR6 = __readdr(6);
1890	__writedr(6, 0xffff0ff0U); /* 31-16 and 4-11 are 1's, 12 and 63-31 are zero. */
1891	# elif RT_INLINE_ASM_GNU_STYLE
1892	RTCCUINTREG uNewValue = 0xffff0ff0U;/* 31-16 and 4-11 are 1's, 12 and 63-31 are zero. */
1893	# ifdef RT_ARCH_AMD64
1894	__asm__ __volatile__("movq %%dr6, %0\n\t"
1895	"movq %1, %%dr6\n\t"
1896	: "=r" (uDR6)
1897	: "r" (uNewValue));
1898	# else
1899	__asm__ __volatile__("movl %%dr6, %0\n\t"
1900	"movl %1, %%dr6\n\t"
1901	: "=r" (uDR6)
1902	: "r" (uNewValue));
1903	# endif
1904	# else
1905	__asm
1906	{
1907	# ifdef RT_ARCH_AMD64
1908	mov rax, dr6
1909	mov [uDR6], rax
1910	mov rcx, rax
1911	mov ecx, 0ffff0ff0h; /* 31-16 and 4-11 are 1's, 12 and 63-31 are zero. */
1912	mov dr6, rcx
1913	# else
1914	mov eax, dr6
1915	mov [uDR6], eax
1916	mov ecx, 0ffff0ff0h; /* 31-16 and 4-11 are 1's, 12 is zero. */
1917	mov dr6, ecx
1918	# endif
1919	}
1920	# endif
1921	return uDR6;
1922	}
1923	#endif
1924
1925
1926	/**
1927	* Gets dr7.
1928	*
1929	* @returns dr7.
1930	*/
1931	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1932	DECLASM(RTCCUINTREG) ASMGetDR7(void);
1933	#else
1934	DECLINLINE(RTCCUINTREG) ASMGetDR7(void)
1935	{
1936	RTCCUINTREG uDR7;
1937	# if RT_INLINE_ASM_USES_INTRIN
1938	uDR7 = __readdr(7);
1939	# elif RT_INLINE_ASM_GNU_STYLE
1940	# ifdef RT_ARCH_AMD64
1941	__asm__ __volatile__("movq %%dr7, %0\n\t" : "=r" (uDR7));
1942	# else
1943	__asm__ __volatile__("movl %%dr7, %0\n\t" : "=r" (uDR7));
1944	# endif
1945	# else
1946	__asm
1947	{
1948	# ifdef RT_ARCH_AMD64
1949	mov rax, dr7
1950	mov [uDR7], rax
1951	# else
1952	mov eax, dr7
1953	mov [uDR7], eax
1954	# endif
1955	}
1956	# endif
1957	return uDR7;
1958	}
1959	#endif
1960
1961
1962	/**
1963	* Sets dr0.
1964	*
1965	* @param uDRVal Debug register value to write
1966	*/
1967	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1968	DECLASM(void) ASMSetDR0(RTCCUINTREG uDRVal);
1969	#else
1970	DECLINLINE(void) ASMSetDR0(RTCCUINTREG uDRVal)
1971	{
1972	# if RT_INLINE_ASM_USES_INTRIN
1973	__writedr(0, uDRVal);
1974	# elif RT_INLINE_ASM_GNU_STYLE
1975	# ifdef RT_ARCH_AMD64
1976	__asm__ __volatile__("movq %0, %%dr0\n\t" : : "r" (uDRVal));
1977	# else
1978	__asm__ __volatile__("movl %0, %%dr0\n\t" : : "r" (uDRVal));
1979	# endif
1980	# else
1981	__asm
1982	{
1983	# ifdef RT_ARCH_AMD64
1984	mov rax, [uDRVal]
1985	mov dr0, rax
1986	# else
1987	mov eax, [uDRVal]
1988	mov dr0, eax
1989	# endif
1990	}
1991	# endif
1992	}
1993	#endif
1994
1995
1996	/**
1997	* Sets dr1.
1998	*
1999	* @param uDRVal Debug register value to write
2000	*/
2001	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2002	DECLASM(void) ASMSetDR1(RTCCUINTREG uDRVal);
2003	#else
2004	DECLINLINE(void) ASMSetDR1(RTCCUINTREG uDRVal)
2005	{
2006	# if RT_INLINE_ASM_USES_INTRIN
2007	__writedr(1, uDRVal);
2008	# elif RT_INLINE_ASM_GNU_STYLE
2009	# ifdef RT_ARCH_AMD64
2010	__asm__ __volatile__("movq %0, %%dr1\n\t" : : "r" (uDRVal));
2011	# else
2012	__asm__ __volatile__("movl %0, %%dr1\n\t" : : "r" (uDRVal));
2013	# endif
2014	# else
2015	__asm
2016	{
2017	# ifdef RT_ARCH_AMD64
2018	mov rax, [uDRVal]
2019	mov dr1, rax
2020	# else
2021	mov eax, [uDRVal]
2022	mov dr1, eax
2023	# endif
2024	}
2025	# endif
2026	}
2027	#endif
2028
2029
2030	/**
2031	* Sets dr2.
2032	*
2033	* @param uDRVal Debug register value to write
2034	*/
2035	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2036	DECLASM(void) ASMSetDR2(RTCCUINTREG uDRVal);
2037	#else
2038	DECLINLINE(void) ASMSetDR2(RTCCUINTREG uDRVal)
2039	{
2040	# if RT_INLINE_ASM_USES_INTRIN
2041	__writedr(2, uDRVal);
2042	# elif RT_INLINE_ASM_GNU_STYLE
2043	# ifdef RT_ARCH_AMD64
2044	__asm__ __volatile__("movq %0, %%dr2\n\t" : : "r" (uDRVal));
2045	# else
2046	__asm__ __volatile__("movl %0, %%dr2\n\t" : : "r" (uDRVal));
2047	# endif
2048	# else
2049	__asm
2050	{
2051	# ifdef RT_ARCH_AMD64
2052	mov rax, [uDRVal]
2053	mov dr2, rax
2054	# else
2055	mov eax, [uDRVal]
2056	mov dr2, eax
2057	# endif
2058	}
2059	# endif
2060	}
2061	#endif
2062
2063
2064	/**
2065	* Sets dr3.
2066	*
2067	* @param uDRVal Debug register value to write
2068	*/
2069	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2070	DECLASM(void) ASMSetDR3(RTCCUINTREG uDRVal);
2071	#else
2072	DECLINLINE(void) ASMSetDR3(RTCCUINTREG uDRVal)
2073	{
2074	# if RT_INLINE_ASM_USES_INTRIN
2075	__writedr(3, uDRVal);
2076	# elif RT_INLINE_ASM_GNU_STYLE
2077	# ifdef RT_ARCH_AMD64
2078	__asm__ __volatile__("movq %0, %%dr3\n\t" : : "r" (uDRVal));
2079	# else
2080	__asm__ __volatile__("movl %0, %%dr3\n\t" : : "r" (uDRVal));
2081	# endif
2082	# else
2083	__asm
2084	{
2085	# ifdef RT_ARCH_AMD64
2086	mov rax, [uDRVal]
2087	mov dr3, rax
2088	# else
2089	mov eax, [uDRVal]
2090	mov dr3, eax
2091	# endif
2092	}
2093	# endif
2094	}
2095	#endif
2096
2097
2098	/**
2099	* Sets dr6.
2100	*
2101	* @param uDRVal Debug register value to write
2102	*/
2103	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2104	DECLASM(void) ASMSetDR6(RTCCUINTREG uDRVal);
2105	#else
2106	DECLINLINE(void) ASMSetDR6(RTCCUINTREG uDRVal)
2107	{
2108	# if RT_INLINE_ASM_USES_INTRIN
2109	__writedr(6, uDRVal);
2110	# elif RT_INLINE_ASM_GNU_STYLE
2111	# ifdef RT_ARCH_AMD64
2112	__asm__ __volatile__("movq %0, %%dr6\n\t" : : "r" (uDRVal));
2113	# else
2114	__asm__ __volatile__("movl %0, %%dr6\n\t" : : "r" (uDRVal));
2115	# endif
2116	# else
2117	__asm
2118	{
2119	# ifdef RT_ARCH_AMD64
2120	mov rax, [uDRVal]
2121	mov dr6, rax
2122	# else
2123	mov eax, [uDRVal]
2124	mov dr6, eax
2125	# endif
2126	}
2127	# endif
2128	}
2129	#endif
2130
2131
2132	/**
2133	* Sets dr7.
2134	*
2135	* @param uDRVal Debug register value to write
2136	*/
2137	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2138	DECLASM(void) ASMSetDR7(RTCCUINTREG uDRVal);
2139	#else
2140	DECLINLINE(void) ASMSetDR7(RTCCUINTREG uDRVal)
2141	{
2142	# if RT_INLINE_ASM_USES_INTRIN
2143	__writedr(7, uDRVal);
2144	# elif RT_INLINE_ASM_GNU_STYLE
2145	# ifdef RT_ARCH_AMD64
2146	__asm__ __volatile__("movq %0, %%dr7\n\t" : : "r" (uDRVal));
2147	# else
2148	__asm__ __volatile__("movl %0, %%dr7\n\t" : : "r" (uDRVal));
2149	# endif
2150	# else
2151	__asm
2152	{
2153	# ifdef RT_ARCH_AMD64
2154	mov rax, [uDRVal]
2155	mov dr7, rax
2156	# else
2157	mov eax, [uDRVal]
2158	mov dr7, eax
2159	# endif
2160	}
2161	# endif
2162	}
2163	#endif
2164
2165
2166	/**
2167	* Compiler memory barrier.
2168	*
2169	* Ensure that the compiler does not use any cached (register/tmp stack) memory
2170	* values or any outstanding writes when returning from this function.
2171	*
2172	* This function must be used if non-volatile data is modified by a
2173	* device or the VMM. Typical cases are port access, MMIO access,
2174	* trapping instruction, etc.
2175	*/
2176	#if RT_INLINE_ASM_GNU_STYLE
2177	# define ASMCompilerBarrier() do { __asm__ __volatile__("" : : : "memory"); } while (0)
2178	#elif RT_INLINE_ASM_USES_INTRIN
2179	# define ASMCompilerBarrier() do { _ReadWriteBarrier(); } while (0)
2180	#else /* 2003 should have _ReadWriteBarrier() but I guess we're at 2002 level then... */
2181	DECLINLINE(void) ASMCompilerBarrier(void)
2182	{
2183	__asm
2184	{
2185	}
2186	}
2187	#endif
2188
2189
2190	/**
2191	* Writes a 8-bit unsigned integer to an I/O port, ordered.
2192	*
2193	* @param Port I/O port to write to.
2194	* @param u8 8-bit integer to write.
2195	*/
2196	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2197	DECLASM(void) ASMOutU8(RTIOPORT Port, uint8_t u8);
2198	#else
2199	DECLINLINE(void) ASMOutU8(RTIOPORT Port, uint8_t u8)
2200	{
2201	# if RT_INLINE_ASM_GNU_STYLE
2202	__asm__ __volatile__("outb %b1, %w0\n\t"
2203	:: "Nd" (Port),
2204	"a" (u8));
2205
2206	# elif RT_INLINE_ASM_USES_INTRIN
2207	__outbyte(Port, u8);
2208
2209	# else
2210	__asm
2211	{
2212	mov dx, [Port]
2213	mov al, [u8]
2214	out dx, al
2215	}
2216	# endif
2217	}
2218	#endif
2219
2220
2221	/**
2222	* Reads a 8-bit unsigned integer from an I/O port, ordered.
2223	*
2224	* @returns 8-bit integer.
2225	* @param Port I/O port to read from.
2226	*/
2227	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2228	DECLASM(uint8_t) ASMInU8(RTIOPORT Port);
2229	#else
2230	DECLINLINE(uint8_t) ASMInU8(RTIOPORT Port)
2231	{
2232	uint8_t u8;
2233	# if RT_INLINE_ASM_GNU_STYLE
2234	__asm__ __volatile__("inb %w1, %b0\n\t"
2235	: "=a" (u8)
2236	: "Nd" (Port));
2237
2238	# elif RT_INLINE_ASM_USES_INTRIN
2239	u8 = __inbyte(Port);
2240
2241	# else
2242	__asm
2243	{
2244	mov dx, [Port]
2245	in al, dx
2246	mov [u8], al
2247	}
2248	# endif
2249	return u8;
2250	}
2251	#endif
2252
2253
2254	/**
2255	* Writes a 16-bit unsigned integer to an I/O port, ordered.
2256	*
2257	* @param Port I/O port to write to.
2258	* @param u16 16-bit integer to write.
2259	*/
2260	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2261	DECLASM(void) ASMOutU16(RTIOPORT Port, uint16_t u16);
2262	#else
2263	DECLINLINE(void) ASMOutU16(RTIOPORT Port, uint16_t u16)
2264	{
2265	# if RT_INLINE_ASM_GNU_STYLE
2266	__asm__ __volatile__("outw %w1, %w0\n\t"
2267	:: "Nd" (Port),
2268	"a" (u16));
2269
2270	# elif RT_INLINE_ASM_USES_INTRIN
2271	__outword(Port, u16);
2272
2273	# else
2274	__asm
2275	{
2276	mov dx, [Port]
2277	mov ax, [u16]
2278	out dx, ax
2279	}
2280	# endif
2281	}
2282	#endif
2283
2284
2285	/**
2286	* Reads a 16-bit unsigned integer from an I/O port, ordered.
2287	*
2288	* @returns 16-bit integer.
2289	* @param Port I/O port to read from.
2290	*/
2291	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2292	DECLASM(uint16_t) ASMInU16(RTIOPORT Port);
2293	#else
2294	DECLINLINE(uint16_t) ASMInU16(RTIOPORT Port)
2295	{
2296	uint16_t u16;
2297	# if RT_INLINE_ASM_GNU_STYLE
2298	__asm__ __volatile__("inw %w1, %w0\n\t"
2299	: "=a" (u16)
2300	: "Nd" (Port));
2301
2302	# elif RT_INLINE_ASM_USES_INTRIN
2303	u16 = __inword(Port);
2304
2305	# else
2306	__asm
2307	{
2308	mov dx, [Port]
2309	in ax, dx
2310	mov [u16], ax
2311	}
2312	# endif
2313	return u16;
2314	}
2315	#endif
2316
2317
2318	/**
2319	* Writes a 32-bit unsigned integer to an I/O port, ordered.
2320	*
2321	* @param Port I/O port to write to.
2322	* @param u32 32-bit integer to write.
2323	*/
2324	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2325	DECLASM(void) ASMOutU32(RTIOPORT Port, uint32_t u32);
2326	#else
2327	DECLINLINE(void) ASMOutU32(RTIOPORT Port, uint32_t u32)
2328	{
2329	# if RT_INLINE_ASM_GNU_STYLE
2330	__asm__ __volatile__("outl %1, %w0\n\t"
2331	:: "Nd" (Port),
2332	"a" (u32));
2333
2334	# elif RT_INLINE_ASM_USES_INTRIN
2335	__outdword(Port, u32);
2336
2337	# else
2338	__asm
2339	{
2340	mov dx, [Port]
2341	mov eax, [u32]
2342	out dx, eax
2343	}
2344	# endif
2345	}
2346	#endif
2347
2348
2349	/**
2350	* Reads a 32-bit unsigned integer from an I/O port, ordered.
2351	*
2352	* @returns 32-bit integer.
2353	* @param Port I/O port to read from.
2354	*/
2355	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2356	DECLASM(uint32_t) ASMInU32(RTIOPORT Port);
2357	#else
2358	DECLINLINE(uint32_t) ASMInU32(RTIOPORT Port)
2359	{
2360	uint32_t u32;
2361	# if RT_INLINE_ASM_GNU_STYLE
2362	__asm__ __volatile__("inl %w1, %0\n\t"
2363	: "=a" (u32)
2364	: "Nd" (Port));
2365
2366	# elif RT_INLINE_ASM_USES_INTRIN
2367	u32 = __indword(Port);
2368
2369	# else
2370	__asm
2371	{
2372	mov dx, [Port]
2373	in eax, dx
2374	mov [u32], eax
2375	}
2376	# endif
2377	return u32;
2378	}
2379	#endif
2380
2381
2382	/**
2383	* Writes a string of 8-bit unsigned integer items to an I/O port, ordered.
2384	*
2385	* @param Port I/O port to write to.
2386	* @param pau8 Pointer to the string buffer.
2387	* @param c The number of items to write.
2388	*/
2389	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2390	DECLASM(void) ASMOutStrU8(RTIOPORT Port, uint8_t const *pau8, size_t c);
2391	#else
2392	DECLINLINE(void) ASMOutStrU8(RTIOPORT Port, uint8_t const *pau8, size_t c)
2393	{
2394	# if RT_INLINE_ASM_GNU_STYLE
2395	__asm__ __volatile__("rep; outsb\n\t"
2396	: "+S" (pau8),
2397	"+c" (c)
2398	: "d" (Port));
2399
2400	# elif RT_INLINE_ASM_USES_INTRIN
2401	__outbytestring(Port, (unsigned char *)pau8, (unsigned long)c);
2402
2403	# else
2404	__asm
2405	{
2406	mov dx, [Port]
2407	mov ecx, [c]
2408	mov eax, [pau8]
2409	xchg esi, eax
2410	rep outsb
2411	xchg esi, eax
2412	}
2413	# endif
2414	}
2415	#endif
2416
2417
2418	/**
2419	* Reads a string of 8-bit unsigned integer items from an I/O port, ordered.
2420	*
2421	* @param Port I/O port to read from.
2422	* @param pau8 Pointer to the string buffer (output).
2423	* @param c The number of items to read.
2424	*/
2425	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2426	DECLASM(void) ASMInStrU8(RTIOPORT Port, uint8_t *pau8, size_t c);
2427	#else
2428	DECLINLINE(void) ASMInStrU8(RTIOPORT Port, uint8_t *pau8, size_t c)
2429	{
2430	# if RT_INLINE_ASM_GNU_STYLE
2431	__asm__ __volatile__("rep; insb\n\t"
2432	: "+D" (pau8),
2433	"+c" (c)
2434	: "d" (Port));
2435
2436	# elif RT_INLINE_ASM_USES_INTRIN
2437	__inbytestring(Port, pau8, (unsigned long)c);
2438
2439	# else
2440	__asm
2441	{
2442	mov dx, [Port]
2443	mov ecx, [c]
2444	mov eax, [pau8]
2445	xchg edi, eax
2446	rep insb
2447	xchg edi, eax
2448	}
2449	# endif
2450	}
2451	#endif
2452
2453
2454	/**
2455	* Writes a string of 16-bit unsigned integer items to an I/O port, ordered.
2456	*
2457	* @param Port I/O port to write to.
2458	* @param pau16 Pointer to the string buffer.
2459	* @param c The number of items to write.
2460	*/
2461	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2462	DECLASM(void) ASMOutStrU16(RTIOPORT Port, uint16_t const *pau16, size_t c);
2463	#else
2464	DECLINLINE(void) ASMOutStrU16(RTIOPORT Port, uint16_t const *pau16, size_t c)
2465	{
2466	# if RT_INLINE_ASM_GNU_STYLE
2467	__asm__ __volatile__("rep; outsw\n\t"
2468	: "+S" (pau16),
2469	"+c" (c)
2470	: "d" (Port));
2471
2472	# elif RT_INLINE_ASM_USES_INTRIN
2473	__outwordstring(Port, (unsigned short *)pau16, (unsigned long)c);
2474
2475	# else
2476	__asm
2477	{
2478	mov dx, [Port]
2479	mov ecx, [c]
2480	mov eax, [pau16]
2481	xchg esi, eax
2482	rep outsw
2483	xchg esi, eax
2484	}
2485	# endif
2486	}
2487	#endif
2488
2489
2490	/**
2491	* Reads a string of 16-bit unsigned integer items from an I/O port, ordered.
2492	*
2493	* @param Port I/O port to read from.
2494	* @param pau16 Pointer to the string buffer (output).
2495	* @param c The number of items to read.
2496	*/
2497	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2498	DECLASM(void) ASMInStrU16(RTIOPORT Port, uint16_t *pau16, size_t c);
2499	#else
2500	DECLINLINE(void) ASMInStrU16(RTIOPORT Port, uint16_t *pau16, size_t c)
2501	{
2502	# if RT_INLINE_ASM_GNU_STYLE
2503	__asm__ __volatile__("rep; insw\n\t"
2504	: "+D" (pau16),
2505	"+c" (c)
2506	: "d" (Port));
2507
2508	# elif RT_INLINE_ASM_USES_INTRIN
2509	__inwordstring(Port, pau16, (unsigned long)c);
2510
2511	# else
2512	__asm
2513	{
2514	mov dx, [Port]
2515	mov ecx, [c]
2516	mov eax, [pau16]
2517	xchg edi, eax
2518	rep insw
2519	xchg edi, eax
2520	}
2521	# endif
2522	}
2523	#endif
2524
2525
2526	/**
2527	* Writes a string of 32-bit unsigned integer items to an I/O port, ordered.
2528	*
2529	* @param Port I/O port to write to.
2530	* @param pau32 Pointer to the string buffer.
2531	* @param c The number of items to write.
2532	*/
2533	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2534	DECLASM(void) ASMOutStrU32(RTIOPORT Port, uint32_t const *pau32, size_t c);
2535	#else
2536	DECLINLINE(void) ASMOutStrU32(RTIOPORT Port, uint32_t const *pau32, size_t c)
2537	{
2538	# if RT_INLINE_ASM_GNU_STYLE
2539	__asm__ __volatile__("rep; outsl\n\t"
2540	: "+S" (pau32),
2541	"+c" (c)
2542	: "d" (Port));
2543
2544	# elif RT_INLINE_ASM_USES_INTRIN
2545	__outdwordstring(Port, (unsigned long *)pau32, (unsigned long)c);
2546
2547	# else
2548	__asm
2549	{
2550	mov dx, [Port]
2551	mov ecx, [c]
2552	mov eax, [pau32]
2553	xchg esi, eax
2554	rep outsd
2555	xchg esi, eax
2556	}
2557	# endif
2558	}
2559	#endif
2560
2561
2562	/**
2563	* Reads a string of 32-bit unsigned integer items from an I/O port, ordered.
2564	*
2565	* @param Port I/O port to read from.
2566	* @param pau32 Pointer to the string buffer (output).
2567	* @param c The number of items to read.
2568	*/
2569	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2570	DECLASM(void) ASMInStrU32(RTIOPORT Port, uint32_t *pau32, size_t c);
2571	#else
2572	DECLINLINE(void) ASMInStrU32(RTIOPORT Port, uint32_t *pau32, size_t c)
2573	{
2574	# if RT_INLINE_ASM_GNU_STYLE
2575	__asm__ __volatile__("rep; insl\n\t"
2576	: "+D" (pau32),
2577	"+c" (c)
2578	: "d" (Port));
2579
2580	# elif RT_INLINE_ASM_USES_INTRIN
2581	__indwordstring(Port, (unsigned long *)pau32, (unsigned long)c);
2582
2583	# else
2584	__asm
2585	{
2586	mov dx, [Port]
2587	mov ecx, [c]
2588	mov eax, [pau32]
2589	xchg edi, eax
2590	rep insd
2591	xchg edi, eax
2592	}
2593	# endif
2594	}
2595	#endif
2596
2597
2598	/**
2599	* Atomically Exchange an unsigned 8-bit value, ordered.
2600	*
2601	* @returns Current *pu8 value
2602	* @param pu8 Pointer to the 8-bit variable to update.
2603	* @param u8 The 8-bit value to assign to *pu8.
2604	*/
2605	#if RT_INLINE_ASM_EXTERNAL
2606	DECLASM(uint8_t) ASMAtomicXchgU8(volatile uint8_t *pu8, uint8_t u8);
2607	#else
2608	DECLINLINE(uint8_t) ASMAtomicXchgU8(volatile uint8_t *pu8, uint8_t u8)
2609	{
2610	# if RT_INLINE_ASM_GNU_STYLE
2611	__asm__ __volatile__("xchgb %0, %1\n\t"
2612	: "=m" (*pu8),
2613	"=q" (u8) /* =r - busted on g++ (GCC) 3.4.4 20050721 (Red Hat 3.4.4-2) */
2614	: "1" (u8),
2615	"m" (*pu8));
2616	# else
2617	__asm
2618	{
2619	# ifdef RT_ARCH_AMD64
2620	mov rdx, [pu8]
2621	mov al, [u8]
2622	xchg [rdx], al
2623	mov [u8], al
2624	# else
2625	mov edx, [pu8]
2626	mov al, [u8]
2627	xchg [edx], al
2628	mov [u8], al
2629	# endif
2630	}
2631	# endif
2632	return u8;
2633	}
2634	#endif
2635
2636
2637	/**
2638	* Atomically Exchange a signed 8-bit value, ordered.
2639	*
2640	* @returns Current *pu8 value
2641	* @param pi8 Pointer to the 8-bit variable to update.
2642	* @param i8 The 8-bit value to assign to *pi8.
2643	*/
2644	DECLINLINE(int8_t) ASMAtomicXchgS8(volatile int8_t *pi8, int8_t i8)
2645	{
2646	return (int8_t)ASMAtomicXchgU8((volatile uint8_t *)pi8, (uint8_t)i8);
2647	}
2648
2649
2650	/**
2651	* Atomically Exchange a bool value, ordered.
2652	*
2653	* @returns Current *pf value
2654	* @param pf Pointer to the 8-bit variable to update.
2655	* @param f The 8-bit value to assign to *pi8.
2656	*/
2657	DECLINLINE(bool) ASMAtomicXchgBool(volatile bool *pf, bool f)
2658	{
2659	#ifdef _MSC_VER
2660	return !!ASMAtomicXchgU8((volatile uint8_t *)pf, (uint8_t)f);
2661	#else
2662	return (bool)ASMAtomicXchgU8((volatile uint8_t *)pf, (uint8_t)f);
2663	#endif
2664	}
2665
2666
2667	/**
2668	* Atomically Exchange an unsigned 16-bit value, ordered.
2669	*
2670	* @returns Current *pu16 value
2671	* @param pu16 Pointer to the 16-bit variable to update.
2672	* @param u16 The 16-bit value to assign to *pu16.
2673	*/
2674	#if RT_INLINE_ASM_EXTERNAL
2675	DECLASM(uint16_t) ASMAtomicXchgU16(volatile uint16_t *pu16, uint16_t u16);
2676	#else
2677	DECLINLINE(uint16_t) ASMAtomicXchgU16(volatile uint16_t *pu16, uint16_t u16)
2678	{
2679	# if RT_INLINE_ASM_GNU_STYLE
2680	__asm__ __volatile__("xchgw %0, %1\n\t"
2681	: "=m" (*pu16),
2682	"=r" (u16)
2683	: "1" (u16),
2684	"m" (*pu16));
2685	# else
2686	__asm
2687	{
2688	# ifdef RT_ARCH_AMD64
2689	mov rdx, [pu16]
2690	mov ax, [u16]
2691	xchg [rdx], ax
2692	mov [u16], ax
2693	# else
2694	mov edx, [pu16]
2695	mov ax, [u16]
2696	xchg [edx], ax
2697	mov [u16], ax
2698	# endif
2699	}
2700	# endif
2701	return u16;
2702	}
2703	#endif
2704
2705
2706	/**
2707	* Atomically Exchange a signed 16-bit value, ordered.
2708	*
2709	* @returns Current *pu16 value
2710	* @param pi16 Pointer to the 16-bit variable to update.
2711	* @param i16 The 16-bit value to assign to *pi16.
2712	*/
2713	DECLINLINE(int16_t) ASMAtomicXchgS16(volatile int16_t *pi16, int16_t i16)
2714	{
2715	return (int16_t)ASMAtomicXchgU16((volatile uint16_t *)pi16, (uint16_t)i16);
2716	}
2717
2718
2719	/**
2720	* Atomically Exchange an unsigned 32-bit value, ordered.
2721	*
2722	* @returns Current *pu32 value
2723	* @param pu32 Pointer to the 32-bit variable to update.
2724	* @param u32 The 32-bit value to assign to *pu32.
2725	*/
2726	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2727	DECLASM(uint32_t) ASMAtomicXchgU32(volatile uint32_t *pu32, uint32_t u32);
2728	#else
2729	DECLINLINE(uint32_t) ASMAtomicXchgU32(volatile uint32_t *pu32, uint32_t u32)
2730	{
2731	# if RT_INLINE_ASM_GNU_STYLE
2732	__asm__ __volatile__("xchgl %0, %1\n\t"
2733	: "=m" (*pu32),
2734	"=r" (u32)
2735	: "1" (u32),
2736	"m" (*pu32));
2737
2738	# elif RT_INLINE_ASM_USES_INTRIN
2739	u32 = _InterlockedExchange((long *)pu32, u32);
2740
2741	# else
2742	__asm
2743	{
2744	# ifdef RT_ARCH_AMD64
2745	mov rdx, [pu32]
2746	mov eax, u32
2747	xchg [rdx], eax
2748	mov [u32], eax
2749	# else
2750	mov edx, [pu32]
2751	mov eax, u32
2752	xchg [edx], eax
2753	mov [u32], eax
2754	# endif
2755	}
2756	# endif
2757	return u32;
2758	}
2759	#endif
2760
2761
2762	/**
2763	* Atomically Exchange a signed 32-bit value, ordered.
2764	*
2765	* @returns Current *pu32 value
2766	* @param pi32 Pointer to the 32-bit variable to update.
2767	* @param i32 The 32-bit value to assign to *pi32.
2768	*/
2769	DECLINLINE(int32_t) ASMAtomicXchgS32(volatile int32_t *pi32, int32_t i32)
2770	{
2771	return (int32_t)ASMAtomicXchgU32((volatile uint32_t *)pi32, (uint32_t)i32);
2772	}
2773
2774
2775	/**
2776	* Atomically Exchange an unsigned 64-bit value, ordered.
2777	*
2778	* @returns Current *pu64 value
2779	* @param pu64 Pointer to the 64-bit variable to update.
2780	* @param u64 The 64-bit value to assign to *pu64.
2781	*/
2782	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2783	DECLASM(uint64_t) ASMAtomicXchgU64(volatile uint64_t *pu64, uint64_t u64);
2784	#else
2785	DECLINLINE(uint64_t) ASMAtomicXchgU64(volatile uint64_t *pu64, uint64_t u64)
2786	{
2787	# if defined(RT_ARCH_AMD64)
2788	# if RT_INLINE_ASM_USES_INTRIN
2789	u64 = _InterlockedExchange64((__int64 *)pu64, u64);
2790
2791	# elif RT_INLINE_ASM_GNU_STYLE
2792	__asm__ __volatile__("xchgq %0, %1\n\t"
2793	: "=m" (*pu64),
2794	"=r" (u64)
2795	: "1" (u64),
2796	"m" (*pu64));
2797	# else
2798	__asm
2799	{
2800	mov rdx, [pu64]
2801	mov rax, [u64]
2802	xchg [rdx], rax
2803	mov [u64], rax
2804	}
2805	# endif
2806	# else /* !RT_ARCH_AMD64 */
2807	# if RT_INLINE_ASM_GNU_STYLE
2808	# if defined(PIC) \|\| defined(__PIC__)
2809	uint32_t u32EBX = (uint32_t)u64;
2810	__asm__ __volatile__(/"xchgl %%esi, %5\n\t"/
2811	"xchgl %%ebx, %3\n\t"
2812	"1:\n\t"
2813	"lock; cmpxchg8b (%5)\n\t"
2814	"jnz 1b\n\t"
2815	"movl %3, %%ebx\n\t"
2816	/"xchgl %%esi, %5\n\t"/
2817	: "=A" (u64),
2818	"=m" (*pu64)
2819	: "0" (*pu64),
2820	"m" ( u32EBX ),
2821	"c" ( (uint32_t)(u64 >> 32) ),
2822	"S" (pu64));
2823	# else /* !PIC */
2824	__asm__ __volatile__("1:\n\t"
2825	"lock; cmpxchg8b %1\n\t"
2826	"jnz 1b\n\t"
2827	: "=A" (u64),
2828	"=m" (*pu64)
2829	: "0" (*pu64),
2830	"b" ( (uint32_t)u64 ),
2831	"c" ( (uint32_t)(u64 >> 32) ));
2832	# endif
2833	# else
2834	__asm
2835	{
2836	mov ebx, dword ptr [u64]
2837	mov ecx, dword ptr [u64 + 4]
2838	mov edi, pu64
2839	mov eax, dword ptr [edi]
2840	mov edx, dword ptr [edi + 4]
2841	retry:
2842	lock cmpxchg8b [edi]
2843	jnz retry
2844	mov dword ptr [u64], eax
2845	mov dword ptr [u64 + 4], edx
2846	}
2847	# endif
2848	# endif /* !RT_ARCH_AMD64 */
2849	return u64;
2850	}
2851	#endif
2852
2853
2854	/**
2855	* Atomically Exchange an signed 64-bit value, ordered.
2856	*
2857	* @returns Current *pi64 value
2858	* @param pi64 Pointer to the 64-bit variable to update.
2859	* @param i64 The 64-bit value to assign to *pi64.
2860	*/
2861	DECLINLINE(int64_t) ASMAtomicXchgS64(volatile int64_t *pi64, int64_t i64)
2862	{
2863	return (int64_t)ASMAtomicXchgU64((volatile uint64_t *)pi64, (uint64_t)i64);
2864	}
2865
2866
2867	/**
2868	* Atomically Exchange a pointer value, ordered.
2869	*
2870	* @returns Current *ppv value
2871	* @param ppv Pointer to the pointer variable to update.
2872	* @param pv The pointer value to assign to *ppv.
2873	*/
2874	DECLINLINE(void ) ASMAtomicXchgPtr(void volatile ppv, const void pv)
2875	{
2876	#if ARCH_BITS == 32
2877	return (void )ASMAtomicXchgU32((volatile uint32_t )(void *)ppv, (uint32_t)pv);
2878	#elif ARCH_BITS == 64
2879	return (void )ASMAtomicXchgU64((volatile uint64_t )(void *)ppv, (uint64_t)pv);
2880	#else
2881	# error "ARCH_BITS is bogus"
2882	#endif
2883	}
2884
2885
2886	/**
2887	* Atomically Exchange a raw-mode context pointer value, ordered.
2888	*
2889	* @returns Current *ppv value
2890	* @param ppvRC Pointer to the pointer variable to update.
2891	* @param pvRC The pointer value to assign to *ppv.
2892	*/
2893	DECLINLINE(RTRCPTR) ASMAtomicXchgRCPtr(RTRCPTR volatile *ppvRC, RTRCPTR pvRC)
2894	{
2895	return (RTRCPTR)ASMAtomicXchgU32((uint32_t volatile )(void )ppvRC, (uint32_t)pvRC);
2896	}
2897
2898
2899	/**
2900	* Atomically Exchange a ring-0 pointer value, ordered.
2901	*
2902	* @returns Current *ppv value
2903	* @param ppvR0 Pointer to the pointer variable to update.
2904	* @param pvR0 The pointer value to assign to *ppv.
2905	*/
2906	DECLINLINE(RTR0PTR) ASMAtomicXchgR0Ptr(RTR0PTR volatile *ppvR0, RTR0PTR pvR0)
2907	{
2908	#if R0_ARCH_BITS == 32
2909	return (RTR0PTR)ASMAtomicXchgU32((volatile uint32_t )(void )ppvR0, (uint32_t)pvR0);
2910	#elif R0_ARCH_BITS == 64
2911	return (RTR0PTR)ASMAtomicXchgU64((volatile uint64_t )(void )ppvR0, (uint64_t)pvR0);
2912	#else
2913	# error "R0_ARCH_BITS is bogus"
2914	#endif
2915	}
2916
2917
2918	/**
2919	* Atomically Exchange a ring-3 pointer value, ordered.
2920	*
2921	* @returns Current *ppv value
2922	* @param ppvR3 Pointer to the pointer variable to update.
2923	* @param pvR3 The pointer value to assign to *ppv.
2924	*/
2925	DECLINLINE(RTR3PTR) ASMAtomicXchgR3Ptr(RTR3PTR volatile *ppvR3, RTR3PTR pvR3)
2926	{
2927	#if R3_ARCH_BITS == 32
2928	return (RTR3PTR)ASMAtomicXchgU32((volatile uint32_t )(void )ppvR3, (uint32_t)pvR3);
2929	#elif R3_ARCH_BITS == 64
2930	return (RTR3PTR)ASMAtomicXchgU64((volatile uint64_t )(void )ppvR3, (uint64_t)pvR3);
2931	#else
2932	# error "R3_ARCH_BITS is bogus"
2933	#endif
2934	}
2935
2936
2937	/** @def ASMAtomicXchgHandle
2938	* Atomically Exchange a typical IPRT handle value, ordered.
2939	*
2940	* @param ph Pointer to the value to update.
2941	* @param hNew The new value to assigned to *pu.
2942	* @param phRes Where to store the current *ph value.
2943	*
2944	* @remarks This doesn't currently work for all handles (like RTFILE).
2945	*/
2946	#if HC_ARCH_BITS == 32
2947	# define ASMAtomicXchgHandle(ph, hNew, phRes) \
2948	do { \
2949	AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
2950	AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
2951	(uint32_t )(phRes) = ASMAtomicXchgU32((uint32_t volatile *)(ph), (const uint32_t)(hNew)); \
2952	} while (0)
2953	#elif HC_ARCH_BITS == 64
2954	# define ASMAtomicXchgHandle(ph, hNew, phRes) \
2955	do { \
2956	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
2957	AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
2958	(uint64_t )(phRes) = ASMAtomicXchgU64((uint64_t volatile *)(ph), (const uint64_t)(hNew)); \
2959	} while (0)
2960	#else
2961	# error HC_ARCH_BITS
2962	#endif
2963
2964
2965	/**
2966	* Atomically Exchange a value which size might differ
2967	* between platforms or compilers, ordered.
2968	*
2969	* @param pu Pointer to the variable to update.
2970	* @param uNew The value to assign to *pu.
2971	* @todo This is busted as its missing the result argument.
2972	*/
2973	#define ASMAtomicXchgSize(pu, uNew) \
2974	do { \
2975	switch (sizeof(*(pu))) { \
2976	case 1: ASMAtomicXchgU8((volatile uint8_t )(void )(pu), (uint8_t)(uNew)); break; \
2977	case 2: ASMAtomicXchgU16((volatile uint16_t )(void )(pu), (uint16_t)(uNew)); break; \
2978	case 4: ASMAtomicXchgU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew)); break; \
2979	case 8: ASMAtomicXchgU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew)); break; \
2980	default: AssertMsgFailed(("ASMAtomicXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
2981	} \
2982	} while (0)
2983
2984	/**
2985	* Atomically Exchange a value which size might differ
2986	* between platforms or compilers, ordered.
2987	*
2988	* @param pu Pointer to the variable to update.
2989	* @param uNew The value to assign to *pu.
2990	* @param puRes Where to store the current *pu value.
2991	*/
2992	#define ASMAtomicXchgSizeCorrect(pu, uNew, puRes) \
2993	do { \
2994	switch (sizeof(*(pu))) { \
2995	case 1: (uint8_t )(puRes) = ASMAtomicXchgU8((volatile uint8_t )(void )(pu), (uint8_t)(uNew)); break; \
2996	case 2: (uint16_t )(puRes) = ASMAtomicXchgU16((volatile uint16_t )(void )(pu), (uint16_t)(uNew)); break; \
2997	case 4: (uint32_t )(puRes) = ASMAtomicXchgU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew)); break; \
2998	case 8: (uint64_t )(puRes) = ASMAtomicXchgU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew)); break; \
2999	default: AssertMsgFailed(("ASMAtomicXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
3000	} \
3001	} while (0)
3002
3003
3004	/**
3005	* Atomically Compare and Exchange an unsigned 32-bit value, ordered.
3006	*
3007	* @returns true if xchg was done.
3008	* @returns false if xchg wasn't done.
3009	*
3010	* @param pu32 Pointer to the value to update.
3011	* @param u32New The new value to assigned to *pu32.
3012	* @param u32Old The old value to *pu32 compare with.
3013	*/
3014	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3015	DECLASM(bool) ASMAtomicCmpXchgU32(volatile uint32_t *pu32, const uint32_t u32New, const uint32_t u32Old);
3016	#else
3017	DECLINLINE(bool) ASMAtomicCmpXchgU32(volatile uint32_t *pu32, const uint32_t u32New, uint32_t u32Old)
3018	{
3019	# if RT_INLINE_ASM_GNU_STYLE
3020	uint8_t u8Ret;
3021	__asm__ __volatile__("lock; cmpxchgl %3, %0\n\t"
3022	"setz %1\n\t"
3023	: "=m" (*pu32),
3024	"=qm" (u8Ret),
3025	"=a" (u32Old)
3026	: "r" (u32New),
3027	"2" (u32Old),
3028	"m" (*pu32));
3029	return (bool)u8Ret;
3030
3031	# elif RT_INLINE_ASM_USES_INTRIN
3032	return _InterlockedCompareExchange((long *)pu32, u32New, u32Old) == u32Old;
3033
3034	# else
3035	uint32_t u32Ret;
3036	__asm
3037	{
3038	# ifdef RT_ARCH_AMD64
3039	mov rdx, [pu32]
3040	# else
3041	mov edx, [pu32]
3042	# endif
3043	mov eax, [u32Old]
3044	mov ecx, [u32New]
3045	# ifdef RT_ARCH_AMD64
3046	lock cmpxchg [rdx], ecx
3047	# else
3048	lock cmpxchg [edx], ecx
3049	# endif
3050	setz al
3051	movzx eax, al
3052	mov [u32Ret], eax
3053	}
3054	return !!u32Ret;
3055	# endif
3056	}
3057	#endif
3058
3059
3060	/**
3061	* Atomically Compare and Exchange a signed 32-bit value, ordered.
3062	*
3063	* @returns true if xchg was done.
3064	* @returns false if xchg wasn't done.
3065	*
3066	* @param pi32 Pointer to the value to update.
3067	* @param i32New The new value to assigned to *pi32.
3068	* @param i32Old The old value to *pi32 compare with.
3069	*/
3070	DECLINLINE(bool) ASMAtomicCmpXchgS32(volatile int32_t *pi32, const int32_t i32New, const int32_t i32Old)
3071	{
3072	return ASMAtomicCmpXchgU32((volatile uint32_t *)pi32, (uint32_t)i32New, (uint32_t)i32Old);
3073	}
3074
3075
3076	/**
3077	* Atomically Compare and exchange an unsigned 64-bit value, ordered.
3078	*
3079	* @returns true if xchg was done.
3080	* @returns false if xchg wasn't done.
3081	*
3082	* @param pu64 Pointer to the 64-bit variable to update.
3083	* @param u64New The 64-bit value to assign to *pu64.
3084	* @param u64Old The value to compare with.
3085	*/
3086	#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
3087	\|\| (RT_INLINE_ASM_GCC_4_3_X_X86 && defined(IN_RING3) && defined(__PIC__))
3088	DECLASM(bool) ASMAtomicCmpXchgU64(volatile uint64_t *pu64, const uint64_t u64New, const uint64_t u64Old);
3089	#else
3090	DECLINLINE(bool) ASMAtomicCmpXchgU64(volatile uint64_t *pu64, const uint64_t u64New, uint64_t u64Old)
3091	{
3092	# if RT_INLINE_ASM_USES_INTRIN
3093	return _InterlockedCompareExchange64((__int64 *)pu64, u64New, u64Old) == u64Old;
3094
3095	# elif defined(RT_ARCH_AMD64)
3096	# if RT_INLINE_ASM_GNU_STYLE
3097	uint8_t u8Ret;
3098	__asm__ __volatile__("lock; cmpxchgq %3, %0\n\t"
3099	"setz %1\n\t"
3100	: "=m" (*pu64),
3101	"=qm" (u8Ret),
3102	"=a" (u64Old)
3103	: "r" (u64New),
3104	"2" (u64Old),
3105	"m" (*pu64));
3106	return (bool)u8Ret;
3107	# else
3108	bool fRet;
3109	__asm
3110	{
3111	mov rdx, [pu32]
3112	mov rax, [u64Old]
3113	mov rcx, [u64New]
3114	lock cmpxchg [rdx], rcx
3115	setz al
3116	mov [fRet], al
3117	}
3118	return fRet;
3119	# endif
3120	# else /* !RT_ARCH_AMD64 */
3121	uint32_t u32Ret;
3122	# if RT_INLINE_ASM_GNU_STYLE
3123	# if defined(PIC) \|\| defined(__PIC__)
3124	uint32_t u32EBX = (uint32_t)u64New;
3125	uint32_t u32Spill;
3126	__asm__ __volatile__("xchgl %%ebx, %4\n\t"
3127	"lock; cmpxchg8b (%6)\n\t"
3128	"setz %%al\n\t"
3129	"movl %4, %%ebx\n\t"
3130	"movzbl %%al, %%eax\n\t"
3131	: "=a" (u32Ret),
3132	"=d" (u32Spill),
3133	# if (__GNUC__ * 100 + __GNUC_MINOR__) >= 403
3134	"+m" (*pu64)
3135	# else
3136	"=m" (*pu64)
3137	# endif
3138	: "A" (u64Old),
3139	"m" ( u32EBX ),
3140	"c" ( (uint32_t)(u64New >> 32) ),
3141	"S" (pu64));
3142	# else /* !PIC */
3143	uint32_t u32Spill;
3144	__asm__ __volatile__("lock; cmpxchg8b %2\n\t"
3145	"setz %%al\n\t"
3146	"movzbl %%al, %%eax\n\t"
3147	: "=a" (u32Ret),
3148	"=d" (u32Spill),
3149	"+m" (*pu64)
3150	: "A" (u64Old),
3151	"b" ( (uint32_t)u64New ),
3152	"c" ( (uint32_t)(u64New >> 32) ));
3153	# endif
3154	return (bool)u32Ret;
3155	# else
3156	__asm
3157	{
3158	mov ebx, dword ptr [u64New]
3159	mov ecx, dword ptr [u64New + 4]
3160	mov edi, [pu64]
3161	mov eax, dword ptr [u64Old]
3162	mov edx, dword ptr [u64Old + 4]
3163	lock cmpxchg8b [edi]
3164	setz al
3165	movzx eax, al
3166	mov dword ptr [u32Ret], eax
3167	}
3168	return !!u32Ret;
3169	# endif
3170	# endif /* !RT_ARCH_AMD64 */
3171	}
3172	#endif
3173
3174
3175	/**
3176	* Atomically Compare and exchange a signed 64-bit value, ordered.
3177	*
3178	* @returns true if xchg was done.
3179	* @returns false if xchg wasn't done.
3180	*
3181	* @param pi64 Pointer to the 64-bit variable to update.
3182	* @param i64 The 64-bit value to assign to *pu64.
3183	* @param i64Old The value to compare with.
3184	*/
3185	DECLINLINE(bool) ASMAtomicCmpXchgS64(volatile int64_t *pi64, const int64_t i64, const int64_t i64Old)
3186	{
3187	return ASMAtomicCmpXchgU64((volatile uint64_t *)pi64, (uint64_t)i64, (uint64_t)i64Old);
3188	}
3189
3190
3191	/**
3192	* Atomically Compare and Exchange a pointer value, ordered.
3193	*
3194	* @returns true if xchg was done.
3195	* @returns false if xchg wasn't done.
3196	*
3197	* @param ppv Pointer to the value to update.
3198	* @param pvNew The new value to assigned to *ppv.
3199	* @param pvOld The old value to *ppv compare with.
3200	*/
3201	DECLINLINE(bool) ASMAtomicCmpXchgPtr(void * volatile ppv, const void pvNew, const void *pvOld)
3202	{
3203	#if ARCH_BITS == 32
3204	return ASMAtomicCmpXchgU32((volatile uint32_t )(void )ppv, (uint32_t)pvNew, (uint32_t)pvOld);
3205	#elif ARCH_BITS == 64
3206	return ASMAtomicCmpXchgU64((volatile uint64_t )(void )ppv, (uint64_t)pvNew, (uint64_t)pvOld);
3207	#else
3208	# error "ARCH_BITS is bogus"
3209	#endif
3210	}
3211
3212
3213	/** @def ASMAtomicCmpXchgHandle
3214	* Atomically Compare and Exchange a typical IPRT handle value, ordered.
3215	*
3216	* @param ph Pointer to the value to update.
3217	* @param hNew The new value to assigned to *pu.
3218	* @param hOld The old value to *pu compare with.
3219	* @param fRc Where to store the result.
3220	*
3221	* @remarks This doesn't currently work for all handles (like RTFILE).
3222	*/
3223	#if HC_ARCH_BITS == 32
3224	# define ASMAtomicCmpXchgHandle(ph, hNew, hOld, fRc) \
3225	do { \
3226	AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
3227	(fRc) = ASMAtomicCmpXchgU32((uint32_t volatile *)(ph), (const uint32_t)(hNew), (const uint32_t)(hOld)); \
3228	} while (0)
3229	#elif HC_ARCH_BITS == 64
3230	# define ASMAtomicCmpXchgHandle(ph, hNew, hOld, fRc) \
3231	do { \
3232	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
3233	(fRc) = ASMAtomicCmpXchgU64((uint64_t volatile *)(ph), (const uint64_t)(hNew), (const uint64_t)(hOld)); \
3234	} while (0)
3235	#else
3236	# error HC_ARCH_BITS
3237	#endif
3238
3239
3240	/** @def ASMAtomicCmpXchgSize
3241	* Atomically Compare and Exchange a value which size might differ
3242	* between platforms or compilers, ordered.
3243	*
3244	* @param pu Pointer to the value to update.
3245	* @param uNew The new value to assigned to *pu.
3246	* @param uOld The old value to *pu compare with.
3247	* @param fRc Where to store the result.
3248	*/
3249	#define ASMAtomicCmpXchgSize(pu, uNew, uOld, fRc) \
3250	do { \
3251	switch (sizeof(*(pu))) { \
3252	case 4: (fRc) = ASMAtomicCmpXchgU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew), (uint32_t)(uOld)); \
3253	break; \
3254	case 8: (fRc) = ASMAtomicCmpXchgU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew), (uint64_t)(uOld)); \
3255	break; \
3256	default: AssertMsgFailed(("ASMAtomicCmpXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
3257	(fRc) = false; \
3258	break; \
3259	} \
3260	} while (0)
3261
3262
3263	/**
3264	* Atomically Compare and Exchange an unsigned 32-bit value, additionally
3265	* passes back old value, ordered.
3266	*
3267	* @returns true if xchg was done.
3268	* @returns false if xchg wasn't done.
3269	*
3270	* @param pu32 Pointer to the value to update.
3271	* @param u32New The new value to assigned to *pu32.
3272	* @param u32Old The old value to *pu32 compare with.
3273	* @param pu32Old Pointer store the old value at.
3274	*/
3275	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3276	DECLASM(bool) ASMAtomicCmpXchgExU32(volatile uint32_t pu32, const uint32_t u32New, const uint32_t u32Old, uint32_t pu32Old);
3277	#else
3278	DECLINLINE(bool) ASMAtomicCmpXchgExU32(volatile uint32_t pu32, const uint32_t u32New, const uint32_t u32Old, uint32_t pu32Old)
3279	{
3280	# if RT_INLINE_ASM_GNU_STYLE
3281	uint8_t u8Ret;
3282	__asm__ __volatile__("lock; cmpxchgl %3, %0\n\t"
3283	"setz %1\n\t"
3284	: "=m" (*pu32),
3285	"=qm" (u8Ret),
3286	"=a" (*pu32Old)
3287	: "r" (u32New),
3288	"a" (u32Old),
3289	"m" (*pu32));
3290	return (bool)u8Ret;
3291
3292	# elif RT_INLINE_ASM_USES_INTRIN
3293	return (pu32Old =_InterlockedCompareExchange((long )pu32, u32New, u32Old)) == u32Old;
3294
3295	# else
3296	uint32_t u32Ret;
3297	__asm
3298	{
3299	# ifdef RT_ARCH_AMD64
3300	mov rdx, [pu32]
3301	# else
3302	mov edx, [pu32]
3303	# endif
3304	mov eax, [u32Old]
3305	mov ecx, [u32New]
3306	# ifdef RT_ARCH_AMD64
3307	lock cmpxchg [rdx], ecx
3308	mov rdx, [pu32Old]
3309	mov [rdx], eax
3310	# else
3311	lock cmpxchg [edx], ecx
3312	mov edx, [pu32Old]
3313	mov [edx], eax
3314	# endif
3315	setz al
3316	movzx eax, al
3317	mov [u32Ret], eax
3318	}
3319	return !!u32Ret;
3320	# endif
3321	}
3322	#endif
3323
3324
3325	/**
3326	* Atomically Compare and Exchange a signed 32-bit value, additionally
3327	* passes back old value, ordered.
3328	*
3329	* @returns true if xchg was done.
3330	* @returns false if xchg wasn't done.
3331	*
3332	* @param pi32 Pointer to the value to update.
3333	* @param i32New The new value to assigned to *pi32.
3334	* @param i32Old The old value to *pi32 compare with.
3335	* @param pi32Old Pointer store the old value at.
3336	*/
3337	DECLINLINE(bool) ASMAtomicCmpXchgExS32(volatile int32_t pi32, const int32_t i32New, const int32_t i32Old, int32_t pi32Old)
3338	{
3339	return ASMAtomicCmpXchgExU32((volatile uint32_t )pi32, (uint32_t)i32New, (uint32_t)i32Old, (uint32_t )pi32Old);
3340	}
3341
3342
3343	/**
3344	* Atomically Compare and exchange an unsigned 64-bit value, additionally
3345	* passing back old value, ordered.
3346	*
3347	* @returns true if xchg was done.
3348	* @returns false if xchg wasn't done.
3349	*
3350	* @param pu64 Pointer to the 64-bit variable to update.
3351	* @param u64New The 64-bit value to assign to *pu64.
3352	* @param u64Old The value to compare with.
3353	* @param pu64Old Pointer store the old value at.
3354	*/
3355	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3356	DECLASM(bool) ASMAtomicCmpXchgExU64(volatile uint64_t pu64, const uint64_t u64New, const uint64_t u64Old, uint64_t pu64Old);
3357	#else
3358	DECLINLINE(bool) ASMAtomicCmpXchgExU64(volatile uint64_t pu64, const uint64_t u64New, const uint64_t u64Old, uint64_t pu64Old)
3359	{
3360	# if RT_INLINE_ASM_USES_INTRIN
3361	return (pu64Old =_InterlockedCompareExchange64((__int64 )pu64, u64New, u64Old)) == u64Old;
3362
3363	# elif defined(RT_ARCH_AMD64)
3364	# if RT_INLINE_ASM_GNU_STYLE
3365	uint8_t u8Ret;
3366	__asm__ __volatile__("lock; cmpxchgq %3, %0\n\t"
3367	"setz %1\n\t"
3368	: "=m" (*pu64),
3369	"=qm" (u8Ret),
3370	"=a" (*pu64Old)
3371	: "r" (u64New),
3372	"a" (u64Old),
3373	"m" (*pu64));
3374	return (bool)u8Ret;
3375	# else
3376	bool fRet;
3377	__asm
3378	{
3379	mov rdx, [pu32]
3380	mov rax, [u64Old]
3381	mov rcx, [u64New]
3382	lock cmpxchg [rdx], rcx
3383	mov rdx, [pu64Old]
3384	mov [rdx], rax
3385	setz al
3386	mov [fRet], al
3387	}
3388	return fRet;
3389	# endif
3390	# else /* !RT_ARCH_AMD64 */
3391	# if RT_INLINE_ASM_GNU_STYLE
3392	uint64_t u64Ret;
3393	# if defined(PIC) \|\| defined(__PIC__)
3394	/* NB: this code uses a memory clobber description, because the clean
3395	* solution with an output value for *pu64 makes gcc run out of registers.
3396	* This will cause suboptimal code, and anyone with a better solution is
3397	* welcome to improve this. */
3398	__asm__ __volatile__("xchgl %%ebx, %1\n\t"
3399	"lock; cmpxchg8b %3\n\t"
3400	"xchgl %%ebx, %1\n\t"
3401	: "=A" (u64Ret)
3402	: "DS" ((uint32_t)u64New),
3403	"c" ((uint32_t)(u64New >> 32)),
3404	"m" (*pu64),
3405	"0" (u64Old)
3406	: "memory" );
3407	# else /* !PIC */
3408	__asm__ __volatile__("lock; cmpxchg8b %4\n\t"
3409	: "=A" (u64Ret),
3410	"=m" (*pu64)
3411	: "b" ((uint32_t)u64New),
3412	"c" ((uint32_t)(u64New >> 32)),
3413	"m" (*pu64),
3414	"0" (u64Old));
3415	# endif
3416	*pu64Old = u64Ret;
3417	return u64Ret == u64Old;
3418	# else
3419	uint32_t u32Ret;
3420	__asm
3421	{
3422	mov ebx, dword ptr [u64New]
3423	mov ecx, dword ptr [u64New + 4]
3424	mov edi, [pu64]
3425	mov eax, dword ptr [u64Old]
3426	mov edx, dword ptr [u64Old + 4]
3427	lock cmpxchg8b [edi]
3428	mov ebx, [pu64Old]
3429	mov [ebx], eax
3430	setz al
3431	movzx eax, al
3432	add ebx, 4
3433	mov [ebx], edx
3434	mov dword ptr [u32Ret], eax
3435	}
3436	return !!u32Ret;
3437	# endif
3438	# endif /* !RT_ARCH_AMD64 */
3439	}
3440	#endif
3441
3442
3443	/**
3444	* Atomically Compare and exchange a signed 64-bit value, additionally
3445	* passing back old value, ordered.
3446	*
3447	* @returns true if xchg was done.
3448	* @returns false if xchg wasn't done.
3449	*
3450	* @param pi64 Pointer to the 64-bit variable to update.
3451	* @param i64 The 64-bit value to assign to *pu64.
3452	* @param i64Old The value to compare with.
3453	* @param pi64Old Pointer store the old value at.
3454	*/
3455	DECLINLINE(bool) ASMAtomicCmpXchgExS64(volatile int64_t pi64, const int64_t i64, const int64_t i64Old, int64_t pi64Old)
3456	{
3457	return ASMAtomicCmpXchgExU64((volatile uint64_t )pi64, (uint64_t)i64, (uint64_t)i64Old, (uint64_t )pi64Old);
3458	}
3459
3460	/** @def ASMAtomicCmpXchgExHandle
3461	* Atomically Compare and Exchange a typical IPRT handle value, ordered.
3462	*
3463	* @param ph Pointer to the value to update.
3464	* @param hNew The new value to assigned to *pu.
3465	* @param hOld The old value to *pu compare with.
3466	* @param fRc Where to store the result.
3467	* @param phOldVal Pointer to where to store the old value.
3468	*
3469	* @remarks This doesn't currently work for all handles (like RTFILE).
3470	*/
3471	#if HC_ARCH_BITS == 32
3472	# define ASMAtomicCmpXchgExHandle(ph, hNew, hOld, fRc, phOldVal) \
3473	do { \
3474	AssertCompile(sizeof(*ph) == sizeof(uint32_t)); \
3475	AssertCompile(sizeof(*phOldVal) == sizeof(uint32_t)); \
3476	(fRc) = ASMAtomicCmpXchgExU32((volatile uint32_t )(pu), (uint32_t)(uNew), (uint32_t)(uOld), (uint32_t )(puOldVal)); \
3477	} while (0)
3478	#elif HC_ARCH_BITS == 64
3479	# define ASMAtomicCmpXchgExHandle(ph, hNew, hOld, fRc, phOldVal) \
3480	do { \
3481	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
3482	AssertCompile(sizeof(*(phOldVal)) == sizeof(uint64_t)); \
3483	(fRc) = ASMAtomicCmpXchgExU64((volatile uint64_t )(pu), (uint64_t)(uNew), (uint64_t)(uOld), (uint64_t )(puOldVal)); \
3484	} while (0)
3485	#else
3486	# error HC_ARCH_BITS
3487	#endif
3488
3489
3490	/** @def ASMAtomicCmpXchgExSize
3491	* Atomically Compare and Exchange a value which size might differ
3492	* between platforms or compilers. Additionally passes back old value.
3493	*
3494	* @param pu Pointer to the value to update.
3495	* @param uNew The new value to assigned to *pu.
3496	* @param uOld The old value to *pu compare with.
3497	* @param fRc Where to store the result.
3498	* @param puOldVal Pointer to where to store the old value.
3499	*/
3500	#define ASMAtomicCmpXchgExSize(pu, uNew, uOld, fRc, puOldVal) \
3501	do { \
3502	switch (sizeof(*(pu))) { \
3503	case 4: (fRc) = ASMAtomicCmpXchgExU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew), (uint32_t)(uOld), (uint32_t *)(uOldVal)); \
3504	break; \
3505	case 8: (fRc) = ASMAtomicCmpXchgExU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew), (uint64_t)(uOld), (uint64_t *)(uOldVal)); \
3506	break; \
3507	default: AssertMsgFailed(("ASMAtomicCmpXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
3508	(fRc) = false; \
3509	(uOldVal) = 0; \
3510	break; \
3511	} \
3512	} while (0)
3513
3514
3515	/**
3516	* Atomically Compare and Exchange a pointer value, additionally
3517	* passing back old value, ordered.
3518	*
3519	* @returns true if xchg was done.
3520	* @returns false if xchg wasn't done.
3521	*
3522	* @param ppv Pointer to the value to update.
3523	* @param pvNew The new value to assigned to *ppv.
3524	* @param pvOld The old value to *ppv compare with.
3525	* @param ppvOld Pointer store the old value at.
3526	*/
3527	DECLINLINE(bool) ASMAtomicCmpXchgExPtr(void * volatile ppv, const void pvNew, const void pvOld, void *ppvOld)
3528	{
3529	#if ARCH_BITS == 32
3530	return ASMAtomicCmpXchgExU32((volatile uint32_t )(void )ppv, (uint32_t)pvNew, (uint32_t)pvOld, (uint32_t *)ppvOld);
3531	#elif ARCH_BITS == 64
3532	return ASMAtomicCmpXchgExU64((volatile uint64_t )(void )ppv, (uint64_t)pvNew, (uint64_t)pvOld, (uint64_t *)ppvOld);
3533	#else
3534	# error "ARCH_BITS is bogus"
3535	#endif
3536	}
3537
3538
3539	/**
3540	* Atomically exchanges and adds to a 32-bit value, ordered.
3541	*
3542	* @returns The old value.
3543	* @param pu32 Pointer to the value.
3544	* @param u32 Number to add.
3545	*/
3546	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3547	DECLASM(uint32_t) ASMAtomicAddU32(uint32_t volatile *pu32, uint32_t u32);
3548	#else
3549	DECLINLINE(uint32_t) ASMAtomicAddU32(uint32_t volatile *pu32, uint32_t u32)
3550	{
3551	# if RT_INLINE_ASM_USES_INTRIN
3552	u32 = _InterlockedExchangeAdd((long *)pu32, u32);
3553	return u32;
3554
3555	# elif RT_INLINE_ASM_GNU_STYLE
3556	__asm__ __volatile__("lock; xaddl %0, %1\n\t"
3557	: "=r" (u32),
3558	"=m" (*pu32)
3559	: "0" (u32),
3560	"m" (*pu32)
3561	: "memory");
3562	return u32;
3563	# else
3564	__asm
3565	{
3566	mov eax, [u32]
3567	# ifdef RT_ARCH_AMD64
3568	mov rdx, [pu32]
3569	lock xadd [rdx], eax
3570	# else
3571	mov edx, [pu32]
3572	lock xadd [edx], eax
3573	# endif
3574	mov [u32], eax
3575	}
3576	return u32;
3577	# endif
3578	}
3579	#endif
3580
3581
3582	/**
3583	* Atomically exchanges and adds to a signed 32-bit value, ordered.
3584	*
3585	* @returns The old value.
3586	* @param pi32 Pointer to the value.
3587	* @param i32 Number to add.
3588	*/
3589	DECLINLINE(int32_t) ASMAtomicAddS32(int32_t volatile *pi32, int32_t i32)
3590	{
3591	return (int32_t)ASMAtomicAddU32((uint32_t volatile *)pi32, (uint32_t)i32);
3592	}
3593
3594
3595	/**
3596	* Atomically exchanges and subtracts to an unsigned 32-bit value, ordered.
3597	*
3598	* @returns The old value.
3599	* @param pu32 Pointer to the value.
3600	* @param u32 Number to subtract.
3601	*/
3602	DECLINLINE(uint32_t) ASMAtomicSubU32(int32_t volatile *pi32, uint32_t u32)
3603	{
3604	return ASMAtomicAddU32((uint32_t volatile *)pi32, (uint32_t)-(int32_t)u32);
3605	}
3606
3607
3608	/**
3609	* Atomically exchanges and subtracts to a signed 32-bit value, ordered.
3610	*
3611	* @returns The old value.
3612	* @param pi32 Pointer to the value.
3613	* @param i32 Number to subtract.
3614	*/
3615	DECLINLINE(int32_t) ASMAtomicSubS32(int32_t volatile *pi32, int32_t i32)
3616	{
3617	return (int32_t)ASMAtomicAddU32((uint32_t volatile *)pi32, (uint32_t)-i32);
3618	}
3619
3620
3621	/**
3622	* Atomically increment a 32-bit value, ordered.
3623	*
3624	* @returns The new value.
3625	* @param pu32 Pointer to the value to increment.
3626	*/
3627	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3628	DECLASM(uint32_t) ASMAtomicIncU32(uint32_t volatile *pu32);
3629	#else
3630	DECLINLINE(uint32_t) ASMAtomicIncU32(uint32_t volatile *pu32)
3631	{
3632	uint32_t u32;
3633	# if RT_INLINE_ASM_USES_INTRIN
3634	u32 = _InterlockedIncrement((long *)pu32);
3635	return u32;
3636
3637	# elif RT_INLINE_ASM_GNU_STYLE
3638	__asm__ __volatile__("lock; xaddl %0, %1\n\t"
3639	: "=r" (u32),
3640	"=m" (*pu32)
3641	: "0" (1),
3642	"m" (*pu32)
3643	: "memory");
3644	return u32+1;
3645	# else
3646	__asm
3647	{
3648	mov eax, 1
3649	# ifdef RT_ARCH_AMD64
3650	mov rdx, [pu32]
3651	lock xadd [rdx], eax
3652	# else
3653	mov edx, [pu32]
3654	lock xadd [edx], eax
3655	# endif
3656	mov u32, eax
3657	}
3658	return u32+1;
3659	# endif
3660	}
3661	#endif
3662
3663
3664	/**
3665	* Atomically increment a signed 32-bit value, ordered.
3666	*
3667	* @returns The new value.
3668	* @param pi32 Pointer to the value to increment.
3669	*/
3670	DECLINLINE(int32_t) ASMAtomicIncS32(int32_t volatile *pi32)
3671	{
3672	return (int32_t)ASMAtomicIncU32((uint32_t volatile *)pi32);
3673	}
3674
3675
3676	/**
3677	* Atomically decrement an unsigned 32-bit value, ordered.
3678	*
3679	* @returns The new value.
3680	* @param pu32 Pointer to the value to decrement.
3681	*/
3682	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3683	DECLASM(uint32_t) ASMAtomicDecU32(uint32_t volatile *pu32);
3684	#else
3685	DECLINLINE(uint32_t) ASMAtomicDecU32(uint32_t volatile *pu32)
3686	{
3687	uint32_t u32;
3688	# if RT_INLINE_ASM_USES_INTRIN
3689	u32 = _InterlockedDecrement((long *)pu32);
3690	return u32;
3691
3692	# elif RT_INLINE_ASM_GNU_STYLE
3693	__asm__ __volatile__("lock; xaddl %0, %1\n\t"
3694	: "=r" (u32),
3695	"=m" (*pu32)
3696	: "0" (-1),
3697	"m" (*pu32)
3698	: "memory");
3699	return u32-1;
3700	# else
3701	__asm
3702	{
3703	mov eax, -1
3704	# ifdef RT_ARCH_AMD64
3705	mov rdx, [pu32]
3706	lock xadd [rdx], eax
3707	# else
3708	mov edx, [pu32]
3709	lock xadd [edx], eax
3710	# endif
3711	mov u32, eax
3712	}
3713	return u32-1;
3714	# endif
3715	}
3716	#endif
3717
3718
3719	/**
3720	* Atomically decrement a signed 32-bit value, ordered.
3721	*
3722	* @returns The new value.
3723	* @param pi32 Pointer to the value to decrement.
3724	*/
3725	DECLINLINE(int32_t) ASMAtomicDecS32(int32_t volatile *pi32)
3726	{
3727	return (int32_t)ASMAtomicDecU32((uint32_t volatile *)pi32);
3728	}
3729
3730
3731	/**
3732	* Atomically Or an unsigned 32-bit value, ordered.
3733	*
3734	* @param pu32 Pointer to the pointer variable to OR u32 with.
3735	* @param u32 The value to OR *pu32 with.
3736	*/
3737	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3738	DECLASM(void) ASMAtomicOrU32(uint32_t volatile *pu32, uint32_t u32);
3739	#else
3740	DECLINLINE(void) ASMAtomicOrU32(uint32_t volatile *pu32, uint32_t u32)
3741	{
3742	# if RT_INLINE_ASM_USES_INTRIN
3743	_InterlockedOr((long volatile *)pu32, (long)u32);
3744
3745	# elif RT_INLINE_ASM_GNU_STYLE
3746	__asm__ __volatile__("lock; orl %1, %0\n\t"
3747	: "=m" (*pu32)
3748	: "ir" (u32),
3749	"m" (*pu32));
3750	# else
3751	__asm
3752	{
3753	mov eax, [u32]
3754	# ifdef RT_ARCH_AMD64
3755	mov rdx, [pu32]
3756	lock or [rdx], eax
3757	# else
3758	mov edx, [pu32]
3759	lock or [edx], eax
3760	# endif
3761	}
3762	# endif
3763	}
3764	#endif
3765
3766
3767	/**
3768	* Atomically Or a signed 32-bit value, ordered.
3769	*
3770	* @param pi32 Pointer to the pointer variable to OR u32 with.
3771	* @param i32 The value to OR *pu32 with.
3772	*/
3773	DECLINLINE(void) ASMAtomicOrS32(int32_t volatile *pi32, int32_t i32)
3774	{
3775	ASMAtomicOrU32((uint32_t volatile *)pi32, i32);
3776	}
3777
3778
3779	/**
3780	* Atomically And an unsigned 32-bit value, ordered.
3781	*
3782	* @param pu32 Pointer to the pointer variable to AND u32 with.
3783	* @param u32 The value to AND *pu32 with.
3784	*/
3785	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3786	DECLASM(void) ASMAtomicAndU32(uint32_t volatile *pu32, uint32_t u32);
3787	#else
3788	DECLINLINE(void) ASMAtomicAndU32(uint32_t volatile *pu32, uint32_t u32)
3789	{
3790	# if RT_INLINE_ASM_USES_INTRIN
3791	_InterlockedAnd((long volatile *)pu32, u32);
3792
3793	# elif RT_INLINE_ASM_GNU_STYLE
3794	__asm__ __volatile__("lock; andl %1, %0\n\t"
3795	: "=m" (*pu32)
3796	: "ir" (u32),
3797	"m" (*pu32));
3798	# else
3799	__asm
3800	{
3801	mov eax, [u32]
3802	# ifdef RT_ARCH_AMD64
3803	mov rdx, [pu32]
3804	lock and [rdx], eax
3805	# else
3806	mov edx, [pu32]
3807	lock and [edx], eax
3808	# endif
3809	}
3810	# endif
3811	}
3812	#endif
3813
3814
3815	/**
3816	* Atomically And a signed 32-bit value, ordered.
3817	*
3818	* @param pi32 Pointer to the pointer variable to AND i32 with.
3819	* @param i32 The value to AND *pi32 with.
3820	*/
3821	DECLINLINE(void) ASMAtomicAndS32(int32_t volatile *pi32, int32_t i32)
3822	{
3823	ASMAtomicAndU32((uint32_t volatile *)pi32, (uint32_t)i32);
3824	}
3825
3826
3827	/**
3828	* Serialize Instruction.
3829	*/
3830	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3831	DECLASM(void) ASMSerializeInstruction(void);
3832	#else
3833	DECLINLINE(void) ASMSerializeInstruction(void)
3834	{
3835	# if RT_INLINE_ASM_GNU_STYLE
3836	RTCCUINTREG xAX = 0;
3837	# ifdef RT_ARCH_AMD64
3838	__asm__ ("cpuid"
3839	: "=a" (xAX)
3840	: "0" (xAX)
3841	: "rbx", "rcx", "rdx");
3842	# elif (defined(PIC) \|\| defined(__PIC__)) && defined(__i386__)
3843	__asm__ ("push %%ebx\n\t"
3844	"cpuid\n\t"
3845	"pop %%ebx\n\t"
3846	: "=a" (xAX)
3847	: "0" (xAX)
3848	: "ecx", "edx");
3849	# else
3850	__asm__ ("cpuid"
3851	: "=a" (xAX)
3852	: "0" (xAX)
3853	: "ebx", "ecx", "edx");
3854	# endif
3855
3856	# elif RT_INLINE_ASM_USES_INTRIN
3857	int aInfo[4];
3858	__cpuid(aInfo, 0);
3859
3860	# else
3861	__asm
3862	{
3863	push ebx
3864	xor eax, eax
3865	cpuid
3866	pop ebx
3867	}
3868	# endif
3869	}
3870	#endif
3871
3872
3873	/**
3874	* Memory load/store fence, waits for any pending writes and reads to complete.
3875	* Requires the X86_CPUID_FEATURE_EDX_SSE2 CPUID bit set.
3876	*/
3877	DECLINLINE(void) ASMMemoryFenceSSE2(void)
3878	{
3879	#if RT_INLINE_ASM_GNU_STYLE
3880	__asm__ __volatile__ (".byte 0x0f,0xae,0xf0\n\t");
3881	#elif RT_INLINE_ASM_USES_INTRIN
3882	_mm_mfence();
3883	#else
3884	__asm
3885	{
3886	_emit 0x0f
3887	_emit 0xae
3888	_emit 0xf0
3889	}
3890	#endif
3891	}
3892
3893
3894	/**
3895	* Memory store fence, waits for any writes to complete.
3896	* Requires the X86_CPUID_FEATURE_EDX_SSE CPUID bit set.
3897	*/
3898	DECLINLINE(void) ASMWriteFenceSSE(void)
3899	{
3900	#if RT_INLINE_ASM_GNU_STYLE
3901	__asm__ __volatile__ (".byte 0x0f,0xae,0xf8\n\t");
3902	#elif RT_INLINE_ASM_USES_INTRIN
3903	_mm_sfence();
3904	#else
3905	__asm
3906	{
3907	_emit 0x0f
3908	_emit 0xae
3909	_emit 0xf8
3910	}
3911	#endif
3912	}
3913
3914
3915	/**
3916	* Memory load fence, waits for any pending reads to complete.
3917	* Requires the X86_CPUID_FEATURE_EDX_SSE2 CPUID bit set.
3918	*/
3919	DECLINLINE(void) ASMReadFenceSSE2(void)
3920	{
3921	#if RT_INLINE_ASM_GNU_STYLE
3922	__asm__ __volatile__ (".byte 0x0f,0xae,0xe8\n\t");
3923	#elif RT_INLINE_ASM_USES_INTRIN
3924	_mm_lfence();
3925	#else
3926	__asm
3927	{
3928	_emit 0x0f
3929	_emit 0xae
3930	_emit 0xe8
3931	}
3932	#endif
3933	}
3934
3935
3936	/**
3937	* Memory fence, waits for any pending writes and reads to complete.
3938	*/
3939	DECLINLINE(void) ASMMemoryFence(void)
3940	{
3941	/** @todo use mfence? check if all cpus we care for support it. */
3942	uint32_t volatile u32;
3943	ASMAtomicXchgU32(&u32, 0);
3944	}
3945
3946
3947	/**
3948	* Write fence, waits for any pending writes to complete.
3949	*/
3950	DECLINLINE(void) ASMWriteFence(void)
3951	{
3952	/** @todo use sfence? check if all cpus we care for support it. */
3953	ASMMemoryFence();
3954	}
3955
3956
3957	/**
3958	* Read fence, waits for any pending reads to complete.
3959	*/
3960	DECLINLINE(void) ASMReadFence(void)
3961	{
3962	/** @todo use lfence? check if all cpus we care for support it. */
3963	ASMMemoryFence();
3964	}
3965
3966
3967	/**
3968	* Atomically reads an unsigned 8-bit value, ordered.
3969	*
3970	* @returns Current *pu8 value
3971	* @param pu8 Pointer to the 8-bit variable to read.
3972	*/
3973	DECLINLINE(uint8_t) ASMAtomicReadU8(volatile uint8_t *pu8)
3974	{
3975	ASMMemoryFence();
3976	return pu8; / byte reads are atomic on x86 */
3977	}
3978
3979
3980	/**
3981	* Atomically reads an unsigned 8-bit value, unordered.
3982	*
3983	* @returns Current *pu8 value
3984	* @param pu8 Pointer to the 8-bit variable to read.
3985	*/
3986	DECLINLINE(uint8_t) ASMAtomicUoReadU8(volatile uint8_t *pu8)
3987	{
3988	return pu8; / byte reads are atomic on x86 */
3989	}
3990
3991
3992	/**
3993	* Atomically reads a signed 8-bit value, ordered.
3994	*
3995	* @returns Current *pi8 value
3996	* @param pi8 Pointer to the 8-bit variable to read.
3997	*/
3998	DECLINLINE(int8_t) ASMAtomicReadS8(volatile int8_t *pi8)
3999	{
4000	ASMMemoryFence();
4001	return pi8; / byte reads are atomic on x86 */
4002	}
4003
4004
4005	/**
4006	* Atomically reads a signed 8-bit value, unordered.
4007	*
4008	* @returns Current *pi8 value
4009	* @param pi8 Pointer to the 8-bit variable to read.
4010	*/
4011	DECLINLINE(int8_t) ASMAtomicUoReadS8(volatile int8_t *pi8)
4012	{
4013	return pi8; / byte reads are atomic on x86 */
4014	}
4015
4016
4017	/**
4018	* Atomically reads an unsigned 16-bit value, ordered.
4019	*
4020	* @returns Current *pu16 value
4021	* @param pu16 Pointer to the 16-bit variable to read.
4022	*/
4023	DECLINLINE(uint16_t) ASMAtomicReadU16(volatile uint16_t *pu16)
4024	{
4025	ASMMemoryFence();
4026	Assert(!((uintptr_t)pu16 & 1));
4027	return *pu16;
4028	}
4029
4030
4031	/**
4032	* Atomically reads an unsigned 16-bit value, unordered.
4033	*
4034	* @returns Current *pu16 value
4035	* @param pu16 Pointer to the 16-bit variable to read.
4036	*/
4037	DECLINLINE(uint16_t) ASMAtomicUoReadU16(volatile uint16_t *pu16)
4038	{
4039	Assert(!((uintptr_t)pu16 & 1));
4040	return *pu16;
4041	}
4042
4043
4044	/**
4045	* Atomically reads a signed 16-bit value, ordered.
4046	*
4047	* @returns Current *pi16 value
4048	* @param pi16 Pointer to the 16-bit variable to read.
4049	*/
4050	DECLINLINE(int16_t) ASMAtomicReadS16(volatile int16_t *pi16)
4051	{
4052	ASMMemoryFence();
4053	Assert(!((uintptr_t)pi16 & 1));
4054	return *pi16;
4055	}
4056
4057
4058	/**
4059	* Atomically reads a signed 16-bit value, unordered.
4060	*
4061	* @returns Current *pi16 value
4062	* @param pi16 Pointer to the 16-bit variable to read.
4063	*/
4064	DECLINLINE(int16_t) ASMAtomicUoReadS16(volatile int16_t *pi16)
4065	{
4066	Assert(!((uintptr_t)pi16 & 1));
4067	return *pi16;
4068	}
4069
4070
4071	/**
4072	* Atomically reads an unsigned 32-bit value, ordered.
4073	*
4074	* @returns Current *pu32 value
4075	* @param pu32 Pointer to the 32-bit variable to read.
4076	*/
4077	DECLINLINE(uint32_t) ASMAtomicReadU32(volatile uint32_t *pu32)
4078	{
4079	ASMMemoryFence();
4080	Assert(!((uintptr_t)pu32 & 3));
4081	return *pu32;
4082	}
4083
4084
4085	/**
4086	* Atomically reads an unsigned 32-bit value, unordered.
4087	*
4088	* @returns Current *pu32 value
4089	* @param pu32 Pointer to the 32-bit variable to read.
4090	*/
4091	DECLINLINE(uint32_t) ASMAtomicUoReadU32(volatile uint32_t *pu32)
4092	{
4093	Assert(!((uintptr_t)pu32 & 3));
4094	return *pu32;
4095	}
4096
4097
4098	/**
4099	* Atomically reads a signed 32-bit value, ordered.
4100	*
4101	* @returns Current *pi32 value
4102	* @param pi32 Pointer to the 32-bit variable to read.
4103	*/
4104	DECLINLINE(int32_t) ASMAtomicReadS32(volatile int32_t *pi32)
4105	{
4106	ASMMemoryFence();
4107	Assert(!((uintptr_t)pi32 & 3));
4108	return *pi32;
4109	}
4110
4111
4112	/**
4113	* Atomically reads a signed 32-bit value, unordered.
4114	*
4115	* @returns Current *pi32 value
4116	* @param pi32 Pointer to the 32-bit variable to read.
4117	*/
4118	DECLINLINE(int32_t) ASMAtomicUoReadS32(volatile int32_t *pi32)
4119	{
4120	Assert(!((uintptr_t)pi32 & 3));
4121	return *pi32;
4122	}
4123
4124
4125	/**
4126	* Atomically reads an unsigned 64-bit value, ordered.
4127	*
4128	* @returns Current *pu64 value
4129	* @param pu64 Pointer to the 64-bit variable to read.
4130	* The memory pointed to must be writable.
4131	* @remark This will fault if the memory is read-only!
4132	*/
4133	#if (RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)) \
4134	\|\| (RT_INLINE_ASM_GCC_4_3_X_X86 && defined(IN_RING3) && defined(__PIC__))
4135	DECLASM(uint64_t) ASMAtomicReadU64(volatile uint64_t *pu64);
4136	#else
4137	DECLINLINE(uint64_t) ASMAtomicReadU64(volatile uint64_t *pu64)
4138	{
4139	uint64_t u64;
4140	# ifdef RT_ARCH_AMD64
4141	Assert(!((uintptr_t)pu64 & 7));
4142	/*# if RT_INLINE_ASM_GNU_STYLE
4143	__asm__ __volatile__( "mfence\n\t"
4144	"movq %1, %0\n\t"
4145	: "=r" (u64)
4146	: "m" (*pu64));
4147	# else
4148	__asm
4149	{
4150	mfence
4151	mov rdx, [pu64]
4152	mov rax, [rdx]
4153	mov [u64], rax
4154	}
4155	# endif*/
4156	ASMMemoryFence();
4157	u64 = *pu64;
4158	# else /* !RT_ARCH_AMD64 */
4159	# if RT_INLINE_ASM_GNU_STYLE
4160	# if defined(PIC) \|\| defined(__PIC__)
4161	uint32_t u32EBX = 0;
4162	Assert(!((uintptr_t)pu64 & 7));
4163	__asm__ __volatile__("xchgl %%ebx, %3\n\t"
4164	"lock; cmpxchg8b (%5)\n\t"
4165	"movl %3, %%ebx\n\t"
4166	: "=A" (u64),
4167	# if (__GNUC__ * 100 + __GNUC_MINOR__) >= 403
4168	"+m" (*pu64)
4169	# else
4170	"=m" (*pu64)
4171	# endif
4172	: "0" (0),
4173	"m" (u32EBX),
4174	"c" (0),
4175	"S" (pu64));
4176	# else /* !PIC */
4177	__asm__ __volatile__("lock; cmpxchg8b %1\n\t"
4178	: "=A" (u64),
4179	"+m" (*pu64)
4180	: "0" (0),
4181	"b" (0),
4182	"c" (0));
4183	# endif
4184	# else
4185	Assert(!((uintptr_t)pu64 & 7));
4186	__asm
4187	{
4188	xor eax, eax
4189	xor edx, edx
4190	mov edi, pu64
4191	xor ecx, ecx
4192	xor ebx, ebx
4193	lock cmpxchg8b [edi]
4194	mov dword ptr [u64], eax
4195	mov dword ptr [u64 + 4], edx
4196	}
4197	# endif
4198	# endif /* !RT_ARCH_AMD64 */
4199	return u64;
4200	}
4201	#endif
4202
4203
4204	/**
4205	* Atomically reads an unsigned 64-bit value, unordered.
4206	*
4207	* @returns Current *pu64 value
4208	* @param pu64 Pointer to the 64-bit variable to read.
4209	* The memory pointed to must be writable.
4210	* @remark This will fault if the memory is read-only!
4211	*/
4212	#if RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)
4213	DECLASM(uint64_t) ASMAtomicUoReadU64(volatile uint64_t *pu64);
4214	#else
4215	DECLINLINE(uint64_t) ASMAtomicUoReadU64(volatile uint64_t *pu64)
4216	{
4217	uint64_t u64;
4218	# ifdef RT_ARCH_AMD64
4219	Assert(!((uintptr_t)pu64 & 7));
4220	/*# if RT_INLINE_ASM_GNU_STYLE
4221	Assert(!((uintptr_t)pu64 & 7));
4222	__asm__ __volatile__("movq %1, %0\n\t"
4223	: "=r" (u64)
4224	: "m" (*pu64));
4225	# else
4226	__asm
4227	{
4228	mov rdx, [pu64]
4229	mov rax, [rdx]
4230	mov [u64], rax
4231	}
4232	# endif */
4233	u64 = *pu64;
4234	# else /* !RT_ARCH_AMD64 */
4235	# if RT_INLINE_ASM_GNU_STYLE
4236	# if defined(PIC) \|\| defined(__PIC__)
4237	uint32_t u32EBX = 0;
4238	uint32_t u32Spill;
4239	Assert(!((uintptr_t)pu64 & 7));
4240	__asm__ __volatile__("xor %%eax,%%eax\n\t"
4241	"xor %%ecx,%%ecx\n\t"
4242	"xor %%edx,%%edx\n\t"
4243	"xchgl %%ebx, %3\n\t"
4244	"lock; cmpxchg8b (%4)\n\t"
4245	"movl %3, %%ebx\n\t"
4246	: "=A" (u64),
4247	# if (__GNUC__ * 100 + __GNUC_MINOR__) >= 403
4248	"+m" (*pu64),
4249	# else
4250	"=m" (*pu64),
4251	# endif
4252	"=c" (u32Spill)
4253	: "m" (u32EBX),
4254	"S" (pu64));
4255	# else /* !PIC */
4256	__asm__ __volatile__("cmpxchg8b %1\n\t"
4257	: "=A" (u64),
4258	"+m" (*pu64)
4259	: "0" (0),
4260	"b" (0),
4261	"c" (0));
4262	# endif
4263	# else
4264	Assert(!((uintptr_t)pu64 & 7));
4265	__asm
4266	{
4267	xor eax, eax
4268	xor edx, edx
4269	mov edi, pu64
4270	xor ecx, ecx
4271	xor ebx, ebx
4272	lock cmpxchg8b [edi]
4273	mov dword ptr [u64], eax
4274	mov dword ptr [u64 + 4], edx
4275	}
4276	# endif
4277	# endif /* !RT_ARCH_AMD64 */
4278	return u64;
4279	}
4280	#endif
4281
4282
4283	/**
4284	* Atomically reads a signed 64-bit value, ordered.
4285	*
4286	* @returns Current *pi64 value
4287	* @param pi64 Pointer to the 64-bit variable to read.
4288	* The memory pointed to must be writable.
4289	* @remark This will fault if the memory is read-only!
4290	*/
4291	DECLINLINE(int64_t) ASMAtomicReadS64(volatile int64_t *pi64)
4292	{
4293	return (int64_t)ASMAtomicReadU64((volatile uint64_t *)pi64);
4294	}
4295
4296
4297	/**
4298	* Atomically reads a signed 64-bit value, unordered.
4299	*
4300	* @returns Current *pi64 value
4301	* @param pi64 Pointer to the 64-bit variable to read.
4302	* The memory pointed to must be writable.
4303	* @remark This will fault if the memory is read-only!
4304	*/
4305	DECLINLINE(int64_t) ASMAtomicUoReadS64(volatile int64_t *pi64)
4306	{
4307	return (int64_t)ASMAtomicUoReadU64((volatile uint64_t *)pi64);
4308	}
4309
4310
4311	/**
4312	* Atomically reads a pointer value, ordered.
4313	*
4314	* @returns Current *pv value
4315	* @param ppv Pointer to the pointer variable to read.
4316	*/
4317	DECLINLINE(void ) ASMAtomicReadPtr(void volatile *ppv)
4318	{
4319	#if ARCH_BITS == 32
4320	return (void )ASMAtomicReadU32((volatile uint32_t )(void *)ppv);
4321	#elif ARCH_BITS == 64
4322	return (void )ASMAtomicReadU64((volatile uint64_t )(void *)ppv);
4323	#else
4324	# error "ARCH_BITS is bogus"
4325	#endif
4326	}
4327
4328
4329	/**
4330	* Atomically reads a pointer value, unordered.
4331	*
4332	* @returns Current *pv value
4333	* @param ppv Pointer to the pointer variable to read.
4334	*/
4335	DECLINLINE(void ) ASMAtomicUoReadPtr(void volatile *ppv)
4336	{
4337	#if ARCH_BITS == 32
4338	return (void )ASMAtomicUoReadU32((volatile uint32_t )(void *)ppv);
4339	#elif ARCH_BITS == 64
4340	return (void )ASMAtomicUoReadU64((volatile uint64_t )(void *)ppv);
4341	#else
4342	# error "ARCH_BITS is bogus"
4343	#endif
4344	}
4345
4346
4347	/**
4348	* Atomically reads a boolean value, ordered.
4349	*
4350	* @returns Current *pf value
4351	* @param pf Pointer to the boolean variable to read.
4352	*/
4353	DECLINLINE(bool) ASMAtomicReadBool(volatile bool *pf)
4354	{
4355	ASMMemoryFence();
4356	return pf; / byte reads are atomic on x86 */
4357	}
4358
4359
4360	/**
4361	* Atomically reads a boolean value, unordered.
4362	*
4363	* @returns Current *pf value
4364	* @param pf Pointer to the boolean variable to read.
4365	*/
4366	DECLINLINE(bool) ASMAtomicUoReadBool(volatile bool *pf)
4367	{
4368	return pf; / byte reads are atomic on x86 */
4369	}
4370
4371
4372	/**
4373	* Atomically read a typical IPRT handle value, ordered.
4374	*
4375	* @param ph Pointer to the handle variable to read.
4376	* @param phRes Where to store the result.
4377	*
4378	* @remarks This doesn't currently work for all handles (like RTFILE).
4379	*/
4380	#if HC_ARCH_BITS == 32
4381	# define ASMAtomicReadHandle(ph, phRes) \
4382	do { \
4383	AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
4384	AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
4385	(uint32_t )(phRes) = ASMAtomicReadU32((uint32_t volatile *)(ph)); \
4386	} while (0)
4387	#elif HC_ARCH_BITS == 64
4388	# define ASMAtomicReadHandle(ph, phRes) \
4389	do { \
4390	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
4391	AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
4392	(uint64_t )(phRes) = ASMAtomicReadU64((uint64_t volatile *)(ph)); \
4393	} while (0)
4394	#else
4395	# error HC_ARCH_BITS
4396	#endif
4397
4398
4399	/**
4400	* Atomically read a typical IPRT handle value, unordered.
4401	*
4402	* @param ph Pointer to the handle variable to read.
4403	* @param phRes Where to store the result.
4404	*
4405	* @remarks This doesn't currently work for all handles (like RTFILE).
4406	*/
4407	#if HC_ARCH_BITS == 32
4408	# define ASMAtomicUoReadHandle(ph, phRes) \
4409	do { \
4410	AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
4411	AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
4412	(uint32_t )(phRes) = ASMAtomicUoReadU32((uint32_t volatile *)(ph)); \
4413	} while (0)
4414	#elif HC_ARCH_BITS == 64
4415	# define ASMAtomicUoReadHandle(ph, phRes) \
4416	do { \
4417	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
4418	AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
4419	(uint64_t )(phRes) = ASMAtomicUoReadU64((uint64_t volatile *)(ph)); \
4420	} while (0)
4421	#else
4422	# error HC_ARCH_BITS
4423	#endif
4424
4425
4426	/**
4427	* Atomically read a value which size might differ
4428	* between platforms or compilers, ordered.
4429	*
4430	* @param pu Pointer to the variable to update.
4431	* @param puRes Where to store the result.
4432	*/
4433	#define ASMAtomicReadSize(pu, puRes) \
4434	do { \
4435	switch (sizeof(*(pu))) { \
4436	case 1: (uint8_t )(puRes) = ASMAtomicReadU8( (volatile uint8_t )(void )(pu)); break; \
4437	case 2: (uint16_t )(puRes) = ASMAtomicReadU16((volatile uint16_t )(void )(pu)); break; \
4438	case 4: (uint32_t )(puRes) = ASMAtomicReadU32((volatile uint32_t )(void )(pu)); break; \
4439	case 8: (uint64_t )(puRes) = ASMAtomicReadU64((volatile uint64_t )(void )(pu)); break; \
4440	default: AssertMsgFailed(("ASMAtomicReadSize: size %d is not supported\n", sizeof(*(pu)))); \
4441	} \
4442	} while (0)
4443
4444
4445	/**
4446	* Atomically read a value which size might differ
4447	* between platforms or compilers, unordered.
4448	*
4449	* @param pu Pointer to the variable to read.
4450	* @param puRes Where to store the result.
4451	*/
4452	#define ASMAtomicUoReadSize(pu, puRes) \
4453	do { \
4454	switch (sizeof(*(pu))) { \
4455	case 1: (uint8_t )(puRes) = ASMAtomicUoReadU8( (volatile uint8_t )(void )(pu)); break; \
4456	case 2: (uint16_t )(puRes) = ASMAtomicUoReadU16((volatile uint16_t )(void )(pu)); break; \
4457	case 4: (uint32_t )(puRes) = ASMAtomicUoReadU32((volatile uint32_t )(void )(pu)); break; \
4458	case 8: (uint64_t )(puRes) = ASMAtomicUoReadU64((volatile uint64_t )(void )(pu)); break; \
4459	default: AssertMsgFailed(("ASMAtomicReadSize: size %d is not supported\n", sizeof(*(pu)))); \
4460	} \
4461	} while (0)
4462
4463
4464	/**
4465	* Atomically writes an unsigned 8-bit value, ordered.
4466	*
4467	* @param pu8 Pointer to the 8-bit variable.
4468	* @param u8 The 8-bit value to assign to *pu8.
4469	*/
4470	DECLINLINE(void) ASMAtomicWriteU8(volatile uint8_t *pu8, uint8_t u8)
4471	{
4472	ASMAtomicXchgU8(pu8, u8);
4473	}
4474
4475
4476	/**
4477	* Atomically writes an unsigned 8-bit value, unordered.
4478	*
4479	* @param pu8 Pointer to the 8-bit variable.
4480	* @param u8 The 8-bit value to assign to *pu8.
4481	*/
4482	DECLINLINE(void) ASMAtomicUoWriteU8(volatile uint8_t *pu8, uint8_t u8)
4483	{
4484	pu8 = u8; / byte writes are atomic on x86 */
4485	}
4486
4487
4488	/**
4489	* Atomically writes a signed 8-bit value, ordered.
4490	*
4491	* @param pi8 Pointer to the 8-bit variable to read.
4492	* @param i8 The 8-bit value to assign to *pi8.
4493	*/
4494	DECLINLINE(void) ASMAtomicWriteS8(volatile int8_t *pi8, int8_t i8)
4495	{
4496	ASMAtomicXchgS8(pi8, i8);
4497	}
4498
4499
4500	/**
4501	* Atomically writes a signed 8-bit value, unordered.
4502	*
4503	* @param pi8 Pointer to the 8-bit variable to read.
4504	* @param i8 The 8-bit value to assign to *pi8.
4505	*/
4506	DECLINLINE(void) ASMAtomicUoWriteS8(volatile int8_t *pi8, int8_t i8)
4507	{
4508	pi8 = i8; / byte writes are atomic on x86 */
4509	}
4510
4511
4512	/**
4513	* Atomically writes an unsigned 16-bit value, ordered.
4514	*
4515	* @param pu16 Pointer to the 16-bit variable.
4516	* @param u16 The 16-bit value to assign to *pu16.
4517	*/
4518	DECLINLINE(void) ASMAtomicWriteU16(volatile uint16_t *pu16, uint16_t u16)
4519	{
4520	ASMAtomicXchgU16(pu16, u16);
4521	}
4522
4523
4524	/**
4525	* Atomically writes an unsigned 16-bit value, unordered.
4526	*
4527	* @param pu16 Pointer to the 16-bit variable.
4528	* @param u16 The 16-bit value to assign to *pu16.
4529	*/
4530	DECLINLINE(void) ASMAtomicUoWriteU16(volatile uint16_t *pu16, uint16_t u16)
4531	{
4532	Assert(!((uintptr_t)pu16 & 1));
4533	*pu16 = u16;
4534	}
4535
4536
4537	/**
4538	* Atomically writes a signed 16-bit value, ordered.
4539	*
4540	* @param pi16 Pointer to the 16-bit variable to read.
4541	* @param i16 The 16-bit value to assign to *pi16.
4542	*/
4543	DECLINLINE(void) ASMAtomicWriteS16(volatile int16_t *pi16, int16_t i16)
4544	{
4545	ASMAtomicXchgS16(pi16, i16);
4546	}
4547
4548
4549	/**
4550	* Atomically writes a signed 16-bit value, unordered.
4551	*
4552	* @param pi16 Pointer to the 16-bit variable to read.
4553	* @param i16 The 16-bit value to assign to *pi16.
4554	*/
4555	DECLINLINE(void) ASMAtomicUoWriteS16(volatile int16_t *pi16, int16_t i16)
4556	{
4557	Assert(!((uintptr_t)pi16 & 1));
4558	*pi16 = i16;
4559	}
4560
4561
4562	/**
4563	* Atomically writes an unsigned 32-bit value, ordered.
4564	*
4565	* @param pu32 Pointer to the 32-bit variable.
4566	* @param u32 The 32-bit value to assign to *pu32.
4567	*/
4568	DECLINLINE(void) ASMAtomicWriteU32(volatile uint32_t *pu32, uint32_t u32)
4569	{
4570	ASMAtomicXchgU32(pu32, u32);
4571	}
4572
4573
4574	/**
4575	* Atomically writes an unsigned 32-bit value, unordered.
4576	*
4577	* @param pu32 Pointer to the 32-bit variable.
4578	* @param u32 The 32-bit value to assign to *pu32.
4579	*/
4580	DECLINLINE(void) ASMAtomicUoWriteU32(volatile uint32_t *pu32, uint32_t u32)
4581	{
4582	Assert(!((uintptr_t)pu32 & 3));
4583	*pu32 = u32;
4584	}
4585
4586
4587	/**
4588	* Atomically writes a signed 32-bit value, ordered.
4589	*
4590	* @param pi32 Pointer to the 32-bit variable to read.
4591	* @param i32 The 32-bit value to assign to *pi32.
4592	*/
4593	DECLINLINE(void) ASMAtomicWriteS32(volatile int32_t *pi32, int32_t i32)
4594	{
4595	ASMAtomicXchgS32(pi32, i32);
4596	}
4597
4598
4599	/**
4600	* Atomically writes a signed 32-bit value, unordered.
4601	*
4602	* @param pi32 Pointer to the 32-bit variable to read.
4603	* @param i32 The 32-bit value to assign to *pi32.
4604	*/
4605	DECLINLINE(void) ASMAtomicUoWriteS32(volatile int32_t *pi32, int32_t i32)
4606	{
4607	Assert(!((uintptr_t)pi32 & 3));
4608	*pi32 = i32;
4609	}
4610
4611
4612	/**
4613	* Atomically writes an unsigned 64-bit value, ordered.
4614	*
4615	* @param pu64 Pointer to the 64-bit variable.
4616	* @param u64 The 64-bit value to assign to *pu64.
4617	*/
4618	DECLINLINE(void) ASMAtomicWriteU64(volatile uint64_t *pu64, uint64_t u64)
4619	{
4620	ASMAtomicXchgU64(pu64, u64);
4621	}
4622
4623
4624	/**
4625	* Atomically writes an unsigned 64-bit value, unordered.
4626	*
4627	* @param pu64 Pointer to the 64-bit variable.
4628	* @param u64 The 64-bit value to assign to *pu64.
4629	*/
4630	DECLINLINE(void) ASMAtomicUoWriteU64(volatile uint64_t *pu64, uint64_t u64)
4631	{
4632	Assert(!((uintptr_t)pu64 & 7));
4633	#if ARCH_BITS == 64
4634	*pu64 = u64;
4635	#else
4636	ASMAtomicXchgU64(pu64, u64);
4637	#endif
4638	}
4639
4640
4641	/**
4642	* Atomically writes a signed 64-bit value, ordered.
4643	*
4644	* @param pi64 Pointer to the 64-bit variable.
4645	* @param i64 The 64-bit value to assign to *pi64.
4646	*/
4647	DECLINLINE(void) ASMAtomicWriteS64(volatile int64_t *pi64, int64_t i64)
4648	{
4649	ASMAtomicXchgS64(pi64, i64);
4650	}
4651
4652
4653	/**
4654	* Atomically writes a signed 64-bit value, unordered.
4655	*
4656	* @param pi64 Pointer to the 64-bit variable.
4657	* @param i64 The 64-bit value to assign to *pi64.
4658	*/
4659	DECLINLINE(void) ASMAtomicUoWriteS64(volatile int64_t *pi64, int64_t i64)
4660	{
4661	Assert(!((uintptr_t)pi64 & 7));
4662	#if ARCH_BITS == 64
4663	*pi64 = i64;
4664	#else
4665	ASMAtomicXchgS64(pi64, i64);
4666	#endif
4667	}
4668
4669
4670	/**
4671	* Atomically writes a boolean value, unordered.
4672	*
4673	* @param pf Pointer to the boolean variable.
4674	* @param f The boolean value to assign to *pf.
4675	*/
4676	DECLINLINE(void) ASMAtomicWriteBool(volatile bool *pf, bool f)
4677	{
4678	ASMAtomicWriteU8((uint8_t volatile *)pf, f);
4679	}
4680
4681
4682	/**
4683	* Atomically writes a boolean value, unordered.
4684	*
4685	* @param pf Pointer to the boolean variable.
4686	* @param f The boolean value to assign to *pf.
4687	*/
4688	DECLINLINE(void) ASMAtomicUoWriteBool(volatile bool *pf, bool f)
4689	{
4690	pf = f; / byte writes are atomic on x86 */
4691	}
4692
4693
4694	/**
4695	* Atomically writes a pointer value, ordered.
4696	*
4697	* @returns Current *pv value
4698	* @param ppv Pointer to the pointer variable.
4699	* @param pv The pointer value to assigne to *ppv.
4700	*/
4701	DECLINLINE(void) ASMAtomicWritePtr(void * volatile ppv, const void pv)
4702	{
4703	#if ARCH_BITS == 32
4704	ASMAtomicWriteU32((volatile uint32_t )(void )ppv, (uint32_t)pv);
4705	#elif ARCH_BITS == 64
4706	ASMAtomicWriteU64((volatile uint64_t )(void )ppv, (uint64_t)pv);
4707	#else
4708	# error "ARCH_BITS is bogus"
4709	#endif
4710	}
4711
4712
4713	/**
4714	* Atomically writes a pointer value, unordered.
4715	*
4716	* @returns Current *pv value
4717	* @param ppv Pointer to the pointer variable.
4718	* @param pv The pointer value to assigne to *ppv.
4719	*/
4720	DECLINLINE(void) ASMAtomicUoWritePtr(void * volatile ppv, const void pv)
4721	{
4722	#if ARCH_BITS == 32
4723	ASMAtomicUoWriteU32((volatile uint32_t )(void )ppv, (uint32_t)pv);
4724	#elif ARCH_BITS == 64
4725	ASMAtomicUoWriteU64((volatile uint64_t )(void )ppv, (uint64_t)pv);
4726	#else
4727	# error "ARCH_BITS is bogus"
4728	#endif
4729	}
4730
4731
4732	/**
4733	* Atomically write a typical IPRT handle value, ordered.
4734	*
4735	* @param ph Pointer to the variable to update.
4736	* @param hNew The value to assign to *ph.
4737	*
4738	* @remarks This doesn't currently work for all handles (like RTFILE).
4739	*/
4740	#if HC_ARCH_BITS == 32
4741	# define ASMAtomicWriteHandle(ph, hNew) \
4742	do { \
4743	AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
4744	ASMAtomicWriteU32((uint32_t volatile *)(ph), (const uint32_t)(hNew)); \
4745	} while (0)
4746	#elif HC_ARCH_BITS == 64
4747	# define ASMAtomicWriteHandle(ph, hNew) \
4748	do { \
4749	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
4750	ASMAtomicWriteU64((uint64_t volatile *)(ph), (const uint64_t)(hNew)); \
4751	} while (0)
4752	#else
4753	# error HC_ARCH_BITS
4754	#endif
4755
4756
4757	/**
4758	* Atomically write a typical IPRT handle value, unordered.
4759	*
4760	* @param ph Pointer to the variable to update.
4761	* @param hNew The value to assign to *ph.
4762	*
4763	* @remarks This doesn't currently work for all handles (like RTFILE).
4764	*/
4765	#if HC_ARCH_BITS == 32
4766	# define ASMAtomicUoWriteHandle(ph, hNew) \
4767	do { \
4768	AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
4769	ASMAtomicUoWriteU32((uint32_t volatile *)(ph), (const uint32_t)hNew); \
4770	} while (0)
4771	#elif HC_ARCH_BITS == 64
4772	# define ASMAtomicUoWriteHandle(ph, hNew) \
4773	do { \
4774	AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
4775	ASMAtomicUoWriteU64((uint64_t volatile *)(ph), (const uint64_t)hNew); \
4776	} while (0)
4777	#else
4778	# error HC_ARCH_BITS
4779	#endif
4780
4781
4782	/**
4783	* Atomically write a value which size might differ
4784	* between platforms or compilers, ordered.
4785	*
4786	* @param pu Pointer to the variable to update.
4787	* @param uNew The value to assign to *pu.
4788	*/
4789	#define ASMAtomicWriteSize(pu, uNew) \
4790	do { \
4791	switch (sizeof(*(pu))) { \
4792	case 1: ASMAtomicWriteU8( (volatile uint8_t )(void )(pu), (uint8_t )(uNew)); break; \
4793	case 2: ASMAtomicWriteU16((volatile uint16_t )(void )(pu), (uint16_t)(uNew)); break; \
4794	case 4: ASMAtomicWriteU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew)); break; \
4795	case 8: ASMAtomicWriteU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew)); break; \
4796	default: AssertMsgFailed(("ASMAtomicWriteSize: size %d is not supported\n", sizeof(*(pu)))); \
4797	} \
4798	} while (0)
4799
4800	/**
4801	* Atomically write a value which size might differ
4802	* between platforms or compilers, unordered.
4803	*
4804	* @param pu Pointer to the variable to update.
4805	* @param uNew The value to assign to *pu.
4806	*/
4807	#define ASMAtomicUoWriteSize(pu, uNew) \
4808	do { \
4809	switch (sizeof(*(pu))) { \
4810	case 1: ASMAtomicUoWriteU8( (volatile uint8_t )(void )(pu), (uint8_t )(uNew)); break; \
4811	case 2: ASMAtomicUoWriteU16((volatile uint16_t )(void )(pu), (uint16_t)(uNew)); break; \
4812	case 4: ASMAtomicUoWriteU32((volatile uint32_t )(void )(pu), (uint32_t)(uNew)); break; \
4813	case 8: ASMAtomicUoWriteU64((volatile uint64_t )(void )(pu), (uint64_t)(uNew)); break; \
4814	default: AssertMsgFailed(("ASMAtomicWriteSize: size %d is not supported\n", sizeof(*(pu)))); \
4815	} \
4816	} while (0)
4817
4818
4819
4820
4821	/**
4822	* Invalidate page.
4823	*
4824	* @param pv Address of the page to invalidate.
4825	*/
4826	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4827	DECLASM(void) ASMInvalidatePage(void *pv);
4828	#else
4829	DECLINLINE(void) ASMInvalidatePage(void *pv)
4830	{
4831	# if RT_INLINE_ASM_USES_INTRIN
4832	__invlpg(pv);
4833
4834	# elif RT_INLINE_ASM_GNU_STYLE
4835	__asm__ __volatile__("invlpg %0\n\t"
4836	: : "m" ((uint8_t )pv));
4837	# else
4838	__asm
4839	{
4840	# ifdef RT_ARCH_AMD64
4841	mov rax, [pv]
4842	invlpg [rax]
4843	# else
4844	mov eax, [pv]
4845	invlpg [eax]
4846	# endif
4847	}
4848	# endif
4849	}
4850	#endif
4851
4852
4853	/**
4854	* Write back the internal caches and invalidate them.
4855	*/
4856	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4857	DECLASM(void) ASMWriteBackAndInvalidateCaches(void);
4858	#else
4859	DECLINLINE(void) ASMWriteBackAndInvalidateCaches(void)
4860	{
4861	# if RT_INLINE_ASM_USES_INTRIN
4862	__wbinvd();
4863
4864	# elif RT_INLINE_ASM_GNU_STYLE
4865	__asm__ __volatile__("wbinvd");
4866	# else
4867	__asm
4868	{
4869	wbinvd
4870	}
4871	# endif
4872	}
4873	#endif
4874
4875
4876	/**
4877	* Invalidate internal and (perhaps) external caches without first
4878	* flushing dirty cache lines. Use with extreme care.
4879	*/
4880	#if RT_INLINE_ASM_EXTERNAL
4881	DECLASM(void) ASMInvalidateInternalCaches(void);
4882	#else
4883	DECLINLINE(void) ASMInvalidateInternalCaches(void)
4884	{
4885	# if RT_INLINE_ASM_GNU_STYLE
4886	__asm__ __volatile__("invd");
4887	# else
4888	__asm
4889	{
4890	invd
4891	}
4892	# endif
4893	}
4894	#endif
4895
4896
4897	#if defined(PAGE_SIZE) && !defined(NT_INCLUDED)
4898	# if PAGE_SIZE != 0x1000
4899	# error "PAGE_SIZE is not 0x1000!"
4900	# endif
4901	#endif
4902
4903	/**
4904	* Zeros a 4K memory page.
4905	*
4906	* @param pv Pointer to the memory block. This must be page aligned.
4907	*/
4908	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4909	DECLASM(void) ASMMemZeroPage(volatile void *pv);
4910	# else
4911	DECLINLINE(void) ASMMemZeroPage(volatile void *pv)
4912	{
4913	# if RT_INLINE_ASM_USES_INTRIN
4914	# ifdef RT_ARCH_AMD64
4915	__stosq((unsigned __int64 )pv, 0, /PAGE_SIZE*/0x1000 / 8);
4916	# else
4917	__stosd((unsigned long )pv, 0, /PAGE_SIZE*/0x1000 / 4);
4918	# endif
4919
4920	# elif RT_INLINE_ASM_GNU_STYLE
4921	RTCCUINTREG uDummy;
4922	# ifdef RT_ARCH_AMD64
4923	__asm__ __volatile__("rep stosq"
4924	: "=D" (pv),
4925	"=c" (uDummy)
4926	: "0" (pv),
4927	"c" (0x1000 >> 3),
4928	"a" (0)
4929	: "memory");
4930	# else
4931	__asm__ __volatile__("rep stosl"
4932	: "=D" (pv),
4933	"=c" (uDummy)
4934	: "0" (pv),
4935	"c" (0x1000 >> 2),
4936	"a" (0)
4937	: "memory");
4938	# endif
4939	# else
4940	__asm
4941	{
4942	# ifdef RT_ARCH_AMD64
4943	xor rax, rax
4944	mov ecx, 0200h
4945	mov rdi, [pv]
4946	rep stosq
4947	# else
4948	xor eax, eax
4949	mov ecx, 0400h
4950	mov edi, [pv]
4951	rep stosd
4952	# endif
4953	}
4954	# endif
4955	}
4956	# endif
4957
4958
4959	/**
4960	* Zeros a memory block with a 32-bit aligned size.
4961	*
4962	* @param pv Pointer to the memory block.
4963	* @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
4964	*/
4965	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4966	DECLASM(void) ASMMemZero32(volatile void *pv, size_t cb);
4967	#else
4968	DECLINLINE(void) ASMMemZero32(volatile void *pv, size_t cb)
4969	{
4970	# if RT_INLINE_ASM_USES_INTRIN
4971	# ifdef RT_ARCH_AMD64
4972	if (!(cb & 7))
4973	__stosq((unsigned __int64 *)pv, 0, cb / 8);
4974	else
4975	# endif
4976	__stosd((unsigned long *)pv, 0, cb / 4);
4977
4978	# elif RT_INLINE_ASM_GNU_STYLE
4979	__asm__ __volatile__("rep stosl"
4980	: "=D" (pv),
4981	"=c" (cb)
4982	: "0" (pv),
4983	"1" (cb >> 2),
4984	"a" (0)
4985	: "memory");
4986	# else
4987	__asm
4988	{
4989	xor eax, eax
4990	# ifdef RT_ARCH_AMD64
4991	mov rcx, [cb]
4992	shr rcx, 2
4993	mov rdi, [pv]
4994	# else
4995	mov ecx, [cb]
4996	shr ecx, 2
4997	mov edi, [pv]
4998	# endif
4999	rep stosd
5000	}
5001	# endif
5002	}
5003	#endif
5004
5005
5006	/**
5007	* Fills a memory block with a 32-bit aligned size.
5008	*
5009	* @param pv Pointer to the memory block.
5010	* @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
5011	* @param u32 The value to fill with.
5012	*/
5013	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5014	DECLASM(void) ASMMemFill32(volatile void *pv, size_t cb, uint32_t u32);
5015	#else
5016	DECLINLINE(void) ASMMemFill32(volatile void *pv, size_t cb, uint32_t u32)
5017	{
5018	# if RT_INLINE_ASM_USES_INTRIN
5019	# ifdef RT_ARCH_AMD64
5020	if (!(cb & 7))
5021	__stosq((unsigned __int64 *)pv, RT_MAKE_U64(u32, u32), cb / 8);
5022	else
5023	# endif
5024	__stosd((unsigned long *)pv, u32, cb / 4);
5025
5026	# elif RT_INLINE_ASM_GNU_STYLE
5027	__asm__ __volatile__("rep stosl"
5028	: "=D" (pv),
5029	"=c" (cb)
5030	: "0" (pv),
5031	"1" (cb >> 2),
5032	"a" (u32)
5033	: "memory");
5034	# else
5035	__asm
5036	{
5037	# ifdef RT_ARCH_AMD64
5038	mov rcx, [cb]
5039	shr rcx, 2
5040	mov rdi, [pv]
5041	# else
5042	mov ecx, [cb]
5043	shr ecx, 2
5044	mov edi, [pv]
5045	# endif
5046	mov eax, [u32]
5047	rep stosd
5048	}
5049	# endif
5050	}
5051	#endif
5052
5053
5054	/**
5055	* Checks if a memory page is all zeros.
5056	*
5057	* @returns true / false.
5058	*
5059	* @param pvPage Pointer to the page. Must be aligned on 16 byte
5060	* boundrary
5061	*/
5062	DECLINLINE(bool) ASMMemIsZeroPage(void const *pvPage)
5063	{
5064	# if 0 /RT_INLINE_ASM_GNU_STYLE - this is actually slower... /
5065	union { RTCCUINTREG r; bool f; } uAX;
5066	RTCCUINTREG xCX, xDI;
5067	Assert(!((uintptr_t)pvPage & 15));
5068	__asm__ __volatile__("repe; "
5069	# ifdef RT_ARCH_AMD64
5070	"scasq\n\t"
5071	# else
5072	"scasl\n\t"
5073	# endif
5074	"setnc %%al\n\t"
5075	: "=&c" (xCX),
5076	"=&D" (xDI),
5077	"=&a" (uAX.r)
5078	: "mr" (pvPage),
5079	# ifdef RT_ARCH_AMD64
5080	"0" (0x1000/8),
5081	# else
5082	"0" (0x1000/4),
5083	# endif
5084	"1" (pvPage),
5085	"2" (0));
5086	return uAX.f;
5087	# else
5088	uintptr_t const puPtr = (uintptr_t const )pvPage;
5089	int cLeft = 0x1000 / sizeof(uintptr_t) / 8;
5090	Assert(!((uintptr_t)pvPage & 15));
5091	for (;;)
5092	{
5093	if (puPtr[0]) return false;
5094	if (puPtr[4]) return false;
5095
5096	if (puPtr[2]) return false;
5097	if (puPtr[6]) return false;
5098
5099	if (puPtr[1]) return false;
5100	if (puPtr[5]) return false;
5101
5102	if (puPtr[3]) return false;
5103	if (puPtr[7]) return false;
5104
5105	if (!--cLeft)
5106	return true;
5107	puPtr += 8;
5108	}
5109	return true;
5110	# endif
5111	}
5112
5113
5114	/**
5115	* Checks if a memory block is filled with the specified byte.
5116	*
5117	* This is a sort of inverted memchr.
5118	*
5119	* @returns Pointer to the byte which doesn't equal u8.
5120	* @returns NULL if all equal to u8.
5121	*
5122	* @param pv Pointer to the memory block.
5123	* @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
5124	* @param u8 The value it's supposed to be filled with.
5125	*/
5126	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5127	DECLASM(void ) ASMMemIsAll8(void const pv, size_t cb, uint8_t u8);
5128	#else
5129	DECLINLINE(void ) ASMMemIsAll8(void const pv, size_t cb, uint8_t u8)
5130	{
5131	/** @todo rewrite this in inline assembly? */
5132	uint8_t const pb = (uint8_t const )pv;
5133	for (; cb; cb--, pb++)
5134	if (RT_UNLIKELY(*pb != u8))
5135	return (void *)pb;
5136	return NULL;
5137	}
5138	#endif
5139
5140
5141	/**
5142	* Checks if a memory block is filled with the specified 32-bit value.
5143	*
5144	* This is a sort of inverted memchr.
5145	*
5146	* @returns Pointer to the first value which doesn't equal u32.
5147	* @returns NULL if all equal to u32.
5148	*
5149	* @param pv Pointer to the memory block.
5150	* @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
5151	* @param u32 The value it's supposed to be filled with.
5152	*/
5153	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5154	DECLASM(uint32_t ) ASMMemIsAllU32(void const pv, size_t cb, uint32_t u32);
5155	#else
5156	DECLINLINE(uint32_t ) ASMMemIsAllU32(void const pv, size_t cb, uint32_t u32)
5157	{
5158	/** @todo rewrite this in inline assembly? */
5159	uint32_t const pu32 = (uint32_t const )pv;
5160	for (; cb; cb -= 4, pu32++)
5161	if (RT_UNLIKELY(*pu32 != u32))
5162	return (uint32_t *)pu32;
5163	return NULL;
5164	}
5165	#endif
5166
5167
5168	/**
5169	* Multiplies two unsigned 32-bit values returning an unsigned 64-bit result.
5170	*
5171	* @returns u32F1 * u32F2.
5172	*/
5173	#if RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)
5174	DECLASM(uint64_t) ASMMult2xU32RetU64(uint32_t u32F1, uint32_t u32F2);
5175	#else
5176	DECLINLINE(uint64_t) ASMMult2xU32RetU64(uint32_t u32F1, uint32_t u32F2)
5177	{
5178	# ifdef RT_ARCH_AMD64
5179	return (uint64_t)u32F1 * u32F2;
5180	# else /* !RT_ARCH_AMD64 */
5181	uint64_t u64;
5182	# if RT_INLINE_ASM_GNU_STYLE
5183	__asm__ __volatile__("mull %%edx"
5184	: "=A" (u64)
5185	: "a" (u32F2), "d" (u32F1));
5186	# else
5187	__asm
5188	{
5189	mov edx, [u32F1]
5190	mov eax, [u32F2]
5191	mul edx
5192	mov dword ptr [u64], eax
5193	mov dword ptr [u64 + 4], edx
5194	}
5195	# endif
5196	return u64;
5197	# endif /* !RT_ARCH_AMD64 */
5198	}
5199	#endif
5200
5201
5202	/**
5203	* Multiplies two signed 32-bit values returning a signed 64-bit result.
5204	*
5205	* @returns u32F1 * u32F2.
5206	*/
5207	#if RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)
5208	DECLASM(int64_t) ASMMult2xS32RetS64(int32_t i32F1, int32_t i32F2);
5209	#else
5210	DECLINLINE(int64_t) ASMMult2xS32RetS64(int32_t i32F1, int32_t i32F2)
5211	{
5212	# ifdef RT_ARCH_AMD64
5213	return (int64_t)i32F1 * i32F2;
5214	# else /* !RT_ARCH_AMD64 */
5215	int64_t i64;
5216	# if RT_INLINE_ASM_GNU_STYLE
5217	__asm__ __volatile__("imull %%edx"
5218	: "=A" (i64)
5219	: "a" (i32F2), "d" (i32F1));
5220	# else
5221	__asm
5222	{
5223	mov edx, [i32F1]
5224	mov eax, [i32F2]
5225	imul edx
5226	mov dword ptr [i64], eax
5227	mov dword ptr [i64 + 4], edx
5228	}
5229	# endif
5230	return i64;
5231	# endif /* !RT_ARCH_AMD64 */
5232	}
5233	#endif
5234
5235
5236	/**
5237	* Divides a 64-bit unsigned by a 32-bit unsigned returning an unsigned 32-bit result.
5238	*
5239	* @returns u64 / u32.
5240	*/
5241	#if RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)
5242	DECLASM(uint32_t) ASMDivU64ByU32RetU32(uint64_t u64, uint32_t u32);
5243	#else
5244	DECLINLINE(uint32_t) ASMDivU64ByU32RetU32(uint64_t u64, uint32_t u32)
5245	{
5246	# ifdef RT_ARCH_AMD64
5247	return (uint32_t)(u64 / u32);
5248	# else /* !RT_ARCH_AMD64 */
5249	# if RT_INLINE_ASM_GNU_STYLE
5250	RTCCUINTREG uDummy;
5251	__asm__ __volatile__("divl %3"
5252	: "=a" (u32), "=d"(uDummy)
5253	: "A" (u64), "r" (u32));
5254	# else
5255	__asm
5256	{
5257	mov eax, dword ptr [u64]
5258	mov edx, dword ptr [u64 + 4]
5259	mov ecx, [u32]
5260	div ecx
5261	mov [u32], eax
5262	}
5263	# endif
5264	return u32;
5265	# endif /* !RT_ARCH_AMD64 */
5266	}
5267	#endif
5268
5269
5270	/**
5271	* Divides a 64-bit signed by a 32-bit signed returning a signed 32-bit result.
5272	*
5273	* @returns u64 / u32.
5274	*/
5275	#if RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)
5276	DECLASM(int32_t) ASMDivS64ByS32RetS32(int64_t i64, int32_t i32);
5277	#else
5278	DECLINLINE(int32_t) ASMDivS64ByS32RetS32(int64_t i64, int32_t i32)
5279	{
5280	# ifdef RT_ARCH_AMD64
5281	return (int32_t)(i64 / i32);
5282	# else /* !RT_ARCH_AMD64 */
5283	# if RT_INLINE_ASM_GNU_STYLE
5284	RTCCUINTREG iDummy;
5285	__asm__ __volatile__("idivl %3"
5286	: "=a" (i32), "=d"(iDummy)
5287	: "A" (i64), "r" (i32));
5288	# else
5289	__asm
5290	{
5291	mov eax, dword ptr [i64]
5292	mov edx, dword ptr [i64 + 4]
5293	mov ecx, [i32]
5294	idiv ecx
5295	mov [i32], eax
5296	}
5297	# endif
5298	return i32;
5299	# endif /* !RT_ARCH_AMD64 */
5300	}
5301	#endif
5302
5303
5304	/**
5305	* Performs 64-bit unsigned by a 32-bit unsigned division with a 32-bit unsigned result,
5306	* returning the rest.
5307	*
5308	* @returns u64 % u32.
5309	*
5310	* @remarks It is important that the result is <= UINT32_MAX or we'll overflow and crash.
5311	*/
5312	#if RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)
5313	DECLASM(uint32_t) ASMModU64ByU32RetU32(uint64_t u64, uint32_t u32);
5314	#else
5315	DECLINLINE(uint32_t) ASMModU64ByU32RetU32(uint64_t u64, uint32_t u32)
5316	{
5317	# ifdef RT_ARCH_AMD64
5318	return (uint32_t)(u64 % u32);
5319	# else /* !RT_ARCH_AMD64 */
5320	# if RT_INLINE_ASM_GNU_STYLE
5321	RTCCUINTREG uDummy;
5322	__asm__ __volatile__("divl %3"
5323	: "=a" (uDummy), "=d"(u32)
5324	: "A" (u64), "r" (u32));
5325	# else
5326	__asm
5327	{
5328	mov eax, dword ptr [u64]
5329	mov edx, dword ptr [u64 + 4]
5330	mov ecx, [u32]
5331	div ecx
5332	mov [u32], edx
5333	}
5334	# endif
5335	return u32;
5336	# endif /* !RT_ARCH_AMD64 */
5337	}
5338	#endif
5339
5340
5341	/**
5342	* Performs 64-bit signed by a 32-bit signed division with a 32-bit signed result,
5343	* returning the rest.
5344	*
5345	* @returns u64 % u32.
5346	*
5347	* @remarks It is important that the result is <= UINT32_MAX or we'll overflow and crash.
5348	*/
5349	#if RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)
5350	DECLASM(int32_t) ASMModS64ByS32RetS32(int64_t i64, int32_t i32);
5351	#else
5352	DECLINLINE(int32_t) ASMModS64ByS32RetS32(int64_t i64, int32_t i32)
5353	{
5354	# ifdef RT_ARCH_AMD64
5355	return (int32_t)(i64 % i32);
5356	# else /* !RT_ARCH_AMD64 */
5357	# if RT_INLINE_ASM_GNU_STYLE
5358	RTCCUINTREG iDummy;
5359	__asm__ __volatile__("idivl %3"
5360	: "=a" (iDummy), "=d"(i32)
5361	: "A" (i64), "r" (i32));
5362	# else
5363	__asm
5364	{
5365	mov eax, dword ptr [i64]
5366	mov edx, dword ptr [i64 + 4]
5367	mov ecx, [i32]
5368	idiv ecx
5369	mov [i32], edx
5370	}
5371	# endif
5372	return i32;
5373	# endif /* !RT_ARCH_AMD64 */
5374	}
5375	#endif
5376
5377
5378	/**
5379	* Multiple a 64-bit by a 32-bit integer and divide the result by a 32-bit integer
5380	* using a 96 bit intermediate result.
5381	* @note Don't use 64-bit C arithmetic here since some gcc compilers generate references to
5382	* __udivdi3 and __umoddi3 even if this inline function is not used.
5383	*
5384	* @returns (u64A * u32B) / u32C.
5385	* @param u64A The 64-bit value.
5386	* @param u32B The 32-bit value to multiple by A.
5387	* @param u32C The 32-bit value to divide A*B by.
5388	*/
5389	#if RT_INLINE_ASM_EXTERNAL \|\| !defined(__GNUC__)
5390	DECLASM(uint64_t) ASMMultU64ByU32DivByU32(uint64_t u64A, uint32_t u32B, uint32_t u32C);
5391	#else
5392	DECLINLINE(uint64_t) ASMMultU64ByU32DivByU32(uint64_t u64A, uint32_t u32B, uint32_t u32C)
5393	{
5394	# if RT_INLINE_ASM_GNU_STYLE
5395	# ifdef RT_ARCH_AMD64
5396	uint64_t u64Result, u64Spill;
5397	__asm__ __volatile__("mulq %2\n\t"
5398	"divq %3\n\t"
5399	: "=a" (u64Result),
5400	"=d" (u64Spill)
5401	: "r" ((uint64_t)u32B),
5402	"r" ((uint64_t)u32C),
5403	"0" (u64A),
5404	"1" (0));
5405	return u64Result;
5406	# else
5407	uint32_t u32Dummy;
5408	uint64_t u64Result;
5409	__asm__ __volatile__("mull %%ecx \n\t" /* eax = u64Lo.lo = (u64A.lo * u32B).lo
5410	edx = u64Lo.hi = (u64A.lo * u32B).hi */
5411	"xchg %%eax,%%esi \n\t" /* esi = u64Lo.lo
5412	eax = u64A.hi */
5413	"xchg %%edx,%%edi \n\t" /* edi = u64Low.hi
5414	edx = u32C */
5415	"xchg %%edx,%%ecx \n\t" /* ecx = u32C
5416	edx = u32B */
5417	"mull %%edx \n\t" /* eax = u64Hi.lo = (u64A.hi * u32B).lo
5418	edx = u64Hi.hi = (u64A.hi * u32B).hi */
5419	"addl %%edi,%%eax \n\t" /* u64Hi.lo += u64Lo.hi */
5420	"adcl $0,%%edx \n\t" /* u64Hi.hi += carry */
5421	"divl %%ecx \n\t" /* eax = u64Hi / u32C
5422	edx = u64Hi % u32C */
5423	"movl %%eax,%%edi \n\t" /* edi = u64Result.hi = u64Hi / u32C */
5424	"movl %%esi,%%eax \n\t" /* eax = u64Lo.lo */
5425	"divl %%ecx \n\t" /* u64Result.lo */
5426	"movl %%edi,%%edx \n\t" /* u64Result.hi */
5427	: "=A"(u64Result), "=c"(u32Dummy),
5428	"=S"(u32Dummy), "=D"(u32Dummy)
5429	: "a"((uint32_t)u64A),
5430	"S"((uint32_t)(u64A >> 32)),
5431	"c"(u32B),
5432	"D"(u32C));
5433	return u64Result;
5434	# endif
5435	# else
5436	RTUINT64U u;
5437	uint64_t u64Lo = (uint64_t)(u64A & 0xffffffff) * u32B;
5438	uint64_t u64Hi = (uint64_t)(u64A >> 32) * u32B;
5439	u64Hi += (u64Lo >> 32);
5440	u.s.Hi = (uint32_t)(u64Hi / u32C);
5441	u.s.Lo = (uint32_t)((((u64Hi % u32C) << 32) + (u64Lo & 0xffffffff)) / u32C);
5442	return u.u;
5443	# endif
5444	}
5445	#endif
5446
5447
5448	/**
5449	* Probes a byte pointer for read access.
5450	*
5451	* While the function will not fault if the byte is not read accessible,
5452	* the idea is to do this in a safe place like before acquiring locks
5453	* and such like.
5454	*
5455	* Also, this functions guarantees that an eager compiler is not going
5456	* to optimize the probing away.
5457	*
5458	* @param pvByte Pointer to the byte.
5459	*/
5460	#if RT_INLINE_ASM_EXTERNAL
5461	DECLASM(uint8_t) ASMProbeReadByte(const void *pvByte);
5462	#else
5463	DECLINLINE(uint8_t) ASMProbeReadByte(const void *pvByte)
5464	{
5465	/** @todo verify that the compiler actually doesn't optimize this away. (intel & gcc) */
5466	uint8_t u8;
5467	# if RT_INLINE_ASM_GNU_STYLE
5468	__asm__ __volatile__("movb (%1), %0\n\t"
5469	: "=r" (u8)
5470	: "r" (pvByte));
5471	# else
5472	__asm
5473	{
5474	# ifdef RT_ARCH_AMD64
5475	mov rax, [pvByte]
5476	mov al, [rax]
5477	# else
5478	mov eax, [pvByte]
5479	mov al, [eax]
5480	# endif
5481	mov [u8], al
5482	}
5483	# endif
5484	return u8;
5485	}
5486	#endif
5487
5488	/**
5489	* Probes a buffer for read access page by page.
5490	*
5491	* While the function will fault if the buffer is not fully read
5492	* accessible, the idea is to do this in a safe place like before
5493	* acquiring locks and such like.
5494	*
5495	* Also, this functions guarantees that an eager compiler is not going
5496	* to optimize the probing away.
5497	*
5498	* @param pvBuf Pointer to the buffer.
5499	* @param cbBuf The size of the buffer in bytes. Must be >= 1.
5500	*/
5501	DECLINLINE(void) ASMProbeReadBuffer(const void *pvBuf, size_t cbBuf)
5502	{
5503	/** @todo verify that the compiler actually doesn't optimize this away. (intel & gcc) */
5504	/* the first byte */
5505	const uint8_t pu8 = (const uint8_t )pvBuf;
5506	ASMProbeReadByte(pu8);
5507
5508	/* the pages in between pages. */
5509	while (cbBuf > /PAGE_SIZE/0x1000)
5510	{
5511	ASMProbeReadByte(pu8);
5512	cbBuf -= /PAGE_SIZE/0x1000;
5513	pu8 += /PAGE_SIZE/0x1000;
5514	}
5515
5516	/* the last byte */
5517	ASMProbeReadByte(pu8 + cbBuf - 1);
5518	}
5519
5520
5521	/** @def ASMBreakpoint
5522	* Debugger Breakpoint.
5523	* @remark In the gnu world we add a nop instruction after the int3 to
5524	* force gdb to remain at the int3 source line.
5525	* @remark The L4 kernel will try make sense of the breakpoint, thus the jmp.
5526	* @internal
5527	*/
5528	#if RT_INLINE_ASM_GNU_STYLE
5529	# ifndef __L4ENV__
5530	# define ASMBreakpoint() do { __asm__ __volatile__("int3\n\tnop"); } while (0)
5531	# else
5532	# define ASMBreakpoint() do { __asm__ __volatile__("int3; jmp 1f; 1:"); } while (0)
5533	# endif
5534	#else
5535	# define ASMBreakpoint() __debugbreak()
5536	#endif
5537
5538
5539
5540	/** @defgroup grp_inline_bits Bit Operations
5541	* @{
5542	*/
5543
5544
5545	/**
5546	* Sets a bit in a bitmap.
5547	*
5548	* @param pvBitmap Pointer to the bitmap. This should be 32-bit aligned.
5549	* @param iBit The bit to set.
5550	*
5551	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
5552	* However, doing so will yield better performance as well as avoiding
5553	* traps accessing the last bits in the bitmap.
5554	*/
5555	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5556	DECLASM(void) ASMBitSet(volatile void *pvBitmap, int32_t iBit);
5557	#else
5558	DECLINLINE(void) ASMBitSet(volatile void *pvBitmap, int32_t iBit)
5559	{
5560	# if RT_INLINE_ASM_USES_INTRIN
5561	_bittestandset((long *)pvBitmap, iBit);
5562
5563	# elif RT_INLINE_ASM_GNU_STYLE
5564	__asm__ __volatile__("btsl %1, %0"
5565	: "=m" ((volatile long )pvBitmap)
5566	: "Ir" (iBit),
5567	"m" ((volatile long )pvBitmap)
5568	: "memory");
5569	# else
5570	__asm
5571	{
5572	# ifdef RT_ARCH_AMD64
5573	mov rax, [pvBitmap]
5574	mov edx, [iBit]
5575	bts [rax], edx
5576	# else
5577	mov eax, [pvBitmap]
5578	mov edx, [iBit]
5579	bts [eax], edx
5580	# endif
5581	}
5582	# endif
5583	}
5584	#endif
5585
5586
5587	/**
5588	* Atomically sets a bit in a bitmap, ordered.
5589	*
5590	* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
5591	* the memory access isn't atomic!
5592	* @param iBit The bit to set.
5593	*/
5594	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5595	DECLASM(void) ASMAtomicBitSet(volatile void *pvBitmap, int32_t iBit);
5596	#else
5597	DECLINLINE(void) ASMAtomicBitSet(volatile void *pvBitmap, int32_t iBit)
5598	{
5599	AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
5600	# if RT_INLINE_ASM_USES_INTRIN
5601	_interlockedbittestandset((long *)pvBitmap, iBit);
5602	# elif RT_INLINE_ASM_GNU_STYLE
5603	__asm__ __volatile__("lock; btsl %1, %0"
5604	: "=m" ((volatile long )pvBitmap)
5605	: "Ir" (iBit),
5606	"m" ((volatile long )pvBitmap)
5607	: "memory");
5608	# else
5609	__asm
5610	{
5611	# ifdef RT_ARCH_AMD64
5612	mov rax, [pvBitmap]
5613	mov edx, [iBit]
5614	lock bts [rax], edx
5615	# else
5616	mov eax, [pvBitmap]
5617	mov edx, [iBit]
5618	lock bts [eax], edx
5619	# endif
5620	}
5621	# endif
5622	}
5623	#endif
5624
5625
5626	/**
5627	* Clears a bit in a bitmap.
5628	*
5629	* @param pvBitmap Pointer to the bitmap.
5630	* @param iBit The bit to clear.
5631	*
5632	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
5633	* However, doing so will yield better performance as well as avoiding
5634	* traps accessing the last bits in the bitmap.
5635	*/
5636	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5637	DECLASM(void) ASMBitClear(volatile void *pvBitmap, int32_t iBit);
5638	#else
5639	DECLINLINE(void) ASMBitClear(volatile void *pvBitmap, int32_t iBit)
5640	{
5641	# if RT_INLINE_ASM_USES_INTRIN
5642	_bittestandreset((long *)pvBitmap, iBit);
5643
5644	# elif RT_INLINE_ASM_GNU_STYLE
5645	__asm__ __volatile__("btrl %1, %0"
5646	: "=m" ((volatile long )pvBitmap)
5647	: "Ir" (iBit),
5648	"m" ((volatile long )pvBitmap)
5649	: "memory");
5650	# else
5651	__asm
5652	{
5653	# ifdef RT_ARCH_AMD64
5654	mov rax, [pvBitmap]
5655	mov edx, [iBit]
5656	btr [rax], edx
5657	# else
5658	mov eax, [pvBitmap]
5659	mov edx, [iBit]
5660	btr [eax], edx
5661	# endif
5662	}
5663	# endif
5664	}
5665	#endif
5666
5667
5668	/**
5669	* Atomically clears a bit in a bitmap, ordered.
5670	*
5671	* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
5672	* the memory access isn't atomic!
5673	* @param iBit The bit to toggle set.
5674	* @remarks No memory barrier, take care on smp.
5675	*/
5676	#if RT_INLINE_ASM_EXTERNAL
5677	DECLASM(void) ASMAtomicBitClear(volatile void *pvBitmap, int32_t iBit);
5678	#else
5679	DECLINLINE(void) ASMAtomicBitClear(volatile void *pvBitmap, int32_t iBit)
5680	{
5681	AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
5682	# if RT_INLINE_ASM_GNU_STYLE
5683	__asm__ __volatile__("lock; btrl %1, %0"
5684	: "=m" ((volatile long )pvBitmap)
5685	: "Ir" (iBit),
5686	"m" ((volatile long )pvBitmap)
5687	: "memory");
5688	# else
5689	__asm
5690	{
5691	# ifdef RT_ARCH_AMD64
5692	mov rax, [pvBitmap]
5693	mov edx, [iBit]
5694	lock btr [rax], edx
5695	# else
5696	mov eax, [pvBitmap]
5697	mov edx, [iBit]
5698	lock btr [eax], edx
5699	# endif
5700	}
5701	# endif
5702	}
5703	#endif
5704
5705
5706	/**
5707	* Toggles a bit in a bitmap.
5708	*
5709	* @param pvBitmap Pointer to the bitmap.
5710	* @param iBit The bit to toggle.
5711	*
5712	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
5713	* However, doing so will yield better performance as well as avoiding
5714	* traps accessing the last bits in the bitmap.
5715	*/
5716	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5717	DECLASM(void) ASMBitToggle(volatile void *pvBitmap, int32_t iBit);
5718	#else
5719	DECLINLINE(void) ASMBitToggle(volatile void *pvBitmap, int32_t iBit)
5720	{
5721	# if RT_INLINE_ASM_USES_INTRIN
5722	_bittestandcomplement((long *)pvBitmap, iBit);
5723	# elif RT_INLINE_ASM_GNU_STYLE
5724	__asm__ __volatile__("btcl %1, %0"
5725	: "=m" ((volatile long )pvBitmap)
5726	: "Ir" (iBit),
5727	"m" ((volatile long )pvBitmap)
5728	: "memory");
5729	# else
5730	__asm
5731	{
5732	# ifdef RT_ARCH_AMD64
5733	mov rax, [pvBitmap]
5734	mov edx, [iBit]
5735	btc [rax], edx
5736	# else
5737	mov eax, [pvBitmap]
5738	mov edx, [iBit]
5739	btc [eax], edx
5740	# endif
5741	}
5742	# endif
5743	}
5744	#endif
5745
5746
5747	/**
5748	* Atomically toggles a bit in a bitmap, ordered.
5749	*
5750	* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
5751	* the memory access isn't atomic!
5752	* @param iBit The bit to test and set.
5753	*/
5754	#if RT_INLINE_ASM_EXTERNAL
5755	DECLASM(void) ASMAtomicBitToggle(volatile void *pvBitmap, int32_t iBit);
5756	#else
5757	DECLINLINE(void) ASMAtomicBitToggle(volatile void *pvBitmap, int32_t iBit)
5758	{
5759	AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
5760	# if RT_INLINE_ASM_GNU_STYLE
5761	__asm__ __volatile__("lock; btcl %1, %0"
5762	: "=m" ((volatile long )pvBitmap)
5763	: "Ir" (iBit),
5764	"m" ((volatile long )pvBitmap)
5765	: "memory");
5766	# else
5767	__asm
5768	{
5769	# ifdef RT_ARCH_AMD64
5770	mov rax, [pvBitmap]
5771	mov edx, [iBit]
5772	lock btc [rax], edx
5773	# else
5774	mov eax, [pvBitmap]
5775	mov edx, [iBit]
5776	lock btc [eax], edx
5777	# endif
5778	}
5779	# endif
5780	}
5781	#endif
5782
5783
5784	/**
5785	* Tests and sets a bit in a bitmap.
5786	*
5787	* @returns true if the bit was set.
5788	* @returns false if the bit was clear.
5789	*
5790	* @param pvBitmap Pointer to the bitmap.
5791	* @param iBit The bit to test and set.
5792	*
5793	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
5794	* However, doing so will yield better performance as well as avoiding
5795	* traps accessing the last bits in the bitmap.
5796	*/
5797	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5798	DECLASM(bool) ASMBitTestAndSet(volatile void *pvBitmap, int32_t iBit);
5799	#else
5800	DECLINLINE(bool) ASMBitTestAndSet(volatile void *pvBitmap, int32_t iBit)
5801	{
5802	union { bool f; uint32_t u32; uint8_t u8; } rc;
5803	# if RT_INLINE_ASM_USES_INTRIN
5804	rc.u8 = _bittestandset((long *)pvBitmap, iBit);
5805
5806	# elif RT_INLINE_ASM_GNU_STYLE
5807	__asm__ __volatile__("btsl %2, %1\n\t"
5808	"setc %b0\n\t"
5809	"andl $1, %0\n\t"
5810	: "=q" (rc.u32),
5811	"=m" ((volatile long )pvBitmap)
5812	: "Ir" (iBit),
5813	"m" ((volatile long )pvBitmap)
5814	: "memory");
5815	# else
5816	__asm
5817	{
5818	mov edx, [iBit]
5819	# ifdef RT_ARCH_AMD64
5820	mov rax, [pvBitmap]
5821	bts [rax], edx
5822	# else
5823	mov eax, [pvBitmap]
5824	bts [eax], edx
5825	# endif
5826	setc al
5827	and eax, 1
5828	mov [rc.u32], eax
5829	}
5830	# endif
5831	return rc.f;
5832	}
5833	#endif
5834
5835
5836	/**
5837	* Atomically tests and sets a bit in a bitmap, ordered.
5838	*
5839	* @returns true if the bit was set.
5840	* @returns false if the bit was clear.
5841	*
5842	* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
5843	* the memory access isn't atomic!
5844	* @param iBit The bit to set.
5845	*/
5846	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5847	DECLASM(bool) ASMAtomicBitTestAndSet(volatile void *pvBitmap, int32_t iBit);
5848	#else
5849	DECLINLINE(bool) ASMAtomicBitTestAndSet(volatile void *pvBitmap, int32_t iBit)
5850	{
5851	union { bool f; uint32_t u32; uint8_t u8; } rc;
5852	AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
5853	# if RT_INLINE_ASM_USES_INTRIN
5854	rc.u8 = _interlockedbittestandset((long *)pvBitmap, iBit);
5855	# elif RT_INLINE_ASM_GNU_STYLE
5856	__asm__ __volatile__("lock; btsl %2, %1\n\t"
5857	"setc %b0\n\t"
5858	"andl $1, %0\n\t"
5859	: "=q" (rc.u32),
5860	"=m" ((volatile long )pvBitmap)
5861	: "Ir" (iBit),
5862	"m" ((volatile long )pvBitmap)
5863	: "memory");
5864	# else
5865	__asm
5866	{
5867	mov edx, [iBit]
5868	# ifdef RT_ARCH_AMD64
5869	mov rax, [pvBitmap]
5870	lock bts [rax], edx
5871	# else
5872	mov eax, [pvBitmap]
5873	lock bts [eax], edx
5874	# endif
5875	setc al
5876	and eax, 1
5877	mov [rc.u32], eax
5878	}
5879	# endif
5880	return rc.f;
5881	}
5882	#endif
5883
5884
5885	/**
5886	* Tests and clears a bit in a bitmap.
5887	*
5888	* @returns true if the bit was set.
5889	* @returns false if the bit was clear.
5890	*
5891	* @param pvBitmap Pointer to the bitmap.
5892	* @param iBit The bit to test and clear.
5893	*
5894	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
5895	* However, doing so will yield better performance as well as avoiding
5896	* traps accessing the last bits in the bitmap.
5897	*/
5898	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5899	DECLASM(bool) ASMBitTestAndClear(volatile void *pvBitmap, int32_t iBit);
5900	#else
5901	DECLINLINE(bool) ASMBitTestAndClear(volatile void *pvBitmap, int32_t iBit)
5902	{
5903	union { bool f; uint32_t u32; uint8_t u8; } rc;
5904	# if RT_INLINE_ASM_USES_INTRIN
5905	rc.u8 = _bittestandreset((long *)pvBitmap, iBit);
5906
5907	# elif RT_INLINE_ASM_GNU_STYLE
5908	__asm__ __volatile__("btrl %2, %1\n\t"
5909	"setc %b0\n\t"
5910	"andl $1, %0\n\t"
5911	: "=q" (rc.u32),
5912	"=m" ((volatile long )pvBitmap)
5913	: "Ir" (iBit),
5914	"m" ((volatile long )pvBitmap)
5915	: "memory");
5916	# else
5917	__asm
5918	{
5919	mov edx, [iBit]
5920	# ifdef RT_ARCH_AMD64
5921	mov rax, [pvBitmap]
5922	btr [rax], edx
5923	# else
5924	mov eax, [pvBitmap]
5925	btr [eax], edx
5926	# endif
5927	setc al
5928	and eax, 1
5929	mov [rc.u32], eax
5930	}
5931	# endif
5932	return rc.f;
5933	}
5934	#endif
5935
5936
5937	/**
5938	* Atomically tests and clears a bit in a bitmap, ordered.
5939	*
5940	* @returns true if the bit was set.
5941	* @returns false if the bit was clear.
5942	*
5943	* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
5944	* the memory access isn't atomic!
5945	* @param iBit The bit to test and clear.
5946	*
5947	* @remarks No memory barrier, take care on smp.
5948	*/
5949	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
5950	DECLASM(bool) ASMAtomicBitTestAndClear(volatile void *pvBitmap, int32_t iBit);
5951	#else
5952	DECLINLINE(bool) ASMAtomicBitTestAndClear(volatile void *pvBitmap, int32_t iBit)
5953	{
5954	union { bool f; uint32_t u32; uint8_t u8; } rc;
5955	AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
5956	# if RT_INLINE_ASM_USES_INTRIN
5957	rc.u8 = _interlockedbittestandreset((long *)pvBitmap, iBit);
5958
5959	# elif RT_INLINE_ASM_GNU_STYLE
5960	__asm__ __volatile__("lock; btrl %2, %1\n\t"
5961	"setc %b0\n\t"
5962	"andl $1, %0\n\t"
5963	: "=q" (rc.u32),
5964	"=m" ((volatile long )pvBitmap)
5965	: "Ir" (iBit),
5966	"m" ((volatile long )pvBitmap)
5967	: "memory");
5968	# else
5969	__asm
5970	{
5971	mov edx, [iBit]
5972	# ifdef RT_ARCH_AMD64
5973	mov rax, [pvBitmap]
5974	lock btr [rax], edx
5975	# else
5976	mov eax, [pvBitmap]
5977	lock btr [eax], edx
5978	# endif
5979	setc al
5980	and eax, 1
5981	mov [rc.u32], eax
5982	}
5983	# endif
5984	return rc.f;
5985	}
5986	#endif
5987
5988
5989	/**
5990	* Tests and toggles a bit in a bitmap.
5991	*
5992	* @returns true if the bit was set.
5993	* @returns false if the bit was clear.
5994	*
5995	* @param pvBitmap Pointer to the bitmap.
5996	* @param iBit The bit to test and toggle.
5997	*
5998	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
5999	* However, doing so will yield better performance as well as avoiding
6000	* traps accessing the last bits in the bitmap.
6001	*/
6002	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
6003	DECLINLINE(bool) ASMBitTestAndToggle(volatile void *pvBitmap, int32_t iBit);
6004	#else
6005	DECLINLINE(bool) ASMBitTestAndToggle(volatile void *pvBitmap, int32_t iBit)
6006	{
6007	union { bool f; uint32_t u32; uint8_t u8; } rc;
6008	# if RT_INLINE_ASM_USES_INTRIN
6009	rc.u8 = _bittestandcomplement((long *)pvBitmap, iBit);
6010
6011	# elif RT_INLINE_ASM_GNU_STYLE
6012	__asm__ __volatile__("btcl %2, %1\n\t"
6013	"setc %b0\n\t"
6014	"andl $1, %0\n\t"
6015	: "=q" (rc.u32),
6016	"=m" ((volatile long )pvBitmap)
6017	: "Ir" (iBit),
6018	"m" ((volatile long )pvBitmap)
6019	: "memory");
6020	# else
6021	__asm
6022	{
6023	mov edx, [iBit]
6024	# ifdef RT_ARCH_AMD64
6025	mov rax, [pvBitmap]
6026	btc [rax], edx
6027	# else
6028	mov eax, [pvBitmap]
6029	btc [eax], edx
6030	# endif
6031	setc al
6032	and eax, 1
6033	mov [rc.u32], eax
6034	}
6035	# endif
6036	return rc.f;
6037	}
6038	#endif
6039
6040
6041	/**
6042	* Atomically tests and toggles a bit in a bitmap, ordered.
6043	*
6044	* @returns true if the bit was set.
6045	* @returns false if the bit was clear.
6046	*
6047	* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
6048	* the memory access isn't atomic!
6049	* @param iBit The bit to test and toggle.
6050	*/
6051	#if RT_INLINE_ASM_EXTERNAL
6052	DECLASM(bool) ASMAtomicBitTestAndToggle(volatile void *pvBitmap, int32_t iBit);
6053	#else
6054	DECLINLINE(bool) ASMAtomicBitTestAndToggle(volatile void *pvBitmap, int32_t iBit)
6055	{
6056	union { bool f; uint32_t u32; uint8_t u8; } rc;
6057	AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
6058	# if RT_INLINE_ASM_GNU_STYLE
6059	__asm__ __volatile__("lock; btcl %2, %1\n\t"
6060	"setc %b0\n\t"
6061	"andl $1, %0\n\t"
6062	: "=q" (rc.u32),
6063	"=m" ((volatile long )pvBitmap)
6064	: "Ir" (iBit),
6065	"m" ((volatile long )pvBitmap)
6066	: "memory");
6067	# else
6068	__asm
6069	{
6070	mov edx, [iBit]
6071	# ifdef RT_ARCH_AMD64
6072	mov rax, [pvBitmap]
6073	lock btc [rax], edx
6074	# else
6075	mov eax, [pvBitmap]
6076	lock btc [eax], edx
6077	# endif
6078	setc al
6079	and eax, 1
6080	mov [rc.u32], eax
6081	}
6082	# endif
6083	return rc.f;
6084	}
6085	#endif
6086
6087
6088	/**
6089	* Tests if a bit in a bitmap is set.
6090	*
6091	* @returns true if the bit is set.
6092	* @returns false if the bit is clear.
6093	*
6094	* @param pvBitmap Pointer to the bitmap.
6095	* @param iBit The bit to test.
6096	*
6097	* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
6098	* However, doing so will yield better performance as well as avoiding
6099	* traps accessing the last bits in the bitmap.
6100	*/
6101	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
6102	DECLASM(bool) ASMBitTest(const volatile void *pvBitmap, int32_t iBit);
6103	#else
6104	DECLINLINE(bool) ASMBitTest(const volatile void *pvBitmap, int32_t iBit)
6105	{
6106	union { bool f; uint32_t u32; uint8_t u8; } rc;
6107	# if RT_INLINE_ASM_USES_INTRIN
6108	rc.u32 = _bittest((long *)pvBitmap, iBit);
6109	# elif RT_INLINE_ASM_GNU_STYLE
6110
6111	__asm__ __volatile__("btl %2, %1\n\t"
6112	"setc %b0\n\t"
6113	"andl $1, %0\n\t"
6114	: "=q" (rc.u32)
6115	: "m" ((const volatile long )pvBitmap),
6116	"Ir" (iBit)
6117	: "memory");
6118	# else
6119	__asm
6120	{
6121	mov edx, [iBit]
6122	# ifdef RT_ARCH_AMD64
6123	mov rax, [pvBitmap]
6124	bt [rax], edx
6125	# else
6126	mov eax, [pvBitmap]
6127	bt [eax], edx
6128	# endif
6129	setc al
6130	and eax, 1
6131	mov [rc.u32], eax
6132	}
6133	# endif
6134	return rc.f;
6135	}
6136	#endif
6137
6138
6139	/**
6140	* Clears a bit range within a bitmap.
6141	*
6142	* @param pvBitmap Pointer to the bitmap.
6143	* @param iBitStart The First bit to clear.
6144	* @param iBitEnd The first bit not to clear.
6145	*/
6146	DECLINLINE(void) ASMBitClearRange(volatile void *pvBitmap, int32_t iBitStart, int32_t iBitEnd)
6147	{
6148	if (iBitStart < iBitEnd)
6149	{
6150	volatile uint32_t pu32 = (volatile uint32_t )pvBitmap + (iBitStart >> 5);
6151	int iStart = iBitStart & ~31;
6152	int iEnd = iBitEnd & ~31;
6153	if (iStart == iEnd)
6154	*pu32 &= ((1 << (iBitStart & 31)) - 1) \| ~((1 << (iBitEnd & 31)) - 1);
6155	else
6156	{
6157	/* bits in first dword. */
6158	if (iBitStart & 31)
6159	{
6160	*pu32 &= (1 << (iBitStart & 31)) - 1;
6161	pu32++;
6162	iBitStart = iStart + 32;
6163	}
6164
6165	/* whole dword. */
6166	if (iBitStart != iEnd)
6167	ASMMemZero32(pu32, (iEnd - iBitStart) >> 3);
6168
6169	/* bits in last dword. */
6170	if (iBitEnd & 31)
6171	{
6172	pu32 = (volatile uint32_t *)pvBitmap + (iBitEnd >> 5);
6173	*pu32 &= ~((1 << (iBitEnd & 31)) - 1);
6174	}
6175	}
6176	}
6177	}
6178
6179
6180	/**
6181	* Sets a bit range within a bitmap.
6182	*
6183	* @param pvBitmap Pointer to the bitmap.
6184	* @param iBitStart The First bit to set.
6185	* @param iBitEnd The first bit not to set.
6186	*/
6187	DECLINLINE(void) ASMBitSetRange(volatile void *pvBitmap, int32_t iBitStart, int32_t iBitEnd)
6188	{
6189	if (iBitStart < iBitEnd)
6190	{
6191	volatile uint32_t pu32 = (volatile uint32_t )pvBitmap + (iBitStart >> 5);
6192	int iStart = iBitStart & ~31;
6193	int iEnd = iBitEnd & ~31;
6194	if (iStart == iEnd)
6195	*pu32 \|= ((1 << (iBitEnd - iBitStart)) - 1) << iBitStart;
6196	else
6197	{
6198	/* bits in first dword. */
6199	if (iBitStart & 31)
6200	{
6201	*pu32 \|= ~((1 << (iBitStart & 31)) - 1);
6202	pu32++;
6203	iBitStart = iStart + 32;
6204	}
6205
6206	/* whole dword. */
6207	if (iBitStart != iEnd)
6208	ASMMemFill32(pu32, (iEnd - iBitStart) >> 3, ~0);
6209
6210	/* bits in last dword. */
6211	if (iBitEnd & 31)
6212	{
6213	pu32 = (volatile uint32_t *)pvBitmap + (iBitEnd >> 5);
6214	*pu32 \|= (1 << (iBitEnd & 31)) - 1;
6215	}
6216	}
6217	}
6218	}
6219
6220
6221	/**
6222	* Finds the first clear bit in a bitmap.
6223	*
6224	* @returns Index of the first zero bit.
6225	* @returns -1 if no clear bit was found.
6226	* @param pvBitmap Pointer to the bitmap.
6227	* @param cBits The number of bits in the bitmap. Multiple of 32.
6228	*/
6229	#if RT_INLINE_ASM_EXTERNAL
6230	DECLASM(int) ASMBitFirstClear(const volatile void *pvBitmap, uint32_t cBits);
6231	#else
6232	DECLINLINE(int) ASMBitFirstClear(const volatile void *pvBitmap, uint32_t cBits)
6233	{
6234	if (cBits)
6235	{
6236	int32_t iBit;
6237	# if RT_INLINE_ASM_GNU_STYLE
6238	RTCCUINTREG uEAX, uECX, uEDI;
6239	cBits = RT_ALIGN_32(cBits, 32);
6240	__asm__ __volatile__("repe; scasl\n\t"
6241	"je 1f\n\t"
6242	# ifdef RT_ARCH_AMD64
6243	"lea -4(%%rdi), %%rdi\n\t"
6244	"xorl (%%rdi), %%eax\n\t"
6245	"subq %5, %%rdi\n\t"
6246	# else
6247	"lea -4(%%edi), %%edi\n\t"
6248	"xorl (%%edi), %%eax\n\t"
6249	"subl %5, %%edi\n\t"
6250	# endif
6251	"shll $3, %%edi\n\t"
6252	"bsfl %%eax, %%edx\n\t"
6253	"addl %%edi, %%edx\n\t"
6254	"1:\t\n"
6255	: "=d" (iBit),
6256	"=&c" (uECX),
6257	"=&D" (uEDI),
6258	"=&a" (uEAX)
6259	: "0" (0xffffffff),
6260	"mr" (pvBitmap),
6261	"1" (cBits >> 5),
6262	"2" (pvBitmap),
6263	"3" (0xffffffff));
6264	# else
6265	cBits = RT_ALIGN_32(cBits, 32);
6266	__asm
6267	{
6268	# ifdef RT_ARCH_AMD64
6269	mov rdi, [pvBitmap]
6270	mov rbx, rdi
6271	# else
6272	mov edi, [pvBitmap]
6273	mov ebx, edi
6274	# endif
6275	mov edx, 0ffffffffh
6276	mov eax, edx
6277	mov ecx, [cBits]
6278	shr ecx, 5
6279	repe scasd
6280	je done
6281
6282	# ifdef RT_ARCH_AMD64
6283	lea rdi, [rdi - 4]
6284	xor eax, [rdi]
6285	sub rdi, rbx
6286	# else
6287	lea edi, [edi - 4]
6288	xor eax, [edi]
6289	sub edi, ebx
6290	# endif
6291	shl edi, 3
6292	bsf edx, eax
6293	add edx, edi
6294	done:
6295	mov [iBit], edx
6296	}
6297	# endif
6298	return iBit;
6299	}
6300	return -1;
6301	}
6302	#endif
6303
6304
6305	/**
6306	* Finds the next clear bit in a bitmap.
6307	*
6308	* @returns Index of the first zero bit.
6309	* @returns -1 if no clear bit was found.
6310	* @param pvBitmap Pointer to the bitmap.
6311	* @param cBits The number of bits in the bitmap. Multiple of 32.
6312	* @param iBitPrev The bit returned from the last search.
6313	* The search will start at iBitPrev + 1.
6314	*/
6315	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
6316	DECLASM(int) ASMBitNextClear(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev);
6317	#else
6318	DECLINLINE(int) ASMBitNextClear(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev)
6319	{
6320	const volatile uint32_t pau32Bitmap = (const volatile uint32_t )pvBitmap;
6321	int iBit = ++iBitPrev & 31;
6322	if (iBit)
6323	{
6324	/*
6325	* Inspect the 32-bit word containing the unaligned bit.
6326	*/
6327	uint32_t u32 = ~pau32Bitmap[iBitPrev / 32] >> iBit;
6328
6329	# if RT_INLINE_ASM_USES_INTRIN
6330	unsigned long ulBit = 0;
6331	if (_BitScanForward(&ulBit, u32))
6332	return ulBit + iBitPrev;
6333	# else
6334	# if RT_INLINE_ASM_GNU_STYLE
6335	__asm__ __volatile__("bsf %1, %0\n\t"
6336	"jnz 1f\n\t"
6337	"movl $-1, %0\n\t"
6338	"1:\n\t"
6339	: "=r" (iBit)
6340	: "r" (u32));
6341	# else
6342	__asm
6343	{
6344	mov edx, [u32]
6345	bsf eax, edx
6346	jnz done
6347	mov eax, 0ffffffffh
6348	done:
6349	mov [iBit], eax
6350	}
6351	# endif
6352	if (iBit >= 0)
6353	return iBit + iBitPrev;
6354	# endif
6355
6356	/*
6357	* Skip ahead and see if there is anything left to search.
6358	*/
6359	iBitPrev \|= 31;
6360	iBitPrev++;
6361	if (cBits <= (uint32_t)iBitPrev)
6362	return -1;
6363	}
6364
6365	/*
6366	* 32-bit aligned search, let ASMBitFirstClear do the dirty work.
6367	*/
6368	iBit = ASMBitFirstClear(&pau32Bitmap[iBitPrev / 32], cBits - iBitPrev);
6369	if (iBit >= 0)
6370	iBit += iBitPrev;
6371	return iBit;
6372	}
6373	#endif
6374
6375
6376	/**
6377	* Finds the first set bit in a bitmap.
6378	*
6379	* @returns Index of the first set bit.
6380	* @returns -1 if no clear bit was found.
6381	* @param pvBitmap Pointer to the bitmap.
6382	* @param cBits The number of bits in the bitmap. Multiple of 32.
6383	*/
6384	#if RT_INLINE_ASM_EXTERNAL
6385	DECLASM(int) ASMBitFirstSet(const volatile void *pvBitmap, uint32_t cBits);
6386	#else
6387	DECLINLINE(int) ASMBitFirstSet(const volatile void *pvBitmap, uint32_t cBits)
6388	{
6389	if (cBits)
6390	{
6391	int32_t iBit;
6392	# if RT_INLINE_ASM_GNU_STYLE
6393	RTCCUINTREG uEAX, uECX, uEDI;
6394	cBits = RT_ALIGN_32(cBits, 32);
6395	__asm__ __volatile__("repe; scasl\n\t"
6396	"je 1f\n\t"
6397	# ifdef RT_ARCH_AMD64
6398	"lea -4(%%rdi), %%rdi\n\t"
6399	"movl (%%rdi), %%eax\n\t"
6400	"subq %5, %%rdi\n\t"
6401	# else
6402	"lea -4(%%edi), %%edi\n\t"
6403	"movl (%%edi), %%eax\n\t"
6404	"subl %5, %%edi\n\t"
6405	# endif
6406	"shll $3, %%edi\n\t"
6407	"bsfl %%eax, %%edx\n\t"
6408	"addl %%edi, %%edx\n\t"
6409	"1:\t\n"
6410	: "=d" (iBit),
6411	"=&c" (uECX),
6412	"=&D" (uEDI),
6413	"=&a" (uEAX)
6414	: "0" (0xffffffff),
6415	"mr" (pvBitmap),
6416	"1" (cBits >> 5),
6417	"2" (pvBitmap),
6418	"3" (0));
6419	# else
6420	cBits = RT_ALIGN_32(cBits, 32);
6421	__asm
6422	{
6423	# ifdef RT_ARCH_AMD64
6424	mov rdi, [pvBitmap]
6425	mov rbx, rdi
6426	# else
6427	mov edi, [pvBitmap]
6428	mov ebx, edi
6429	# endif
6430	mov edx, 0ffffffffh
6431	xor eax, eax
6432	mov ecx, [cBits]
6433	shr ecx, 5
6434	repe scasd
6435	je done
6436	# ifdef RT_ARCH_AMD64
6437	lea rdi, [rdi - 4]
6438	mov eax, [rdi]
6439	sub rdi, rbx
6440	# else
6441	lea edi, [edi - 4]
6442	mov eax, [edi]
6443	sub edi, ebx
6444	# endif
6445	shl edi, 3
6446	bsf edx, eax
6447	add edx, edi
6448	done:
6449	mov [iBit], edx
6450	}
6451	# endif
6452	return iBit;
6453	}
6454	return -1;
6455	}
6456	#endif
6457
6458
6459	/**
6460	* Finds the next set bit in a bitmap.
6461	*
6462	* @returns Index of the next set bit.
6463	* @returns -1 if no set bit was found.
6464	* @param pvBitmap Pointer to the bitmap.
6465	* @param cBits The number of bits in the bitmap. Multiple of 32.
6466	* @param iBitPrev The bit returned from the last search.
6467	* The search will start at iBitPrev + 1.
6468	*/
6469	#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
6470	DECLASM(int) ASMBitNextSet(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev);
6471	#else
6472	DECLINLINE(int) ASMBitNextSet(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev)
6473	{
6474	const volatile uint32_t pau32Bitmap = (const volatile uint32_t )pvBitmap;
6475	int iBit = ++iBitPrev & 31;
6476	if (iBit)
6477	{
6478	/*
6479	* Inspect the 32-bit word containing the unaligned bit.
6480	*/
6481	uint32_t u32 = pau32Bitmap[iBitPrev / 32] >> iBit;
6482
6483	# if RT_INLINE_ASM_USES_INTRIN
6484	unsigned long ulBit = 0;
6485	if (_BitScanForward(&ulBit, u32))
6486	return ulBit + iBitPrev;
6487	# else
6488	# if RT_INLINE_ASM_GNU_STYLE
6489	__asm__ __volatile__("bsf %1, %0\n\t"
6490	"jnz 1f\n\t"
6491	"movl $-1, %0\n\t"
6492	"1:\n\t"
6493	: "=r" (iBit)
6494	: "r" (u32));
6495	# else
6496	__asm
6497	{
6498	mov edx, [u32]
6499	bsf eax, edx
6500	jnz done
6501	mov eax, 0ffffffffh
6502	done:
6503	mov [iBit], eax
6504	}
6505	# endif
6506	if (iBit >= 0)
6507	return iBit + iBitPrev;
6508	# endif
6509
6510	/*
6511	* Skip ahead and see if there is anything left to search.
6512	*/
6513	iBitPrev \|= 31;
6514	iBitPrev++;
6515	if (cBits <= (uint32_t)iBitPrev)
6516	return -1;
6517	}
6518
6519	/*
6520	* 32-bit aligned search, let ASMBitFirstClear do the dirty work.
6521	*/
6522	iBit = ASMBitFirstSet(&pau32Bitmap[iBitPrev / 32], cBits - iBitPrev);
6523	if (iBit >= 0)
6524	iBit += iBitPrev;
6525	return iBit;
6526	}
6527	#endif
6528
6529
6530	/**
6531	* Finds the first bit which is set in the given 32-bit integer.
6532	* Bits are numbered from 1 (least significant) to 32.
6533	*
6534	* @returns index [1..32] of the first set bit.
6535	* @returns 0 if all bits are cleared.
6536	* @param u32 Integer to search for set bits.
6537	* @remark Similar to ffs() in BSD.
6538	*/
6539	DECLINLINE(unsigned) ASMBitFirstSetU32(uint32_t u32)
6540	{
6541	# if RT_INLINE_ASM_USES_INTRIN
6542	unsigned long iBit;
6543	if (_BitScanForward(&iBit, u32))
6544	iBit++;
6545	else
6546	iBit = 0;
6547	# elif RT_INLINE_ASM_GNU_STYLE
6548	uint32_t iBit;
6549	__asm__ __volatile__("bsf %1, %0\n\t"
6550	"jnz 1f\n\t"
6551	"xorl %0, %0\n\t"
6552	"jmp 2f\n"
6553	"1:\n\t"
6554	"incl %0\n"
6555	"2:\n\t"
6556	: "=r" (iBit)
6557	: "rm" (u32));
6558	# else
6559	uint32_t iBit;
6560	_asm
6561	{
6562	bsf eax, [u32]
6563	jnz found
6564	xor eax, eax
6565	jmp done
6566	found:
6567	inc eax
6568	done:
6569	mov [iBit], eax
6570	}
6571	# endif
6572	return iBit;
6573	}
6574
6575
6576	/**
6577	* Finds the first bit which is set in the given 32-bit integer.
6578	* Bits are numbered from 1 (least significant) to 32.
6579	*
6580	* @returns index [1..32] of the first set bit.
6581	* @returns 0 if all bits are cleared.
6582	* @param i32 Integer to search for set bits.
6583	* @remark Similar to ffs() in BSD.
6584	*/
6585	DECLINLINE(unsigned) ASMBitFirstSetS32(int32_t i32)
6586	{
6587	return ASMBitFirstSetU32((uint32_t)i32);
6588	}
6589
6590
6591	/**
6592	* Finds the last bit which is set in the given 32-bit integer.
6593	* Bits are numbered from 1 (least significant) to 32.
6594	*
6595	* @returns index [1..32] of the last set bit.
6596	* @returns 0 if all bits are cleared.
6597	* @param u32 Integer to search for set bits.
6598	* @remark Similar to fls() in BSD.
6599	*/
6600	DECLINLINE(unsigned) ASMBitLastSetU32(uint32_t u32)
6601	{
6602	# if RT_INLINE_ASM_USES_INTRIN
6603	unsigned long iBit;
6604	if (_BitScanReverse(&iBit, u32))
6605	iBit++;
6606	else
6607	iBit = 0;
6608	# elif RT_INLINE_ASM_GNU_STYLE
6609	uint32_t iBit;
6610	__asm__ __volatile__("bsrl %1, %0\n\t"
6611	"jnz 1f\n\t"
6612	"xorl %0, %0\n\t"
6613	"jmp 2f\n"
6614	"1:\n\t"
6615	"incl %0\n"
6616	"2:\n\t"
6617	: "=r" (iBit)
6618	: "rm" (u32));
6619	# else
6620	uint32_t iBit;
6621	_asm
6622	{
6623	bsr eax, [u32]
6624	jnz found
6625	xor eax, eax
6626	jmp done
6627	found:
6628	inc eax
6629	done:
6630	mov [iBit], eax
6631	}
6632	# endif
6633	return iBit;
6634	}
6635
6636
6637	/**
6638	* Finds the last bit which is set in the given 32-bit integer.
6639	* Bits are numbered from 1 (least significant) to 32.
6640	*
6641	* @returns index [1..32] of the last set bit.
6642	* @returns 0 if all bits are cleared.
6643	* @param i32 Integer to search for set bits.
6644	* @remark Similar to fls() in BSD.
6645	*/
6646	DECLINLINE(unsigned) ASMBitLastSetS32(int32_t i32)
6647	{
6648	return ASMBitLastSetU32((uint32_t)i32);
6649	}
6650
6651	/**
6652	* Reverse the byte order of the given 16-bit integer.
6653	*
6654	* @returns Revert
6655	* @param u16 16-bit integer value.
6656	*/
6657	DECLINLINE(uint16_t) ASMByteSwapU16(uint16_t u16)
6658	{
6659	#if RT_INLINE_ASM_USES_INTRIN
6660	u16 = _byteswap_ushort(u16);
6661	#elif RT_INLINE_ASM_GNU_STYLE
6662	__asm__ ("rorw $8, %0" : "=r" (u16) : "0" (u16));
6663	#else
6664	_asm
6665	{
6666	mov ax, [u16]
6667	ror ax, 8
6668	mov [u16], ax
6669	}
6670	#endif
6671	return u16;
6672	}
6673
6674	/**
6675	* Reverse the byte order of the given 32-bit integer.
6676	*
6677	* @returns Revert
6678	* @param u32 32-bit integer value.
6679	*/
6680	DECLINLINE(uint32_t) ASMByteSwapU32(uint32_t u32)
6681	{
6682	#if RT_INLINE_ASM_USES_INTRIN
6683	u32 = _byteswap_ulong(u32);
6684	#elif RT_INLINE_ASM_GNU_STYLE
6685	__asm__ ("bswapl %0" : "=r" (u32) : "0" (u32));
6686	#else
6687	_asm
6688	{
6689	mov eax, [u32]
6690	bswap eax
6691	mov [u32], eax
6692	}
6693	#endif
6694	return u32;
6695	}
6696
6697
6698	/**
6699	* Reverse the byte order of the given 64-bit integer.
6700	*
6701	* @returns Revert
6702	* @param u64 64-bit integer value.
6703	*/
6704	DECLINLINE(uint64_t) ASMByteSwapU64(uint64_t u64)
6705	{
6706	#if defined(RT_ARCH_AMD64) && RT_INLINE_ASM_USES_INTRIN
6707	u64 = _byteswap_uint64(u64);
6708	#else
6709	u64 = (uint64_t)ASMByteSwapU32((uint32_t)u64) << 32
6710	\| (uint64_t)ASMByteSwapU32((uint32_t)(u64 >> 32));
6711	#endif
6712	return u64;
6713	}
6714
6715
6716	/** @} */
6717
6718
6719	/** @} */
6720	#endif
6721

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/include/iprt/asm.h@ 22279

Download in other formats: