semrw-lockless-generic.cpp@ 55747

Last change on this file since 55747 was 45309, checked in by vboxsync, 12 years ago
RTCritSectRw: Two fixes.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 33.7 KB

Line
1	/* $Id: semrw-lockless-generic.cpp 45309 2013-04-03 14:40:35Z vboxsync $ */
2	/** @file
3	* IPRT - Read-Write Semaphore, Generic, lockless variant.
4	*/
5
6	/*
7	* Copyright (C) 2009-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*******************************************************************************
29	* Header Files *
30	*******************************************************************************/
31	#define RTSEMRW_WITHOUT_REMAPPING
32	#define RTASSERT_QUIET
33	#include <iprt/semaphore.h>
34	#include "internal/iprt.h"
35
36	#include <iprt/asm.h>
37	#include <iprt/assert.h>
38	#include <iprt/err.h>
39	#include <iprt/lockvalidator.h>
40	#include <iprt/mem.h>
41	#include <iprt/thread.h>
42
43	#include "internal/magics.h"
44	#include "internal/strict.h"
45
46
47	/*******************************************************************************
48	* Structures and Typedefs *
49	*******************************************************************************/
50	typedef struct RTSEMRWINTERNAL
51	{
52	/** Magic value (RTSEMRW_MAGIC). */
53	uint32_t volatile u32Magic;
54	/** Indicates whether hEvtRead needs resetting. */
55	bool volatile fNeedReset;
56
57	/** The state variable.
58	* All accesses are atomic and it bits are defined like this:
59	* Bits 0..14 - cReads.
60	* Bit 15 - Unused.
61	* Bits 16..31 - cWrites. - doesn't make sense here
62	* Bit 31 - fDirection; 0=Read, 1=Write.
63	* Bits 32..46 - cWaitingReads
64	* Bit 47 - Unused.
65	* Bits 48..62 - cWaitingWrites
66	* Bit 63 - Unused.
67	*/
68	uint64_t volatile u64State;
69	/** The write owner. */
70	RTNATIVETHREAD volatile hNativeWriter;
71	/** The number of reads made by the current writer. */
72	uint32_t volatile cWriterReads;
73	/** The number of recursions made by the current writer. (The initial grabbing
74	* of the lock counts as the first one.) */
75	uint32_t volatile cWriteRecursions;
76
77	/** What the writer threads are blocking on. */
78	RTSEMEVENT hEvtWrite;
79	/** What the read threads are blocking on when waiting for the writer to
80	* finish. */
81	RTSEMEVENTMULTI hEvtRead;
82
83	#ifdef RTSEMRW_STRICT
84	/** The validator record for the writer. */
85	RTLOCKVALRECEXCL ValidatorWrite;
86	/** The validator record for the readers. */
87	RTLOCKVALRECSHRD ValidatorRead;
88	#endif
89	} RTSEMRWINTERNAL;
90
91
92	/*******************************************************************************
93	* Defined Constants And Macros *
94	*******************************************************************************/
95	#define RTSEMRW_CNT_BITS 15
96	#define RTSEMRW_CNT_MASK UINT64_C(0x00007fff)
97
98	#define RTSEMRW_CNT_RD_SHIFT 0
99	#define RTSEMRW_CNT_RD_MASK (RTSEMRW_CNT_MASK << RTSEMRW_CNT_RD_SHIFT)
100	#define RTSEMRW_CNT_WR_SHIFT 16
101	#define RTSEMRW_CNT_WR_MASK (RTSEMRW_CNT_MASK << RTSEMRW_CNT_WR_SHIFT)
102	#define RTSEMRW_DIR_SHIFT 31
103	#define RTSEMRW_DIR_MASK RT_BIT_64(RTSEMRW_DIR_SHIFT)
104	#define RTSEMRW_DIR_READ UINT64_C(0)
105	#define RTSEMRW_DIR_WRITE UINT64_C(1)
106
107	#define RTSEMRW_WAIT_CNT_RD_SHIFT 32
108	#define RTSEMRW_WAIT_CNT_RD_MASK (RTSEMRW_CNT_MASK << RTSEMRW_WAIT_CNT_RD_SHIFT)
109	//#define RTSEMRW_WAIT_CNT_WR_SHIFT 48
110	//#define RTSEMRW_WAIT_CNT_WR_MASK (RTSEMRW_CNT_MASK << RTSEMRW_WAIT_CNT_WR_SHIFT)
111
112
113	RTDECL(int) RTSemRWCreate(PRTSEMRW phRWSem)
114	{
115	return RTSemRWCreateEx(phRWSem, 0 /fFlags/, NIL_RTLOCKVALCLASS, RTLOCKVAL_SUB_CLASS_NONE, "RTSemRW");
116	}
117	RT_EXPORT_SYMBOL(RTSemRWCreate);
118
119
120	RTDECL(int) RTSemRWCreateEx(PRTSEMRW phRWSem, uint32_t fFlags,
121	RTLOCKVALCLASS hClass, uint32_t uSubClass, const char *pszNameFmt, ...)
122	{
123	AssertReturn(!(fFlags & ~RTSEMRW_FLAGS_NO_LOCK_VAL), VERR_INVALID_PARAMETER);
124
125	RTSEMRWINTERNAL pThis = (RTSEMRWINTERNAL )RTMemAlloc(sizeof(*pThis));
126	if (!pThis)
127	return VERR_NO_MEMORY;
128
129	int rc = RTSemEventMultiCreate(&pThis->hEvtRead);
130	if (RT_SUCCESS(rc))
131	{
132	rc = RTSemEventCreate(&pThis->hEvtWrite);
133	if (RT_SUCCESS(rc))
134	{
135	pThis->u32Magic = RTSEMRW_MAGIC;
136	pThis->u32Padding = 0;
137	pThis->u64State = 0;
138	pThis->hNativeWriter = NIL_RTNATIVETHREAD;
139	pThis->cWriterReads = 0;
140	pThis->cWriteRecursions = 0;
141	pThis->fNeedReset = false;
142	#ifdef RTSEMRW_STRICT
143	bool const fLVEnabled = !(fFlags & RTSEMRW_FLAGS_NO_LOCK_VAL);
144	if (!pszNameFmt)
145	{
146	static uint32_t volatile s_iSemRWAnon = 0;
147	uint32_t i = ASMAtomicIncU32(&s_iSemRWAnon) - 1;
148	RTLockValidatorRecExclInit(&pThis->ValidatorWrite, hClass, uSubClass, pThis,
149	fLVEnabled, "RTSemRW-%u", i);
150	RTLockValidatorRecSharedInit(&pThis->ValidatorRead, hClass, uSubClass, pThis,
151	false /fSignaller/, fLVEnabled, "RTSemRW-%u", i);
152	}
153	else
154	{
155	va_list va;
156	va_start(va, pszNameFmt);
157	RTLockValidatorRecExclInitV(&pThis->ValidatorWrite, hClass, uSubClass, pThis,
158	fLVEnabled, pszNameFmt, va);
159	va_end(va);
160	va_start(va, pszNameFmt);
161	RTLockValidatorRecSharedInitV(&pThis->ValidatorRead, hClass, uSubClass, pThis,
162	false /fSignaller/, fLVEnabled, pszNameFmt, va);
163	va_end(va);
164	}
165	RTLockValidatorRecMakeSiblings(&pThis->ValidatorWrite.Core, &pThis->ValidatorRead.Core);
166	#endif
167
168	*phRWSem = pThis;
169	return VINF_SUCCESS;
170	}
171	RTSemEventMultiDestroy(pThis->hEvtRead);
172	}
173	return rc;
174	}
175	RT_EXPORT_SYMBOL(RTSemRWCreateEx);
176
177
178	RTDECL(int) RTSemRWDestroy(RTSEMRW hRWSem)
179	{
180	/*
181	* Validate input.
182	*/
183	RTSEMRWINTERNAL *pThis = hRWSem;
184	if (pThis == NIL_RTSEMRW)
185	return VINF_SUCCESS;
186	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
187	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, VERR_INVALID_HANDLE);
188	Assert(!(ASMAtomicReadU64(&pThis->u64State) & (RTSEMRW_CNT_RD_MASK \| RTSEMRW_CNT_WR_MASK)));
189
190	/*
191	* Invalidate the object and free up the resources.
192	*/
193	AssertReturn(ASMAtomicCmpXchgU32(&pThis->u32Magic, ~RTSEMRW_MAGIC, RTSEMRW_MAGIC), VERR_INVALID_HANDLE);
194
195	RTSEMEVENTMULTI hEvtRead;
196	ASMAtomicXchgHandle(&pThis->hEvtRead, NIL_RTSEMEVENTMULTI, &hEvtRead);
197	int rc = RTSemEventMultiDestroy(hEvtRead);
198	AssertRC(rc);
199
200	RTSEMEVENT hEvtWrite;
201	ASMAtomicXchgHandle(&pThis->hEvtWrite, NIL_RTSEMEVENT, &hEvtWrite);
202	rc = RTSemEventDestroy(hEvtWrite);
203	AssertRC(rc);
204
205	#ifdef RTSEMRW_STRICT
206	RTLockValidatorRecSharedDelete(&pThis->ValidatorRead);
207	RTLockValidatorRecExclDelete(&pThis->ValidatorWrite);
208	#endif
209	RTMemFree(pThis);
210	return VINF_SUCCESS;
211	}
212	RT_EXPORT_SYMBOL(RTSemRWDestroy);
213
214
215	RTDECL(uint32_t) RTSemRWSetSubClass(RTSEMRW hRWSem, uint32_t uSubClass)
216	{
217	#ifdef RTSEMRW_STRICT
218	/*
219	* Validate handle.
220	*/
221	struct RTSEMRWINTERNAL *pThis = hRWSem;
222	AssertPtrReturn(pThis, RTLOCKVAL_SUB_CLASS_INVALID);
223	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, RTLOCKVAL_SUB_CLASS_INVALID);
224
225	RTLockValidatorRecSharedSetSubClass(&pThis->ValidatorRead, uSubClass);
226	return RTLockValidatorRecExclSetSubClass(&pThis->ValidatorWrite, uSubClass);
227	#else
228	return RTLOCKVAL_SUB_CLASS_INVALID;
229	#endif
230	}
231	RT_EXPORT_SYMBOL(RTSemRWSetSubClass);
232
233
234	static int rtSemRWRequestRead(RTSEMRW hRWSem, RTMSINTERVAL cMillies, bool fInterruptible, PCRTLOCKVALSRCPOS pSrcPos)
235	{
236	/*
237	* Validate input.
238	*/
239	RTSEMRWINTERNAL *pThis = hRWSem;
240	if (pThis == NIL_RTSEMRW)
241	return VINF_SUCCESS;
242	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
243	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, VERR_INVALID_HANDLE);
244
245	#ifdef RTSEMRW_STRICT
246	RTTHREAD hThreadSelf = RTThreadSelfAutoAdopt();
247	if (cMillies > 0)
248	{
249	int rc9;
250	RTNATIVETHREAD hNativeWriter;
251	ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
252	if (hNativeWriter != NIL_RTTHREAD && hNativeWriter == RTThreadNativeSelf())
253	rc9 = RTLockValidatorRecExclCheckOrder(&pThis->ValidatorWrite, hThreadSelf, pSrcPos, cMillies);
254	else
255	rc9 = RTLockValidatorRecSharedCheckOrder(&pThis->ValidatorRead, hThreadSelf, pSrcPos, cMillies);
256	if (RT_FAILURE(rc9))
257	return rc9;
258	}
259	#endif
260
261	/*
262	* Get cracking...
263	*/
264	uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
265	uint64_t u64OldState = u64State;
266
267	for (;;)
268	{
269	if ((u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT))
270	{
271	/* It flows in the right direction, try follow it before it changes. */
272	uint64_t c = (u64State & RTSEMRW_CNT_RD_MASK) >> RTSEMRW_CNT_RD_SHIFT;
273	c++;
274	Assert(c < RTSEMRW_CNT_MASK / 2);
275	u64State &= ~RTSEMRW_CNT_RD_MASK;
276	u64State \|= c << RTSEMRW_CNT_RD_SHIFT;
277	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
278	{
279	#ifdef RTSEMRW_STRICT
280	RTLockValidatorRecSharedAddOwner(&pThis->ValidatorRead, hThreadSelf, pSrcPos);
281	#endif
282	break;
283	}
284	}
285	else if ((u64State & (RTSEMRW_CNT_RD_MASK \| RTSEMRW_CNT_WR_MASK)) == 0)
286	{
287	/* Wrong direction, but we're alone here and can simply try switch the direction. */
288	u64State &= ~(RTSEMRW_CNT_RD_MASK \| RTSEMRW_CNT_WR_MASK \| RTSEMRW_DIR_MASK);
289	u64State \|= (UINT64_C(1) << RTSEMRW_CNT_RD_SHIFT) \| (RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT);
290	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
291	{
292	Assert(!pThis->fNeedReset);
293	#ifdef RTSEMRW_STRICT
294	RTLockValidatorRecSharedAddOwner(&pThis->ValidatorRead, hThreadSelf, pSrcPos);
295	#endif
296	break;
297	}
298	}
299	else
300	{
301	/* Is the writer perhaps doing a read recursion? */
302	RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
303	RTNATIVETHREAD hNativeWriter;
304	ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
305	if (hNativeSelf == hNativeWriter)
306	{
307	#ifdef RTSEMRW_STRICT
308	int rc9 = RTLockValidatorRecExclRecursionMixed(&pThis->ValidatorWrite, &pThis->ValidatorRead.Core, pSrcPos);
309	if (RT_FAILURE(rc9))
310	return rc9;
311	#endif
312	Assert(pThis->cWriterReads < UINT32_MAX / 2);
313	ASMAtomicIncU32(&pThis->cWriterReads);
314	return VINF_SUCCESS; /* don't break! */
315	}
316
317	/* If the timeout is 0, return already. */
318	if (!cMillies)
319	return VERR_TIMEOUT;
320
321	/* Add ourselves to the queue and wait for the direction to change. */
322	uint64_t c = (u64State & RTSEMRW_CNT_RD_MASK) >> RTSEMRW_CNT_RD_SHIFT;
323	c++;
324	Assert(c < RTSEMRW_CNT_MASK / 2);
325
326	uint64_t cWait = (u64State & RTSEMRW_WAIT_CNT_RD_MASK) >> RTSEMRW_WAIT_CNT_RD_SHIFT;
327	cWait++;
328	Assert(cWait <= c);
329	Assert(cWait < RTSEMRW_CNT_MASK / 2);
330
331	u64State &= ~(RTSEMRW_CNT_RD_MASK \| RTSEMRW_WAIT_CNT_RD_MASK);
332	u64State \|= (c << RTSEMRW_CNT_RD_SHIFT) \| (cWait << RTSEMRW_WAIT_CNT_RD_SHIFT);
333
334	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
335	{
336	for (uint32_t iLoop = 0; ; iLoop++)
337	{
338	int rc;
339	#ifdef RTSEMRW_STRICT
340	rc = RTLockValidatorRecSharedCheckBlocking(&pThis->ValidatorRead, hThreadSelf, pSrcPos, true,
341	cMillies, RTTHREADSTATE_RW_READ, false);
342	if (RT_SUCCESS(rc))
343	#else
344	RTTHREAD hThreadSelf = RTThreadSelf();
345	RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_READ, false);
346	#endif
347	{
348	if (fInterruptible)
349	rc = RTSemEventMultiWaitNoResume(pThis->hEvtRead, cMillies);
350	else
351	rc = RTSemEventMultiWait(pThis->hEvtRead, cMillies);
352	RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_READ);
353	if (pThis->u32Magic != RTSEMRW_MAGIC)
354	return VERR_SEM_DESTROYED;
355	}
356	if (RT_FAILURE(rc))
357	{
358	/* Decrement the counts and return the error. */
359	for (;;)
360	{
361	u64OldState = u64State = ASMAtomicReadU64(&pThis->u64State);
362	c = (u64State & RTSEMRW_CNT_RD_MASK) >> RTSEMRW_CNT_RD_SHIFT; Assert(c > 0);
363	c--;
364	cWait = (u64State & RTSEMRW_WAIT_CNT_RD_MASK) >> RTSEMRW_WAIT_CNT_RD_SHIFT; Assert(cWait > 0);
365	cWait--;
366	u64State &= ~(RTSEMRW_CNT_RD_MASK \| RTSEMRW_WAIT_CNT_RD_MASK);
367	u64State \|= (c << RTSEMRW_CNT_RD_SHIFT) \| (cWait << RTSEMRW_WAIT_CNT_RD_SHIFT);
368	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
369	break;
370	}
371	return rc;
372	}
373
374	Assert(pThis->fNeedReset);
375	u64State = ASMAtomicReadU64(&pThis->u64State);
376	if ((u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT))
377	break;
378	AssertMsg(iLoop < 1, ("%u\n", iLoop));
379	}
380
381	/* Decrement the wait count and maybe reset the semaphore (if we're last). */
382	for (;;)
383	{
384	u64OldState = u64State;
385
386	cWait = (u64State & RTSEMRW_WAIT_CNT_RD_MASK) >> RTSEMRW_WAIT_CNT_RD_SHIFT;
387	Assert(cWait > 0);
388	cWait--;
389	u64State &= ~RTSEMRW_WAIT_CNT_RD_MASK;
390	u64State \|= cWait << RTSEMRW_WAIT_CNT_RD_SHIFT;
391
392	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
393	{
394	if (cWait == 0)
395	{
396	if (ASMAtomicXchgBool(&pThis->fNeedReset, false))
397	{
398	int rc = RTSemEventMultiReset(pThis->hEvtRead);
399	AssertRCReturn(rc, rc);
400	}
401	}
402	break;
403	}
404	u64State = ASMAtomicReadU64(&pThis->u64State);
405	}
406
407	#ifdef RTSEMRW_STRICT
408	RTLockValidatorRecSharedAddOwner(&pThis->ValidatorRead, hThreadSelf, pSrcPos);
409	#endif
410	break;
411	}
412	}
413
414	if (pThis->u32Magic != RTSEMRW_MAGIC)
415	return VERR_SEM_DESTROYED;
416
417	ASMNopPause();
418	u64State = ASMAtomicReadU64(&pThis->u64State);
419	u64OldState = u64State;
420	}
421
422	/* got it! */
423	Assert((ASMAtomicReadU64(&pThis->u64State) & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT));
424	return VINF_SUCCESS;
425
426	}
427
428
429	RTDECL(int) RTSemRWRequestRead(RTSEMRW hRWSem, RTMSINTERVAL cMillies)
430	{
431	#ifndef RTSEMRW_STRICT
432	return rtSemRWRequestRead(hRWSem, cMillies, false, NULL);
433	#else
434	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
435	return rtSemRWRequestRead(hRWSem, cMillies, false, &SrcPos);
436	#endif
437	}
438	RT_EXPORT_SYMBOL(RTSemRWRequestRead);
439
440
441	RTDECL(int) RTSemRWRequestReadDebug(RTSEMRW hRWSem, RTMSINTERVAL cMillies, RTHCUINTPTR uId, RT_SRC_POS_DECL)
442	{
443	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
444	return rtSemRWRequestRead(hRWSem, cMillies, false, &SrcPos);
445	}
446	RT_EXPORT_SYMBOL(RTSemRWRequestReadDebug);
447
448
449	RTDECL(int) RTSemRWRequestReadNoResume(RTSEMRW hRWSem, RTMSINTERVAL cMillies)
450	{
451	#ifndef RTSEMRW_STRICT
452	return rtSemRWRequestRead(hRWSem, cMillies, true, NULL);
453	#else
454	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
455	return rtSemRWRequestRead(hRWSem, cMillies, true, &SrcPos);
456	#endif
457	}
458	RT_EXPORT_SYMBOL(RTSemRWRequestReadNoResume);
459
460
461	RTDECL(int) RTSemRWRequestReadNoResumeDebug(RTSEMRW hRWSem, RTMSINTERVAL cMillies, RTHCUINTPTR uId, RT_SRC_POS_DECL)
462	{
463	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
464	return rtSemRWRequestRead(hRWSem, cMillies, true, &SrcPos);
465	}
466	RT_EXPORT_SYMBOL(RTSemRWRequestReadNoResumeDebug);
467
468
469
470	RTDECL(int) RTSemRWReleaseRead(RTSEMRW hRWSem)
471	{
472	/*
473	* Validate handle.
474	*/
475	RTSEMRWINTERNAL *pThis = hRWSem;
476	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
477	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, VERR_INVALID_HANDLE);
478
479	/*
480	* Check the direction and take action accordingly.
481	*/
482	uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
483	uint64_t u64OldState = u64State;
484	if ((u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT))
485	{
486	#ifdef RTSEMRW_STRICT
487	int rc9 = RTLockValidatorRecSharedCheckAndRelease(&pThis->ValidatorRead, NIL_RTTHREAD);
488	if (RT_FAILURE(rc9))
489	return rc9;
490	#endif
491	for (;;)
492	{
493	uint64_t c = (u64State & RTSEMRW_CNT_RD_MASK) >> RTSEMRW_CNT_RD_SHIFT;
494	AssertReturn(c > 0, VERR_NOT_OWNER);
495	c--;
496
497	if ( c > 0
498	\|\| (u64State & RTSEMRW_CNT_WD_MASK) == 0)
499	{
500	/* Don't change the direction. */
501	u64State &= ~RTSEMRW_CNT_RD_MASK;
502	u64State \|= c << RTSEMRW_CNT_RD_SHIFT;
503	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
504	break;
505	}
506	else
507	{
508	/* Reverse the direction and signal the reader threads. */
509	u64State &= ~(RTSEMRW_CNT_RD_MASK \| RTSEMRW_DIR_MASK);
510	u64State \|= RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT;
511	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
512	{
513	int rc = RTSemEventSignal(pThis->hEvtWrite);
514	AssertRC(rc);
515	break;
516	}
517	}
518
519	ASMNopPause();
520	u64State = ASMAtomicReadU64(&pThis->u64State);
521	u64OldState = u64State;
522	}
523	}
524	else
525	{
526	RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
527	RTNATIVETHREAD hNativeWriter;
528	ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
529	AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER);
530	AssertReturn(pThis->cWriterReads > 0, VERR_NOT_OWNER);
531	#ifdef RTSEMRW_STRICT
532	int rc = RTLockValidatorRecExclUnwindMixed(&pThis->ValidatorWrite, &pThis->ValidatorRead.Core);
533	if (RT_FAILURE(rc))
534	return rc;
535	#endif
536	ASMAtomicDecU32(&pThis->cWriterReads);
537	}
538
539	return VINF_SUCCESS;
540	}
541	RT_EXPORT_SYMBOL(RTSemRWReleaseRead);
542
543
544	DECL_FORCE_INLINE(int) rtSemRWRequestWrite(RTSEMRW hRWSem, RTMSINTERVAL cMillies, bool fInterruptible, PCRTLOCKVALSRCPOS pSrcPos)
545	{
546	/*
547	* Validate input.
548	*/
549	RTSEMRWINTERNAL *pThis = hRWSem;
550	if (pThis == NIL_RTSEMRW)
551	return VINF_SUCCESS;
552	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
553	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, VERR_INVALID_HANDLE);
554
555	#ifdef RTSEMRW_STRICT
556	RTTHREAD hThreadSelf = NIL_RTTHREAD;
557	if (cMillies)
558	{
559	hThreadSelf = RTThreadSelfAutoAdopt();
560	int rc9 = RTLockValidatorRecExclCheckOrder(&pThis->ValidatorWrite, hThreadSelf, pSrcPos, cMillies);
561	if (RT_FAILURE(rc9))
562	return rc9;
563	}
564	#endif
565
566	/*
567	* Check if we're already the owner and just recursing.
568	*/
569	RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
570	RTNATIVETHREAD hNativeWriter;
571	ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
572	if (hNativeSelf == hNativeWriter)
573	{
574	Assert((ASMAtomicReadU64(&pThis->u64State) & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT));
575	#ifdef RTSEMRW_STRICT
576	int rc9 = RTLockValidatorRecExclRecursion(&pThis->ValidatorWrite, pSrcPos);
577	if (RT_FAILURE(rc9))
578	return rc9;
579	#endif
580	Assert(pThis->cWriteRecursions < UINT32_MAX / 2);
581	ASMAtomicIncU32(&pThis->cWriteRecursions);
582	return VINF_SUCCESS;
583	}
584
585	/*
586	* Get cracking.
587	*/
588	uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
589	uint64_t u64OldState = u64State;
590
591	for (;;)
592	{
593	if ( (u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT)
594	\|\| (u64State & (RTSEMRW_CNT_RD_MASK \| RTSEMRW_CNT_WR_MASK)) != 0)
595	{
596	/* It flows in the right direction, try follow it before it changes. */
597	uint64_t c = (u64State & RTSEMRW_CNT_WR_MASK) >> RTSEMRW_CNT_WR_SHIFT;
598	c++;
599	Assert(c < RTSEMRW_CNT_MASK / 2);
600	u64State &= ~RTSEMRW_CNT_WR_MASK;
601	u64State \|= c << RTSEMRW_CNT_WR_SHIFT;
602	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
603	break;
604	}
605	else if ((u64State & (RTSEMRW_CNT_RD_MASK \| RTSEMRW_CNT_WR_MASK)) == 0)
606	{
607	/* Wrong direction, but we're alone here and can simply try switch the direction. */
608	u64State &= ~(RTSEMRW_CNT_RD_MASK \| RTSEMRW_CNT_WR_MASK \| RTSEMRW_DIR_MASK);
609	u64State \|= (UINT64_C(1) << RTSEMRW_CNT_WR_SHIFT) \| (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT);
610	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
611	break;
612	}
613	else if (!cMillies)
614	/* Wrong direction and we're not supposed to wait, just return. */
615	return VERR_TIMEOUT;
616	else
617	{
618	/* Add ourselves to the write count and break out to do the wait. */
619	uint64_t c = (u64State & RTSEMRW_CNT_WR_MASK) >> RTSEMRW_CNT_WR_SHIFT;
620	c++;
621	Assert(c < RTSEMRW_CNT_MASK / 2);
622	u64State &= ~RTSEMRW_CNT_WR_MASK;
623	u64State \|= c << RTSEMRW_CNT_WR_SHIFT;
624	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
625	break;
626	}
627
628	if (pThis->u32Magic != RTSEMRW_MAGIC)
629	return VERR_SEM_DESTROYED;
630
631	ASMNopPause();
632	u64State = ASMAtomicReadU64(&pThis->u64State);
633	u64OldState = u64State;
634	}
635
636	/*
637	* If we're in write mode now try grab the ownership. Play fair if there
638	* are threads already waiting.
639	*/
640	bool fDone = (u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT)
641	&& ( ((u64State & RTSEMRW_CNT_WR_MASK) >> RTSEMRW_CNT_WR_SHIFT) == 1
642	\|\| cMillies == 0);
643	if (fDone)
644	ASMAtomicCmpXchgHandle(&pThis->hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone);
645	if (!fDone)
646	{
647	/*
648	* Wait for our turn.
649	*/
650	for (uint32_t iLoop = 0; ; iLoop++)
651	{
652	int rc;
653	#ifdef RTSEMRW_STRICT
654	if (cMillies)
655	{
656	if (hThreadSelf == NIL_RTTHREAD)
657	hThreadSelf = RTThreadSelfAutoAdopt();
658	rc = RTLockValidatorRecExclCheckBlocking(&pThis->ValidatorWrite, hThreadSelf, pSrcPos, true,
659	cMillies, RTTHREADSTATE_RW_WRITE, false);
660	}
661	else
662	rc = VINF_SUCCESS;
663	if (RT_SUCCESS(rc))
664	#else
665	RTTHREAD hThreadSelf = RTThreadSelf();
666	RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_WRITE, false);
667	#endif
668	{
669	if (fInterruptible)
670	rc = RTSemEventWaitNoResume(pThis->hEvtWrite, cMillies);
671	else
672	rc = RTSemEventWait(pThis->hEvtWrite, cMillies);
673	RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
674	if (pThis->u32Magic != RTSEMRW_MAGIC)
675	return VERR_SEM_DESTROYED;
676	}
677	if (RT_FAILURE(rc))
678	{
679	/* Decrement the counts and return the error. */
680	for (;;)
681	{
682	u64OldState = u64State = ASMAtomicReadU64(&pThis->u64State);
683	uint64_t c = (u64State & RTSEMRW_CNT_WR_MASK) >> RTSEMRW_CNT_WR_SHIFT; Assert(c > 0);
684	c--;
685	u64State &= ~RTSEMRW_CNT_WR_MASK;
686	u64State \|= c << RTSEMRW_CNT_WR_SHIFT;
687	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
688	break;
689	}
690	return rc;
691	}
692
693	u64State = ASMAtomicReadU64(&pThis->u64State);
694	if ((u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT))
695	{
696	ASMAtomicCmpXchgHandle(&pThis->hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone);
697	if (fDone)
698	break;
699	}
700	AssertMsg(iLoop < 1000, ("%u\n", iLoop)); /* may loop a few times here... */
701	}
702	}
703
704	/*
705	* Got it!
706	*/
707	Assert((ASMAtomicReadU64(&pThis->u64State) & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT));
708	ASMAtomicWriteU32(&pThis->cWriteRecursions, 1);
709	Assert(pThis->cWriterReads == 0);
710	#ifdef RTSEMRW_STRICT
711	RTLockValidatorRecExclSetOwner(&pThis->ValidatorWrite, hThreadSelf, pSrcPos, true);
712	#endif
713
714	return VINF_SUCCESS;
715	}
716
717
718	RTDECL(int) RTSemRWRequestWrite(RTSEMRW hRWSem, RTMSINTERVAL cMillies)
719	{
720	#ifndef RTSEMRW_STRICT
721	return rtSemRWRequestWrite(hRWSem, cMillies, false, NULL);
722	#else
723	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
724	return rtSemRWRequestWrite(hRWSem, cMillies, false, &SrcPos);
725	#endif
726	}
727	RT_EXPORT_SYMBOL(RTSemRWRequestWrite);
728
729
730	RTDECL(int) RTSemRWRequestWriteDebug(RTSEMRW hRWSem, RTMSINTERVAL cMillies, RTHCUINTPTR uId, RT_SRC_POS_DECL)
731	{
732	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
733	return rtSemRWRequestWrite(hRWSem, cMillies, false, &SrcPos);
734	}
735	RT_EXPORT_SYMBOL(RTSemRWRequestWriteDebug);
736
737
738	RTDECL(int) RTSemRWRequestWriteNoResume(RTSEMRW hRWSem, RTMSINTERVAL cMillies)
739	{
740	#ifndef RTSEMRW_STRICT
741	return rtSemRWRequestWrite(hRWSem, cMillies, true, NULL);
742	#else
743	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
744	return rtSemRWRequestWrite(hRWSem, cMillies, true, &SrcPos);
745	#endif
746	}
747	RT_EXPORT_SYMBOL(RTSemRWRequestWriteNoResume);
748
749
750	RTDECL(int) RTSemRWRequestWriteNoResumeDebug(RTSEMRW hRWSem, RTMSINTERVAL cMillies, RTHCUINTPTR uId, RT_SRC_POS_DECL)
751	{
752	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
753	return rtSemRWRequestWrite(hRWSem, cMillies, true, &SrcPos);
754	}
755	RT_EXPORT_SYMBOL(RTSemRWRequestWriteNoResumeDebug);
756
757
758	RTDECL(int) RTSemRWReleaseWrite(RTSEMRW hRWSem)
759	{
760
761	/*
762	* Validate handle.
763	*/
764	struct RTSEMRWINTERNAL *pThis = hRWSem;
765	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
766	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, VERR_INVALID_HANDLE);
767
768	RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
769	RTNATIVETHREAD hNativeWriter;
770	ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
771	AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER);
772
773	/*
774	* Unwind a recursion.
775	*/
776	if (pThis->cWriteRecursions == 1)
777	{
778	AssertReturn(pThis->cWriterReads == 0, VERR_WRONG_ORDER); /* (must release all read recursions before the final write.) */
779	#ifdef RTSEMRW_STRICT
780	int rc9 = RTLockValidatorRecExclReleaseOwner(&pThis->ValidatorWrite, true);
781	if (RT_FAILURE(rc9))
782	return rc9;
783	#endif
784	/*
785	* Update the state.
786	*/
787	ASMAtomicWriteU32(&pThis->cWriteRecursions, 0);
788	ASMAtomicWriteHandle(&pThis->hNativeWriter, NIL_RTNATIVETHREAD);
789
790	for (;;)
791	{
792	uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
793	uint64_t u64OldState = u64State;
794
795	uint64_t c = (u64State & RTSEMRW_CNT_WR_MASK) >> RTSEMRW_CNT_WR_SHIFT;
796	Assert(c > 0);
797	c--;
798
799	if ( c > 0
800	\|\| (u64State & RTSEMRW_CNT_RD_MASK) == 0)
801	{
802	/* Don't change the direction, wait up the next writer if any. */
803	u64State &= ~RTSEMRW_CNT_WR_MASK;
804	u64State \|= c << RTSEMRW_CNT_WR_SHIFT;
805	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
806	{
807	if (c > 0)
808	{
809	int rc = RTSemEventSignal(pThis->hEvtWrite);
810	AssertRC(rc);
811	}
812	break;
813	}
814	}
815	else
816	{
817	/* Reverse the direction and signal the reader threads. */
818	u64State &= ~(RTSEMRW_CNT_WR_MASK \| RTSEMRW_DIR_MASK);
819	u64State \|= RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT;
820	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
821	{
822	Assert(!pThis->fNeedReset);
823	ASMAtomicWriteBool(&pThis->fNeedReset, true);
824	int rc = RTSemEventMultiSignal(pThis->hEvtRead);
825	AssertRC(rc);
826	break;
827	}
828	}
829
830	ASMNopPause();
831	if (pThis->u32Magic != RTSEMRW_MAGIC)
832	return VERR_SEM_DESTROYED;
833	}
834	}
835	else
836	{
837	Assert(pThis->cWriteRecursions != 0);
838	#ifdef RTSEMRW_STRICT
839	int rc9 = RTLockValidatorRecExclUnwind(&pThis->ValidatorWrite);
840	if (RT_FAILURE(rc9))
841	return rc9;
842	#endif
843	ASMAtomicDecU32(&pThis->cWriteRecursions);
844	}
845
846	return VINF_SUCCESS;
847	}
848	RT_EXPORT_SYMBOL(RTSemRWReleaseWrite);
849
850
851	RTDECL(bool) RTSemRWIsWriteOwner(RTSEMRW hRWSem)
852	{
853	/*
854	* Validate handle.
855	*/
856	struct RTSEMRWINTERNAL *pThis = hRWSem;
857	AssertPtrReturn(pThis, false);
858	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, false);
859
860	/*
861	* Check ownership.
862	*/
863	RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
864	RTNATIVETHREAD hNativeWriter;
865	ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
866	return hNativeWriter == hNativeSelf;
867	}
868	RT_EXPORT_SYMBOL(RTSemRWIsWriteOwner);
869
870
871	RTDECL(bool) RTSemRWIsReadOwner(RTSEMRW hRWSem, bool fWannaHear)
872	{
873	/*
874	* Validate handle.
875	*/
876	struct RTSEMRWINTERNAL *pThis = hRWSem;
877	AssertPtrReturn(pThis, false);
878	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, false);
879
880	/*
881	* Inspect the state.
882	*/
883	uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
884	if ((u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT))
885	{
886	/*
887	* It's in write mode, so we can only be a reader if we're also the
888	* current writer.
889	*/
890	RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
891	RTNATIVETHREAD hWriter;
892	ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hWriter);
893	return hWriter == hNativeSelf;
894	}
895
896	/*
897	* Read mode. If there are no current readers, then we cannot be a reader.
898	*/
899	if (!(u64State & RTSEMRW_CNT_RD_MASK))
900	return false;
901
902	#ifdef RTSEMRW_STRICT
903	/*
904	* Ask the lock validator.
905	*/
906	return RTLockValidatorRecSharedIsOwner(&pThis->ValidatorRead, NIL_RTTHREAD);
907	#else
908	/*
909	* Ok, we don't know, just tell the caller what he want to hear.
910	*/
911	return fWannaHear;
912	#endif
913	}
914	RT_EXPORT_SYMBOL(RTSemRWIsReadOwner);
915
916
917	RTDECL(uint32_t) RTSemRWGetWriteRecursion(RTSEMRW hRWSem)
918	{
919	/*
920	* Validate handle.
921	*/
922	struct RTSEMRWINTERNAL *pThis = hRWSem;
923	AssertPtrReturn(pThis, 0);
924	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, 0);
925
926	/*
927	* Return the requested data.
928	*/
929	return pThis->cWriteRecursions;
930	}
931	RT_EXPORT_SYMBOL(RTSemRWGetWriteRecursion);
932
933
934	RTDECL(uint32_t) RTSemRWGetWriterReadRecursion(RTSEMRW hRWSem)
935	{
936	/*
937	* Validate handle.
938	*/
939	struct RTSEMRWINTERNAL *pThis = hRWSem;
940	AssertPtrReturn(pThis, 0);
941	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, 0);
942
943	/*
944	* Return the requested data.
945	*/
946	return pThis->cWriterReads;
947	}
948	RT_EXPORT_SYMBOL(RTSemRWGetWriterReadRecursion);
949
950
951	RTDECL(uint32_t) RTSemRWGetReadCount(RTSEMRW hRWSem)
952	{
953	/*
954	* Validate input.
955	*/
956	struct RTSEMRWINTERNAL *pThis = hRWSem;
957	AssertPtrReturn(pThis, 0);
958	AssertMsgReturn(pThis->u32Magic == RTSEMRW_MAGIC,
959	("pThis=%p u32Magic=%#x\n", pThis, pThis->u32Magic),
960	0);
961
962	/*
963	* Return the requested data.
964	*/
965	uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
966	if ((u64State & RTSEMRW_DIR_MASK) != (RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT))
967	return 0;
968	return (u64State & RTSEMRW_CNT_RD_MASK) >> RTSEMRW_CNT_RD_SHIFT;
969	}
970	RT_EXPORT_SYMBOL(RTSemRWGetReadCount);
971

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/generic/semrw-lockless-generic.cpp@ 55747

Download in other formats: