semrw-lockless-generic.cpp@ 100762

Last change on this file since 100762 was 98103, checked in by vboxsync, 2 years ago
Copyright year updates by scm.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 34.4 KB

Line
1	/* $Id: semrw-lockless-generic.cpp 98103 2023-01-17 14:15:46Z vboxsync $ */
2	/** @file
3	* IPRT - Read-Write Semaphore, Generic, lockless variant.
4	*/
5
6	/*
7	* Copyright (C) 2009-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* The contents of this file may alternatively be used under the terms
26	* of the Common Development and Distribution License Version 1.0
27	* (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
28	* in the VirtualBox distribution, in which case the provisions of the
29	* CDDL are applicable instead of those of the GPL.
30	*
31	* You may elect to license modified versions of this file under the
32	* terms and conditions of either the GPL or the CDDL or both.
33	*
34	* SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
35	*/
36
37
38	/*********************************************************************************************************************************
39	* Header Files *
40	*********************************************************************************************************************************/
41	#define RTSEMRW_WITHOUT_REMAPPING
42	#define RTASSERT_QUIET
43	#include <iprt/semaphore.h>
44	#include "internal/iprt.h"
45
46	#include <iprt/asm.h>
47	#include <iprt/assert.h>
48	#include <iprt/err.h>
49	#include <iprt/lockvalidator.h>
50	#include <iprt/mem.h>
51	#include <iprt/thread.h>
52
53	#include "internal/magics.h"
54	#include "internal/strict.h"
55
56
57	/*********************************************************************************************************************************
58	* Structures and Typedefs *
59	*********************************************************************************************************************************/
60	typedef struct RTSEMRWINTERNAL
61	{
62	/** Magic value (RTSEMRW_MAGIC). */
63	uint32_t volatile u32Magic;
64	/** Indicates whether hEvtRead needs resetting. */
65	bool volatile fNeedReset;
66
67	/** The state variable.
68	* All accesses are atomic and it bits are defined like this:
69	* Bits 0..14 - cReads.
70	* Bit 15 - Unused.
71	* Bits 16..31 - cWrites. - doesn't make sense here
72	* Bit 31 - fDirection; 0=Read, 1=Write.
73	* Bits 32..46 - cWaitingReads
74	* Bit 47 - Unused.
75	* Bits 48..62 - cWaitingWrites
76	* Bit 63 - Unused.
77	*/
78	uint64_t volatile u64State;
79	/** The write owner. */
80	RTNATIVETHREAD volatile hNativeWriter;
81	/** The number of reads made by the current writer. */
82	uint32_t volatile cWriterReads;
83	/** The number of recursions made by the current writer. (The initial grabbing
84	* of the lock counts as the first one.) */
85	uint32_t volatile cWriteRecursions;
86
87	/** What the writer threads are blocking on. */
88	RTSEMEVENT hEvtWrite;
89	/** What the read threads are blocking on when waiting for the writer to
90	* finish. */
91	RTSEMEVENTMULTI hEvtRead;
92
93	#ifdef RTSEMRW_STRICT
94	/** The validator record for the writer. */
95	RTLOCKVALRECEXCL ValidatorWrite;
96	/** The validator record for the readers. */
97	RTLOCKVALRECSHRD ValidatorRead;
98	#endif
99	} RTSEMRWINTERNAL;
100
101
102	/*********************************************************************************************************************************
103	* Defined Constants And Macros *
104	*********************************************************************************************************************************/
105	#define RTSEMRW_CNT_BITS 15
106	#define RTSEMRW_CNT_MASK UINT64_C(0x00007fff)
107
108	#define RTSEMRW_CNT_RD_SHIFT 0
109	#define RTSEMRW_CNT_RD_MASK (RTSEMRW_CNT_MASK << RTSEMRW_CNT_RD_SHIFT)
110	#define RTSEMRW_CNT_WR_SHIFT 16
111	#define RTSEMRW_CNT_WR_MASK (RTSEMRW_CNT_MASK << RTSEMRW_CNT_WR_SHIFT)
112	#define RTSEMRW_DIR_SHIFT 31
113	#define RTSEMRW_DIR_MASK RT_BIT_64(RTSEMRW_DIR_SHIFT)
114	#define RTSEMRW_DIR_READ UINT64_C(0)
115	#define RTSEMRW_DIR_WRITE UINT64_C(1)
116
117	#define RTSEMRW_WAIT_CNT_RD_SHIFT 32
118	#define RTSEMRW_WAIT_CNT_RD_MASK (RTSEMRW_CNT_MASK << RTSEMRW_WAIT_CNT_RD_SHIFT)
119	//#define RTSEMRW_WAIT_CNT_WR_SHIFT 48
120	//#define RTSEMRW_WAIT_CNT_WR_MASK (RTSEMRW_CNT_MASK << RTSEMRW_WAIT_CNT_WR_SHIFT)
121
122
123	RTDECL(int) RTSemRWCreate(PRTSEMRW phRWSem)
124	{
125	return RTSemRWCreateEx(phRWSem, 0 /fFlags/, NIL_RTLOCKVALCLASS, RTLOCKVAL_SUB_CLASS_NONE, "RTSemRW");
126	}
127	RT_EXPORT_SYMBOL(RTSemRWCreate);
128
129
130	RTDECL(int) RTSemRWCreateEx(PRTSEMRW phRWSem, uint32_t fFlags,
131	RTLOCKVALCLASS hClass, uint32_t uSubClass, const char *pszNameFmt, ...)
132	{
133	AssertReturn(!(fFlags & ~RTSEMRW_FLAGS_NO_LOCK_VAL), VERR_INVALID_PARAMETER);
134
135	RTSEMRWINTERNAL pThis = (RTSEMRWINTERNAL )RTMemAlloc(sizeof(*pThis));
136	if (!pThis)
137	return VERR_NO_MEMORY;
138
139	int rc = RTSemEventMultiCreate(&pThis->hEvtRead);
140	if (RT_SUCCESS(rc))
141	{
142	rc = RTSemEventCreate(&pThis->hEvtWrite);
143	if (RT_SUCCESS(rc))
144	{
145	pThis->u32Magic = RTSEMRW_MAGIC;
146	pThis->u32Padding = 0;
147	pThis->u64State = 0;
148	pThis->hNativeWriter = NIL_RTNATIVETHREAD;
149	pThis->cWriterReads = 0;
150	pThis->cWriteRecursions = 0;
151	pThis->fNeedReset = false;
152	#ifdef RTSEMRW_STRICT
153	bool const fLVEnabled = !(fFlags & RTSEMRW_FLAGS_NO_LOCK_VAL);
154	if (!pszNameFmt)
155	{
156	static uint32_t volatile s_iSemRWAnon = 0;
157	uint32_t i = ASMAtomicIncU32(&s_iSemRWAnon) - 1;
158	RTLockValidatorRecExclInit(&pThis->ValidatorWrite, hClass, uSubClass, pThis,
159	fLVEnabled, "RTSemRW-%u", i);
160	RTLockValidatorRecSharedInit(&pThis->ValidatorRead, hClass, uSubClass, pThis,
161	false /fSignaller/, fLVEnabled, "RTSemRW-%u", i);
162	}
163	else
164	{
165	va_list va;
166	va_start(va, pszNameFmt);
167	RTLockValidatorRecExclInitV(&pThis->ValidatorWrite, hClass, uSubClass, pThis,
168	fLVEnabled, pszNameFmt, va);
169	va_end(va);
170	va_start(va, pszNameFmt);
171	RTLockValidatorRecSharedInitV(&pThis->ValidatorRead, hClass, uSubClass, pThis,
172	false /fSignaller/, fLVEnabled, pszNameFmt, va);
173	va_end(va);
174	}
175	RTLockValidatorRecMakeSiblings(&pThis->ValidatorWrite.Core, &pThis->ValidatorRead.Core);
176	#endif
177
178	*phRWSem = pThis;
179	return VINF_SUCCESS;
180	}
181	RTSemEventMultiDestroy(pThis->hEvtRead);
182	}
183	return rc;
184	}
185	RT_EXPORT_SYMBOL(RTSemRWCreateEx);
186
187
188	RTDECL(int) RTSemRWDestroy(RTSEMRW hRWSem)
189	{
190	/*
191	* Validate input.
192	*/
193	RTSEMRWINTERNAL *pThis = hRWSem;
194	if (pThis == NIL_RTSEMRW)
195	return VINF_SUCCESS;
196	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
197	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, VERR_INVALID_HANDLE);
198	Assert(!(ASMAtomicReadU64(&pThis->u64State) & (RTSEMRW_CNT_RD_MASK \| RTSEMRW_CNT_WR_MASK)));
199
200	/*
201	* Invalidate the object and free up the resources.
202	*/
203	AssertReturn(ASMAtomicCmpXchgU32(&pThis->u32Magic, ~RTSEMRW_MAGIC, RTSEMRW_MAGIC), VERR_INVALID_HANDLE);
204
205	RTSEMEVENTMULTI hEvtRead;
206	ASMAtomicXchgHandle(&pThis->hEvtRead, NIL_RTSEMEVENTMULTI, &hEvtRead);
207	int rc = RTSemEventMultiDestroy(hEvtRead);
208	AssertRC(rc);
209
210	RTSEMEVENT hEvtWrite;
211	ASMAtomicXchgHandle(&pThis->hEvtWrite, NIL_RTSEMEVENT, &hEvtWrite);
212	rc = RTSemEventDestroy(hEvtWrite);
213	AssertRC(rc);
214
215	#ifdef RTSEMRW_STRICT
216	RTLockValidatorRecSharedDelete(&pThis->ValidatorRead);
217	RTLockValidatorRecExclDelete(&pThis->ValidatorWrite);
218	#endif
219	RTMemFree(pThis);
220	return VINF_SUCCESS;
221	}
222	RT_EXPORT_SYMBOL(RTSemRWDestroy);
223
224
225	RTDECL(uint32_t) RTSemRWSetSubClass(RTSEMRW hRWSem, uint32_t uSubClass)
226	{
227	#ifdef RTSEMRW_STRICT
228	/*
229	* Validate handle.
230	*/
231	struct RTSEMRWINTERNAL *pThis = hRWSem;
232	AssertPtrReturn(pThis, RTLOCKVAL_SUB_CLASS_INVALID);
233	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, RTLOCKVAL_SUB_CLASS_INVALID);
234
235	RTLockValidatorRecSharedSetSubClass(&pThis->ValidatorRead, uSubClass);
236	return RTLockValidatorRecExclSetSubClass(&pThis->ValidatorWrite, uSubClass);
237	#else
238	return RTLOCKVAL_SUB_CLASS_INVALID;
239	#endif
240	}
241	RT_EXPORT_SYMBOL(RTSemRWSetSubClass);
242
243
244	static int rtSemRWRequestRead(RTSEMRW hRWSem, RTMSINTERVAL cMillies, bool fInterruptible, PCRTLOCKVALSRCPOS pSrcPos)
245	{
246	/*
247	* Validate input.
248	*/
249	RTSEMRWINTERNAL *pThis = hRWSem;
250	if (pThis == NIL_RTSEMRW)
251	return VINF_SUCCESS;
252	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
253	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, VERR_INVALID_HANDLE);
254
255	#ifdef RTSEMRW_STRICT
256	RTTHREAD hThreadSelf = RTThreadSelfAutoAdopt();
257	if (cMillies > 0)
258	{
259	int rc9;
260	RTNATIVETHREAD hNativeWriter;
261	ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
262	if (hNativeWriter != NIL_RTTHREAD && hNativeWriter == RTThreadNativeSelf())
263	rc9 = RTLockValidatorRecExclCheckOrder(&pThis->ValidatorWrite, hThreadSelf, pSrcPos, cMillies);
264	else
265	rc9 = RTLockValidatorRecSharedCheckOrder(&pThis->ValidatorRead, hThreadSelf, pSrcPos, cMillies);
266	if (RT_FAILURE(rc9))
267	return rc9;
268	}
269	#endif
270
271	/*
272	* Get cracking...
273	*/
274	uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
275	uint64_t u64OldState = u64State;
276
277	for (;;)
278	{
279	if ((u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT))
280	{
281	/* It flows in the right direction, try follow it before it changes. */
282	uint64_t c = (u64State & RTSEMRW_CNT_RD_MASK) >> RTSEMRW_CNT_RD_SHIFT;
283	c++;
284	Assert(c < RTSEMRW_CNT_MASK / 2);
285	u64State &= ~RTSEMRW_CNT_RD_MASK;
286	u64State \|= c << RTSEMRW_CNT_RD_SHIFT;
287	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
288	{
289	#ifdef RTSEMRW_STRICT
290	RTLockValidatorRecSharedAddOwner(&pThis->ValidatorRead, hThreadSelf, pSrcPos);
291	#endif
292	break;
293	}
294	}
295	else if ((u64State & (RTSEMRW_CNT_RD_MASK \| RTSEMRW_CNT_WR_MASK)) == 0)
296	{
297	/* Wrong direction, but we're alone here and can simply try switch the direction. */
298	u64State &= ~(RTSEMRW_CNT_RD_MASK \| RTSEMRW_CNT_WR_MASK \| RTSEMRW_DIR_MASK);
299	u64State \|= (UINT64_C(1) << RTSEMRW_CNT_RD_SHIFT) \| (RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT);
300	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
301	{
302	Assert(!pThis->fNeedReset);
303	#ifdef RTSEMRW_STRICT
304	RTLockValidatorRecSharedAddOwner(&pThis->ValidatorRead, hThreadSelf, pSrcPos);
305	#endif
306	break;
307	}
308	}
309	else
310	{
311	/* Is the writer perhaps doing a read recursion? */
312	RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
313	RTNATIVETHREAD hNativeWriter;
314	ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
315	if (hNativeSelf == hNativeWriter)
316	{
317	#ifdef RTSEMRW_STRICT
318	int rc9 = RTLockValidatorRecExclRecursionMixed(&pThis->ValidatorWrite, &pThis->ValidatorRead.Core, pSrcPos);
319	if (RT_FAILURE(rc9))
320	return rc9;
321	#endif
322	Assert(pThis->cWriterReads < UINT32_MAX / 2);
323	ASMAtomicIncU32(&pThis->cWriterReads);
324	return VINF_SUCCESS; /* don't break! */
325	}
326
327	/* If the timeout is 0, return already. */
328	if (!cMillies)
329	return VERR_TIMEOUT;
330
331	/* Add ourselves to the queue and wait for the direction to change. */
332	uint64_t c = (u64State & RTSEMRW_CNT_RD_MASK) >> RTSEMRW_CNT_RD_SHIFT;
333	c++;
334	Assert(c < RTSEMRW_CNT_MASK / 2);
335
336	uint64_t cWait = (u64State & RTSEMRW_WAIT_CNT_RD_MASK) >> RTSEMRW_WAIT_CNT_RD_SHIFT;
337	cWait++;
338	Assert(cWait <= c);
339	Assert(cWait < RTSEMRW_CNT_MASK / 2);
340
341	u64State &= ~(RTSEMRW_CNT_RD_MASK \| RTSEMRW_WAIT_CNT_RD_MASK);
342	u64State \|= (c << RTSEMRW_CNT_RD_SHIFT) \| (cWait << RTSEMRW_WAIT_CNT_RD_SHIFT);
343
344	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
345	{
346	for (uint32_t iLoop = 0; ; iLoop++)
347	{
348	int rc;
349	#ifdef RTSEMRW_STRICT
350	rc = RTLockValidatorRecSharedCheckBlocking(&pThis->ValidatorRead, hThreadSelf, pSrcPos, true,
351	cMillies, RTTHREADSTATE_RW_READ, false);
352	if (RT_SUCCESS(rc))
353	#else
354	RTTHREAD hThreadSelf = RTThreadSelf();
355	RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_READ, false);
356	#endif
357	{
358	if (fInterruptible)
359	rc = RTSemEventMultiWaitNoResume(pThis->hEvtRead, cMillies);
360	else
361	rc = RTSemEventMultiWait(pThis->hEvtRead, cMillies);
362	RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_READ);
363	if (pThis->u32Magic != RTSEMRW_MAGIC)
364	return VERR_SEM_DESTROYED;
365	}
366	if (RT_FAILURE(rc))
367	{
368	/* Decrement the counts and return the error. */
369	for (;;)
370	{
371	u64OldState = u64State = ASMAtomicReadU64(&pThis->u64State);
372	c = (u64State & RTSEMRW_CNT_RD_MASK) >> RTSEMRW_CNT_RD_SHIFT; Assert(c > 0);
373	c--;
374	cWait = (u64State & RTSEMRW_WAIT_CNT_RD_MASK) >> RTSEMRW_WAIT_CNT_RD_SHIFT; Assert(cWait > 0);
375	cWait--;
376	u64State &= ~(RTSEMRW_CNT_RD_MASK \| RTSEMRW_WAIT_CNT_RD_MASK);
377	u64State \|= (c << RTSEMRW_CNT_RD_SHIFT) \| (cWait << RTSEMRW_WAIT_CNT_RD_SHIFT);
378	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
379	break;
380	}
381	return rc;
382	}
383
384	Assert(pThis->fNeedReset);
385	u64State = ASMAtomicReadU64(&pThis->u64State);
386	if ((u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT))
387	break;
388	AssertMsg(iLoop < 1, ("%u\n", iLoop));
389	}
390
391	/* Decrement the wait count and maybe reset the semaphore (if we're last). */
392	for (;;)
393	{
394	u64OldState = u64State;
395
396	cWait = (u64State & RTSEMRW_WAIT_CNT_RD_MASK) >> RTSEMRW_WAIT_CNT_RD_SHIFT;
397	Assert(cWait > 0);
398	cWait--;
399	u64State &= ~RTSEMRW_WAIT_CNT_RD_MASK;
400	u64State \|= cWait << RTSEMRW_WAIT_CNT_RD_SHIFT;
401
402	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
403	{
404	if (cWait == 0)
405	{
406	if (ASMAtomicXchgBool(&pThis->fNeedReset, false))
407	{
408	int rc = RTSemEventMultiReset(pThis->hEvtRead);
409	AssertRCReturn(rc, rc);
410	}
411	}
412	break;
413	}
414	u64State = ASMAtomicReadU64(&pThis->u64State);
415	}
416
417	#ifdef RTSEMRW_STRICT
418	RTLockValidatorRecSharedAddOwner(&pThis->ValidatorRead, hThreadSelf, pSrcPos);
419	#endif
420	break;
421	}
422	}
423
424	if (pThis->u32Magic != RTSEMRW_MAGIC)
425	return VERR_SEM_DESTROYED;
426
427	ASMNopPause();
428	u64State = ASMAtomicReadU64(&pThis->u64State);
429	u64OldState = u64State;
430	}
431
432	/* got it! */
433	Assert((ASMAtomicReadU64(&pThis->u64State) & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT));
434	return VINF_SUCCESS;
435
436	}
437
438
439	RTDECL(int) RTSemRWRequestRead(RTSEMRW hRWSem, RTMSINTERVAL cMillies)
440	{
441	#ifndef RTSEMRW_STRICT
442	return rtSemRWRequestRead(hRWSem, cMillies, false, NULL);
443	#else
444	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
445	return rtSemRWRequestRead(hRWSem, cMillies, false, &SrcPos);
446	#endif
447	}
448	RT_EXPORT_SYMBOL(RTSemRWRequestRead);
449
450
451	RTDECL(int) RTSemRWRequestReadDebug(RTSEMRW hRWSem, RTMSINTERVAL cMillies, RTHCUINTPTR uId, RT_SRC_POS_DECL)
452	{
453	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
454	return rtSemRWRequestRead(hRWSem, cMillies, false, &SrcPos);
455	}
456	RT_EXPORT_SYMBOL(RTSemRWRequestReadDebug);
457
458
459	RTDECL(int) RTSemRWRequestReadNoResume(RTSEMRW hRWSem, RTMSINTERVAL cMillies)
460	{
461	#ifndef RTSEMRW_STRICT
462	return rtSemRWRequestRead(hRWSem, cMillies, true, NULL);
463	#else
464	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
465	return rtSemRWRequestRead(hRWSem, cMillies, true, &SrcPos);
466	#endif
467	}
468	RT_EXPORT_SYMBOL(RTSemRWRequestReadNoResume);
469
470
471	RTDECL(int) RTSemRWRequestReadNoResumeDebug(RTSEMRW hRWSem, RTMSINTERVAL cMillies, RTHCUINTPTR uId, RT_SRC_POS_DECL)
472	{
473	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
474	return rtSemRWRequestRead(hRWSem, cMillies, true, &SrcPos);
475	}
476	RT_EXPORT_SYMBOL(RTSemRWRequestReadNoResumeDebug);
477
478
479
480	RTDECL(int) RTSemRWReleaseRead(RTSEMRW hRWSem)
481	{
482	/*
483	* Validate handle.
484	*/
485	RTSEMRWINTERNAL *pThis = hRWSem;
486	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
487	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, VERR_INVALID_HANDLE);
488
489	/*
490	* Check the direction and take action accordingly.
491	*/
492	uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
493	uint64_t u64OldState = u64State;
494	if ((u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT))
495	{
496	#ifdef RTSEMRW_STRICT
497	int rc9 = RTLockValidatorRecSharedCheckAndRelease(&pThis->ValidatorRead, NIL_RTTHREAD);
498	if (RT_FAILURE(rc9))
499	return rc9;
500	#endif
501	for (;;)
502	{
503	uint64_t c = (u64State & RTSEMRW_CNT_RD_MASK) >> RTSEMRW_CNT_RD_SHIFT;
504	AssertReturn(c > 0, VERR_NOT_OWNER);
505	c--;
506
507	if ( c > 0
508	\|\| (u64State & RTSEMRW_CNT_WD_MASK) == 0)
509	{
510	/* Don't change the direction. */
511	u64State &= ~RTSEMRW_CNT_RD_MASK;
512	u64State \|= c << RTSEMRW_CNT_RD_SHIFT;
513	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
514	break;
515	}
516	else
517	{
518	/* Reverse the direction and signal the reader threads. */
519	u64State &= ~(RTSEMRW_CNT_RD_MASK \| RTSEMRW_DIR_MASK);
520	u64State \|= RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT;
521	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
522	{
523	int rc = RTSemEventSignal(pThis->hEvtWrite);
524	AssertRC(rc);
525	break;
526	}
527	}
528
529	ASMNopPause();
530	u64State = ASMAtomicReadU64(&pThis->u64State);
531	u64OldState = u64State;
532	}
533	}
534	else
535	{
536	RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
537	RTNATIVETHREAD hNativeWriter;
538	ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
539	AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER);
540	AssertReturn(pThis->cWriterReads > 0, VERR_NOT_OWNER);
541	#ifdef RTSEMRW_STRICT
542	int rc = RTLockValidatorRecExclUnwindMixed(&pThis->ValidatorWrite, &pThis->ValidatorRead.Core);
543	if (RT_FAILURE(rc))
544	return rc;
545	#endif
546	ASMAtomicDecU32(&pThis->cWriterReads);
547	}
548
549	return VINF_SUCCESS;
550	}
551	RT_EXPORT_SYMBOL(RTSemRWReleaseRead);
552
553
554	DECL_FORCE_INLINE(int) rtSemRWRequestWrite(RTSEMRW hRWSem, RTMSINTERVAL cMillies, bool fInterruptible, PCRTLOCKVALSRCPOS pSrcPos)
555	{
556	/*
557	* Validate input.
558	*/
559	RTSEMRWINTERNAL *pThis = hRWSem;
560	if (pThis == NIL_RTSEMRW)
561	return VINF_SUCCESS;
562	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
563	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, VERR_INVALID_HANDLE);
564
565	#ifdef RTSEMRW_STRICT
566	RTTHREAD hThreadSelf = NIL_RTTHREAD;
567	if (cMillies)
568	{
569	hThreadSelf = RTThreadSelfAutoAdopt();
570	int rc9 = RTLockValidatorRecExclCheckOrder(&pThis->ValidatorWrite, hThreadSelf, pSrcPos, cMillies);
571	if (RT_FAILURE(rc9))
572	return rc9;
573	}
574	#endif
575
576	/*
577	* Check if we're already the owner and just recursing.
578	*/
579	RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
580	RTNATIVETHREAD hNativeWriter;
581	ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
582	if (hNativeSelf == hNativeWriter)
583	{
584	Assert((ASMAtomicReadU64(&pThis->u64State) & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT));
585	#ifdef RTSEMRW_STRICT
586	int rc9 = RTLockValidatorRecExclRecursion(&pThis->ValidatorWrite, pSrcPos);
587	if (RT_FAILURE(rc9))
588	return rc9;
589	#endif
590	Assert(pThis->cWriteRecursions < UINT32_MAX / 2);
591	ASMAtomicIncU32(&pThis->cWriteRecursions);
592	return VINF_SUCCESS;
593	}
594
595	/*
596	* Get cracking.
597	*/
598	uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
599	uint64_t u64OldState = u64State;
600
601	for (;;)
602	{
603	if ( (u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT)
604	\|\| (u64State & (RTSEMRW_CNT_RD_MASK \| RTSEMRW_CNT_WR_MASK)) != 0)
605	{
606	/* It flows in the right direction, try follow it before it changes. */
607	uint64_t c = (u64State & RTSEMRW_CNT_WR_MASK) >> RTSEMRW_CNT_WR_SHIFT;
608	c++;
609	Assert(c < RTSEMRW_CNT_MASK / 2);
610	u64State &= ~RTSEMRW_CNT_WR_MASK;
611	u64State \|= c << RTSEMRW_CNT_WR_SHIFT;
612	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
613	break;
614	}
615	else if ((u64State & (RTSEMRW_CNT_RD_MASK \| RTSEMRW_CNT_WR_MASK)) == 0)
616	{
617	/* Wrong direction, but we're alone here and can simply try switch the direction. */
618	u64State &= ~(RTSEMRW_CNT_RD_MASK \| RTSEMRW_CNT_WR_MASK \| RTSEMRW_DIR_MASK);
619	u64State \|= (UINT64_C(1) << RTSEMRW_CNT_WR_SHIFT) \| (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT);
620	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
621	break;
622	}
623	else if (!cMillies)
624	/* Wrong direction and we're not supposed to wait, just return. */
625	return VERR_TIMEOUT;
626	else
627	{
628	/* Add ourselves to the write count and break out to do the wait. */
629	uint64_t c = (u64State & RTSEMRW_CNT_WR_MASK) >> RTSEMRW_CNT_WR_SHIFT;
630	c++;
631	Assert(c < RTSEMRW_CNT_MASK / 2);
632	u64State &= ~RTSEMRW_CNT_WR_MASK;
633	u64State \|= c << RTSEMRW_CNT_WR_SHIFT;
634	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
635	break;
636	}
637
638	if (pThis->u32Magic != RTSEMRW_MAGIC)
639	return VERR_SEM_DESTROYED;
640
641	ASMNopPause();
642	u64State = ASMAtomicReadU64(&pThis->u64State);
643	u64OldState = u64State;
644	}
645
646	/*
647	* If we're in write mode now try grab the ownership. Play fair if there
648	* are threads already waiting.
649	*/
650	bool fDone = (u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT)
651	&& ( ((u64State & RTSEMRW_CNT_WR_MASK) >> RTSEMRW_CNT_WR_SHIFT) == 1
652	\|\| cMillies == 0);
653	if (fDone)
654	ASMAtomicCmpXchgHandle(&pThis->hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone);
655	if (!fDone)
656	{
657	/*
658	* Wait for our turn.
659	*/
660	for (uint32_t iLoop = 0; ; iLoop++)
661	{
662	int rc;
663	#ifdef RTSEMRW_STRICT
664	if (cMillies)
665	{
666	if (hThreadSelf == NIL_RTTHREAD)
667	hThreadSelf = RTThreadSelfAutoAdopt();
668	rc = RTLockValidatorRecExclCheckBlocking(&pThis->ValidatorWrite, hThreadSelf, pSrcPos, true,
669	cMillies, RTTHREADSTATE_RW_WRITE, false);
670	}
671	else
672	rc = VINF_SUCCESS;
673	if (RT_SUCCESS(rc))
674	#else
675	RTTHREAD hThreadSelf = RTThreadSelf();
676	RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_WRITE, false);
677	#endif
678	{
679	if (fInterruptible)
680	rc = RTSemEventWaitNoResume(pThis->hEvtWrite, cMillies);
681	else
682	rc = RTSemEventWait(pThis->hEvtWrite, cMillies);
683	RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
684	if (pThis->u32Magic != RTSEMRW_MAGIC)
685	return VERR_SEM_DESTROYED;
686	}
687	if (RT_FAILURE(rc))
688	{
689	/* Decrement the counts and return the error. */
690	for (;;)
691	{
692	u64OldState = u64State = ASMAtomicReadU64(&pThis->u64State);
693	uint64_t c = (u64State & RTSEMRW_CNT_WR_MASK) >> RTSEMRW_CNT_WR_SHIFT; Assert(c > 0);
694	c--;
695	u64State &= ~RTSEMRW_CNT_WR_MASK;
696	u64State \|= c << RTSEMRW_CNT_WR_SHIFT;
697	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
698	break;
699	}
700	return rc;
701	}
702
703	u64State = ASMAtomicReadU64(&pThis->u64State);
704	if ((u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT))
705	{
706	ASMAtomicCmpXchgHandle(&pThis->hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone);
707	if (fDone)
708	break;
709	}
710	AssertMsg(iLoop < 1000, ("%u\n", iLoop)); /* may loop a few times here... */
711	}
712	}
713
714	/*
715	* Got it!
716	*/
717	Assert((ASMAtomicReadU64(&pThis->u64State) & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT));
718	ASMAtomicWriteU32(&pThis->cWriteRecursions, 1);
719	Assert(pThis->cWriterReads == 0);
720	#ifdef RTSEMRW_STRICT
721	RTLockValidatorRecExclSetOwner(&pThis->ValidatorWrite, hThreadSelf, pSrcPos, true);
722	#endif
723
724	return VINF_SUCCESS;
725	}
726
727
728	RTDECL(int) RTSemRWRequestWrite(RTSEMRW hRWSem, RTMSINTERVAL cMillies)
729	{
730	#ifndef RTSEMRW_STRICT
731	return rtSemRWRequestWrite(hRWSem, cMillies, false, NULL);
732	#else
733	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
734	return rtSemRWRequestWrite(hRWSem, cMillies, false, &SrcPos);
735	#endif
736	}
737	RT_EXPORT_SYMBOL(RTSemRWRequestWrite);
738
739
740	RTDECL(int) RTSemRWRequestWriteDebug(RTSEMRW hRWSem, RTMSINTERVAL cMillies, RTHCUINTPTR uId, RT_SRC_POS_DECL)
741	{
742	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
743	return rtSemRWRequestWrite(hRWSem, cMillies, false, &SrcPos);
744	}
745	RT_EXPORT_SYMBOL(RTSemRWRequestWriteDebug);
746
747
748	RTDECL(int) RTSemRWRequestWriteNoResume(RTSEMRW hRWSem, RTMSINTERVAL cMillies)
749	{
750	#ifndef RTSEMRW_STRICT
751	return rtSemRWRequestWrite(hRWSem, cMillies, true, NULL);
752	#else
753	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
754	return rtSemRWRequestWrite(hRWSem, cMillies, true, &SrcPos);
755	#endif
756	}
757	RT_EXPORT_SYMBOL(RTSemRWRequestWriteNoResume);
758
759
760	RTDECL(int) RTSemRWRequestWriteNoResumeDebug(RTSEMRW hRWSem, RTMSINTERVAL cMillies, RTHCUINTPTR uId, RT_SRC_POS_DECL)
761	{
762	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
763	return rtSemRWRequestWrite(hRWSem, cMillies, true, &SrcPos);
764	}
765	RT_EXPORT_SYMBOL(RTSemRWRequestWriteNoResumeDebug);
766
767
768	RTDECL(int) RTSemRWReleaseWrite(RTSEMRW hRWSem)
769	{
770
771	/*
772	* Validate handle.
773	*/
774	struct RTSEMRWINTERNAL *pThis = hRWSem;
775	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
776	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, VERR_INVALID_HANDLE);
777
778	RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
779	RTNATIVETHREAD hNativeWriter;
780	ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
781	AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER);
782
783	/*
784	* Unwind a recursion.
785	*/
786	if (pThis->cWriteRecursions == 1)
787	{
788	AssertReturn(pThis->cWriterReads == 0, VERR_WRONG_ORDER); /* (must release all read recursions before the final write.) */
789	#ifdef RTSEMRW_STRICT
790	int rc9 = RTLockValidatorRecExclReleaseOwner(&pThis->ValidatorWrite, true);
791	if (RT_FAILURE(rc9))
792	return rc9;
793	#endif
794	/*
795	* Update the state.
796	*/
797	ASMAtomicWriteU32(&pThis->cWriteRecursions, 0);
798	ASMAtomicWriteHandle(&pThis->hNativeWriter, NIL_RTNATIVETHREAD);
799
800	for (;;)
801	{
802	uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
803	uint64_t u64OldState = u64State;
804
805	uint64_t c = (u64State & RTSEMRW_CNT_WR_MASK) >> RTSEMRW_CNT_WR_SHIFT;
806	Assert(c > 0);
807	c--;
808
809	if ( c > 0
810	\|\| (u64State & RTSEMRW_CNT_RD_MASK) == 0)
811	{
812	/* Don't change the direction, wait up the next writer if any. */
813	u64State &= ~RTSEMRW_CNT_WR_MASK;
814	u64State \|= c << RTSEMRW_CNT_WR_SHIFT;
815	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
816	{
817	if (c > 0)
818	{
819	int rc = RTSemEventSignal(pThis->hEvtWrite);
820	AssertRC(rc);
821	}
822	break;
823	}
824	}
825	else
826	{
827	/* Reverse the direction and signal the reader threads. */
828	u64State &= ~(RTSEMRW_CNT_WR_MASK \| RTSEMRW_DIR_MASK);
829	u64State \|= RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT;
830	if (ASMAtomicCmpXchgU64(&pThis->u64State, u64State, u64OldState))
831	{
832	Assert(!pThis->fNeedReset);
833	ASMAtomicWriteBool(&pThis->fNeedReset, true);
834	int rc = RTSemEventMultiSignal(pThis->hEvtRead);
835	AssertRC(rc);
836	break;
837	}
838	}
839
840	ASMNopPause();
841	if (pThis->u32Magic != RTSEMRW_MAGIC)
842	return VERR_SEM_DESTROYED;
843	}
844	}
845	else
846	{
847	Assert(pThis->cWriteRecursions != 0);
848	#ifdef RTSEMRW_STRICT
849	int rc9 = RTLockValidatorRecExclUnwind(&pThis->ValidatorWrite);
850	if (RT_FAILURE(rc9))
851	return rc9;
852	#endif
853	ASMAtomicDecU32(&pThis->cWriteRecursions);
854	}
855
856	return VINF_SUCCESS;
857	}
858	RT_EXPORT_SYMBOL(RTSemRWReleaseWrite);
859
860
861	RTDECL(bool) RTSemRWIsWriteOwner(RTSEMRW hRWSem)
862	{
863	/*
864	* Validate handle.
865	*/
866	struct RTSEMRWINTERNAL *pThis = hRWSem;
867	AssertPtrReturn(pThis, false);
868	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, false);
869
870	/*
871	* Check ownership.
872	*/
873	RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
874	RTNATIVETHREAD hNativeWriter;
875	ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hNativeWriter);
876	return hNativeWriter == hNativeSelf;
877	}
878	RT_EXPORT_SYMBOL(RTSemRWIsWriteOwner);
879
880
881	RTDECL(bool) RTSemRWIsReadOwner(RTSEMRW hRWSem, bool fWannaHear)
882	{
883	/*
884	* Validate handle.
885	*/
886	struct RTSEMRWINTERNAL *pThis = hRWSem;
887	AssertPtrReturn(pThis, false);
888	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, false);
889
890	/*
891	* Inspect the state.
892	*/
893	uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
894	if ((u64State & RTSEMRW_DIR_MASK) == (RTSEMRW_DIR_WRITE << RTSEMRW_DIR_SHIFT))
895	{
896	/*
897	* It's in write mode, so we can only be a reader if we're also the
898	* current writer.
899	*/
900	RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
901	RTNATIVETHREAD hWriter;
902	ASMAtomicUoReadHandle(&pThis->hNativeWriter, &hWriter);
903	return hWriter == hNativeSelf;
904	}
905
906	/*
907	* Read mode. If there are no current readers, then we cannot be a reader.
908	*/
909	if (!(u64State & RTSEMRW_CNT_RD_MASK))
910	return false;
911
912	#ifdef RTSEMRW_STRICT
913	/*
914	* Ask the lock validator.
915	*/
916	return RTLockValidatorRecSharedIsOwner(&pThis->ValidatorRead, NIL_RTTHREAD);
917	#else
918	/*
919	* Ok, we don't know, just tell the caller what he want to hear.
920	*/
921	return fWannaHear;
922	#endif
923	}
924	RT_EXPORT_SYMBOL(RTSemRWIsReadOwner);
925
926
927	RTDECL(uint32_t) RTSemRWGetWriteRecursion(RTSEMRW hRWSem)
928	{
929	/*
930	* Validate handle.
931	*/
932	struct RTSEMRWINTERNAL *pThis = hRWSem;
933	AssertPtrReturn(pThis, 0);
934	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, 0);
935
936	/*
937	* Return the requested data.
938	*/
939	return pThis->cWriteRecursions;
940	}
941	RT_EXPORT_SYMBOL(RTSemRWGetWriteRecursion);
942
943
944	RTDECL(uint32_t) RTSemRWGetWriterReadRecursion(RTSEMRW hRWSem)
945	{
946	/*
947	* Validate handle.
948	*/
949	struct RTSEMRWINTERNAL *pThis = hRWSem;
950	AssertPtrReturn(pThis, 0);
951	AssertReturn(pThis->u32Magic == RTSEMRW_MAGIC, 0);
952
953	/*
954	* Return the requested data.
955	*/
956	return pThis->cWriterReads;
957	}
958	RT_EXPORT_SYMBOL(RTSemRWGetWriterReadRecursion);
959
960
961	RTDECL(uint32_t) RTSemRWGetReadCount(RTSEMRW hRWSem)
962	{
963	/*
964	* Validate input.
965	*/
966	struct RTSEMRWINTERNAL *pThis = hRWSem;
967	AssertPtrReturn(pThis, 0);
968	AssertMsgReturn(pThis->u32Magic == RTSEMRW_MAGIC,
969	("pThis=%p u32Magic=%#x\n", pThis, pThis->u32Magic),
970	0);
971
972	/*
973	* Return the requested data.
974	*/
975	uint64_t u64State = ASMAtomicReadU64(&pThis->u64State);
976	if ((u64State & RTSEMRW_DIR_MASK) != (RTSEMRW_DIR_READ << RTSEMRW_DIR_SHIFT))
977	return 0;
978	return (u64State & RTSEMRW_CNT_RD_MASK) >> RTSEMRW_CNT_RD_SHIFT;
979	}
980	RT_EXPORT_SYMBOL(RTSemRWGetReadCount);
981

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/generic/semrw-lockless-generic.cpp@ 100762

Download in other formats: