PDMAllCritSectRw.cpp@ 101505

Last change on this file since 101505 was 98103, checked in by vboxsync, 2 years ago
Copyright year updates by scm.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 84.1 KB

Line
1	/* $Id: PDMAllCritSectRw.cpp 98103 2023-01-17 14:15:46Z vboxsync $ */
2	/** @file
3	* IPRT - Read/Write Critical Section, Generic.
4	*/
5
6	/*
7	* Copyright (C) 2009-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_PDM_CRITSECTRW
33	#include "PDMInternal.h"
34	#include <VBox/vmm/pdmcritsectrw.h>
35	#include <VBox/vmm/mm.h>
36	#include <VBox/vmm/vmm.h>
37	#include <VBox/vmm/vmcc.h>
38	#include <VBox/err.h>
39	#include <VBox/vmm/hm.h>
40
41	#include <VBox/log.h>
42	#include <iprt/asm.h>
43	#include <iprt/assert.h>
44	#ifdef IN_RING3
45	# include <iprt/lockvalidator.h>
46	#endif
47	#if defined(IN_RING3) \|\| defined(IN_RING0)
48	# include <iprt/semaphore.h>
49	# include <iprt/thread.h>
50	#endif
51	#ifdef IN_RING0
52	# include <iprt/time.h>
53	#endif
54	#ifdef RT_ARCH_AMD64
55	# include <iprt/x86.h>
56	#endif
57
58
59	/*********************************************************************************************************************************
60	* Defined Constants And Macros *
61	*********************************************************************************************************************************/
62	#if 0 /* unused */
63	/** The number loops to spin for shared access in ring-3. */
64	#define PDMCRITSECTRW_SHRD_SPIN_COUNT_R3 20
65	/** The number loops to spin for shared access in ring-0. */
66	#define PDMCRITSECTRW_SHRD_SPIN_COUNT_R0 128
67	/** The number loops to spin for shared access in the raw-mode context. */
68	#define PDMCRITSECTRW_SHRD_SPIN_COUNT_RC 128
69
70	/** The number loops to spin for exclusive access in ring-3. */
71	#define PDMCRITSECTRW_EXCL_SPIN_COUNT_R3 20
72	/** The number loops to spin for exclusive access in ring-0. */
73	#define PDMCRITSECTRW_EXCL_SPIN_COUNT_R0 256
74	/** The number loops to spin for exclusive access in the raw-mode context. */
75	#define PDMCRITSECTRW_EXCL_SPIN_COUNT_RC 256
76	#endif
77
78	/** Max number of write or write/read recursions. */
79	#define PDM_CRITSECTRW_MAX_RECURSIONS _1M
80
81	/** Skips some of the overly paranoid atomic reads and updates.
82	* Makes some assumptions about cache coherence, though not brave enough not to
83	* always end with an atomic update. */
84	#define PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
85
86	/** For reading RTCRITSECTRWSTATE::s::u64State. */
87	#ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
88	# define PDMCRITSECTRW_READ_STATE(a_pu64State) ASMAtomicUoReadU64(a_pu64State)
89	#else
90	# define PDMCRITSECTRW_READ_STATE(a_pu64State) ASMAtomicReadU64(a_pu64State)
91	#endif
92
93
94	/* Undefine the automatic VBOX_STRICT API mappings. */
95	#undef PDMCritSectRwEnterExcl
96	#undef PDMCritSectRwTryEnterExcl
97	#undef PDMCritSectRwEnterShared
98	#undef PDMCritSectRwTryEnterShared
99
100
101	/*********************************************************************************************************************************
102	* Defined Constants And Macros *
103	*********************************************************************************************************************************/
104	#if defined(RTASM_HAVE_CMP_WRITE_U128) && defined(RT_ARCH_AMD64)
105	static int32_t g_fCmpWriteSupported = -1;
106	#endif
107
108
109	/*********************************************************************************************************************************
110	* Internal Functions *
111	*********************************************************************************************************************************/
112	static int pdmCritSectRwLeaveSharedWorker(PVMCC pVM, PPDMCRITSECTRW pThis, bool fNoVal);
113
114
115	#ifdef RTASM_HAVE_CMP_WRITE_U128
116
117	# ifdef RT_ARCH_AMD64
118	/**
119	* Called once to initialize g_fCmpWriteSupported.
120	*/
121	DECL_NO_INLINE(static, bool) pdmCritSectRwIsCmpWriteU128SupportedSlow(void)
122	{
123	bool const fCmpWriteSupported = RT_BOOL(ASMCpuId_ECX(1) & X86_CPUID_FEATURE_ECX_CX16);
124	ASMAtomicWriteS32(&g_fCmpWriteSupported, fCmpWriteSupported);
125	return fCmpWriteSupported;
126	}
127	# endif
128
129
130	/**
131	* Indicates whether hardware actually supports 128-bit compare & write.
132	*/
133	DECL_FORCE_INLINE(bool) pdmCritSectRwIsCmpWriteU128Supported(void)
134	{
135	# ifdef RT_ARCH_AMD64
136	int32_t const fCmpWriteSupported = g_fCmpWriteSupported;
137	if (RT_LIKELY(fCmpWriteSupported >= 0))
138	return fCmpWriteSupported != 0;
139	return pdmCritSectRwIsCmpWriteU128SupportedSlow();
140	# else
141	return true;
142	# endif
143	}
144
145	#endif /* RTASM_HAVE_CMP_WRITE_U128 */
146
147	/**
148	* Gets the ring-3 native thread handle of the calling thread.
149	*
150	* @returns native thread handle (ring-3).
151	* @param pVM The cross context VM structure.
152	* @param pThis The read/write critical section. This is only used in
153	* R0 and RC.
154	*/
155	DECL_FORCE_INLINE(RTNATIVETHREAD) pdmCritSectRwGetNativeSelf(PVMCC pVM, PCPDMCRITSECTRW pThis)
156	{
157	#ifdef IN_RING3
158	RT_NOREF(pVM, pThis);
159	RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf();
160
161	#elif defined(IN_RING0)
162	AssertMsgReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, ("%RX32\n", pThis->s.Core.u32Magic),
163	NIL_RTNATIVETHREAD);
164	RTNATIVETHREAD hNativeSelf = GVMMR0GetRing3ThreadForSelf(pVM);
165	Assert(hNativeSelf != NIL_RTNATIVETHREAD);
166
167	#else
168	# error "invalid context"
169	#endif
170	return hNativeSelf;
171	}
172
173
174	DECL_NO_INLINE(static, int) pdmCritSectRwCorrupted(PPDMCRITSECTRW pThis, const char *pszMsg)
175	{
176	ASMAtomicWriteU32(&pThis->s.Core.u32Magic, PDMCRITSECTRW_MAGIC_CORRUPT);
177	LogRel(("PDMCritSect: %s pCritSect=%p\n", pszMsg, pThis));
178	return VERR_PDM_CRITSECTRW_IPE;
179	}
180
181
182
183	#ifdef IN_RING3
184	/**
185	* Changes the lock validator sub-class of the read/write critical section.
186	*
187	* It is recommended to try make sure that nobody is using this critical section
188	* while changing the value.
189	*
190	* @returns The old sub-class. RTLOCKVAL_SUB_CLASS_INVALID is returns if the
191	* lock validator isn't compiled in or either of the parameters are
192	* invalid.
193	* @param pThis Pointer to the read/write critical section.
194	* @param uSubClass The new sub-class value.
195	*/
196	VMMDECL(uint32_t) PDMR3CritSectRwSetSubClass(PPDMCRITSECTRW pThis, uint32_t uSubClass)
197	{
198	AssertPtrReturn(pThis, RTLOCKVAL_SUB_CLASS_INVALID);
199	AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, RTLOCKVAL_SUB_CLASS_INVALID);
200	# if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
201	AssertReturn(!(pThis->s.Core.fFlags & RTCRITSECT_FLAGS_NOP), RTLOCKVAL_SUB_CLASS_INVALID);
202
203	RTLockValidatorRecSharedSetSubClass(pThis->s.Core.pValidatorRead, uSubClass);
204	return RTLockValidatorRecExclSetSubClass(pThis->s.Core.pValidatorWrite, uSubClass);
205	# else
206	NOREF(uSubClass);
207	return RTLOCKVAL_SUB_CLASS_INVALID;
208	# endif
209	}
210	#endif /* IN_RING3 */
211
212
213	/**
214	* Worker for pdmCritSectRwEnterShared returning with read-ownership of the CS.
215	*/
216	DECL_FORCE_INLINE(int) pdmCritSectRwEnterSharedGotIt(PPDMCRITSECTRW pThis, PCRTLOCKVALSRCPOS pSrcPos,
217	bool fNoVal, RTTHREAD hThreadSelf)
218	{
219	#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
220	if (!fNoVal)
221	RTLockValidatorRecSharedAddOwner(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos);
222	#else
223	RT_NOREF(pSrcPos, fNoVal, hThreadSelf);
224	#endif
225
226	/* got it! */
227	STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterShared));
228	Assert((PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT));
229	return VINF_SUCCESS;
230	}
231
232	/**
233	* Worker for pdmCritSectRwEnterShared and pdmCritSectRwEnterSharedBailOut
234	* that decrement the wait count and maybe resets the semaphore.
235	*/
236	DECLINLINE(int) pdmCritSectRwEnterSharedGotItAfterWaiting(PVMCC pVM, PPDMCRITSECTRW pThis, uint64_t u64State,
237	PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, RTTHREAD hThreadSelf)
238	{
239	for (;;)
240	{
241	uint64_t const u64OldState = u64State;
242	uint64_t cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT;
243	AssertReturn(cWait > 0, pdmCritSectRwCorrupted(pThis, "Invalid waiting read count"));
244	AssertReturn((u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT > 0,
245	pdmCritSectRwCorrupted(pThis, "Invalid read count"));
246	cWait--;
247	u64State &= ~RTCSRW_WAIT_CNT_RD_MASK;
248	u64State \|= cWait << RTCSRW_WAIT_CNT_RD_SHIFT;
249
250	if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
251	{
252	if (cWait == 0)
253	{
254	if (ASMAtomicXchgBool(&pThis->s.Core.fNeedReset, false))
255	{
256	int rc = SUPSemEventMultiReset(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead);
257	AssertRCReturn(rc, rc);
258	}
259	}
260	return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf);
261	}
262
263	ASMNopPause();
264	AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
265	ASMNopPause();
266
267	u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
268	}
269	/* not reached */
270	}
271
272
273	#if defined(IN_RING0) \|\| (defined(IN_RING3) && defined(PDMCRITSECTRW_STRICT))
274	/**
275	* Worker for pdmCritSectRwEnterSharedContended that decrements both read counts
276	* and returns @a rc.
277	*
278	* @note May return VINF_SUCCESS if we race the exclusive leave function and
279	* come out on the bottom.
280	*
281	* Ring-3 only calls in a case where it is _not_ acceptable to take the
282	* lock, so even if we get the lock we'll have to leave. In the ring-0
283	* contexts, we can safely return VINF_SUCCESS in case of a race.
284	*/
285	DECL_NO_INLINE(static, int) pdmCritSectRwEnterSharedBailOut(PVMCC pVM, PPDMCRITSECTRW pThis, int rc,
286	PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, RTTHREAD hThreadSelf)
287	{
288	#ifdef IN_RING0
289	uint64_t const tsStart = RTTimeNanoTS();
290	uint64_t cNsElapsed = 0;
291	#endif
292	for (;;)
293	{
294	uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
295	uint64_t u64OldState = u64State;
296
297	uint64_t cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT;
298	AssertReturn(cWait > 0, pdmCritSectRwCorrupted(pThis, "Invalid waiting read count on bailout"));
299	cWait--;
300
301	uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
302	AssertReturn(c > 0, pdmCritSectRwCorrupted(pThis, "Invalid read count on bailout"));
303
304	if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT))
305	{
306	c--;
307	u64State &= ~(RTCSRW_CNT_RD_MASK \| RTCSRW_WAIT_CNT_RD_MASK);
308	u64State \|= (c << RTCSRW_CNT_RD_SHIFT) \| (cWait << RTCSRW_WAIT_CNT_RD_SHIFT);
309	if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
310	return rc;
311	}
312	else
313	{
314	/*
315	* The direction changed, so we can actually get the lock now.
316	*
317	* This means that we _have_ to wait on the semaphore to be signalled
318	* so we can properly reset it. Otherwise the stuff gets out of wack,
319	* because signalling and resetting will race one another. An
320	* exception would be if we're not the last reader waiting and don't
321	* need to worry about the resetting.
322	*
323	* An option would be to do the resetting in PDMCritSectRwEnterExcl,
324	* but that would still leave a racing PDMCritSectRwEnterShared
325	* spinning hard for a little bit, which isn't great...
326	*/
327	if (cWait == 0)
328	{
329	# ifdef IN_RING0
330	/* Do timeout processing first to avoid redoing the above. */
331	uint32_t cMsWait;
332	if (cNsElapsed <= RT_NS_10SEC)
333	cMsWait = 32;
334	else
335	{
336	u64State &= ~RTCSRW_WAIT_CNT_RD_MASK;
337	u64State \|= cWait << RTCSRW_WAIT_CNT_RD_SHIFT;
338	if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
339	{
340	LogFunc(("%p: giving up\n", pThis));
341	return rc;
342	}
343	cMsWait = 2;
344	}
345
346	int rcWait = SUPSemEventMultiWait(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead, cMsWait);
347	Log11Func(("%p: rc=%Rrc %'RU64 ns (hNativeWriter=%p u64State=%#RX64)\n", pThis, rcWait,
348	RTTimeNanoTS() - tsStart, pThis->s.Core.u.s.hNativeWriter, pThis->s.Core.u.s.u64State));
349	# else
350	RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_READ, false);
351	int rcWait = SUPSemEventMultiWaitNoResume(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead, RT_MS_5SEC);
352	RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
353	# endif
354	if (rcWait == VINF_SUCCESS)
355	{
356	# ifdef IN_RING0
357	return pdmCritSectRwEnterSharedGotItAfterWaiting(pVM, pThis, u64State, pSrcPos, fNoVal, hThreadSelf);
358	# else
359	/* ring-3: Cannot return VINF_SUCCESS. */
360	Assert(RT_FAILURE_NP(rc));
361	int rc2 = pdmCritSectRwEnterSharedGotItAfterWaiting(pVM, pThis, u64State, pSrcPos, fNoVal, hThreadSelf);
362	if (RT_SUCCESS(rc2))
363	rc2 = pdmCritSectRwLeaveSharedWorker(pVM, pThis, fNoVal);
364	return rc;
365	# endif
366	}
367	AssertMsgReturn(rcWait == VERR_TIMEOUT \|\| rcWait == VERR_INTERRUPTED,
368	("%p: rcWait=%Rrc rc=%Rrc", pThis, rcWait, rc),
369	RT_FAILURE_NP(rcWait) ? rcWait : -rcWait);
370	}
371	else
372	{
373	u64State &= ~RTCSRW_WAIT_CNT_RD_MASK;
374	u64State \|= cWait << RTCSRW_WAIT_CNT_RD_SHIFT;
375	if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
376	return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf);
377	}
378
379	# ifdef IN_RING0
380	/* Calculate the elapsed time here to avoid redoing state work. */
381	cNsElapsed = RTTimeNanoTS() - tsStart;
382	# endif
383	}
384
385	ASMNopPause();
386	AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
387	ASMNopPause();
388	}
389	}
390	#endif /* IN_RING0 \|\| (IN_RING3 && PDMCRITSECTRW_STRICT) */
391
392
393	/**
394	* Worker for pdmCritSectRwEnterShared that handles waiting for a contended CS.
395	* Caller has already added us to the read and read-wait counters.
396	*/
397	static int pdmCritSectRwEnterSharedContended(PVMCC pVM, PVMCPUCC pVCpu, PPDMCRITSECTRW pThis,
398	int rcBusy, PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, RTTHREAD hThreadSelf)
399	{
400	PSUPDRVSESSION const pSession = pVM->pSession;
401	SUPSEMEVENTMULTI const hEventMulti = (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead;
402	# ifdef IN_RING0
403	uint64_t const tsStart = RTTimeNanoTS();
404	uint64_t const cNsMaxTotalDef = RT_NS_5MIN;
405	uint64_t cNsMaxTotal = cNsMaxTotalDef;
406	uint32_t cMsMaxOne = RT_MS_5SEC;
407	bool fNonInterruptible = false;
408	# endif
409
410	for (uint32_t iLoop = 0; ; iLoop++)
411	{
412	/*
413	* Wait for the direction to switch.
414	*/
415	int rc;
416	# ifdef IN_RING3
417	# if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
418	rc = RTLockValidatorRecSharedCheckBlocking(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos, true,
419	RT_INDEFINITE_WAIT, RTTHREADSTATE_RW_READ, false);
420	if (RT_FAILURE(rc))
421	return pdmCritSectRwEnterSharedBailOut(pVM, pThis, rc, pSrcPos, fNoVal, hThreadSelf);
422	# else
423	RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_READ, false);
424	# endif
425	# endif
426
427	for (;;)
428	{
429	/*
430	* We always wait with a timeout so we can re-check the structure sanity
431	* and not get stuck waiting on a corrupt or deleted section.
432	*/
433	# ifdef IN_RING3
434	rc = SUPSemEventMultiWaitNoResume(pSession, hEventMulti, RT_MS_5SEC);
435	# else
436	rc = !fNonInterruptible
437	? SUPSemEventMultiWaitNoResume(pSession, hEventMulti, cMsMaxOne)
438	: SUPSemEventMultiWait(pSession, hEventMulti, cMsMaxOne);
439	Log11Func(("%p: rc=%Rrc %'RU64 ns (cMsMaxOne=%RU64 hNativeWriter=%p u64State=%#RX64)\n", pThis, rc,
440	RTTimeNanoTS() - tsStart, cMsMaxOne, pThis->s.Core.u.s.hNativeWriter, pThis->s.Core.u.s.u64State));
441	# endif
442	if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC))
443	{ /* likely */ }
444	else
445	{
446	# ifdef IN_RING3
447	RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
448	# endif
449	return VERR_SEM_DESTROYED;
450	}
451	if (RT_LIKELY(rc == VINF_SUCCESS))
452	break;
453
454	/*
455	* Timeout and interrupted waits needs careful handling in ring-0
456	* because we're cooperating with ring-3 on this critical section
457	* and thus need to make absolutely sure we won't get stuck here.
458	*
459	* The r0 interrupted case means something is pending (termination,
460	* signal, APC, debugger, whatever), so we must try our best to
461	* return to the caller and to ring-3 so it can be dealt with.
462	*/
463	if (rc == VERR_TIMEOUT \|\| rc == VERR_INTERRUPTED)
464	{
465	# ifdef IN_RING0
466	uint64_t const cNsElapsed = RTTimeNanoTS() - tsStart;
467	int const rcTerm = RTThreadQueryTerminationStatus(NIL_RTTHREAD);
468	AssertMsg(rcTerm == VINF_SUCCESS \|\| rcTerm == VERR_NOT_SUPPORTED \|\| rcTerm == VINF_THREAD_IS_TERMINATING,
469	("rcTerm=%Rrc\n", rcTerm));
470	if (rcTerm == VERR_NOT_SUPPORTED && cNsMaxTotal == cNsMaxTotalDef)
471	cNsMaxTotal = RT_NS_1MIN;
472
473	if (rc == VERR_TIMEOUT)
474	{
475	/* Try return get out of here with a non-VINF_SUCCESS status if
476	the thread is terminating or if the timeout has been exceeded. */
477	STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwSharedVerrTimeout);
478	if ( rcTerm == VINF_THREAD_IS_TERMINATING
479	\|\| cNsElapsed > cNsMaxTotal)
480	return pdmCritSectRwEnterSharedBailOut(pVM, pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc,
481	pSrcPos, fNoVal, hThreadSelf);
482	}
483	else
484	{
485	/* For interrupt cases, we must return if we can. If rcBusy is VINF_SUCCESS,
486	we will try non-interruptible sleep for a while to help resolve the issue
487	w/o guru'ing. */
488	STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwSharedVerrInterrupted);
489	if ( rcTerm != VINF_THREAD_IS_TERMINATING
490	&& rcBusy == VINF_SUCCESS
491	&& pVCpu != NULL
492	&& cNsElapsed <= cNsMaxTotal)
493	{
494	if (!fNonInterruptible)
495	{
496	STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwSharedNonInterruptibleWaits);
497	fNonInterruptible = true;
498	cMsMaxOne = 32;
499	uint64_t cNsLeft = cNsMaxTotal - cNsElapsed;
500	if (cNsLeft > RT_NS_10SEC)
501	cNsMaxTotal = cNsElapsed + RT_NS_10SEC;
502	}
503	}
504	else
505	return pdmCritSectRwEnterSharedBailOut(pVM, pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc,
506	pSrcPos, fNoVal, hThreadSelf);
507	}
508	# else /* IN_RING3 */
509	RT_NOREF(pVM, pVCpu, rcBusy);
510	# endif /* IN_RING3 */
511	}
512	/*
513	* Any other return code is fatal.
514	*/
515	else
516	{
517	# ifdef IN_RING3
518	RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
519	# endif
520	AssertMsgFailed(("rc=%Rrc\n", rc));
521	return RT_FAILURE_NP(rc) ? rc : -rc;
522	}
523	}
524
525	# ifdef IN_RING3
526	RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_READ);
527	# endif
528
529	/*
530	* Check the direction.
531	*/
532	Assert(pThis->s.Core.fNeedReset);
533	uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
534	if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
535	{
536	/*
537	* Decrement the wait count and maybe reset the semaphore (if we're last).
538	*/
539	return pdmCritSectRwEnterSharedGotItAfterWaiting(pVM, pThis, u64State, pSrcPos, fNoVal, hThreadSelf);
540	}
541
542	AssertMsg(iLoop < 1,
543	("%p: %u u64State=%#RX64 hNativeWriter=%p\n", pThis, iLoop, u64State, pThis->s.Core.u.s.hNativeWriter));
544	RTThreadYield();
545	}
546
547	/* not reached */
548	}
549
550
551	/**
552	* Worker that enters a read/write critical section with shard access.
553	*
554	* @returns VBox status code.
555	* @param pVM The cross context VM structure.
556	* @param pThis Pointer to the read/write critical section.
557	* @param rcBusy The busy return code for ring-0 and ring-3.
558	* @param fTryOnly Only try enter it, don't wait.
559	* @param pSrcPos The source position. (Can be NULL.)
560	* @param fNoVal No validation records.
561	*/
562	static int pdmCritSectRwEnterShared(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, bool fTryOnly,
563	PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal)
564	{
565	/*
566	* Validate input.
567	*/
568	AssertPtr(pThis);
569	AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
570
571	#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
572	RTTHREAD hThreadSelf = RTThreadSelfAutoAdopt();
573	if (!fTryOnly)
574	{
575	int rc9;
576	RTNATIVETHREAD hNativeWriter;
577	ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter);
578	if (hNativeWriter != NIL_RTTHREAD && hNativeWriter == pdmCritSectRwGetNativeSelf(pVM, pThis))
579	rc9 = RTLockValidatorRecExclCheckOrder(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT);
580	else
581	rc9 = RTLockValidatorRecSharedCheckOrder(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT);
582	if (RT_FAILURE(rc9))
583	return rc9;
584	}
585	#else
586	RTTHREAD hThreadSelf = NIL_RTTHREAD;
587	#endif
588
589	/*
590	* Work the state.
591	*/
592	uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
593	uint64_t u64OldState = u64State;
594	for (;;)
595	{
596	if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
597	{
598	/* It flows in the right direction, try follow it before it changes. */
599	uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
600	c++;
601	Assert(c < RTCSRW_CNT_MASK / 4);
602	AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_READERS);
603	u64State &= ~RTCSRW_CNT_RD_MASK;
604	u64State \|= c << RTCSRW_CNT_RD_SHIFT;
605	if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
606	return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf);
607	}
608	else if ((u64State & (RTCSRW_CNT_RD_MASK \| RTCSRW_CNT_WR_MASK)) == 0)
609	{
610	/* Wrong direction, but we're alone here and can simply try switch the direction. */
611	u64State &= ~(RTCSRW_CNT_RD_MASK \| RTCSRW_CNT_WR_MASK \| RTCSRW_DIR_MASK);
612	u64State \|= (UINT64_C(1) << RTCSRW_CNT_RD_SHIFT) \| (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT);
613	if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
614	{
615	Assert(!pThis->s.Core.fNeedReset);
616	return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf);
617	}
618	}
619	else
620	{
621	/* Is the writer perhaps doing a read recursion? */
622	RTNATIVETHREAD hNativeWriter;
623	ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter);
624	if (hNativeWriter != NIL_RTNATIVETHREAD)
625	{
626	RTNATIVETHREAD hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis);
627	if (hNativeSelf == hNativeWriter)
628	{
629	#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
630	if (!fNoVal)
631	{
632	int rc9 = RTLockValidatorRecExclRecursionMixed(pThis->s.Core.pValidatorWrite, &pThis->s.Core.pValidatorRead->Core, pSrcPos);
633	if (RT_FAILURE(rc9))
634	return rc9;
635	}
636	#endif
637	uint32_t const cReads = ASMAtomicIncU32(&pThis->s.Core.cWriterReads);
638	Assert(cReads < _16K);
639	AssertReturnStmt(cReads < PDM_CRITSECTRW_MAX_RECURSIONS, ASMAtomicDecU32(&pThis->s.Core.cWriterReads),
640	VERR_PDM_CRITSECTRW_TOO_MANY_RECURSIONS);
641	STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterShared));
642	return VINF_SUCCESS; /* don't break! */
643	}
644	}
645
646	/*
647	* If we're only trying, return already.
648	*/
649	if (fTryOnly)
650	{
651	STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterShared));
652	return VERR_SEM_BUSY;
653	}
654
655	#if defined(IN_RING3) \|\| defined(IN_RING0)
656	/*
657	* Add ourselves to the queue and wait for the direction to change.
658	*/
659	uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
660	c++;
661	Assert(c < RTCSRW_CNT_MASK / 2);
662	AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_READERS);
663
664	uint64_t cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT;
665	cWait++;
666	Assert(cWait <= c);
667	Assert(cWait < RTCSRW_CNT_MASK / 2);
668	AssertReturn(cWait < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_READERS);
669
670	u64State &= ~(RTCSRW_CNT_RD_MASK \| RTCSRW_WAIT_CNT_RD_MASK);
671	u64State \|= (c << RTCSRW_CNT_RD_SHIFT) \| (cWait << RTCSRW_WAIT_CNT_RD_SHIFT);
672
673	if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
674	{
675	/*
676	* In ring-3 it's straight forward, just optimize the RTThreadSelf() call.
677	*/
678	# if defined(IN_RING3) && defined(PDMCRITSECTRW_STRICT)
679	return pdmCritSectRwEnterSharedContended(pVM, NULL, pThis, rcBusy, pSrcPos, fNoVal, hThreadSelf);
680	# elif defined(IN_RING3)
681	return pdmCritSectRwEnterSharedContended(pVM, NULL, pThis, rcBusy, pSrcPos, fNoVal, RTThreadSelf());
682	# else /* IN_RING0 */
683	/*
684	* In ring-0 context we have to take the special VT-x/AMD-V HM context into
685	* account when waiting on contended locks.
686	*/
687	PVMCPUCC pVCpu = VMMGetCpu(pVM);
688	if (pVCpu)
689	{
690	VMMR0EMTBLOCKCTX Ctx;
691	int rc = VMMR0EmtPrepareToBlock(pVCpu, rcBusy, __FUNCTION__, pThis, &Ctx);
692	if (rc == VINF_SUCCESS)
693	{
694	Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD));
695
696	rc = pdmCritSectRwEnterSharedContended(pVM, pVCpu, pThis, rcBusy, pSrcPos, fNoVal, hThreadSelf);
697
698	VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx);
699	}
700	else
701	{
702	//STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLockBusy);
703	rc = pdmCritSectRwEnterSharedBailOut(pVM, pThis, rc, pSrcPos, fNoVal, hThreadSelf);
704	}
705	return rc;
706	}
707
708	/* Non-EMT. */
709	Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD));
710	return pdmCritSectRwEnterSharedContended(pVM, NULL, pThis, rcBusy, pSrcPos, fNoVal, hThreadSelf);
711	# endif /* IN_RING0 */
712	}
713
714	#else /* !IN_RING3 && !IN_RING0 */
715	/*
716	* We cannot call SUPSemEventMultiWaitNoResume in this context. Go
717	* back to ring-3 and do it there or return rcBusy.
718	*/
719	# error "Unused code."
720	STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterShared));
721	if (rcBusy == VINF_SUCCESS)
722	{
723	PVMCPUCC pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu);
724	/** @todo Should actually do this in via VMMR0.cpp instead of going all the way
725	* back to ring-3. Goes for both kind of crit sects. */
726	return VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_SHARED, MMHyperCCToR3(pVM, pThis));
727	}
728	return rcBusy;
729	#endif /* !IN_RING3 && !IN_RING0 */
730	}
731
732	ASMNopPause();
733	if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC))
734	{ /* likely */ }
735	else
736	return VERR_SEM_DESTROYED;
737	ASMNopPause();
738
739	u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
740	u64OldState = u64State;
741	}
742	/* not reached */
743	}
744
745
746	/**
747	* Enter a critical section with shared (read) access.
748	*
749	* @returns VBox status code.
750	* @retval VINF_SUCCESS on success.
751	* @retval rcBusy if in ring-0 or raw-mode context and it is busy.
752	* @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
753	* @retval VERR_SEM_DESTROYED if the critical section is delete before or
754	* during the operation.
755	*
756	* @param pVM The cross context VM structure.
757	* @param pThis Pointer to the read/write critical section.
758	* @param rcBusy The status code to return when we're in RC or R0 and the
759	* section is busy. Pass VINF_SUCCESS to acquired the
760	* critical section thru a ring-3 call if necessary.
761	* @sa PDMCritSectRwEnterSharedDebug, PDMCritSectRwTryEnterShared,
762	* PDMCritSectRwTryEnterSharedDebug, PDMCritSectRwLeaveShared,
763	* RTCritSectRwEnterShared.
764	*/
765	VMMDECL(int) PDMCritSectRwEnterShared(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy)
766	{
767	#if !defined(PDMCRITSECTRW_STRICT) \|\| !defined(IN_RING3)
768	return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /fTryOnly/, NULL, false /fNoVal/);
769	#else
770	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
771	return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /fTryOnly/, &SrcPos, false /fNoVal/);
772	#endif
773	}
774
775
776	/**
777	* Enter a critical section with shared (read) access.
778	*
779	* @returns VBox status code.
780	* @retval VINF_SUCCESS on success.
781	* @retval rcBusy if in ring-0 or raw-mode context and it is busy.
782	* @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
783	* @retval VERR_SEM_DESTROYED if the critical section is delete before or
784	* during the operation.
785	*
786	* @param pVM The cross context VM structure.
787	* @param pThis Pointer to the read/write critical section.
788	* @param rcBusy The status code to return when we're in RC or R0 and the
789	* section is busy. Pass VINF_SUCCESS to acquired the
790	* critical section thru a ring-3 call if necessary.
791	* @param uId Where we're entering the section.
792	* @param SRC_POS The source position.
793	* @sa PDMCritSectRwEnterShared, PDMCritSectRwTryEnterShared,
794	* PDMCritSectRwTryEnterSharedDebug, PDMCritSectRwLeaveShared,
795	* RTCritSectRwEnterSharedDebug.
796	*/
797	VMMDECL(int) PDMCritSectRwEnterSharedDebug(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, RTHCUINTPTR uId, RT_SRC_POS_DECL)
798	{
799	NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction);
800	#if !defined(PDMCRITSECTRW_STRICT) \|\| !defined(IN_RING3)
801	return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /fTryOnly/, NULL, false /fNoVal/);
802	#else
803	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
804	return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /fTryOnly/, &SrcPos, false /fNoVal/);
805	#endif
806	}
807
808
809	/**
810	* Try enter a critical section with shared (read) access.
811	*
812	* @returns VBox status code.
813	* @retval VINF_SUCCESS on success.
814	* @retval VERR_SEM_BUSY if the critsect was owned.
815	* @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
816	* @retval VERR_SEM_DESTROYED if the critical section is delete before or
817	* during the operation.
818	*
819	* @param pVM The cross context VM structure.
820	* @param pThis Pointer to the read/write critical section.
821	* @sa PDMCritSectRwTryEnterSharedDebug, PDMCritSectRwEnterShared,
822	* PDMCritSectRwEnterSharedDebug, PDMCritSectRwLeaveShared,
823	* RTCritSectRwTryEnterShared.
824	*/
825	VMMDECL(int) PDMCritSectRwTryEnterShared(PVMCC pVM, PPDMCRITSECTRW pThis)
826	{
827	#if !defined(PDMCRITSECTRW_STRICT) \|\| !defined(IN_RING3)
828	return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /fTryOnly/, NULL, false /fNoVal/);
829	#else
830	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
831	return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /fTryOnly/, &SrcPos, false /fNoVal/);
832	#endif
833	}
834
835
836	/**
837	* Try enter a critical section with shared (read) access.
838	*
839	* @returns VBox status code.
840	* @retval VINF_SUCCESS on success.
841	* @retval VERR_SEM_BUSY if the critsect was owned.
842	* @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
843	* @retval VERR_SEM_DESTROYED if the critical section is delete before or
844	* during the operation.
845	*
846	* @param pVM The cross context VM structure.
847	* @param pThis Pointer to the read/write critical section.
848	* @param uId Where we're entering the section.
849	* @param SRC_POS The source position.
850	* @sa PDMCritSectRwTryEnterShared, PDMCritSectRwEnterShared,
851	* PDMCritSectRwEnterSharedDebug, PDMCritSectRwLeaveShared,
852	* RTCritSectRwTryEnterSharedDebug.
853	*/
854	VMMDECL(int) PDMCritSectRwTryEnterSharedDebug(PVMCC pVM, PPDMCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL)
855	{
856	NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction);
857	#if !defined(PDMCRITSECTRW_STRICT) \|\| !defined(IN_RING3)
858	return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /fTryOnly/, NULL, false /fNoVal/);
859	#else
860	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
861	return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /fTryOnly/, &SrcPos, false /fNoVal/);
862	#endif
863	}
864
865
866	#ifdef IN_RING3
867	/**
868	* Enters a PDM read/write critical section with shared (read) access.
869	*
870	* @returns VINF_SUCCESS if entered successfully.
871	* @retval VERR_SEM_DESTROYED if the critical section is delete before or
872	* during the operation.
873	*
874	* @param pVM The cross context VM structure.
875	* @param pThis Pointer to the read/write critical section.
876	* @param fCallRing3 Whether this is a VMMRZCallRing3()request.
877	*/
878	VMMR3DECL(int) PDMR3CritSectRwEnterSharedEx(PVM pVM, PPDMCRITSECTRW pThis, bool fCallRing3)
879	{
880	return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, false /fTryAgain/, NULL, fCallRing3);
881	}
882	#endif
883
884
885	/**
886	* Leave a critical section held with shared access.
887	*
888	* @returns VBox status code.
889	* @retval VERR_SEM_DESTROYED if the critical section is delete before or
890	* during the operation.
891	* @param pVM The cross context VM structure.
892	* @param pThis Pointer to the read/write critical section.
893	* @param fNoVal No validation records (i.e. queued release).
894	* @sa PDMCritSectRwEnterShared, PDMCritSectRwTryEnterShared,
895	* PDMCritSectRwEnterSharedDebug, PDMCritSectRwTryEnterSharedDebug,
896	* PDMCritSectRwLeaveExcl, RTCritSectRwLeaveShared.
897	*/
898	static int pdmCritSectRwLeaveSharedWorker(PVMCC pVM, PPDMCRITSECTRW pThis, bool fNoVal)
899	{
900	/*
901	* Validate handle.
902	*/
903	AssertPtr(pThis);
904	AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
905
906	#if !defined(PDMCRITSECTRW_STRICT) \|\| !defined(IN_RING3)
907	NOREF(fNoVal);
908	#endif
909
910	/*
911	* Check the direction and take action accordingly.
912	*/
913	#ifdef IN_RING0
914	PVMCPUCC pVCpu = NULL;
915	#endif
916	uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
917	uint64_t u64OldState = u64State;
918	if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
919	{
920	#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
921	if (fNoVal)
922	Assert(!RTLockValidatorRecSharedIsOwner(pThis->s.Core.pValidatorRead, NIL_RTTHREAD));
923	else
924	{
925	int rc9 = RTLockValidatorRecSharedCheckAndRelease(pThis->s.Core.pValidatorRead, NIL_RTTHREAD);
926	if (RT_FAILURE(rc9))
927	return rc9;
928	}
929	#endif
930	for (;;)
931	{
932	uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
933	AssertReturn(c > 0, VERR_NOT_OWNER);
934	c--;
935
936	if ( c > 0
937	\|\| (u64State & RTCSRW_CNT_WR_MASK) == 0)
938	{
939	/* Don't change the direction. */
940	u64State &= ~RTCSRW_CNT_RD_MASK;
941	u64State \|= c << RTCSRW_CNT_RD_SHIFT;
942	if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
943	break;
944	}
945	else
946	{
947	#if defined(IN_RING3) \|\| defined(IN_RING0)
948	# ifdef IN_RING0
949	Assert(RTSemEventIsSignalSafe() == RTSemEventMultiIsSignalSafe());
950	if (!pVCpu)
951	pVCpu = VMMGetCpu(pVM);
952	if ( pVCpu == NULL /* non-EMT access, if we implement it must be able to block */
953	\|\| VMMRZCallRing3IsEnabled(pVCpu)
954	\|\| RTSemEventIsSignalSafe()
955	\|\| ( VMMR0ThreadCtxHookIsEnabled(pVCpu) /* Doesn't matter if Signal() blocks if we have hooks, ... */
956	&& RTThreadPreemptIsEnabled(NIL_RTTHREAD) /* ... and preemption is still enabled, */
957	&& ASMIntAreEnabled()) /* ... and interrupts hasn't yet been disabled. Special pre-GC HM env. */
958	)
959	# endif
960	{
961	/* Reverse the direction and signal the writer threads. */
962	u64State &= ~(RTCSRW_CNT_RD_MASK \| RTCSRW_DIR_MASK);
963	u64State \|= RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT;
964	if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
965	{
966	int rc;
967	# ifdef IN_RING0
968	STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLeaveShared);
969	if (!RTSemEventIsSignalSafe() && pVCpu != NULL)
970	{
971	VMMR0EMTBLOCKCTX Ctx;
972	rc = VMMR0EmtPrepareToBlock(pVCpu, VINF_SUCCESS, __FUNCTION__, pThis, &Ctx);
973	VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_SUCCESS(rc), ("rc=%Rrc\n", rc), rc);
974
975	rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite);
976
977	VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx);
978	}
979	else
980	# endif
981	rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite);
982	AssertRC(rc);
983	return rc;
984	}
985	}
986	#endif /* IN_RING3 \|\| IN_RING0 */
987	#ifndef IN_RING3
988	# ifdef IN_RING0
989	else
990	# endif
991	{
992	/* Queue the exit request (ring-3). */
993	# ifndef IN_RING0
994	PVMCPUCC pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu);
995	# endif
996	uint32_t i = pVCpu->pdm.s.cQueuedCritSectRwShrdLeaves++;
997	LogFlow(("PDMCritSectRwLeaveShared: [%d]=%p => R3 c=%d (%#llx)\n", i, pThis, c, u64State));
998	VMM_ASSERT_RELEASE_MSG_RETURN(pVM, i < RT_ELEMENTS(pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves),
999	("i=%u\n", i), VERR_PDM_CRITSECTRW_IPE);
1000	pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i] = pThis->s.pSelfR3;
1001	VMM_ASSERT_RELEASE_MSG_RETURN(pVM,
1002	RT_VALID_PTR(pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i])
1003	&& ((uintptr_t)pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i] & HOST_PAGE_OFFSET_MASK)
1004	== ((uintptr_t)pThis & HOST_PAGE_OFFSET_MASK),
1005	("%p vs %p\n", pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i], pThis),
1006	pdmCritSectRwCorrupted(pThis, "Invalid self pointer"));
1007	VMCPU_FF_SET(pVCpu, VMCPU_FF_PDM_CRITSECT);
1008	VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
1009	STAM_REL_COUNTER_INC(&pVM->pdm.s.StatQueuedCritSectLeaves);
1010	STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLeaveShared);
1011	break;
1012	}
1013	#endif
1014	}
1015
1016	ASMNopPause();
1017	if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC))
1018	{ }
1019	else
1020	return VERR_SEM_DESTROYED;
1021	ASMNopPause();
1022
1023	u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
1024	u64OldState = u64State;
1025	}
1026	}
1027	else
1028	{
1029	/*
1030	* Write direction. Check that it's the owner calling and that it has reads to undo.
1031	*/
1032	RTNATIVETHREAD hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis);
1033	AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_VM_THREAD_NOT_EMT);
1034
1035	RTNATIVETHREAD hNativeWriter;
1036	ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter);
1037	AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER);
1038	AssertReturn(pThis->s.Core.cWriterReads > 0, VERR_NOT_OWNER);
1039	#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
1040	if (!fNoVal)
1041	{
1042	int rc = RTLockValidatorRecExclUnwindMixed(pThis->s.Core.pValidatorWrite, &pThis->s.Core.pValidatorRead->Core);
1043	if (RT_FAILURE(rc))
1044	return rc;
1045	}
1046	#endif
1047	uint32_t cDepth = ASMAtomicDecU32(&pThis->s.Core.cWriterReads);
1048	AssertReturn(cDepth < PDM_CRITSECTRW_MAX_RECURSIONS, pdmCritSectRwCorrupted(pThis, "too many writer-read recursions"));
1049	}
1050
1051	return VINF_SUCCESS;
1052	}
1053
1054
1055	/**
1056	* Leave a critical section held with shared access.
1057	*
1058	* @returns VBox status code.
1059	* @retval VERR_SEM_DESTROYED if the critical section is delete before or
1060	* during the operation.
1061	* @param pVM The cross context VM structure.
1062	* @param pThis Pointer to the read/write critical section.
1063	* @sa PDMCritSectRwEnterShared, PDMCritSectRwTryEnterShared,
1064	* PDMCritSectRwEnterSharedDebug, PDMCritSectRwTryEnterSharedDebug,
1065	* PDMCritSectRwLeaveExcl, RTCritSectRwLeaveShared.
1066	*/
1067	VMMDECL(int) PDMCritSectRwLeaveShared(PVMCC pVM, PPDMCRITSECTRW pThis)
1068	{
1069	return pdmCritSectRwLeaveSharedWorker(pVM, pThis, false /fNoVal/);
1070	}
1071
1072
1073	#if defined(IN_RING3) \|\| defined(IN_RING0)
1074	/**
1075	* PDMCritSectBothFF interface.
1076	*
1077	* @param pVM The cross context VM structure.
1078	* @param pThis Pointer to the read/write critical section.
1079	*/
1080	void pdmCritSectRwLeaveSharedQueued(PVMCC pVM, PPDMCRITSECTRW pThis)
1081	{
1082	pdmCritSectRwLeaveSharedWorker(pVM, pThis, true /fNoVal/);
1083	}
1084	#endif
1085
1086
1087	/**
1088	* Worker for pdmCritSectRwEnterExcl that bails out on wait failure.
1089	*
1090	* @returns @a rc unless corrupted.
1091	* @param pThis Pointer to the read/write critical section.
1092	* @param rc The status to return.
1093	*/
1094	DECL_NO_INLINE(static, int) pdmCritSectRwEnterExclBailOut(PPDMCRITSECTRW pThis, int rc)
1095	{
1096	/*
1097	* Decrement the counts and return the error.
1098	*/
1099	for (;;)
1100	{
1101	uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
1102	uint64_t const u64OldState = u64State;
1103	uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT;
1104	AssertReturn(c > 0, pdmCritSectRwCorrupted(pThis, "Invalid write count on bailout"));
1105	c--;
1106	u64State &= ~RTCSRW_CNT_WR_MASK;
1107	u64State \|= c << RTCSRW_CNT_WR_SHIFT;
1108	if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
1109	return rc;
1110
1111	ASMNopPause();
1112	AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
1113	ASMNopPause();
1114	}
1115	}
1116
1117
1118	/**
1119	* Worker for pdmCritSectRwEnterExcl that handles the red tape after we've
1120	* gotten exclusive ownership of the critical section.
1121	*/
1122	DECL_FORCE_INLINE(int) pdmCritSectRwEnterExclFirst(PPDMCRITSECTRW pThis, PCRTLOCKVALSRCPOS pSrcPos,
1123	bool fNoVal, RTTHREAD hThreadSelf)
1124	{
1125	RT_NOREF(hThreadSelf, fNoVal, pSrcPos);
1126	Assert((PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT));
1127
1128	#ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
1129	pThis->s.Core.cWriteRecursions = 1;
1130	#else
1131	ASMAtomicWriteU32(&pThis->s.Core.cWriteRecursions, 1);
1132	#endif
1133	Assert(pThis->s.Core.cWriterReads == 0);
1134
1135	#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
1136	if (!fNoVal)
1137	{
1138	if (hThreadSelf == NIL_RTTHREAD)
1139	hThreadSelf = RTThreadSelfAutoAdopt();
1140	RTLockValidatorRecExclSetOwner(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, true);
1141	}
1142	#endif
1143	STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterExcl));
1144	STAM_PROFILE_ADV_START(&pThis->s.StatWriteLocked, swl);
1145	return VINF_SUCCESS;
1146	}
1147
1148
1149	#if defined(IN_RING3) \|\| defined(IN_RING0)
1150	/**
1151	* Worker for pdmCritSectRwEnterExcl that handles waiting when the section is
1152	* contended.
1153	*/
1154	static int pdmR3R0CritSectRwEnterExclContended(PVMCC pVM, PVMCPUCC pVCpu, PPDMCRITSECTRW pThis, RTNATIVETHREAD hNativeSelf,
1155	PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, int rcBusy, RTTHREAD hThreadSelf)
1156	{
1157	RT_NOREF(hThreadSelf, rcBusy, pSrcPos, fNoVal, pVCpu);
1158
1159	PSUPDRVSESSION const pSession = pVM->pSession;
1160	SUPSEMEVENT const hEvent = (SUPSEMEVENT)pThis->s.Core.hEvtWrite;
1161	# ifdef IN_RING0
1162	uint64_t const tsStart = RTTimeNanoTS();
1163	uint64_t const cNsMaxTotalDef = RT_NS_5MIN;
1164	uint64_t cNsMaxTotal = cNsMaxTotalDef;
1165	uint32_t cMsMaxOne = RT_MS_5SEC;
1166	bool fNonInterruptible = false;
1167	# endif
1168
1169	for (uint32_t iLoop = 0; ; iLoop++)
1170	{
1171	/*
1172	* Wait for our turn.
1173	*/
1174	int rc;
1175	# ifdef IN_RING3
1176	# ifdef PDMCRITSECTRW_STRICT
1177	rc = RTLockValidatorRecExclCheckBlocking(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, true,
1178	RT_INDEFINITE_WAIT, RTTHREADSTATE_RW_WRITE, false);
1179	if (RT_SUCCESS(rc))
1180	{ /* likely */ }
1181	else
1182	return pdmCritSectRwEnterExclBailOut(pThis, rc);
1183	# else
1184	RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_WRITE, false);
1185	# endif
1186	# endif
1187
1188	for (;;)
1189	{
1190	/*
1191	* We always wait with a timeout so we can re-check the structure sanity
1192	* and not get stuck waiting on a corrupt or deleted section.
1193	*/
1194	# ifdef IN_RING3
1195	rc = SUPSemEventWaitNoResume(pSession, hEvent, RT_MS_5SEC);
1196	# else
1197	rc = !fNonInterruptible
1198	? SUPSemEventWaitNoResume(pSession, hEvent, cMsMaxOne)
1199	: SUPSemEventWait(pSession, hEvent, cMsMaxOne);
1200	Log11Func(("%p: rc=%Rrc %'RU64 ns (cMsMaxOne=%RU64 hNativeWriter=%p)\n",
1201	pThis, rc, RTTimeNanoTS() - tsStart, cMsMaxOne, pThis->s.Core.u.s.hNativeWriter));
1202	# endif
1203	if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC))
1204	{ /* likely */ }
1205	else
1206	{
1207	# ifdef IN_RING3
1208	RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
1209	# endif
1210	return VERR_SEM_DESTROYED;
1211	}
1212	if (RT_LIKELY(rc == VINF_SUCCESS))
1213	break;
1214
1215	/*
1216	* Timeout and interrupted waits needs careful handling in ring-0
1217	* because we're cooperating with ring-3 on this critical section
1218	* and thus need to make absolutely sure we won't get stuck here.
1219	*
1220	* The r0 interrupted case means something is pending (termination,
1221	* signal, APC, debugger, whatever), so we must try our best to
1222	* return to the caller and to ring-3 so it can be dealt with.
1223	*/
1224	if (rc == VERR_TIMEOUT \|\| rc == VERR_INTERRUPTED)
1225	{
1226	# ifdef IN_RING0
1227	uint64_t const cNsElapsed = RTTimeNanoTS() - tsStart;
1228	int const rcTerm = RTThreadQueryTerminationStatus(NIL_RTTHREAD);
1229	AssertMsg(rcTerm == VINF_SUCCESS \|\| rcTerm == VERR_NOT_SUPPORTED \|\| rcTerm == VINF_THREAD_IS_TERMINATING,
1230	("rcTerm=%Rrc\n", rcTerm));
1231	if (rcTerm == VERR_NOT_SUPPORTED && cNsMaxTotal == cNsMaxTotalDef)
1232	cNsMaxTotal = RT_NS_1MIN;
1233
1234	if (rc == VERR_TIMEOUT)
1235	{
1236	/* Try return get out of here with a non-VINF_SUCCESS status if
1237	the thread is terminating or if the timeout has been exceeded. */
1238	STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwExclVerrTimeout);
1239	if ( rcTerm == VINF_THREAD_IS_TERMINATING
1240	\|\| cNsElapsed > cNsMaxTotal)
1241	return pdmCritSectRwEnterExclBailOut(pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc);
1242	}
1243	else
1244	{
1245	/* For interrupt cases, we must return if we can. If rcBusy is VINF_SUCCESS,
1246	we will try non-interruptible sleep for a while to help resolve the issue
1247	w/o guru'ing. */
1248	STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwExclVerrInterrupted);
1249	if ( rcTerm != VINF_THREAD_IS_TERMINATING
1250	&& rcBusy == VINF_SUCCESS
1251	&& pVCpu != NULL
1252	&& cNsElapsed <= cNsMaxTotal)
1253	{
1254	if (!fNonInterruptible)
1255	{
1256	STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwExclNonInterruptibleWaits);
1257	fNonInterruptible = true;
1258	cMsMaxOne = 32;
1259	uint64_t cNsLeft = cNsMaxTotal - cNsElapsed;
1260	if (cNsLeft > RT_NS_10SEC)
1261	cNsMaxTotal = cNsElapsed + RT_NS_10SEC;
1262	}
1263	}
1264	else
1265	return pdmCritSectRwEnterExclBailOut(pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc);
1266	}
1267	# else /* IN_RING3 */
1268	RT_NOREF(pVM, pVCpu, rcBusy);
1269	# endif /* IN_RING3 */
1270	}
1271	/*
1272	* Any other return code is fatal.
1273	*/
1274	else
1275	{
1276	# ifdef IN_RING3
1277	RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
1278	# endif
1279	AssertMsgFailed(("rc=%Rrc\n", rc));
1280	return RT_FAILURE_NP(rc) ? rc : -rc;
1281	}
1282	}
1283
1284	# ifdef IN_RING3
1285	RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
1286	# endif
1287
1288	/*
1289	* Try take exclusive write ownership.
1290	*/
1291	uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
1292	if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT))
1293	{
1294	bool fDone;
1295	ASMAtomicCmpXchgHandle(&pThis->s.Core.u.s.hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone);
1296	if (fDone)
1297	return pdmCritSectRwEnterExclFirst(pThis, pSrcPos, fNoVal, hThreadSelf);
1298	}
1299	AssertMsg(iLoop < 1000, ("%u\n", iLoop)); /* may loop a few times here... */
1300	}
1301	}
1302	#endif /* IN_RING3 \|\| IN_RING0 */
1303
1304
1305	/**
1306	* Worker that enters a read/write critical section with exclusive access.
1307	*
1308	* @returns VBox status code.
1309	* @param pVM The cross context VM structure.
1310	* @param pThis Pointer to the read/write critical section.
1311	* @param rcBusy The busy return code for ring-0 and ring-3.
1312	* @param fTryOnly Only try enter it, don't wait.
1313	* @param pSrcPos The source position. (Can be NULL.)
1314	* @param fNoVal No validation records.
1315	*/
1316	static int pdmCritSectRwEnterExcl(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, bool fTryOnly,
1317	PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal)
1318	{
1319	/*
1320	* Validate input.
1321	*/
1322	AssertPtr(pThis);
1323	AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
1324
1325	RTTHREAD hThreadSelf = NIL_RTTHREAD;
1326	#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
1327	if (!fTryOnly)
1328	{
1329	hThreadSelf = RTThreadSelfAutoAdopt();
1330	int rc9 = RTLockValidatorRecExclCheckOrder(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT);
1331	if (RT_FAILURE(rc9))
1332	return rc9;
1333	}
1334	#endif
1335
1336	/*
1337	* Check if we're already the owner and just recursing.
1338	*/
1339	RTNATIVETHREAD const hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis);
1340	AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_VM_THREAD_NOT_EMT);
1341	RTNATIVETHREAD hNativeWriter;
1342	ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter);
1343	if (hNativeSelf == hNativeWriter)
1344	{
1345	Assert((PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT));
1346	#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
1347	if (!fNoVal)
1348	{
1349	int rc9 = RTLockValidatorRecExclRecursion(pThis->s.Core.pValidatorWrite, pSrcPos);
1350	if (RT_FAILURE(rc9))
1351	return rc9;
1352	}
1353	#endif
1354	STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterExcl));
1355	#ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
1356	uint32_t const cDepth = ++pThis->s.Core.cWriteRecursions;
1357	#else
1358	uint32_t const cDepth = ASMAtomicIncU32(&pThis->s.Core.cWriteRecursions);
1359	#endif
1360	AssertReturnStmt(cDepth > 1 && cDepth <= PDM_CRITSECTRW_MAX_RECURSIONS,
1361	ASMAtomicDecU32(&pThis->s.Core.cWriteRecursions),
1362	VERR_PDM_CRITSECTRW_TOO_MANY_RECURSIONS);
1363	return VINF_SUCCESS;
1364	}
1365
1366	/*
1367	* First we try grab an idle critical section using 128-bit atomics.
1368	*/
1369	/** @todo This could be moved up before the recursion check. */
1370	uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
1371	#ifdef RTASM_HAVE_CMP_WRITE_U128
1372	if ( (u64State & ~RTCSRW_DIR_MASK) == 0
1373	&& pdmCritSectRwIsCmpWriteU128Supported())
1374	{
1375	RTCRITSECTRWSTATE OldState;
1376	OldState.s.u64State = u64State;
1377	OldState.s.hNativeWriter = NIL_RTNATIVETHREAD;
1378	AssertCompile(sizeof(OldState.s.hNativeWriter) == sizeof(OldState.u128.s.Lo));
1379
1380	RTCRITSECTRWSTATE NewState;
1381	NewState.s.u64State = (UINT64_C(1) << RTCSRW_CNT_WR_SHIFT) \| (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT);
1382	NewState.s.hNativeWriter = hNativeSelf;
1383
1384	if (ASMAtomicCmpWriteU128U(&pThis->s.Core.u.u128, NewState.u128, OldState.u128))
1385	return pdmCritSectRwEnterExclFirst(pThis, pSrcPos, fNoVal, hThreadSelf);
1386
1387	u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
1388	}
1389	#endif
1390
1391	/*
1392	* Do it step by step. Update the state to reflect our desire.
1393	*/
1394	uint64_t u64OldState = u64State;
1395
1396	for (;;)
1397	{
1398	if ( (u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)
1399	\|\| (u64State & (RTCSRW_CNT_RD_MASK \| RTCSRW_CNT_WR_MASK)) != 0)
1400	{
1401	/* It flows in the right direction, try follow it before it changes. */
1402	uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT;
1403	AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_WRITERS);
1404	c++;
1405	Assert(c < RTCSRW_CNT_WR_MASK / 4);
1406	u64State &= ~RTCSRW_CNT_WR_MASK;
1407	u64State \|= c << RTCSRW_CNT_WR_SHIFT;
1408	if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
1409	break;
1410	}
1411	else if ((u64State & (RTCSRW_CNT_RD_MASK \| RTCSRW_CNT_WR_MASK)) == 0)
1412	{
1413	/* Wrong direction, but we're alone here and can simply try switch the direction. */
1414	u64State &= ~(RTCSRW_CNT_RD_MASK \| RTCSRW_CNT_WR_MASK \| RTCSRW_DIR_MASK);
1415	u64State \|= (UINT64_C(1) << RTCSRW_CNT_WR_SHIFT) \| (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT);
1416	if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
1417	break;
1418	}
1419	else if (fTryOnly)
1420	{
1421	/* Wrong direction and we're not supposed to wait, just return. */
1422	STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterExcl));
1423	return VERR_SEM_BUSY;
1424	}
1425	else
1426	{
1427	/* Add ourselves to the write count and break out to do the wait. */
1428	uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT;
1429	AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_WRITERS);
1430	c++;
1431	Assert(c < RTCSRW_CNT_WR_MASK / 4);
1432	u64State &= ~RTCSRW_CNT_WR_MASK;
1433	u64State \|= c << RTCSRW_CNT_WR_SHIFT;
1434	if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
1435	break;
1436	}
1437
1438	ASMNopPause();
1439
1440	if (pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC)
1441	{ /* likely */ }
1442	else
1443	return VERR_SEM_DESTROYED;
1444
1445	ASMNopPause();
1446	u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
1447	u64OldState = u64State;
1448	}
1449
1450	/*
1451	* If we're in write mode now try grab the ownership. Play fair if there
1452	* are threads already waiting.
1453	*/
1454	bool fDone = (u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)
1455	&& ( ((u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT) == 1
1456	\|\| fTryOnly);
1457	if (fDone)
1458	{
1459	ASMAtomicCmpXchgHandle(&pThis->s.Core.u.s.hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone);
1460	if (fDone)
1461	return pdmCritSectRwEnterExclFirst(pThis, pSrcPos, fNoVal, hThreadSelf);
1462	}
1463
1464	/*
1465	* Okay, we have contention and will have to wait unless we're just trying.
1466	*/
1467	if (fTryOnly)
1468	{
1469	STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterExcl)); /** @todo different statistics for this */
1470	return pdmCritSectRwEnterExclBailOut(pThis, VERR_SEM_BUSY);
1471	}
1472
1473	STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterExcl));
1474
1475	/*
1476	* Ring-3 is pretty straight forward.
1477	*/
1478	#if defined(IN_RING3) && defined(PDMCRITSECTRW_STRICT)
1479	return pdmR3R0CritSectRwEnterExclContended(pVM, NULL, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, hThreadSelf);
1480	#elif defined(IN_RING3)
1481	return pdmR3R0CritSectRwEnterExclContended(pVM, NULL, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, RTThreadSelf());
1482
1483	#elif defined(IN_RING0)
1484	/*
1485	* In ring-0 context we have to take the special VT-x/AMD-V HM context into
1486	* account when waiting on contended locks.
1487	*/
1488	PVMCPUCC pVCpu = VMMGetCpu(pVM);
1489	if (pVCpu)
1490	{
1491	VMMR0EMTBLOCKCTX Ctx;
1492	int rc = VMMR0EmtPrepareToBlock(pVCpu, rcBusy, __FUNCTION__, pThis, &Ctx);
1493	if (rc == VINF_SUCCESS)
1494	{
1495	Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1496
1497	rc = pdmR3R0CritSectRwEnterExclContended(pVM, pVCpu, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, NIL_RTTHREAD);
1498
1499	VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx);
1500	}
1501	else
1502	{
1503	//STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLockBusy);
1504	rc = pdmCritSectRwEnterExclBailOut(pThis, rc);
1505	}
1506	return rc;
1507	}
1508
1509	/* Non-EMT. */
1510	Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1511	return pdmR3R0CritSectRwEnterExclContended(pVM, NULL, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, NIL_RTTHREAD);
1512
1513	#else
1514	# error "Unused."
1515	/*
1516	* Raw-mode: Call host and take it there if rcBusy is VINF_SUCCESS.
1517	*/
1518	rcBusy = pdmCritSectRwEnterExclBailOut(pThis, rcBusy);
1519	if (rcBusy == VINF_SUCCESS)
1520	{
1521	Assert(!fTryOnly);
1522	PVMCPUCC pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu);
1523	/** @todo Should actually do this in via VMMR0.cpp instead of going all the way
1524	* back to ring-3. Goes for both kind of crit sects. */
1525	return VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_EXCL, MMHyperCCToR3(pVM, pThis));
1526	}
1527	return rcBusy;
1528	#endif
1529	}
1530
1531
1532	/**
1533	* Try enter a critical section with exclusive (write) access.
1534	*
1535	* @returns VBox status code.
1536	* @retval VINF_SUCCESS on success.
1537	* @retval rcBusy if in ring-0 or raw-mode context and it is busy.
1538	* @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
1539	* @retval VERR_SEM_DESTROYED if the critical section is delete before or
1540	* during the operation.
1541	*
1542	* @param pVM The cross context VM structure.
1543	* @param pThis Pointer to the read/write critical section.
1544	* @param rcBusy The status code to return when we're in RC or R0 and the
1545	* section is busy. Pass VINF_SUCCESS to acquired the
1546	* critical section thru a ring-3 call if necessary.
1547	* @sa PDMCritSectRwEnterExclDebug, PDMCritSectRwTryEnterExcl,
1548	* PDMCritSectRwTryEnterExclDebug,
1549	* PDMCritSectEnterDebug, PDMCritSectEnter,
1550	* RTCritSectRwEnterExcl.
1551	*/
1552	VMMDECL(int) PDMCritSectRwEnterExcl(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy)
1553	{
1554	#if !defined(PDMCRITSECTRW_STRICT) \|\| !defined(IN_RING3)
1555	return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /fTryAgain/, NULL, false /fNoVal/);
1556	#else
1557	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
1558	return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /fTryAgain/, &SrcPos, false /fNoVal/);
1559	#endif
1560	}
1561
1562
1563	/**
1564	* Try enter a critical section with exclusive (write) access.
1565	*
1566	* @returns VBox status code.
1567	* @retval VINF_SUCCESS on success.
1568	* @retval rcBusy if in ring-0 or raw-mode context and it is busy.
1569	* @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
1570	* @retval VERR_SEM_DESTROYED if the critical section is delete before or
1571	* during the operation.
1572	*
1573	* @param pVM The cross context VM structure.
1574	* @param pThis Pointer to the read/write critical section.
1575	* @param rcBusy The status code to return when we're in RC or R0 and the
1576	* section is busy. Pass VINF_SUCCESS to acquired the
1577	* critical section thru a ring-3 call if necessary.
1578	* @param uId Where we're entering the section.
1579	* @param SRC_POS The source position.
1580	* @sa PDMCritSectRwEnterExcl, PDMCritSectRwTryEnterExcl,
1581	* PDMCritSectRwTryEnterExclDebug,
1582	* PDMCritSectEnterDebug, PDMCritSectEnter,
1583	* RTCritSectRwEnterExclDebug.
1584	*/
1585	VMMDECL(int) PDMCritSectRwEnterExclDebug(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, RTHCUINTPTR uId, RT_SRC_POS_DECL)
1586	{
1587	NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction);
1588	#if !defined(PDMCRITSECTRW_STRICT) \|\| !defined(IN_RING3)
1589	return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /fTryAgain/, NULL, false /fNoVal/);
1590	#else
1591	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
1592	return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /fTryAgain/, &SrcPos, false /fNoVal/);
1593	#endif
1594	}
1595
1596
1597	/**
1598	* Try enter a critical section with exclusive (write) access.
1599	*
1600	* @retval VINF_SUCCESS on success.
1601	* @retval VERR_SEM_BUSY if the critsect was owned.
1602	* @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
1603	* @retval VERR_SEM_DESTROYED if the critical section is delete before or
1604	* during the operation.
1605	*
1606	* @param pVM The cross context VM structure.
1607	* @param pThis Pointer to the read/write critical section.
1608	* @sa PDMCritSectRwEnterExcl, PDMCritSectRwTryEnterExclDebug,
1609	* PDMCritSectRwEnterExclDebug,
1610	* PDMCritSectTryEnter, PDMCritSectTryEnterDebug,
1611	* RTCritSectRwTryEnterExcl.
1612	*/
1613	VMMDECL(int) PDMCritSectRwTryEnterExcl(PVMCC pVM, PPDMCRITSECTRW pThis)
1614	{
1615	#if !defined(PDMCRITSECTRW_STRICT) \|\| !defined(IN_RING3)
1616	return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /fTryAgain/, NULL, false /fNoVal/);
1617	#else
1618	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
1619	return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /fTryAgain/, &SrcPos, false /fNoVal/);
1620	#endif
1621	}
1622
1623
1624	/**
1625	* Try enter a critical section with exclusive (write) access.
1626	*
1627	* @retval VINF_SUCCESS on success.
1628	* @retval VERR_SEM_BUSY if the critsect was owned.
1629	* @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
1630	* @retval VERR_SEM_DESTROYED if the critical section is delete before or
1631	* during the operation.
1632	*
1633	* @param pVM The cross context VM structure.
1634	* @param pThis Pointer to the read/write critical section.
1635	* @param uId Where we're entering the section.
1636	* @param SRC_POS The source position.
1637	* @sa PDMCritSectRwTryEnterExcl, PDMCritSectRwEnterExcl,
1638	* PDMCritSectRwEnterExclDebug,
1639	* PDMCritSectTryEnterDebug, PDMCritSectTryEnter,
1640	* RTCritSectRwTryEnterExclDebug.
1641	*/
1642	VMMDECL(int) PDMCritSectRwTryEnterExclDebug(PVMCC pVM, PPDMCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL)
1643	{
1644	NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction);
1645	#if !defined(PDMCRITSECTRW_STRICT) \|\| !defined(IN_RING3)
1646	return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /fTryAgain/, NULL, false /fNoVal/);
1647	#else
1648	RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
1649	return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /fTryAgain/, &SrcPos, false /fNoVal/);
1650	#endif
1651	}
1652
1653
1654	#ifdef IN_RING3
1655	/**
1656	* Enters a PDM read/write critical section with exclusive (write) access.
1657	*
1658	* @returns VINF_SUCCESS if entered successfully.
1659	* @retval VERR_SEM_DESTROYED if the critical section is delete before or
1660	* during the operation.
1661	*
1662	* @param pVM The cross context VM structure.
1663	* @param pThis Pointer to the read/write critical section.
1664	* @param fCallRing3 Whether this is a VMMRZCallRing3()request.
1665	*/
1666	VMMR3DECL(int) PDMR3CritSectRwEnterExclEx(PVM pVM, PPDMCRITSECTRW pThis, bool fCallRing3)
1667	{
1668	return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, false /fTryAgain/, NULL, fCallRing3 /fNoVal/);
1669	}
1670	#endif /* IN_RING3 */
1671
1672
1673	/**
1674	* Leave a critical section held exclusively.
1675	*
1676	* @returns VBox status code.
1677	* @retval VERR_SEM_DESTROYED if the critical section is delete before or
1678	* during the operation.
1679	* @param pVM The cross context VM structure.
1680	* @param pThis Pointer to the read/write critical section.
1681	* @param fNoVal No validation records (i.e. queued release).
1682	* @sa PDMCritSectRwLeaveShared, RTCritSectRwLeaveExcl.
1683	*/
1684	static int pdmCritSectRwLeaveExclWorker(PVMCC pVM, PPDMCRITSECTRW pThis, bool fNoVal)
1685	{
1686	/*
1687	* Validate handle.
1688	*/
1689	AssertPtr(pThis);
1690	AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
1691
1692	#if !defined(PDMCRITSECTRW_STRICT) \|\| !defined(IN_RING3)
1693	NOREF(fNoVal);
1694	#endif
1695
1696	/*
1697	* Check ownership.
1698	*/
1699	RTNATIVETHREAD hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis);
1700	AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_VM_THREAD_NOT_EMT);
1701
1702	RTNATIVETHREAD hNativeWriter;
1703	ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter);
1704	AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER);
1705
1706
1707	/*
1708	* Unwind one recursion. Not the last?
1709	*/
1710	if (pThis->s.Core.cWriteRecursions != 1)
1711	{
1712	#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
1713	if (fNoVal)
1714	Assert(pThis->s.Core.pValidatorWrite->hThread == NIL_RTTHREAD);
1715	else
1716	{
1717	int rc9 = RTLockValidatorRecExclUnwind(pThis->s.Core.pValidatorWrite);
1718	if (RT_FAILURE(rc9))
1719	return rc9;
1720	}
1721	#endif
1722	#ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
1723	uint32_t const cDepth = --pThis->s.Core.cWriteRecursions;
1724	#else
1725	uint32_t const cDepth = ASMAtomicDecU32(&pThis->s.Core.cWriteRecursions);
1726	#endif
1727	AssertReturn(cDepth != 0 && cDepth < UINT32_MAX, pdmCritSectRwCorrupted(pThis, "Invalid write recursion value on leave"));
1728	return VINF_SUCCESS;
1729	}
1730
1731
1732	/*
1733	* Final recursion.
1734	*/
1735	AssertReturn(pThis->s.Core.cWriterReads == 0, VERR_WRONG_ORDER); /* (must release all read recursions before the final write.) */
1736	#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
1737	if (fNoVal)
1738	Assert(pThis->s.Core.pValidatorWrite->hThread == NIL_RTTHREAD);
1739	else
1740	{
1741	int rc9 = RTLockValidatorRecExclReleaseOwner(pThis->s.Core.pValidatorWrite, true);
1742	if (RT_FAILURE(rc9))
1743	return rc9;
1744	}
1745	#endif
1746
1747
1748	#ifdef RTASM_HAVE_CMP_WRITE_U128
1749	/*
1750	* See if we can get out w/o any signalling as this is a common case.
1751	*/
1752	if (pdmCritSectRwIsCmpWriteU128Supported())
1753	{
1754	RTCRITSECTRWSTATE OldState;
1755	OldState.s.u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
1756	if (OldState.s.u64State == ((UINT64_C(1) << RTCSRW_CNT_WR_SHIFT) \| (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)))
1757	{
1758	OldState.s.hNativeWriter = hNativeSelf;
1759	AssertCompile(sizeof(OldState.s.hNativeWriter) == sizeof(OldState.u128.s.Lo));
1760
1761	RTCRITSECTRWSTATE NewState;
1762	NewState.s.u64State = RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT;
1763	NewState.s.hNativeWriter = NIL_RTNATIVETHREAD;
1764
1765	# ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
1766	pThis->s.Core.cWriteRecursions = 0;
1767	# else
1768	ASMAtomicWriteU32(&pThis->s.Core.cWriteRecursions, 0);
1769	# endif
1770	STAM_PROFILE_ADV_STOP(&pThis->s.StatWriteLocked, swl);
1771
1772	if (ASMAtomicCmpWriteU128U(&pThis->s.Core.u.u128, NewState.u128, OldState.u128))
1773	return VINF_SUCCESS;
1774
1775	/* bail out. */
1776	pThis->s.Core.cWriteRecursions = 1;
1777	}
1778	}
1779	#endif /* RTASM_HAVE_CMP_WRITE_U128 */
1780
1781
1782	#if defined(IN_RING3) \|\| defined(IN_RING0)
1783	/*
1784	* Ring-3: Straight forward, just update the state and if necessary signal waiters.
1785	* Ring-0: Try leave for real, depends on host and context.
1786	*/
1787	# ifdef IN_RING0
1788	Assert(RTSemEventIsSignalSafe() == RTSemEventMultiIsSignalSafe());
1789	PVMCPUCC pVCpu = VMMGetCpu(pVM);
1790	if ( pVCpu == NULL /* non-EMT access, if we implement it must be able to block */
1791	\|\| VMMRZCallRing3IsEnabled(pVCpu)
1792	\|\| RTSemEventIsSignalSafe()
1793	\|\| ( VMMR0ThreadCtxHookIsEnabled(pVCpu) /* Doesn't matter if Signal() blocks if we have hooks, ... */
1794	&& RTThreadPreemptIsEnabled(NIL_RTTHREAD) /* ... and preemption is still enabled, */
1795	&& ASMIntAreEnabled()) /* ... and interrupts hasn't yet been disabled. Special pre-GC HM env. */
1796	)
1797	# endif
1798	{
1799	# ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
1800	pThis->s.Core.cWriteRecursions = 0;
1801	# else
1802	ASMAtomicWriteU32(&pThis->s.Core.cWriteRecursions, 0);
1803	# endif
1804	STAM_PROFILE_ADV_STOP(&pThis->s.StatWriteLocked, swl);
1805	ASMAtomicWriteHandle(&pThis->s.Core.u.s.hNativeWriter, NIL_RTNATIVETHREAD);
1806
1807	for (;;)
1808	{
1809	uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
1810	uint64_t u64OldState = u64State;
1811
1812	uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT;
1813	AssertReturn(c > 0, pdmCritSectRwCorrupted(pThis, "Invalid write count on leave"));
1814	c--;
1815
1816	if ( c > 0
1817	\|\| (u64State & RTCSRW_CNT_RD_MASK) == 0)
1818	{
1819	/*
1820	* Don't change the direction, wake up the next writer if any.
1821	*/
1822	u64State &= ~RTCSRW_CNT_WR_MASK;
1823	u64State \|= c << RTCSRW_CNT_WR_SHIFT;
1824	if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
1825	{
1826	STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,LeaveExcl));
1827	int rc;
1828	if (c == 0)
1829	rc = VINF_SUCCESS;
1830	# ifdef IN_RING0
1831	else if (!RTSemEventIsSignalSafe() && pVCpu != NULL)
1832	{
1833	VMMR0EMTBLOCKCTX Ctx;
1834	rc = VMMR0EmtPrepareToBlock(pVCpu, VINF_SUCCESS, __FUNCTION__, pThis, &Ctx);
1835	VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_SUCCESS(rc), ("rc=%Rrc\n", rc), rc);
1836
1837	rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite);
1838
1839	VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx);
1840	}
1841	# endif
1842	else
1843	rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite);
1844	AssertRC(rc);
1845	return rc;
1846	}
1847	}
1848	else
1849	{
1850	/*
1851	* Reverse the direction and signal the reader threads.
1852	*/
1853	u64State &= ~(RTCSRW_CNT_WR_MASK \| RTCSRW_DIR_MASK);
1854	u64State \|= RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT;
1855	if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
1856	{
1857	Assert(!pThis->s.Core.fNeedReset);
1858	ASMAtomicWriteBool(&pThis->s.Core.fNeedReset, true);
1859	STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,LeaveExcl));
1860
1861	int rc;
1862	# ifdef IN_RING0
1863	if (!RTSemEventMultiIsSignalSafe() && pVCpu != NULL)
1864	{
1865	VMMR0EMTBLOCKCTX Ctx;
1866	rc = VMMR0EmtPrepareToBlock(pVCpu, VINF_SUCCESS, __FUNCTION__, pThis, &Ctx);
1867	VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_SUCCESS(rc), ("rc=%Rrc\n", rc), rc);
1868
1869	rc = SUPSemEventMultiSignal(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead);
1870
1871	VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx);
1872	}
1873	else
1874	# endif
1875	rc = SUPSemEventMultiSignal(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead);
1876	AssertRC(rc);
1877	return rc;
1878	}
1879	}
1880
1881	ASMNopPause();
1882	if (pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC)
1883	{ /likely/ }
1884	else
1885	return VERR_SEM_DESTROYED;
1886	ASMNopPause();
1887	}
1888	/* not reached! */
1889	}
1890	#endif /* IN_RING3 \|\| IN_RING0 */
1891
1892
1893	#ifndef IN_RING3
1894	/*
1895	* Queue the requested exit for ring-3 execution.
1896	*/
1897	# ifndef IN_RING0
1898	PVMCPUCC pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu);
1899	# endif
1900	uint32_t i = pVCpu->pdm.s.cQueuedCritSectRwExclLeaves++;
1901	LogFlow(("PDMCritSectRwLeaveShared: [%d]=%p => R3\n", i, pThis));
1902	VMM_ASSERT_RELEASE_MSG_RETURN(pVM, i < RT_ELEMENTS(pVCpu->pdm.s.apQueuedCritSectRwExclLeaves),
1903	("i=%u\n", i), VERR_PDM_CRITSECTRW_IPE);
1904	pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i] = pThis->s.pSelfR3;
1905	VMM_ASSERT_RELEASE_MSG_RETURN(pVM,
1906	RT_VALID_PTR(pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i])
1907	&& ((uintptr_t)pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i] & HOST_PAGE_OFFSET_MASK)
1908	== ((uintptr_t)pThis & HOST_PAGE_OFFSET_MASK),
1909	("%p vs %p\n", pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i], pThis),
1910	pdmCritSectRwCorrupted(pThis, "Invalid self pointer on queue (excl)"));
1911	VMCPU_FF_SET(pVCpu, VMCPU_FF_PDM_CRITSECT);
1912	VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
1913	STAM_REL_COUNTER_INC(&pVM->pdm.s.StatQueuedCritSectLeaves);
1914	STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLeaveExcl);
1915	return VINF_SUCCESS;
1916	#endif
1917	}
1918
1919
1920	/**
1921	* Leave a critical section held exclusively.
1922	*
1923	* @returns VBox status code.
1924	* @retval VERR_SEM_DESTROYED if the critical section is delete before or
1925	* during the operation.
1926	* @param pVM The cross context VM structure.
1927	* @param pThis Pointer to the read/write critical section.
1928	* @sa PDMCritSectRwLeaveShared, RTCritSectRwLeaveExcl.
1929	*/
1930	VMMDECL(int) PDMCritSectRwLeaveExcl(PVMCC pVM, PPDMCRITSECTRW pThis)
1931	{
1932	return pdmCritSectRwLeaveExclWorker(pVM, pThis, false /fNoVal/);
1933	}
1934
1935
1936	#if defined(IN_RING3) \|\| defined(IN_RING0)
1937	/**
1938	* PDMCritSectBothFF interface.
1939	*
1940	* @param pVM The cross context VM structure.
1941	* @param pThis Pointer to the read/write critical section.
1942	*/
1943	void pdmCritSectRwLeaveExclQueued(PVMCC pVM, PPDMCRITSECTRW pThis)
1944	{
1945	pdmCritSectRwLeaveExclWorker(pVM, pThis, true /fNoVal/);
1946	}
1947	#endif
1948
1949
1950	/**
1951	* Checks the caller is the exclusive (write) owner of the critical section.
1952	*
1953	* @retval true if owner.
1954	* @retval false if not owner.
1955	* @param pVM The cross context VM structure.
1956	* @param pThis Pointer to the read/write critical section.
1957	* @sa PDMCritSectRwIsReadOwner, PDMCritSectIsOwner,
1958	* RTCritSectRwIsWriteOwner.
1959	*/
1960	VMMDECL(bool) PDMCritSectRwIsWriteOwner(PVMCC pVM, PPDMCRITSECTRW pThis)
1961	{
1962	/*
1963	* Validate handle.
1964	*/
1965	AssertPtr(pThis);
1966	AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, false);
1967
1968	/*
1969	* Check ownership.
1970	*/
1971	RTNATIVETHREAD hNativeWriter;
1972	ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter);
1973	if (hNativeWriter == NIL_RTNATIVETHREAD)
1974	return false;
1975	return hNativeWriter == pdmCritSectRwGetNativeSelf(pVM, pThis);
1976	}
1977
1978
1979	/**
1980	* Checks if the caller is one of the read owners of the critical section.
1981	*
1982	* @note !CAUTION! This API doesn't work reliably if lock validation isn't
1983	* enabled. Meaning, the answer is not trustworhty unless
1984	* RT_LOCK_STRICT or PDMCRITSECTRW_STRICT was defined at build time.
1985	* Also, make sure you do not use RTCRITSECTRW_FLAGS_NO_LOCK_VAL when
1986	* creating the semaphore. And finally, if you used a locking class,
1987	* don't disable deadlock detection by setting cMsMinDeadlock to
1988	* RT_INDEFINITE_WAIT.
1989	*
1990	* In short, only use this for assertions.
1991	*
1992	* @returns @c true if reader, @c false if not.
1993	* @param pVM The cross context VM structure.
1994	* @param pThis Pointer to the read/write critical section.
1995	* @param fWannaHear What you'd like to hear when lock validation is not
1996	* available. (For avoiding asserting all over the place.)
1997	* @sa PDMCritSectRwIsWriteOwner, RTCritSectRwIsReadOwner.
1998	*/
1999	VMMDECL(bool) PDMCritSectRwIsReadOwner(PVMCC pVM, PPDMCRITSECTRW pThis, bool fWannaHear)
2000	{
2001	/*
2002	* Validate handle.
2003	*/
2004	AssertPtr(pThis);
2005	AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, false);
2006
2007	/*
2008	* Inspect the state.
2009	*/
2010	uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
2011	if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT))
2012	{
2013	/*
2014	* It's in write mode, so we can only be a reader if we're also the
2015	* current writer.
2016	*/
2017	RTNATIVETHREAD hWriter;
2018	ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hWriter);
2019	if (hWriter == NIL_RTNATIVETHREAD)
2020	return false;
2021	return hWriter == pdmCritSectRwGetNativeSelf(pVM, pThis);
2022	}
2023
2024	/*
2025	* Read mode. If there are no current readers, then we cannot be a reader.
2026	*/
2027	if (!(u64State & RTCSRW_CNT_RD_MASK))
2028	return false;
2029
2030	#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
2031	/*
2032	* Ask the lock validator.
2033	* Note! It doesn't know everything, let's deal with that if it becomes an issue...
2034	*/
2035	NOREF(fWannaHear);
2036	return RTLockValidatorRecSharedIsOwner(pThis->s.Core.pValidatorRead, NIL_RTTHREAD);
2037	#else
2038	/*
2039	* Ok, we don't know, just tell the caller what he want to hear.
2040	*/
2041	return fWannaHear;
2042	#endif
2043	}
2044
2045
2046	/**
2047	* Gets the write recursion count.
2048	*
2049	* @returns The write recursion count (0 if bad critsect).
2050	* @param pThis Pointer to the read/write critical section.
2051	* @sa PDMCritSectRwGetWriterReadRecursion, PDMCritSectRwGetReadCount,
2052	* RTCritSectRwGetWriteRecursion.
2053	*/
2054	VMMDECL(uint32_t) PDMCritSectRwGetWriteRecursion(PPDMCRITSECTRW pThis)
2055	{
2056	/*
2057	* Validate handle.
2058	*/
2059	AssertPtr(pThis);
2060	AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, 0);
2061
2062	/*
2063	* Return the requested data.
2064	*/
2065	return pThis->s.Core.cWriteRecursions;
2066	}
2067
2068
2069	/**
2070	* Gets the read recursion count of the current writer.
2071	*
2072	* @returns The read recursion count (0 if bad critsect).
2073	* @param pThis Pointer to the read/write critical section.
2074	* @sa PDMCritSectRwGetWriteRecursion, PDMCritSectRwGetReadCount,
2075	* RTCritSectRwGetWriterReadRecursion.
2076	*/
2077	VMMDECL(uint32_t) PDMCritSectRwGetWriterReadRecursion(PPDMCRITSECTRW pThis)
2078	{
2079	/*
2080	* Validate handle.
2081	*/
2082	AssertPtr(pThis);
2083	AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, 0);
2084
2085	/*
2086	* Return the requested data.
2087	*/
2088	return pThis->s.Core.cWriterReads;
2089	}
2090
2091
2092	/**
2093	* Gets the current number of reads.
2094	*
2095	* This includes all read recursions, so it might be higher than the number of
2096	* read owners. It does not include reads done by the current writer.
2097	*
2098	* @returns The read count (0 if bad critsect).
2099	* @param pThis Pointer to the read/write critical section.
2100	* @sa PDMCritSectRwGetWriteRecursion, PDMCritSectRwGetWriterReadRecursion,
2101	* RTCritSectRwGetReadCount.
2102	*/
2103	VMMDECL(uint32_t) PDMCritSectRwGetReadCount(PPDMCRITSECTRW pThis)
2104	{
2105	/*
2106	* Validate input.
2107	*/
2108	AssertPtr(pThis);
2109	AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, 0);
2110
2111	/*
2112	* Return the requested data.
2113	*/
2114	uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
2115	if ((u64State & RTCSRW_DIR_MASK) != (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
2116	return 0;
2117	return (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
2118	}
2119
2120
2121	/**
2122	* Checks if the read/write critical section is initialized or not.
2123	*
2124	* @retval true if initialized.
2125	* @retval false if not initialized.
2126	* @param pThis Pointer to the read/write critical section.
2127	* @sa PDMCritSectIsInitialized, RTCritSectRwIsInitialized.
2128	*/
2129	VMMDECL(bool) PDMCritSectRwIsInitialized(PCPDMCRITSECTRW pThis)
2130	{
2131	AssertPtr(pThis);
2132	return pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC;
2133	}
2134

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PDMAllCritSectRw.cpp@ 101505

Download in other formats: