mp-r0drv-nt.cpp@ 54410

Last change on this file since 54410 was 54410, checked in by vboxsync, 10 years ago
IPRT/r0drv: Implemented RTMpOnPair for NT (windows).
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 28.0 KB

Line
1	/* $Id: mp-r0drv-nt.cpp 54410 2015-02-24 02:52:10Z vboxsync $ */
2	/** @file
3	* IPRT - Multiprocessor, Ring-0 Driver, NT.
4	*/
5
6	/*
7	* Copyright (C) 2008-2014 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*******************************************************************************
29	* Header Files *
30	*******************************************************************************/
31	#include "the-nt-kernel.h"
32
33	#include <iprt/mp.h>
34	#include <iprt/cpuset.h>
35	#include <iprt/err.h>
36	#include <iprt/asm.h>
37	#include <iprt/log.h>
38	#include <iprt/time.h>
39	#include "r0drv/mp-r0drv.h"
40	#include "internal-r0drv-nt.h"
41
42
43	/*******************************************************************************
44	* Structures and Typedefs *
45	*******************************************************************************/
46	typedef enum
47	{
48	RT_NT_CPUID_SPECIFIC,
49	RT_NT_CPUID_PAIR,
50	RT_NT_CPUID_OTHERS,
51	RT_NT_CPUID_ALL
52	} RT_NT_CPUID;
53
54
55	/**
56	* Used by the RTMpOnSpecific.
57	*/
58	typedef struct RTMPNTONSPECIFICARGS
59	{
60	/** Set if we're executing. */
61	bool volatile fExecuting;
62	/** Set when done executing. */
63	bool volatile fDone;
64	/** Number of references to this heap block. */
65	uint32_t volatile cRefs;
66	/** Event that the calling thread is waiting on. */
67	KEVENT DoneEvt;
68	/** The deferred procedure call object. */
69	KDPC Dpc;
70	/** The callback argument package. */
71	RTMPARGS CallbackArgs;
72	} RTMPNTONSPECIFICARGS;
73	/** Pointer to an argument/state structure for RTMpOnSpecific on NT. */
74	typedef RTMPNTONSPECIFICARGS *PRTMPNTONSPECIFICARGS;
75
76
77
78	/* test a couple of assumption. */
79	AssertCompile(MAXIMUM_PROCESSORS <= RTCPUSET_MAX_CPUS);
80	AssertCompile(NIL_RTCPUID >= MAXIMUM_PROCESSORS);
81
82	/** @todo
83	* We cannot do other than assume a 1:1 relationship between the
84	* affinity mask and the process despite the vagueness/warnings in
85	* the docs. If someone knows a better way to get this done, please
86	* let bird know.
87	*/
88
89
90	RTDECL(RTCPUID) RTMpCpuId(void)
91	{
92	/* WDK upgrade warning: PCR->Number changed from BYTE to WORD. */
93	return KeGetCurrentProcessorNumber();
94	}
95
96
97	RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu)
98	{
99	return idCpu < MAXIMUM_PROCESSORS ? (int)idCpu : -1;
100	}
101
102
103	RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu)
104	{
105	return (unsigned)iCpu < MAXIMUM_PROCESSORS ? iCpu : NIL_RTCPUID;
106	}
107
108
109	RTDECL(RTCPUID) RTMpGetMaxCpuId(void)
110	{
111	/** @todo use KeQueryMaximumProcessorCount on vista+ */
112	return MAXIMUM_PROCESSORS - 1;
113	}
114
115
116	RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
117	{
118	if (idCpu >= MAXIMUM_PROCESSORS)
119	return false;
120
121	#if 0 /* this isn't safe at all IRQLs (great work guys) */
122	KAFFINITY Mask = KeQueryActiveProcessors();
123	return !!(Mask & RT_BIT_64(idCpu));
124	#else
125	return RTCpuSetIsMember(&g_rtMpNtCpuSet, idCpu);
126	#endif
127	}
128
129
130	RTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu)
131	{
132	/* Cannot easily distinguish between online and offline cpus. */
133	/** @todo online/present cpu stuff must be corrected for proper W2K8 support
134	* (KeQueryMaximumProcessorCount). */
135	return RTMpIsCpuOnline(idCpu);
136	}
137
138
139
140	RTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet)
141	{
142	/** @todo online/present cpu stuff must be corrected for proper W2K8 support
143	* (KeQueryMaximumProcessorCount). */
144	return RTMpGetOnlineSet(pSet);
145	}
146
147
148	RTDECL(RTCPUID) RTMpGetCount(void)
149	{
150	/** @todo online/present cpu stuff must be corrected for proper W2K8 support
151	* (KeQueryMaximumProcessorCount). */
152	return RTMpGetOnlineCount();
153	}
154
155
156	RTDECL(PRTCPUSET) RTMpGetOnlineSet(PRTCPUSET pSet)
157	{
158	#if 0 /* this isn't safe at all IRQLs (great work guys) */
159	KAFFINITY Mask = KeQueryActiveProcessors();
160	return RTCpuSetFromU64(pSet, Mask);
161	#else
162	*pSet = g_rtMpNtCpuSet;
163	return pSet;
164	#endif
165	}
166
167
168	RTDECL(RTCPUID) RTMpGetOnlineCount(void)
169	{
170	RTCPUSET Set;
171	RTMpGetOnlineSet(&Set);
172	return RTCpuSetCount(&Set);
173	}
174
175
176	#if 0
177	/* Experiment with checking the undocumented KPRCB structure
178	* 'dt nt!_kprcb 0xaddress' shows the layout
179	*/
180	typedef struct
181	{
182	LIST_ENTRY DpcListHead;
183	ULONG_PTR DpcLock;
184	volatile ULONG DpcQueueDepth;
185	ULONG DpcQueueCount;
186	} KDPC_DATA, *PKDPC_DATA;
187
188	RTDECL(bool) RTMpIsCpuWorkPending(void)
189	{
190	uint8_t *pkprcb;
191	PKDPC_DATA pDpcData;
192
193	_asm {
194	mov eax, fs:0x20
195	mov pkprcb, eax
196	}
197	pDpcData = (PKDPC_DATA)(pkprcb + 0x19e0);
198	if (pDpcData->DpcQueueDepth)
199	return true;
200
201	pDpcData++;
202	if (pDpcData->DpcQueueDepth)
203	return true;
204	return false;
205	}
206	#else
207	RTDECL(bool) RTMpIsCpuWorkPending(void)
208	{
209	/** @todo not implemented */
210	return false;
211	}
212	#endif
213
214
215	/**
216	* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
217	* the RTMpOnAll case.
218	*
219	* @param uUserCtx The user context argument (PRTMPARGS).
220	*/
221	static ULONG_PTR __stdcall rtmpNtOnAllBroadcastIpiWrapper(ULONG_PTR uUserCtx)
222	{
223	PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
224	/ASMAtomicIncU32(&pArgs->cHits); - not needed /
225	pArgs->pfnWorker(KeGetCurrentProcessorNumber(), pArgs->pvUser1, pArgs->pvUser2);
226	return 0;
227	}
228
229
230	/**
231	* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
232	* the RTMpOnOthers case.
233	*
234	* @param uUserCtx The user context argument (PRTMPARGS).
235	*/
236	static ULONG_PTR __stdcall rtmpNtOnOthersBroadcastIpiWrapper(ULONG_PTR uUserCtx)
237	{
238	PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
239	RTCPUID idCpu = KeGetCurrentProcessorNumber();
240	if (pArgs->idCpu != idCpu)
241	{
242	/ASMAtomicIncU32(&pArgs->cHits); - not needed /
243	pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
244	}
245	return 0;
246	}
247
248
249	/**
250	* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
251	* the RTMpOnPair case.
252	*
253	* @param uUserCtx The user context argument (PRTMPARGS).
254	*/
255	static ULONG_PTR __stdcall rtmpNtOnPairBroadcastIpiWrapper(ULONG_PTR uUserCtx)
256	{
257	PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
258	RTCPUID idCpu = KeGetCurrentProcessorNumber();
259	if ( pArgs->idCpu == idCpu
260	\|\| pArgs->idCpu2 == idCpu)
261	{
262	ASMAtomicIncU32(&pArgs->cHits);
263	pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
264	}
265	return 0;
266	}
267
268
269	/**
270	* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
271	* the RTMpOnSpecific case.
272	*
273	* @param uUserCtx The user context argument (PRTMPARGS).
274	*/
275	static ULONG_PTR __stdcall rtmpNtOnSpecificBroadcastIpiWrapper(ULONG_PTR uUserCtx)
276	{
277	PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
278	RTCPUID idCpu = KeGetCurrentProcessorNumber();
279	if (pArgs->idCpu == idCpu)
280	{
281	ASMAtomicIncU32(&pArgs->cHits);
282	pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
283	}
284	return 0;
285	}
286
287
288	/**
289	* Internal worker for the RTMpOn* APIs using KeIpiGenericCall.
290	*
291	* @returns VINF_SUCCESS.
292	* @param pfnWorker The callback.
293	* @param pvUser1 User argument 1.
294	* @param pvUser2 User argument 2.
295	* @param idCpu First CPU to match, ultimately specific to the
296	* pfnNativeWrapper used.
297	* @param idCpu2 Second CPU to match, ultimately specific to the
298	* pfnNativeWrapper used.
299	* @param pcHits Where to return the number of this. Optional.
300	*/
301	static int rtMpCallUsingBroadcastIpi(PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2,
302	PKIPI_BROADCAST_WORKER pfnNativeWrapper, RTCPUID idCpu, RTCPUID idCpu2,
303	uint32_t *pcHits)
304	{
305	RTMPARGS Args;
306	Args.pfnWorker = pfnWorker;
307	Args.pvUser1 = pvUser1;
308	Args.pvUser2 = pvUser2;
309	Args.idCpu = idCpu;
310	Args.idCpu2 = idCpu2;
311	Args.cRefs = 0;
312	Args.cHits = 0;
313
314	AssertPtr(g_pfnrtKeIpiGenericCall);
315	g_pfnrtKeIpiGenericCall(pfnNativeWrapper, (uintptr_t)&Args);
316	if (pcHits)
317	*pcHits = Args.cHits;
318	return VINF_SUCCESS;
319	}
320
321
322	/**
323	* Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
324	*
325	* @param Dpc DPC object
326	* @param DeferredContext Context argument specified by KeInitializeDpc
327	* @param SystemArgument1 Argument specified by KeInsertQueueDpc
328	* @param SystemArgument2 Argument specified by KeInsertQueueDpc
329	*/
330	static VOID __stdcall rtmpNtDPCWrapper(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
331	{
332	PRTMPARGS pArgs = (PRTMPARGS)DeferredContext;
333
334	ASMAtomicIncU32(&pArgs->cHits);
335	pArgs->pfnWorker(KeGetCurrentProcessorNumber(), pArgs->pvUser1, pArgs->pvUser2);
336
337	/* Dereference the argument structure. */
338	int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
339	Assert(cRefs >= 0);
340	if (cRefs == 0)
341	ExFreePool(pArgs);
342	}
343
344
345	/**
346	* Internal worker for the RTMpOn* APIs.
347	*
348	* @returns IPRT status code.
349	* @param pfnWorker The callback.
350	* @param pvUser1 User argument 1.
351	* @param pvUser2 User argument 2.
352	* @param enmCpuid What to do / is idCpu valid.
353	* @param idCpu Used if enmCpuid is RT_NT_CPUID_SPECIFIC or
354	* RT_NT_CPUID_PAIR, otherwise ignored.
355	* @param idCpu2 Used if enmCpuid is RT_NT_CPUID_PAIR, otherwise ignored.
356	* @param pcHits Where to return the number of this. Optional.
357	*/
358	static int rtMpCallUsingDpcs(PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2,
359	RT_NT_CPUID enmCpuid, RTCPUID idCpu, RTCPUID idCpu2, uint32_t *pcHits)
360	{
361	PRTMPARGS pArgs;
362	KDPC *paExecCpuDpcs;
363
364	#if 0
365	/* KeFlushQueuedDpcs must be run at IRQL PASSIVE_LEVEL according to MSDN, but the
366	* driver verifier doesn't complain...
367	*/
368	AssertMsg(KeGetCurrentIrql() == PASSIVE_LEVEL, ("%d != %d (PASSIVE_LEVEL)\n", KeGetCurrentIrql(), PASSIVE_LEVEL));
369	#endif
370
371	#ifdef IPRT_TARGET_NT4
372	KAFFINITY Mask;
373	/* g_pfnrtNt* are not present on NT anyway. */
374	return VERR_NOT_SUPPORTED;
375	#else
376	KAFFINITY Mask = KeQueryActiveProcessors();
377	#endif
378
379	/* KeFlushQueuedDpcs is not present in Windows 2000; import it dynamically so we can just fail this call. */
380	if (!g_pfnrtNtKeFlushQueuedDpcs)
381	return VERR_NOT_SUPPORTED;
382
383	pArgs = (PRTMPARGS)ExAllocatePoolWithTag(NonPagedPool, MAXIMUM_PROCESSORS*sizeof(KDPC) + sizeof(RTMPARGS), (ULONG)'RTMp');
384	if (!pArgs)
385	return VERR_NO_MEMORY;
386
387	pArgs->pfnWorker = pfnWorker;
388	pArgs->pvUser1 = pvUser1;
389	pArgs->pvUser2 = pvUser2;
390	pArgs->idCpu = NIL_RTCPUID;
391	pArgs->idCpu2 = NIL_RTCPUID;
392	pArgs->cHits = 0;
393	pArgs->cRefs = 1;
394
395	paExecCpuDpcs = (KDPC *)(pArgs + 1);
396
397	if (enmCpuid == RT_NT_CPUID_SPECIFIC)
398	{
399	KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
400	KeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
401	KeSetTargetProcessorDpc(&paExecCpuDpcs[0], (int)idCpu);
402	pArgs->idCpu = idCpu;
403	}
404	else if (enmCpuid == RT_NT_CPUID_SPECIFIC)
405	{
406	KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
407	KeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
408	KeSetTargetProcessorDpc(&paExecCpuDpcs[0], (int)idCpu);
409	pArgs->idCpu = idCpu;
410
411	KeInitializeDpc(&paExecCpuDpcs[1], rtmpNtDPCWrapper, pArgs);
412	KeSetImportanceDpc(&paExecCpuDpcs[1], HighImportance);
413	KeSetTargetProcessorDpc(&paExecCpuDpcs[1], (int)idCpu2);
414	pArgs->idCpu2 = idCpu2;
415	}
416	else
417	{
418	for (unsigned i = 0; i < MAXIMUM_PROCESSORS; i++)
419	{
420	KeInitializeDpc(&paExecCpuDpcs[i], rtmpNtDPCWrapper, pArgs);
421	KeSetImportanceDpc(&paExecCpuDpcs[i], HighImportance);
422	KeSetTargetProcessorDpc(&paExecCpuDpcs[i], i);
423	}
424	}
425
426	/* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
427	* KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL.
428	*/
429	KIRQL oldIrql;
430	KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
431
432	/*
433	* We cannot do other than assume a 1:1 relationship between the
434	* affinity mask and the process despite the warnings in the docs.
435	* If someone knows a better way to get this done, please let bird know.
436	*/
437	ASMCompilerBarrier(); /* paranoia */
438	if (enmCpuid == RT_NT_CPUID_SPECIFIC)
439	{
440	ASMAtomicIncS32(&pArgs->cRefs);
441	BOOLEAN ret = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
442	Assert(ret);
443	}
444	else if (enmCpuid == RT_NT_CPUID_PAIR)
445	{
446	ASMAtomicIncS32(&pArgs->cRefs);
447	BOOLEAN ret = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
448	Assert(ret);
449
450	ASMAtomicIncS32(&pArgs->cRefs);
451	ret = KeInsertQueueDpc(&paExecCpuDpcs[1], 0, 0);
452	Assert(ret);
453	}
454	else
455	{
456	unsigned iSelf = KeGetCurrentProcessorNumber();
457
458	for (unsigned i = 0; i < MAXIMUM_PROCESSORS; i++)
459	{
460	if ( (i != iSelf)
461	&& (Mask & RT_BIT_64(i)))
462	{
463	ASMAtomicIncS32(&pArgs->cRefs);
464	BOOLEAN ret = KeInsertQueueDpc(&paExecCpuDpcs[i], 0, 0);
465	Assert(ret);
466	}
467	}
468	if (enmCpuid != RT_NT_CPUID_OTHERS)
469	pfnWorker(iSelf, pvUser1, pvUser2);
470	}
471
472	KeLowerIrql(oldIrql);
473
474	/* Flush all DPCs and wait for completion. (can take long!) */
475	/** @todo Consider changing this to an active wait using some atomic inc/dec
476	* stuff (and check for the current cpu above in the specific case). */
477	/** @todo Seems KeFlushQueuedDpcs doesn't wait for the DPCs to be completely
478	* executed. Seen pArgs being freed while some CPU was using it before
479	* cRefs was added. */
480	g_pfnrtNtKeFlushQueuedDpcs();
481
482	if (pcHits)
483	*pcHits = pArgs->cHits;
484
485	/* Dereference the argument structure. */
486	int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
487	Assert(cRefs >= 0);
488	if (cRefs == 0)
489	ExFreePool(pArgs);
490
491	return VINF_SUCCESS;
492	}
493
494
495	RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2)
496	{
497	if (g_pfnrtKeIpiGenericCall)
498	return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnAllBroadcastIpiWrapper,
499	NIL_RTCPUID, NIL_RTCPUID, NULL);
500	return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_ALL, NIL_RTCPUID, NIL_RTCPUID, NULL);
501	}
502
503
504	RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2)
505	{
506	if (g_pfnrtKeIpiGenericCall)
507	return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnOthersBroadcastIpiWrapper,
508	NIL_RTCPUID, NIL_RTCPUID, NULL);
509	return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_OTHERS, NIL_RTCPUID, NIL_RTCPUID, NULL);
510	}
511
512
513	RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2)
514	{
515	int rc;
516	AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
517	AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
518	if ((fFlags & RTMPON_F_CONCURRENT_EXEC) && !g_pfnrtKeIpiGenericCall)
519	return VERR_NOT_SUPPORTED;
520
521	/*
522	* Check that both CPUs are online before doing the broadcast call.
523	*/
524	if ( RTMpIsCpuOnline(idCpu1)
525	&& RTMpIsCpuOnline(idCpu2))
526	{
527	/*
528	* The broadcast IPI isn't quite as bad as it could have been, because
529	* it looks like windows doesn't synchronize CPUs on the way out, they
530	* seems to get back to normal work while the pair is still busy.
531	*/
532	uint32_t cHits = 0;
533	if (g_pfnrtKeIpiGenericCall)
534	rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnPairBroadcastIpiWrapper, idCpu1, idCpu2, &cHits);
535	else
536	rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_PAIR, idCpu1, idCpu2, &cHits);
537	if (RT_SUCCESS(rc))
538	{
539	Assert(cHits <= 2);
540	if (cHits == 2)
541	rc = VINF_SUCCESS;
542	else if (cHits == 1)
543	rc = VERR_NOT_ALL_CPUS_SHOWED;
544	else if (cHits == 0)
545	rc = VERR_CPU_OFFLINE;
546	else
547	rc = VERR_CPU_IPE_1;
548	}
549	}
550	/*
551	* A CPU must be present to be considered just offline.
552	*/
553	else if ( RTMpIsCpuPresent(idCpu1)
554	&& RTMpIsCpuPresent(idCpu2))
555	rc = VERR_CPU_OFFLINE;
556	else
557	rc = VERR_CPU_NOT_FOUND;
558	return rc;
559	}
560
561
562	RTDECL(bool) RTMpOnPairIsConcurrentExecSupported(void)
563	{
564	return RTMpOnAllIsConcurrentSafe();
565	}
566
567
568	/**
569	* Releases a reference to a RTMPNTONSPECIFICARGS heap allocation, freeing it
570	* when the last reference is released.
571	*/
572	DECLINLINE(void) rtMpNtOnSpecificRelease(PRTMPNTONSPECIFICARGS pArgs)
573	{
574	uint32_t cRefs = ASMAtomicDecU32(&pArgs->cRefs);
575	AssertMsg(cRefs <= 1, ("cRefs=%#x\n", cRefs));
576	if (cRefs == 0)
577	ExFreePool(pArgs);
578	}
579
580
581	/**
582	* Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
583	*
584	* @param Dpc DPC object
585	* @param DeferredContext Context argument specified by KeInitializeDpc
586	* @param SystemArgument1 Argument specified by KeInsertQueueDpc
587	* @param SystemArgument2 Argument specified by KeInsertQueueDpc
588	*/
589	static VOID __stdcall rtMpNtOnSpecificDpcWrapper(IN PKDPC Dpc, IN PVOID DeferredContext,
590	IN PVOID SystemArgument1, IN PVOID SystemArgument2)
591	{
592	PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)DeferredContext;
593	ASMAtomicWriteBool(&pArgs->fExecuting, true);
594
595	pArgs->CallbackArgs.pfnWorker(KeGetCurrentProcessorNumber(), pArgs->CallbackArgs.pvUser1, pArgs->CallbackArgs.pvUser2);
596
597	ASMAtomicWriteBool(&pArgs->fDone, true);
598	KeSetEvent(&pArgs->DoneEvt, 1 /PriorityIncrement/, FALSE /Wait/);
599
600	rtMpNtOnSpecificRelease(pArgs);
601	}
602
603
604	RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2)
605	{
606	/*
607	* Don't try mess with an offline CPU.
608	*/
609	if (!RTMpIsCpuOnline(idCpu))
610	return !RTMpIsCpuPossible(idCpu)
611	? VERR_CPU_NOT_FOUND
612	: VERR_CPU_OFFLINE;
613
614	/*
615	* Use the broadcast IPI routine if there are no more than two CPUs online,
616	* or if the current IRQL is unsuitable for KeWaitForSingleObject.
617	*/
618	int rc;
619	uint32_t cHits = 0;
620	if ( g_pfnrtKeIpiGenericCall
621	&& ( RTMpGetOnlineCount() <= 2
622	\|\| KeGetCurrentIrql() > APC_LEVEL) )
623	rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnSpecificBroadcastIpiWrapper,
624	idCpu, NIL_RTCPUID, &cHits);
625	else
626	rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_SPECIFIC, idCpu, NIL_RTCPUID, &cHits);
627	if (RT_SUCCESS(rc))
628	{
629	if (cHits == 1)
630	return VINF_SUCCESS;
631	rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
632	}
633	return rc;
634
635	#if 0 /** @todo Untested code replacing the rtMpCallUsingDpcs caste. Needs some tuning too, I guess. */
636	/*
637	* Initialize the argument package and the objects within it.
638	* The package is referenced counted to avoid unnecessary spinning to
639	* synchronize cleanup and prevent stack corruption.
640	*/
641	PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)ExAllocatePoolWithTag(NonPagedPool, sizeof(*pArgs), (ULONG)'RTMp');
642	if (!pArgs)
643	return VERR_NO_MEMORY;
644	pArgs->cRefs = 2;
645	pArgs->fExecuting = false;
646	pArgs->fDone = false;
647	pArgs->CallbackArgs.pfnWorker = pfnWorker;
648	pArgs->CallbackArgs.pvUser1 = pvUser1;
649	pArgs->CallbackArgs.pvUser2 = pvUser2;
650	pArgs->CallbackArgs.idCpu = idCpu;
651	pArgs->CallbackArgs.cHits = 0;
652	pArgs->CallbackArgs.cRefs = 2;
653	KeInitializeEvent(&pArgs->DoneEvt, SynchronizationEvent, FALSE /* not signalled */);
654	KeInitializeDpc(&pArgs->Dpc, rtMpNtOnSpecificDpcWrapper, pArgs);
655	KeSetImportanceDpc(&pArgs->Dpc, HighImportance);
656	KeSetTargetProcessorDpc(&pArgs->Dpc, (int)idCpu);
657
658	/*
659	* Disable preemption while we check the current processor and inserts the DPC.
660	*/
661	KIRQL bOldIrql;
662	KeRaiseIrql(DISPATCH_LEVEL, &bOldIrql);
663	ASMCompilerBarrier(); /* paranoia */
664
665	if (RTMpCpuId() == idCpu)
666	{
667	/* Just execute the callback on the current CPU. */
668	pfnWorker(idCpu, pvUser1, pvUser2);
669	KeLowerIrql(bOldIrql);
670
671	ExFreePool(pArgs);
672	return VINF_SUCCESS;
673	}
674
675	/* Different CPU, so queue it if the CPU is still online. */
676	int rc;
677	if (RTMpIsCpuOnline(idCpu))
678	{
679	BOOLEAN fRc = KeInsertQueueDpc(&pArgs->Dpc, 0, 0);
680	Assert(fRc);
681	KeLowerIrql(bOldIrql);
682
683	uint64_t const nsRealWaitTS = RTTimeNanoTS();
684
685	/*
686	* Wait actively for a while in case the CPU/thread responds quickly.
687	*/
688	uint32_t cLoopsLeft = 0x20000;
689	while (cLoopsLeft-- > 0)
690	{
691	if (pArgs->fDone)
692	{
693	rtMpNtOnSpecificRelease(pArgs);
694	return VINF_SUCCESS;
695	}
696	ASMNopPause();
697	}
698
699	/*
700	* It didn't respond, so wait on the event object, poking the CPU if it's slow.
701	*/
702	LARGE_INTEGER Timeout;
703	Timeout.QuadPart = -10000; /* 1ms */
704	NTSTATUS rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
705	if (rcNt == STATUS_SUCCESS)
706	{
707	rtMpNtOnSpecificRelease(pArgs);
708	return VINF_SUCCESS;
709	}
710
711	/* If it hasn't respondend yet, maybe poke it and wait some more. */
712	if (rcNt == STATUS_TIMEOUT)
713	{
714	if ( !pArgs->fExecuting
715	&& ( g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalSendSoftwareInterrupt
716	\|\| g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalReqestIpiW7Plus
717	\|\| g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalReqestIpiPreW7))
718	RTMpPokeCpu(idCpu);
719
720	Timeout.QuadPart = -1280000; /* 128ms */
721	rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
722	if (rcNt == STATUS_SUCCESS)
723	{
724	rtMpNtOnSpecificRelease(pArgs);
725	return VINF_SUCCESS;
726	}
727	}
728
729	/*
730	* Something weird is happening, try bail out.
731	*/
732	if (KeRemoveQueueDpc(&pArgs->Dpc))
733	{
734	ExFreePool(pArgs); /* DPC was still queued, so we can return without further ado. */
735	LogRel(("RTMpOnSpecific(%#x): Not processed after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
736	}
737	else
738	{
739	/* DPC is running, wait a good while for it to complete. */
740	LogRel(("RTMpOnSpecific(%#x): Still running after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
741
742	Timeout.QuadPart = -301000100010; / 30 seconds */
743	rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
744	if (rcNt != STATUS_SUCCESS)
745	LogRel(("RTMpOnSpecific(%#x): Giving up on running worker after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
746	}
747	rc = RTErrConvertFromNtStatus(rcNt);
748	}
749	else
750	{
751	/* CPU is offline.*/
752	KeLowerIrql(bOldIrql);
753	rc = !RTMpIsCpuPossible(idCpu) ? VERR_CPU_NOT_FOUND : VERR_CPU_OFFLINE;
754	}
755
756	rtMpNtOnSpecificRelease(pArgs);
757	return rc;
758	#endif
759	}
760
761
762
763
764	static VOID rtMpNtPokeCpuDummy(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
765	{
766	NOREF(Dpc);
767	NOREF(DeferredContext);
768	NOREF(SystemArgument1);
769	NOREF(SystemArgument2);
770	}
771
772	#ifndef IPRT_TARGET_NT4
773
774	/** Callback used by rtMpPokeCpuUsingBroadcastIpi. */
775	static ULONG_PTR __stdcall rtMpIpiGenericCall(ULONG_PTR Argument)
776	{
777	NOREF(Argument);
778	return 0;
779	}
780
781
782	/**
783	* RTMpPokeCpu worker that uses broadcast IPIs for doing the work.
784	*
785	* @returns VINF_SUCCESS
786	* @param idCpu The CPU identifier.
787	*/
788	int rtMpPokeCpuUsingBroadcastIpi(RTCPUID idCpu)
789	{
790	g_pfnrtKeIpiGenericCall(rtMpIpiGenericCall, 0);
791	return VINF_SUCCESS;
792	}
793
794
795	/**
796	* RTMpPokeCpu worker that uses HalSendSoftwareInterrupt to get the job done.
797	*
798	* This is only really available on AMD64, at least at the time of writing.
799	*
800	* @returns VINF_SUCCESS
801	* @param idCpu The CPU identifier.
802	*/
803	int rtMpPokeCpuUsingHalSendSoftwareInterrupt(RTCPUID idCpu)
804	{
805	g_pfnrtNtHalSendSoftwareInterrupt(idCpu, DISPATCH_LEVEL);
806	return VINF_SUCCESS;
807	}
808
809
810	/**
811	* RTMpPokeCpu worker that uses the Windows 7 and later version of
812	* HalRequestIpip to get the job done.
813	*
814	* @returns VINF_SUCCESS
815	* @param idCpu The CPU identifier.
816	*/
817	int rtMpPokeCpuUsingHalReqestIpiW7Plus(RTCPUID idCpu)
818	{
819	/*
820	* I think we'll let idCpu be an NT processor number and not a HAL processor
821	* index. KeAddProcessorAffinityEx is for HAL and uses HAL processor
822	* indexes as input from what I can tell.
823	*/
824	PROCESSOR_NUMBER ProcNumber = { /Group=/ idCpu / 64, /Number=/ idCpu % 64, /* Reserved=*/ 0};
825	KAFFINITY_EX Target;
826	g_pfnrtKeInitializeAffinityEx(&Target);
827	g_pfnrtKeAddProcessorAffinityEx(&Target, g_pfnrtKeGetProcessorIndexFromNumber(&ProcNumber));
828
829	g_pfnrtHalRequestIpiW7Plus(0, &Target);
830	return VINF_SUCCESS;
831	}
832
833
834	/**
835	* RTMpPokeCpu worker that uses the Vista and earlier version of HalRequestIpip
836	* to get the job done.
837	*
838	* @returns VINF_SUCCESS
839	* @param idCpu The CPU identifier.
840	*/
841	int rtMpPokeCpuUsingHalReqestIpiPreW7(RTCPUID idCpu)
842	{
843	__debugbreak(); /** @todo this code needs testing!! */
844	KAFFINITY Target = 1;
845	Target <<= idCpu;
846	g_pfnrtHalRequestIpiPreW7(Target);
847	return VINF_SUCCESS;
848	}
849
850	#endif /* !IPRT_TARGET_NT4 */
851
852
853	int rtMpPokeCpuUsingDpc(RTCPUID idCpu)
854	{
855	/*
856	* APC fallback.
857	*/
858	static KDPC s_aPokeDpcs[MAXIMUM_PROCESSORS] = {0};
859	static bool s_fPokeDPCsInitialized = false;
860
861	if (!s_fPokeDPCsInitialized)
862	{
863	for (unsigned i = 0; i < RT_ELEMENTS(s_aPokeDpcs); i++)
864	{
865	KeInitializeDpc(&s_aPokeDpcs[i], rtMpNtPokeCpuDummy, NULL);
866	KeSetImportanceDpc(&s_aPokeDpcs[i], HighImportance);
867	KeSetTargetProcessorDpc(&s_aPokeDpcs[i], (int)i);
868	}
869	s_fPokeDPCsInitialized = true;
870	}
871
872	/* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
873	* KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL.
874	*/
875	KIRQL oldIrql;
876	KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
877
878	KeSetImportanceDpc(&s_aPokeDpcs[idCpu], HighImportance);
879	KeSetTargetProcessorDpc(&s_aPokeDpcs[idCpu], (int)idCpu);
880
881	/* Assuming here that high importance DPCs will be delivered immediately; or at least an IPI will be sent immediately.
882	* @note: not true on at least Vista & Windows 7
883	*/
884	BOOLEAN bRet = KeInsertQueueDpc(&s_aPokeDpcs[idCpu], 0, 0);
885
886	KeLowerIrql(oldIrql);
887	return (bRet == TRUE) ? VINF_SUCCESS : VERR_ACCESS_DENIED /* already queued */;
888	}
889
890
891	RTDECL(int) RTMpPokeCpu(RTCPUID idCpu)
892	{
893	if (!RTMpIsCpuOnline(idCpu))
894	return !RTMpIsCpuPossible(idCpu)
895	? VERR_CPU_NOT_FOUND
896	: VERR_CPU_OFFLINE;
897	/* Calls rtMpSendIpiFallback, rtMpSendIpiWin7AndLater or rtMpSendIpiVista. */
898	return g_pfnrtMpPokeCpuWorker(idCpu);
899	}
900
901
902	RTDECL(bool) RTMpOnAllIsConcurrentSafe(void)
903	{
904	return false;
905	}
906

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/nt/mp-r0drv-nt.cpp@ 54410

Download in other formats: