VirtualBox

source: vbox/trunk/src/VBox/Runtime/r0drv/nt/mp-r0drv-nt.cpp@ 61572

Last change on this file since 61572 was 60771, checked in by vboxsync, 9 years ago

RTMpPoke/nt: Removed all the HalSendSoftwareInterrupt code, don't want to spend time trying to figure out why it doesn't work. Just use HalRequestIpi, then no 32/64 differences either.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 28.2 KB
Line 
1/* $Id: mp-r0drv-nt.cpp 60771 2016-04-29 20:49:59Z vboxsync $ */
2/** @file
3 * IPRT - Multiprocessor, Ring-0 Driver, NT.
4 */
5
6/*
7 * Copyright (C) 2008-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 */
26
27
28/*********************************************************************************************************************************
29* Header Files *
30*********************************************************************************************************************************/
31#include "the-nt-kernel.h"
32
33#include <iprt/mp.h>
34#include <iprt/cpuset.h>
35#include <iprt/err.h>
36#include <iprt/asm.h>
37#include <iprt/log.h>
38#include <iprt/time.h>
39#include "r0drv/mp-r0drv.h"
40#include "internal-r0drv-nt.h"
41
42
43/*********************************************************************************************************************************
44* Structures and Typedefs *
45*********************************************************************************************************************************/
46typedef enum
47{
48 RT_NT_CPUID_SPECIFIC,
49 RT_NT_CPUID_PAIR,
50 RT_NT_CPUID_OTHERS,
51 RT_NT_CPUID_ALL
52} RT_NT_CPUID;
53
54
55/**
56 * Used by the RTMpOnSpecific.
57 */
58typedef struct RTMPNTONSPECIFICARGS
59{
60 /** Set if we're executing. */
61 bool volatile fExecuting;
62 /** Set when done executing. */
63 bool volatile fDone;
64 /** Number of references to this heap block. */
65 uint32_t volatile cRefs;
66 /** Event that the calling thread is waiting on. */
67 KEVENT DoneEvt;
68 /** The deferred procedure call object. */
69 KDPC Dpc;
70 /** The callback argument package. */
71 RTMPARGS CallbackArgs;
72} RTMPNTONSPECIFICARGS;
73/** Pointer to an argument/state structure for RTMpOnSpecific on NT. */
74typedef RTMPNTONSPECIFICARGS *PRTMPNTONSPECIFICARGS;
75
76
77
78/* test a couple of assumption. */
79AssertCompile(MAXIMUM_PROCESSORS <= RTCPUSET_MAX_CPUS);
80AssertCompile(NIL_RTCPUID >= MAXIMUM_PROCESSORS);
81
82/** @todo
83 * We cannot do other than assume a 1:1 relationship between the
84 * affinity mask and the process despite the vagueness/warnings in
85 * the docs. If someone knows a better way to get this done, please
86 * let bird know.
87 */
88
89
90RTDECL(RTCPUID) RTMpCpuId(void)
91{
92 /* WDK upgrade warning: PCR->Number changed from BYTE to WORD. */
93 return KeGetCurrentProcessorNumber();
94}
95
96
97RTDECL(int) RTMpCurSetIndex(void)
98{
99 /* WDK upgrade warning: PCR->Number changed from BYTE to WORD. */
100 return KeGetCurrentProcessorNumber();
101}
102
103
104RTDECL(int) RTMpCurSetIndexAndId(PRTCPUID pidCpu)
105{
106 return *pidCpu = KeGetCurrentProcessorNumber();
107}
108
109
110RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu)
111{
112 return idCpu < MAXIMUM_PROCESSORS ? (int)idCpu : -1;
113}
114
115
116RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu)
117{
118 return (unsigned)iCpu < MAXIMUM_PROCESSORS ? iCpu : NIL_RTCPUID;
119}
120
121
122RTDECL(RTCPUID) RTMpGetMaxCpuId(void)
123{
124 /** @todo use KeQueryMaximumProcessorCount on vista+ */
125 return MAXIMUM_PROCESSORS - 1;
126}
127
128
129RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
130{
131 if (idCpu >= MAXIMUM_PROCESSORS)
132 return false;
133
134#if 0 /* this isn't safe at all IRQLs (great work guys) */
135 KAFFINITY Mask = KeQueryActiveProcessors();
136 return !!(Mask & RT_BIT_64(idCpu));
137#else
138 return RTCpuSetIsMember(&g_rtMpNtCpuSet, idCpu);
139#endif
140}
141
142
143RTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu)
144{
145 /* Cannot easily distinguish between online and offline cpus. */
146 /** @todo online/present cpu stuff must be corrected for proper W2K8 support
147 * (KeQueryMaximumProcessorCount). */
148 return RTMpIsCpuOnline(idCpu);
149}
150
151
152
153RTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet)
154{
155 /** @todo online/present cpu stuff must be corrected for proper W2K8 support
156 * (KeQueryMaximumProcessorCount). */
157 return RTMpGetOnlineSet(pSet);
158}
159
160
161RTDECL(RTCPUID) RTMpGetCount(void)
162{
163 /** @todo online/present cpu stuff must be corrected for proper W2K8 support
164 * (KeQueryMaximumProcessorCount). */
165 return RTMpGetOnlineCount();
166}
167
168
169RTDECL(PRTCPUSET) RTMpGetOnlineSet(PRTCPUSET pSet)
170{
171#if 0 /* this isn't safe at all IRQLs (great work guys) */
172 KAFFINITY Mask = KeQueryActiveProcessors();
173 return RTCpuSetFromU64(pSet, Mask);
174#else
175 *pSet = g_rtMpNtCpuSet;
176 return pSet;
177#endif
178}
179
180
181RTDECL(RTCPUID) RTMpGetOnlineCount(void)
182{
183 RTCPUSET Set;
184 RTMpGetOnlineSet(&Set);
185 return RTCpuSetCount(&Set);
186}
187
188
189#if 0
190/* Experiment with checking the undocumented KPRCB structure
191 * 'dt nt!_kprcb 0xaddress' shows the layout
192 */
193typedef struct
194{
195 LIST_ENTRY DpcListHead;
196 ULONG_PTR DpcLock;
197 volatile ULONG DpcQueueDepth;
198 ULONG DpcQueueCount;
199} KDPC_DATA, *PKDPC_DATA;
200
201RTDECL(bool) RTMpIsCpuWorkPending(void)
202{
203 uint8_t *pkprcb;
204 PKDPC_DATA pDpcData;
205
206 _asm {
207 mov eax, fs:0x20
208 mov pkprcb, eax
209 }
210 pDpcData = (PKDPC_DATA)(pkprcb + 0x19e0);
211 if (pDpcData->DpcQueueDepth)
212 return true;
213
214 pDpcData++;
215 if (pDpcData->DpcQueueDepth)
216 return true;
217 return false;
218}
219#else
220RTDECL(bool) RTMpIsCpuWorkPending(void)
221{
222 /** @todo not implemented */
223 return false;
224}
225#endif
226
227
228/**
229 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
230 * the RTMpOnAll case.
231 *
232 * @param uUserCtx The user context argument (PRTMPARGS).
233 */
234static ULONG_PTR rtmpNtOnAllBroadcastIpiWrapper(ULONG_PTR uUserCtx)
235{
236 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
237 /*ASMAtomicIncU32(&pArgs->cHits); - not needed */
238 pArgs->pfnWorker(KeGetCurrentProcessorNumber(), pArgs->pvUser1, pArgs->pvUser2);
239 return 0;
240}
241
242
243/**
244 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
245 * the RTMpOnOthers case.
246 *
247 * @param uUserCtx The user context argument (PRTMPARGS).
248 */
249static ULONG_PTR rtmpNtOnOthersBroadcastIpiWrapper(ULONG_PTR uUserCtx)
250{
251 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
252 RTCPUID idCpu = KeGetCurrentProcessorNumber();
253 if (pArgs->idCpu != idCpu)
254 {
255 /*ASMAtomicIncU32(&pArgs->cHits); - not needed */
256 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
257 }
258 return 0;
259}
260
261
262/**
263 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
264 * the RTMpOnPair case.
265 *
266 * @param uUserCtx The user context argument (PRTMPARGS).
267 */
268static ULONG_PTR rtmpNtOnPairBroadcastIpiWrapper(ULONG_PTR uUserCtx)
269{
270 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
271 RTCPUID idCpu = KeGetCurrentProcessorNumber();
272 if ( pArgs->idCpu == idCpu
273 || pArgs->idCpu2 == idCpu)
274 {
275 ASMAtomicIncU32(&pArgs->cHits);
276 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
277 }
278 return 0;
279}
280
281
282/**
283 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
284 * the RTMpOnSpecific case.
285 *
286 * @param uUserCtx The user context argument (PRTMPARGS).
287 */
288static ULONG_PTR rtmpNtOnSpecificBroadcastIpiWrapper(ULONG_PTR uUserCtx)
289{
290 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
291 RTCPUID idCpu = KeGetCurrentProcessorNumber();
292 if (pArgs->idCpu == idCpu)
293 {
294 ASMAtomicIncU32(&pArgs->cHits);
295 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
296 }
297 return 0;
298}
299
300
301/**
302 * Internal worker for the RTMpOn* APIs using KeIpiGenericCall.
303 *
304 * @returns VINF_SUCCESS.
305 * @param pfnWorker The callback.
306 * @param pvUser1 User argument 1.
307 * @param pvUser2 User argument 2.
308 * @param pfnNativeWrapper The wrapper between the NT and IPRT callbacks.
309 * @param idCpu First CPU to match, ultimately specific to the
310 * pfnNativeWrapper used.
311 * @param idCpu2 Second CPU to match, ultimately specific to the
312 * pfnNativeWrapper used.
313 * @param pcHits Where to return the number of this. Optional.
314 */
315static int rtMpCallUsingBroadcastIpi(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2,
316 PKIPI_BROADCAST_WORKER pfnNativeWrapper, RTCPUID idCpu, RTCPUID idCpu2,
317 uint32_t *pcHits)
318{
319 RTMPARGS Args;
320 Args.pfnWorker = pfnWorker;
321 Args.pvUser1 = pvUser1;
322 Args.pvUser2 = pvUser2;
323 Args.idCpu = idCpu;
324 Args.idCpu2 = idCpu2;
325 Args.cRefs = 0;
326 Args.cHits = 0;
327
328 AssertPtr(g_pfnrtKeIpiGenericCall);
329 g_pfnrtKeIpiGenericCall(pfnNativeWrapper, (uintptr_t)&Args);
330 if (pcHits)
331 *pcHits = Args.cHits;
332 return VINF_SUCCESS;
333}
334
335
336/**
337 * Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
338 *
339 * @param Dpc DPC object
340 * @param DeferredContext Context argument specified by KeInitializeDpc
341 * @param SystemArgument1 Argument specified by KeInsertQueueDpc
342 * @param SystemArgument2 Argument specified by KeInsertQueueDpc
343 */
344static VOID rtmpNtDPCWrapper(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
345{
346 PRTMPARGS pArgs = (PRTMPARGS)DeferredContext;
347
348 ASMAtomicIncU32(&pArgs->cHits);
349 pArgs->pfnWorker(KeGetCurrentProcessorNumber(), pArgs->pvUser1, pArgs->pvUser2);
350
351 /* Dereference the argument structure. */
352 int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
353 Assert(cRefs >= 0);
354 if (cRefs == 0)
355 ExFreePool(pArgs);
356}
357
358
359/**
360 * Internal worker for the RTMpOn* APIs.
361 *
362 * @returns IPRT status code.
363 * @param pfnWorker The callback.
364 * @param pvUser1 User argument 1.
365 * @param pvUser2 User argument 2.
366 * @param enmCpuid What to do / is idCpu valid.
367 * @param idCpu Used if enmCpuid is RT_NT_CPUID_SPECIFIC or
368 * RT_NT_CPUID_PAIR, otherwise ignored.
369 * @param idCpu2 Used if enmCpuid is RT_NT_CPUID_PAIR, otherwise ignored.
370 * @param pcHits Where to return the number of this. Optional.
371 */
372static int rtMpCallUsingDpcs(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2,
373 RT_NT_CPUID enmCpuid, RTCPUID idCpu, RTCPUID idCpu2, uint32_t *pcHits)
374{
375 PRTMPARGS pArgs;
376 KDPC *paExecCpuDpcs;
377
378#if 0
379 /* KeFlushQueuedDpcs must be run at IRQL PASSIVE_LEVEL according to MSDN, but the
380 * driver verifier doesn't complain...
381 */
382 AssertMsg(KeGetCurrentIrql() == PASSIVE_LEVEL, ("%d != %d (PASSIVE_LEVEL)\n", KeGetCurrentIrql(), PASSIVE_LEVEL));
383#endif
384
385#ifdef IPRT_TARGET_NT4
386 KAFFINITY Mask;
387 /* g_pfnrtNt* are not present on NT anyway. */
388 return VERR_NOT_SUPPORTED;
389#else
390 KAFFINITY Mask = KeQueryActiveProcessors();
391#endif
392
393 /* KeFlushQueuedDpcs is not present in Windows 2000; import it dynamically so we can just fail this call. */
394 if (!g_pfnrtNtKeFlushQueuedDpcs)
395 return VERR_NOT_SUPPORTED;
396
397 pArgs = (PRTMPARGS)ExAllocatePoolWithTag(NonPagedPool, MAXIMUM_PROCESSORS*sizeof(KDPC) + sizeof(RTMPARGS), (ULONG)'RTMp');
398 if (!pArgs)
399 return VERR_NO_MEMORY;
400
401 pArgs->pfnWorker = pfnWorker;
402 pArgs->pvUser1 = pvUser1;
403 pArgs->pvUser2 = pvUser2;
404 pArgs->idCpu = NIL_RTCPUID;
405 pArgs->idCpu2 = NIL_RTCPUID;
406 pArgs->cHits = 0;
407 pArgs->cRefs = 1;
408
409 paExecCpuDpcs = (KDPC *)(pArgs + 1);
410
411 if (enmCpuid == RT_NT_CPUID_SPECIFIC)
412 {
413 KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
414 KeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
415 KeSetTargetProcessorDpc(&paExecCpuDpcs[0], (int)idCpu);
416 pArgs->idCpu = idCpu;
417 }
418 else if (enmCpuid == RT_NT_CPUID_PAIR)
419 {
420 KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
421 KeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
422 KeSetTargetProcessorDpc(&paExecCpuDpcs[0], (int)idCpu);
423 pArgs->idCpu = idCpu;
424
425 KeInitializeDpc(&paExecCpuDpcs[1], rtmpNtDPCWrapper, pArgs);
426 KeSetImportanceDpc(&paExecCpuDpcs[1], HighImportance);
427 KeSetTargetProcessorDpc(&paExecCpuDpcs[1], (int)idCpu2);
428 pArgs->idCpu2 = idCpu2;
429 }
430 else
431 {
432 for (unsigned i = 0; i < MAXIMUM_PROCESSORS; i++)
433 {
434 KeInitializeDpc(&paExecCpuDpcs[i], rtmpNtDPCWrapper, pArgs);
435 KeSetImportanceDpc(&paExecCpuDpcs[i], HighImportance);
436 KeSetTargetProcessorDpc(&paExecCpuDpcs[i], i);
437 }
438 }
439
440 /* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
441 * KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL.
442 */
443 KIRQL oldIrql;
444 KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
445
446 /*
447 * We cannot do other than assume a 1:1 relationship between the
448 * affinity mask and the process despite the warnings in the docs.
449 * If someone knows a better way to get this done, please let bird know.
450 */
451 ASMCompilerBarrier(); /* paranoia */
452 if (enmCpuid == RT_NT_CPUID_SPECIFIC)
453 {
454 ASMAtomicIncS32(&pArgs->cRefs);
455 BOOLEAN ret = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
456 Assert(ret);
457 }
458 else if (enmCpuid == RT_NT_CPUID_PAIR)
459 {
460 ASMAtomicIncS32(&pArgs->cRefs);
461 BOOLEAN ret = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
462 Assert(ret);
463
464 ASMAtomicIncS32(&pArgs->cRefs);
465 ret = KeInsertQueueDpc(&paExecCpuDpcs[1], 0, 0);
466 Assert(ret);
467 }
468 else
469 {
470 unsigned iSelf = KeGetCurrentProcessorNumber();
471
472 for (unsigned i = 0; i < MAXIMUM_PROCESSORS; i++)
473 {
474 if ( (i != iSelf)
475 && (Mask & RT_BIT_64(i)))
476 {
477 ASMAtomicIncS32(&pArgs->cRefs);
478 BOOLEAN ret = KeInsertQueueDpc(&paExecCpuDpcs[i], 0, 0);
479 Assert(ret);
480 }
481 }
482 if (enmCpuid != RT_NT_CPUID_OTHERS)
483 pfnWorker(iSelf, pvUser1, pvUser2);
484 }
485
486 KeLowerIrql(oldIrql);
487
488 /* Flush all DPCs and wait for completion. (can take long!) */
489 /** @todo Consider changing this to an active wait using some atomic inc/dec
490 * stuff (and check for the current cpu above in the specific case). */
491 /** @todo Seems KeFlushQueuedDpcs doesn't wait for the DPCs to be completely
492 * executed. Seen pArgs being freed while some CPU was using it before
493 * cRefs was added. */
494 g_pfnrtNtKeFlushQueuedDpcs();
495
496 if (pcHits)
497 *pcHits = pArgs->cHits;
498
499 /* Dereference the argument structure. */
500 int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
501 Assert(cRefs >= 0);
502 if (cRefs == 0)
503 ExFreePool(pArgs);
504
505 return VINF_SUCCESS;
506}
507
508
509RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
510{
511 if (g_pfnrtKeIpiGenericCall)
512 return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnAllBroadcastIpiWrapper,
513 NIL_RTCPUID, NIL_RTCPUID, NULL);
514 return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_ALL, NIL_RTCPUID, NIL_RTCPUID, NULL);
515}
516
517
518RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
519{
520 if (g_pfnrtKeIpiGenericCall)
521 return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnOthersBroadcastIpiWrapper,
522 NIL_RTCPUID, NIL_RTCPUID, NULL);
523 return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_OTHERS, NIL_RTCPUID, NIL_RTCPUID, NULL);
524}
525
526
527RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
528{
529 int rc;
530 AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
531 AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
532 if ((fFlags & RTMPON_F_CONCURRENT_EXEC) && !g_pfnrtKeIpiGenericCall)
533 return VERR_NOT_SUPPORTED;
534
535 /*
536 * Check that both CPUs are online before doing the broadcast call.
537 */
538 if ( RTMpIsCpuOnline(idCpu1)
539 && RTMpIsCpuOnline(idCpu2))
540 {
541 /*
542 * The broadcast IPI isn't quite as bad as it could have been, because
543 * it looks like windows doesn't synchronize CPUs on the way out, they
544 * seems to get back to normal work while the pair is still busy.
545 */
546 uint32_t cHits = 0;
547 if (g_pfnrtKeIpiGenericCall)
548 rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnPairBroadcastIpiWrapper, idCpu1, idCpu2, &cHits);
549 else
550 rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_PAIR, idCpu1, idCpu2, &cHits);
551 if (RT_SUCCESS(rc))
552 {
553 Assert(cHits <= 2);
554 if (cHits == 2)
555 rc = VINF_SUCCESS;
556 else if (cHits == 1)
557 rc = VERR_NOT_ALL_CPUS_SHOWED;
558 else if (cHits == 0)
559 rc = VERR_CPU_OFFLINE;
560 else
561 rc = VERR_CPU_IPE_1;
562 }
563 }
564 /*
565 * A CPU must be present to be considered just offline.
566 */
567 else if ( RTMpIsCpuPresent(idCpu1)
568 && RTMpIsCpuPresent(idCpu2))
569 rc = VERR_CPU_OFFLINE;
570 else
571 rc = VERR_CPU_NOT_FOUND;
572 return rc;
573}
574
575
576RTDECL(bool) RTMpOnPairIsConcurrentExecSupported(void)
577{
578 return g_pfnrtKeIpiGenericCall != NULL;
579}
580
581
582/**
583 * Releases a reference to a RTMPNTONSPECIFICARGS heap allocation, freeing it
584 * when the last reference is released.
585 */
586DECLINLINE(void) rtMpNtOnSpecificRelease(PRTMPNTONSPECIFICARGS pArgs)
587{
588 uint32_t cRefs = ASMAtomicDecU32(&pArgs->cRefs);
589 AssertMsg(cRefs <= 1, ("cRefs=%#x\n", cRefs));
590 if (cRefs == 0)
591 ExFreePool(pArgs);
592}
593
594
595/**
596 * Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
597 *
598 * @param Dpc DPC object
599 * @param DeferredContext Context argument specified by KeInitializeDpc
600 * @param SystemArgument1 Argument specified by KeInsertQueueDpc
601 * @param SystemArgument2 Argument specified by KeInsertQueueDpc
602 */
603static VOID rtMpNtOnSpecificDpcWrapper(IN PKDPC Dpc, IN PVOID DeferredContext,
604 IN PVOID SystemArgument1, IN PVOID SystemArgument2)
605{
606 PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)DeferredContext;
607 ASMAtomicWriteBool(&pArgs->fExecuting, true);
608
609 pArgs->CallbackArgs.pfnWorker(KeGetCurrentProcessorNumber(), pArgs->CallbackArgs.pvUser1, pArgs->CallbackArgs.pvUser2);
610
611 ASMAtomicWriteBool(&pArgs->fDone, true);
612 KeSetEvent(&pArgs->DoneEvt, 1 /*PriorityIncrement*/, FALSE /*Wait*/);
613
614 rtMpNtOnSpecificRelease(pArgs);
615}
616
617
618RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
619{
620 /*
621 * Don't try mess with an offline CPU.
622 */
623 if (!RTMpIsCpuOnline(idCpu))
624 return !RTMpIsCpuPossible(idCpu)
625 ? VERR_CPU_NOT_FOUND
626 : VERR_CPU_OFFLINE;
627
628 /*
629 * Use the broadcast IPI routine if there are no more than two CPUs online,
630 * or if the current IRQL is unsuitable for KeWaitForSingleObject.
631 */
632 int rc;
633 uint32_t cHits = 0;
634 if ( g_pfnrtKeIpiGenericCall
635 && ( RTMpGetOnlineCount() <= 2
636 || KeGetCurrentIrql() > APC_LEVEL)
637 )
638 {
639 rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnSpecificBroadcastIpiWrapper,
640 idCpu, NIL_RTCPUID, &cHits);
641 if (RT_SUCCESS(rc))
642 {
643 if (cHits == 1)
644 return VINF_SUCCESS;
645 rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
646 }
647 return rc;
648 }
649
650#if 0
651 rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_SPECIFIC, idCpu, NIL_RTCPUID, &cHits);
652 if (RT_SUCCESS(rc))
653 {
654 if (cHits == 1)
655 return VINF_SUCCESS;
656 rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
657 }
658 return rc;
659
660#else
661 /*
662 * Initialize the argument package and the objects within it.
663 * The package is referenced counted to avoid unnecessary spinning to
664 * synchronize cleanup and prevent stack corruption.
665 */
666 PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)ExAllocatePoolWithTag(NonPagedPool, sizeof(*pArgs), (ULONG)'RTMp');
667 if (!pArgs)
668 return VERR_NO_MEMORY;
669 pArgs->cRefs = 2;
670 pArgs->fExecuting = false;
671 pArgs->fDone = false;
672 pArgs->CallbackArgs.pfnWorker = pfnWorker;
673 pArgs->CallbackArgs.pvUser1 = pvUser1;
674 pArgs->CallbackArgs.pvUser2 = pvUser2;
675 pArgs->CallbackArgs.idCpu = idCpu;
676 pArgs->CallbackArgs.cHits = 0;
677 pArgs->CallbackArgs.cRefs = 2;
678 KeInitializeEvent(&pArgs->DoneEvt, SynchronizationEvent, FALSE /* not signalled */);
679 KeInitializeDpc(&pArgs->Dpc, rtMpNtOnSpecificDpcWrapper, pArgs);
680 KeSetImportanceDpc(&pArgs->Dpc, HighImportance);
681 KeSetTargetProcessorDpc(&pArgs->Dpc, (int)idCpu);
682
683 /*
684 * Disable preemption while we check the current processor and inserts the DPC.
685 */
686 KIRQL bOldIrql;
687 KeRaiseIrql(DISPATCH_LEVEL, &bOldIrql);
688 ASMCompilerBarrier(); /* paranoia */
689
690 if (RTMpCpuId() == idCpu)
691 {
692 /* Just execute the callback on the current CPU. */
693 pfnWorker(idCpu, pvUser1, pvUser2);
694 KeLowerIrql(bOldIrql);
695
696 ExFreePool(pArgs);
697 return VINF_SUCCESS;
698 }
699
700 /* Different CPU, so queue it if the CPU is still online. */
701 if (RTMpIsCpuOnline(idCpu))
702 {
703 BOOLEAN fRc = KeInsertQueueDpc(&pArgs->Dpc, 0, 0);
704 Assert(fRc);
705 KeLowerIrql(bOldIrql);
706
707 uint64_t const nsRealWaitTS = RTTimeNanoTS();
708
709 /*
710 * Wait actively for a while in case the CPU/thread responds quickly.
711 */
712 uint32_t cLoopsLeft = 0x20000;
713 while (cLoopsLeft-- > 0)
714 {
715 if (pArgs->fDone)
716 {
717 rtMpNtOnSpecificRelease(pArgs);
718 return VINF_SUCCESS;
719 }
720 ASMNopPause();
721 }
722
723 /*
724 * It didn't respond, so wait on the event object, poking the CPU if it's slow.
725 */
726 LARGE_INTEGER Timeout;
727 Timeout.QuadPart = -10000; /* 1ms */
728 NTSTATUS rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
729 if (rcNt == STATUS_SUCCESS)
730 {
731 rtMpNtOnSpecificRelease(pArgs);
732 return VINF_SUCCESS;
733 }
734
735 /* If it hasn't respondend yet, maybe poke it and wait some more. */
736 if (rcNt == STATUS_TIMEOUT)
737 {
738#ifndef IPRT_TARGET_NT4
739 if ( !pArgs->fExecuting
740 && ( g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalReqestIpiW7Plus
741 || g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalReqestIpiPreW7))
742 RTMpPokeCpu(idCpu);
743#endif
744
745 Timeout.QuadPart = -1280000; /* 128ms */
746 rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
747 if (rcNt == STATUS_SUCCESS)
748 {
749 rtMpNtOnSpecificRelease(pArgs);
750 return VINF_SUCCESS;
751 }
752 }
753
754 /*
755 * Something weird is happening, try bail out.
756 */
757 if (KeRemoveQueueDpc(&pArgs->Dpc))
758 {
759 ExFreePool(pArgs); /* DPC was still queued, so we can return without further ado. */
760 LogRel(("RTMpOnSpecific(%#x): Not processed after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
761 }
762 else
763 {
764 /* DPC is running, wait a good while for it to complete. */
765 LogRel(("RTMpOnSpecific(%#x): Still running after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
766
767 Timeout.QuadPart = -30*1000*1000*10; /* 30 seconds */
768 rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
769 if (rcNt != STATUS_SUCCESS)
770 LogRel(("RTMpOnSpecific(%#x): Giving up on running worker after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
771 }
772 rc = RTErrConvertFromNtStatus(rcNt);
773 }
774 else
775 {
776 /* CPU is offline.*/
777 KeLowerIrql(bOldIrql);
778 rc = !RTMpIsCpuPossible(idCpu) ? VERR_CPU_NOT_FOUND : VERR_CPU_OFFLINE;
779 }
780
781 rtMpNtOnSpecificRelease(pArgs);
782 return rc;
783#endif
784}
785
786
787
788
789static VOID rtMpNtPokeCpuDummy(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
790{
791 NOREF(Dpc);
792 NOREF(DeferredContext);
793 NOREF(SystemArgument1);
794 NOREF(SystemArgument2);
795}
796
797#ifndef IPRT_TARGET_NT4
798
799/** Callback used by rtMpPokeCpuUsingBroadcastIpi. */
800static ULONG_PTR rtMpIpiGenericCall(ULONG_PTR Argument)
801{
802 NOREF(Argument);
803 return 0;
804}
805
806
807/**
808 * RTMpPokeCpu worker that uses broadcast IPIs for doing the work.
809 *
810 * @returns VINF_SUCCESS
811 * @param idCpu The CPU identifier.
812 */
813int rtMpPokeCpuUsingBroadcastIpi(RTCPUID idCpu)
814{
815 g_pfnrtKeIpiGenericCall(rtMpIpiGenericCall, 0);
816 return VINF_SUCCESS;
817}
818
819
820/**
821 * RTMpPokeCpu worker that uses the Windows 7 and later version of
822 * HalRequestIpip to get the job done.
823 *
824 * @returns VINF_SUCCESS
825 * @param idCpu The CPU identifier.
826 */
827int rtMpPokeCpuUsingHalReqestIpiW7Plus(RTCPUID idCpu)
828{
829 /*
830 * I think we'll let idCpu be an NT processor number and not a HAL processor
831 * index. KeAddProcessorAffinityEx is for HAL and uses HAL processor
832 * indexes as input from what I can tell.
833 */
834 PROCESSOR_NUMBER ProcNumber = { /*Group=*/ idCpu / 64, /*Number=*/ idCpu % 64, /* Reserved=*/ 0};
835 KAFFINITY_EX Target;
836 g_pfnrtKeInitializeAffinityEx(&Target);
837 g_pfnrtKeAddProcessorAffinityEx(&Target, g_pfnrtKeGetProcessorIndexFromNumber(&ProcNumber));
838
839 g_pfnrtHalRequestIpiW7Plus(0, &Target);
840 return VINF_SUCCESS;
841}
842
843
844/**
845 * RTMpPokeCpu worker that uses the Vista and earlier version of HalRequestIpip
846 * to get the job done.
847 *
848 * @returns VINF_SUCCESS
849 * @param idCpu The CPU identifier.
850 */
851int rtMpPokeCpuUsingHalReqestIpiPreW7(RTCPUID idCpu)
852{
853 __debugbreak(); /** @todo this code needs testing!! */
854 KAFFINITY Target = 1;
855 Target <<= idCpu;
856 g_pfnrtHalRequestIpiPreW7(Target);
857 return VINF_SUCCESS;
858}
859
860#endif /* !IPRT_TARGET_NT4 */
861
862
863int rtMpPokeCpuUsingDpc(RTCPUID idCpu)
864{
865 /*
866 * APC fallback.
867 */
868 static KDPC s_aPokeDpcs[MAXIMUM_PROCESSORS] = {0};
869 static bool s_fPokeDPCsInitialized = false;
870
871 if (!s_fPokeDPCsInitialized)
872 {
873 for (unsigned i = 0; i < RT_ELEMENTS(s_aPokeDpcs); i++)
874 {
875 KeInitializeDpc(&s_aPokeDpcs[i], rtMpNtPokeCpuDummy, NULL);
876 KeSetImportanceDpc(&s_aPokeDpcs[i], HighImportance);
877 KeSetTargetProcessorDpc(&s_aPokeDpcs[i], (int)i);
878 }
879 s_fPokeDPCsInitialized = true;
880 }
881
882 /* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
883 * KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL.
884 */
885 KIRQL oldIrql;
886 KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
887
888 KeSetImportanceDpc(&s_aPokeDpcs[idCpu], HighImportance);
889 KeSetTargetProcessorDpc(&s_aPokeDpcs[idCpu], (int)idCpu);
890
891 /* Assuming here that high importance DPCs will be delivered immediately; or at least an IPI will be sent immediately.
892 * @note: not true on at least Vista & Windows 7
893 */
894 BOOLEAN bRet = KeInsertQueueDpc(&s_aPokeDpcs[idCpu], 0, 0);
895
896 KeLowerIrql(oldIrql);
897 return (bRet == TRUE) ? VINF_SUCCESS : VERR_ACCESS_DENIED /* already queued */;
898}
899
900
901RTDECL(int) RTMpPokeCpu(RTCPUID idCpu)
902{
903 if (!RTMpIsCpuOnline(idCpu))
904 return !RTMpIsCpuPossible(idCpu)
905 ? VERR_CPU_NOT_FOUND
906 : VERR_CPU_OFFLINE;
907 /* Calls rtMpSendIpiFallback, rtMpSendIpiWin7AndLater or rtMpSendIpiVista. */
908 return g_pfnrtMpPokeCpuWorker(idCpu);
909}
910
911
912RTDECL(bool) RTMpOnAllIsConcurrentSafe(void)
913{
914 return false;
915}
916
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette