VirtualBox

source: vbox/trunk/src/VBox/Runtime/r0drv/nt/mp-r0drv-nt.cpp@ 70206

Last change on this file since 70206 was 70153, checked in by vboxsync, 7 years ago

IPRT: Don't call ExAllocatePool[WithTag] directly unless you must, use RTMemAlloc et al instead. Simplified the NT4 ExAllocatePool/ExAllocatePoolWithTag unwrapping.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 74.1 KB
Line 
1/* $Id: mp-r0drv-nt.cpp 70153 2017-12-15 15:07:27Z vboxsync $ */
2/** @file
3 * IPRT - Multiprocessor, Ring-0 Driver, NT.
4 */
5
6/*
7 * Copyright (C) 2008-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 */
26
27
28/*********************************************************************************************************************************
29* Header Files *
30*********************************************************************************************************************************/
31#include "the-nt-kernel.h"
32
33#include <iprt/mp.h>
34#include <iprt/cpuset.h>
35#include <iprt/err.h>
36#include <iprt/asm.h>
37#include <iprt/log.h>
38#include <iprt/mem.h>
39#include <iprt/time.h>
40#include "r0drv/mp-r0drv.h"
41#include "symdb.h"
42#include "internal-r0drv-nt.h"
43#include "internal/mp.h"
44
45
46/*********************************************************************************************************************************
47* Structures and Typedefs *
48*********************************************************************************************************************************/
49typedef enum
50{
51 RT_NT_CPUID_SPECIFIC,
52 RT_NT_CPUID_PAIR,
53 RT_NT_CPUID_OTHERS,
54 RT_NT_CPUID_ALL
55} RT_NT_CPUID;
56
57
58/**
59 * Used by the RTMpOnSpecific.
60 */
61typedef struct RTMPNTONSPECIFICARGS
62{
63 /** Set if we're executing. */
64 bool volatile fExecuting;
65 /** Set when done executing. */
66 bool volatile fDone;
67 /** Number of references to this heap block. */
68 uint32_t volatile cRefs;
69 /** Event that the calling thread is waiting on. */
70 KEVENT DoneEvt;
71 /** The deferred procedure call object. */
72 KDPC Dpc;
73 /** The callback argument package. */
74 RTMPARGS CallbackArgs;
75} RTMPNTONSPECIFICARGS;
76/** Pointer to an argument/state structure for RTMpOnSpecific on NT. */
77typedef RTMPNTONSPECIFICARGS *PRTMPNTONSPECIFICARGS;
78
79
80/*********************************************************************************************************************************
81* Defined Constants And Macros *
82*********************************************************************************************************************************/
83/** Inactive bit for g_aidRtMpNtByCpuSetIdx. */
84#define RTMPNT_ID_F_INACTIVE RT_BIT_32(31)
85
86
87/*********************************************************************************************************************************
88* Global Variables *
89*********************************************************************************************************************************/
90/** Maximum number of processor groups. */
91uint32_t g_cRtMpNtMaxGroups;
92/** Maximum number of processors. */
93uint32_t g_cRtMpNtMaxCpus;
94/** Number of active processors. */
95uint32_t volatile g_cRtMpNtActiveCpus;
96/** The NT CPU set.
97 * KeQueryActiveProcssors() cannot be called at all IRQLs and therefore we'll
98 * have to cache it. Fortunately, NT doesn't really support taking CPUs offline,
99 * and taking them online was introduced with W2K8 where it is intended for virtual
100 * machines and not real HW. We update this, g_cRtMpNtActiveCpus and
101 * g_aidRtMpNtByCpuSetIdx from the rtR0NtMpProcessorChangeCallback.
102 */
103RTCPUSET g_rtMpNtCpuSet;
104
105/** Static per group info.
106 * @remarks With RTCPUSET_MAX_CPUS as 256, this takes up 33KB. */
107static struct
108{
109 /** The max CPUs in the group. */
110 uint16_t cMaxCpus;
111 /** The number of active CPUs at the time of initialization. */
112 uint16_t cActiveCpus;
113 /** CPU set indexes for each CPU in the group. */
114 int16_t aidxCpuSetMembers[64];
115} g_aRtMpNtCpuGroups[RTCPUSET_MAX_CPUS];
116/** Maps CPU set indexes to RTCPUID.
117 * Inactive CPUs has bit 31 set (RTMPNT_ID_F_INACTIVE) so we can identify them
118 * and shuffle duplicates during CPU hotplugging. We assign temporary IDs to
119 * the inactive CPUs starting at g_cRtMpNtMaxCpus - 1, ASSUMING that active
120 * CPUs has IDs from 0 to g_cRtMpNtActiveCpus. */
121RTCPUID g_aidRtMpNtByCpuSetIdx[RTCPUSET_MAX_CPUS];
122/** The handle of the rtR0NtMpProcessorChangeCallback registration. */
123static PVOID g_pvMpCpuChangeCallback = NULL;
124
125
126/*********************************************************************************************************************************
127* Internal Functions *
128*********************************************************************************************************************************/
129static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,
130 PNTSTATUS prcOperationStatus);
131static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo);
132
133
134
135/**
136 * Initalizes multiprocessor globals (called by rtR0InitNative).
137 *
138 * @returns IPRT status code.
139 * @param pOsVerInfo Version information.
140 */
141DECLHIDDEN(int) rtR0MpNtInit(RTNTSDBOSVER const *pOsVerInfo)
142{
143#define MY_CHECK_BREAK(a_Check, a_DbgPrintArgs) \
144 AssertMsgBreakStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
145#define MY_CHECK_RETURN(a_Check, a_DbgPrintArgs, a_rcRet) \
146 AssertMsgReturnStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs, a_rcRet)
147#define MY_CHECK(a_Check, a_DbgPrintArgs) \
148 AssertMsgStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
149
150 /*
151 * API combination checks.
152 */
153 MY_CHECK_RETURN(!g_pfnrtKeSetTargetProcessorDpcEx || g_pfnrtKeGetProcessorNumberFromIndex,
154 ("IPRT: Fatal: Missing KeSetTargetProcessorDpcEx without KeGetProcessorNumberFromIndex!\n"),
155 VERR_SYMBOL_NOT_FOUND);
156
157 /*
158 * Get max number of processor groups.
159 *
160 * We may need to upadjust this number below, because windows likes to keep
161 * all options open when it comes to hotplugged CPU group assignments. A
162 * server advertising up to 64 CPUs in the ACPI table will get a result of
163 * 64 from KeQueryMaximumGroupCount. That makes sense. However, when windows
164 * server 2012 does a two processor group setup for it, the sum of the
165 * GroupInfo[*].MaximumProcessorCount members below is 128. This is probably
166 * because windows doesn't want to make decisions grouping of hotpluggable CPUs.
167 * So, we need to bump the maximum count to 128 below do deal with this as we
168 * want to have valid CPU set indexes for all potential CPUs - how could we
169 * otherwise use the RTMpGetSet() result and also RTCpuSetCount(RTMpGetSet())
170 * should equal RTMpGetCount().
171 */
172 if (g_pfnrtKeQueryMaximumGroupCount)
173 {
174 g_cRtMpNtMaxGroups = g_pfnrtKeQueryMaximumGroupCount();
175 MY_CHECK_RETURN(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,
176 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
177 VERR_MP_TOO_MANY_CPUS);
178 }
179 else
180 g_cRtMpNtMaxGroups = 1;
181
182 /*
183 * Get max number CPUs.
184 * This also defines the range of NT CPU indexes, RTCPUID and index into RTCPUSET.
185 */
186 if (g_pfnrtKeQueryMaximumProcessorCountEx)
187 {
188 g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS);
189 MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
190 ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCountEx]\n",
191 g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
192 VERR_MP_TOO_MANY_CPUS);
193 }
194 else if (g_pfnrtKeQueryMaximumProcessorCount)
195 {
196 g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCount();
197 MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
198 ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCount]\n",
199 g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
200 VERR_MP_TOO_MANY_CPUS);
201 }
202 else if (g_pfnrtKeQueryActiveProcessors)
203 {
204 KAFFINITY fActiveProcessors = g_pfnrtKeQueryActiveProcessors();
205 MY_CHECK_RETURN(fActiveProcessors != 0,
206 ("IPRT: Fatal: KeQueryActiveProcessors returned 0!\n"),
207 VERR_INTERNAL_ERROR_2);
208 g_cRtMpNtMaxCpus = 0;
209 do
210 {
211 g_cRtMpNtMaxCpus++;
212 fActiveProcessors >>= 1;
213 } while (fActiveProcessors);
214 }
215 else
216 g_cRtMpNtMaxCpus = KeNumberProcessors;
217
218 /*
219 * Just because we're a bit paranoid about getting something wrong wrt to the
220 * kernel interfaces, we try 16 times to get the KeQueryActiveProcessorCountEx
221 * and KeQueryLogicalProcessorRelationship information to match up.
222 */
223 for (unsigned cTries = 0;; cTries++)
224 {
225 /*
226 * Get number of active CPUs.
227 */
228 if (g_pfnrtKeQueryActiveProcessorCountEx)
229 {
230 g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS);
231 MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0,
232 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCountEx]\n",
233 g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus),
234 VERR_MP_TOO_MANY_CPUS);
235 }
236 else if (g_pfnrtKeQueryActiveProcessorCount)
237 {
238 g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCount(NULL);
239 MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0,
240 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCount]\n",
241 g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus),
242 VERR_MP_TOO_MANY_CPUS);
243 }
244 else
245 g_cRtMpNtActiveCpus = g_cRtMpNtMaxCpus;
246
247 /*
248 * Query the details for the groups to figure out which CPUs are online as
249 * well as the NT index limit.
250 */
251 for (unsigned i = 0; i < RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx); i++)
252#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
253 g_aidRtMpNtByCpuSetIdx[i] = NIL_RTCPUID;
254#else
255 g_aidRtMpNtByCpuSetIdx[i] = i < g_cRtMpNtMaxCpus ? i : NIL_RTCPUID;
256#endif
257 for (unsigned idxGroup = 0; idxGroup < RT_ELEMENTS(g_aRtMpNtCpuGroups); idxGroup++)
258 {
259 g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = 0;
260 g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0;
261 for (unsigned idxMember = 0; idxMember < RT_ELEMENTS(g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers); idxMember++)
262 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = -1;
263 }
264
265 if (g_pfnrtKeQueryLogicalProcessorRelationship)
266 {
267 MY_CHECK_RETURN(g_pfnrtKeGetProcessorIndexFromNumber,
268 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
269 VERR_SYMBOL_NOT_FOUND);
270 MY_CHECK_RETURN(g_pfnrtKeGetProcessorNumberFromIndex,
271 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
272 VERR_SYMBOL_NOT_FOUND);
273 MY_CHECK_RETURN(g_pfnrtKeSetTargetProcessorDpcEx,
274 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeSetTargetProcessorDpcEx!\n"),
275 VERR_SYMBOL_NOT_FOUND);
276
277 SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = NULL;
278 int rc = rtR0NtInitQueryGroupRelations(&pInfo);
279 if (RT_FAILURE(rc))
280 return rc;
281
282 MY_CHECK(pInfo->Group.MaximumGroupCount == g_cRtMpNtMaxGroups,
283 ("IPRT: Fatal: MaximumGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n",
284 pInfo->Group.MaximumGroupCount, g_cRtMpNtMaxGroups));
285 MY_CHECK(pInfo->Group.ActiveGroupCount > 0 && pInfo->Group.ActiveGroupCount <= g_cRtMpNtMaxGroups,
286 ("IPRT: Fatal: ActiveGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n",
287 pInfo->Group.ActiveGroupCount, g_cRtMpNtMaxGroups));
288
289 /*
290 * First we need to recalc g_cRtMpNtMaxCpus (see above).
291 */
292 uint32_t cMaxCpus = 0;
293 uint32_t idxGroup;
294 for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)
295 {
296 const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];
297 MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,
298 ("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));
299 MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,
300 ("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",
301 pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));
302 cMaxCpus += pGrpInfo->MaximumProcessorCount;
303 }
304 if (cMaxCpus > g_cRtMpNtMaxCpus && RT_SUCCESS(rc))
305 {
306 DbgPrint("IPRT: g_cRtMpNtMaxCpus=%u -> %u\n", g_cRtMpNtMaxCpus, cMaxCpus);
307#ifndef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
308 uint32_t i = RT_MIN(cMaxCpus, RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx));
309 while (i-- > g_cRtMpNtMaxCpus)
310 g_aidRtMpNtByCpuSetIdx[i] = i;
311#endif
312 g_cRtMpNtMaxCpus = cMaxCpus;
313 if (g_cRtMpNtMaxGroups > RTCPUSET_MAX_CPUS)
314 {
315 MY_CHECK(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,
316 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS));
317 rc = VERR_MP_TOO_MANY_CPUS;
318 }
319 }
320
321 /*
322 * Calc online mask, partition IDs and such.
323 *
324 * Also check ASSUMPTIONS:
325 *
326 * 1. Processor indexes going from 0 and up to
327 * KeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1.
328 *
329 * 2. Currently valid processor indexes, i.e. accepted by
330 * KeGetProcessorIndexFromNumber & KeGetProcessorNumberFromIndex, goes
331 * from 0 thru KeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1.
332 *
333 * 3. PROCESSOR_GROUP_INFO::MaximumProcessorCount gives the number of
334 * relevant bits in the ActiveProcessorMask (from LSB).
335 *
336 * 4. Active processor count found in KeQueryLogicalProcessorRelationship
337 * output matches what KeQueryActiveProcessorCountEx(ALL) returns.
338 *
339 * 5. Active + inactive processor counts in same does not exceed
340 * KeQueryMaximumProcessorCountEx(ALL).
341 *
342 * Note! Processor indexes are assigned as CPUs come online and are not
343 * preallocated according to group maximums. Since CPUS are only taken
344 * online and never offlined, this means that internal CPU bitmaps are
345 * never sparse and no time is wasted scanning unused bits.
346 *
347 * Unfortunately, it means that ring-3 cannot easily guess the index
348 * assignments when hotswapping is used, and must use GIP when available.
349 */
350 RTCpuSetEmpty(&g_rtMpNtCpuSet);
351 uint32_t cInactive = 0;
352 uint32_t cActive = 0;
353 uint32_t idxCpuMax = 0;
354 uint32_t idxCpuSetNextInactive = g_cRtMpNtMaxCpus - 1;
355 for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)
356 {
357 const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];
358 MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,
359 ("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));
360 MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,
361 ("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",
362 pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));
363
364 g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = pGrpInfo->MaximumProcessorCount;
365 g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = pGrpInfo->ActiveProcessorCount;
366
367 for (uint32_t idxMember = 0; idxMember < pGrpInfo->MaximumProcessorCount; idxMember++)
368 {
369 PROCESSOR_NUMBER ProcNum;
370 ProcNum.Group = (USHORT)idxGroup;
371 ProcNum.Number = (UCHAR)idxMember;
372 ProcNum.Reserved = 0;
373 ULONG idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
374 if (idxCpu != INVALID_PROCESSOR_INDEX)
375 {
376 MY_CHECK_BREAK(idxCpu < g_cRtMpNtMaxCpus && idxCpu < RTCPUSET_MAX_CPUS, /* ASSUMPTION #1 */
377 ("IPRT: Fatal: idxCpu=%u >= g_cRtMpNtMaxCpus=%u (RTCPUSET_MAX_CPUS=%u)\n",
378 idxCpu, g_cRtMpNtMaxCpus, RTCPUSET_MAX_CPUS));
379 if (idxCpu > idxCpuMax)
380 idxCpuMax = idxCpu;
381 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpu;
382#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
383 g_aidRtMpNtByCpuSetIdx[idxCpu] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember);
384#endif
385
386 ProcNum.Group = UINT16_MAX;
387 ProcNum.Number = UINT8_MAX;
388 ProcNum.Reserved = UINT8_MAX;
389 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(idxCpu, &ProcNum);
390 MY_CHECK_BREAK(NT_SUCCESS(rcNt),
391 ("IPRT: Fatal: KeGetProcessorNumberFromIndex(%u,) -> %#x!\n", idxCpu, rcNt));
392 MY_CHECK_BREAK(ProcNum.Group == idxGroup && ProcNum.Number == idxMember,
393 ("IPRT: Fatal: KeGetProcessorXxxxFromYyyy roundtrip error for %#x! Group: %u vs %u, Number: %u vs %u\n",
394 idxCpu, ProcNum.Group, idxGroup, ProcNum.Number, idxMember));
395
396 if (pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember))
397 {
398 RTCpuSetAddByIndex(&g_rtMpNtCpuSet, idxCpu);
399 cActive++;
400 }
401 else
402 cInactive++; /* (This is a little unexpected, but not important as long as things add up below.) */
403 }
404 else
405 {
406 /* Must be not present / inactive when KeGetProcessorIndexFromNumber fails. */
407 MY_CHECK_BREAK(!(pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember)),
408 ("IPRT: Fatal: KeGetProcessorIndexFromNumber(%u/%u) failed but CPU is active! cMax=%u cActive=%u fActive=%p\n",
409 idxGroup, idxMember, pGrpInfo->MaximumProcessorCount, pGrpInfo->ActiveProcessorCount,
410 pGrpInfo->ActiveProcessorMask));
411 cInactive++;
412 if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus)
413 {
414 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
415#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
416 g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember)
417 | RTMPNT_ID_F_INACTIVE;
418#endif
419 idxCpuSetNextInactive--;
420 }
421 }
422 }
423 }
424
425 MY_CHECK(cInactive + cActive <= g_cRtMpNtMaxCpus, /* ASSUMPTION #5 (not '==' because of inactive groups) */
426 ("IPRT: Fatal: cInactive=%u + cActive=%u > g_cRtMpNtMaxCpus=%u\n", cInactive, cActive, g_cRtMpNtMaxCpus));
427
428 /* Deal with inactive groups using KeQueryMaximumProcessorCountEx or as
429 best as we can by as best we can by stipulating maximum member counts
430 from the previous group. */
431 if ( RT_SUCCESS(rc)
432 && idxGroup < pInfo->Group.MaximumGroupCount)
433 {
434 uint16_t cInactiveLeft = g_cRtMpNtMaxCpus - (cInactive + cActive);
435 while (idxGroup < pInfo->Group.MaximumGroupCount)
436 {
437 uint32_t cMaxMembers = 0;
438 if (g_pfnrtKeQueryMaximumProcessorCountEx)
439 cMaxMembers = g_pfnrtKeQueryMaximumProcessorCountEx(idxGroup);
440 if (cMaxMembers != 0 || cInactiveLeft == 0)
441 AssertStmt(cMaxMembers <= cInactiveLeft, cMaxMembers = cInactiveLeft);
442 else
443 {
444 uint16_t cGroupsLeft = pInfo->Group.MaximumGroupCount - idxGroup;
445 cMaxMembers = pInfo->Group.GroupInfo[idxGroup - 1].MaximumProcessorCount;
446 while (cMaxMembers * cGroupsLeft < cInactiveLeft)
447 cMaxMembers++;
448 if (cMaxMembers > cInactiveLeft)
449 cMaxMembers = cInactiveLeft;
450 }
451
452 g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = (uint16_t)cMaxMembers;
453 g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0;
454 for (uint16_t idxMember = 0; idxMember < cMaxMembers; idxMember++)
455 if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus)
456 {
457 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
458#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
459 g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember)
460 | RTMPNT_ID_F_INACTIVE;
461#endif
462 idxCpuSetNextInactive--;
463 }
464 cInactiveLeft -= cMaxMembers;
465 idxGroup++;
466 }
467 }
468
469 /* We're done with pInfo now, free it so we can start returning when assertions fail. */
470 RTMemFree(pInfo);
471 if (RT_FAILURE(rc)) /* MY_CHECK_BREAK sets rc. */
472 return rc;
473 MY_CHECK_RETURN(cActive >= g_cRtMpNtActiveCpus,
474 ("IPRT: Fatal: cActive=%u < g_cRtMpNtActiveCpus=%u - CPUs removed?\n", cActive, g_cRtMpNtActiveCpus),
475 VERR_INTERNAL_ERROR_3);
476 MY_CHECK_RETURN(idxCpuMax < cActive, /* ASSUMPTION #2 */
477 ("IPRT: Fatal: idCpuMax=%u >= cActive=%u! Unexpected CPU index allocation. CPUs removed?\n",
478 idxCpuMax, cActive),
479 VERR_INTERNAL_ERROR_4);
480
481 /* Retry if CPUs were added. */
482 if ( cActive != g_cRtMpNtActiveCpus
483 && cTries < 16)
484 continue;
485 MY_CHECK_RETURN(cActive == g_cRtMpNtActiveCpus, /* ASSUMPTION #4 */
486 ("IPRT: Fatal: cActive=%u != g_cRtMpNtActiveCpus=%u\n", cActive, g_cRtMpNtActiveCpus),
487 VERR_INTERNAL_ERROR_5);
488 }
489 else
490 {
491 /* Legacy: */
492 MY_CHECK_RETURN(g_cRtMpNtMaxGroups == 1, ("IPRT: Fatal: Missing KeQueryLogicalProcessorRelationship!\n"),
493 VERR_SYMBOL_NOT_FOUND);
494
495 /** @todo Is it possible that the affinity mask returned by
496 * KeQueryActiveProcessors is sparse? */
497 if (g_pfnrtKeQueryActiveProcessors)
498 RTCpuSetFromU64(&g_rtMpNtCpuSet, g_pfnrtKeQueryActiveProcessors());
499 else if (g_cRtMpNtMaxCpus < 64)
500 RTCpuSetFromU64(&g_rtMpNtCpuSet, (UINT64_C(1) << g_cRtMpNtMaxCpus) - 1);
501 else
502 {
503 MY_CHECK_RETURN(g_cRtMpNtMaxCpus == 64, ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, expect 64 or less\n", g_cRtMpNtMaxCpus),
504 VERR_MP_TOO_MANY_CPUS);
505 RTCpuSetFromU64(&g_rtMpNtCpuSet, UINT64_MAX);
506 }
507
508 g_aRtMpNtCpuGroups[0].cMaxCpus = g_cRtMpNtMaxCpus;
509 g_aRtMpNtCpuGroups[0].cActiveCpus = g_cRtMpNtMaxCpus;
510 for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++)
511 {
512 g_aRtMpNtCpuGroups[0].aidxCpuSetMembers[i] = i;
513#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
514 g_aidRtMpNtByCpuSetIdx[i] = RTMPCPUID_FROM_GROUP_AND_NUMBER(0, i);
515#endif
516 }
517 }
518
519 /*
520 * Register CPU hot plugging callback (it also counts active CPUs).
521 */
522 Assert(g_pvMpCpuChangeCallback == NULL);
523 if (g_pfnrtKeRegisterProcessorChangeCallback)
524 {
525 MY_CHECK_RETURN(g_pfnrtKeDeregisterProcessorChangeCallback,
526 ("IPRT: Fatal: KeRegisterProcessorChangeCallback without KeDeregisterProcessorChangeCallback!\n"),
527 VERR_SYMBOL_NOT_FOUND);
528
529 RTCPUSET const ActiveSetCopy = g_rtMpNtCpuSet;
530 RTCpuSetEmpty(&g_rtMpNtCpuSet);
531 uint32_t const cActiveCpus = g_cRtMpNtActiveCpus;
532 g_cRtMpNtActiveCpus = 0;
533
534 g_pvMpCpuChangeCallback = g_pfnrtKeRegisterProcessorChangeCallback(rtR0NtMpProcessorChangeCallback, NULL /*pvUser*/,
535 KE_PROCESSOR_CHANGE_ADD_EXISTING);
536 if (g_pvMpCpuChangeCallback)
537 {
538 if (cActiveCpus == g_cRtMpNtActiveCpus)
539 { /* likely */ }
540 else
541 {
542 g_pfnrtKeDeregisterProcessorChangeCallback(g_pvMpCpuChangeCallback);
543 if (cTries < 16)
544 {
545 /* Retry if CPUs were added. */
546 MY_CHECK_RETURN(g_cRtMpNtActiveCpus >= cActiveCpus,
547 ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u < cActiveCpus=%u! CPUs removed?\n",
548 g_cRtMpNtActiveCpus, cActiveCpus),
549 VERR_INTERNAL_ERROR_2);
550 MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus,
551 ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u > g_cRtMpNtMaxCpus=%u!\n",
552 g_cRtMpNtActiveCpus, g_cRtMpNtMaxCpus),
553 VERR_INTERNAL_ERROR_2);
554 continue;
555 }
556 MY_CHECK_RETURN(0, ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u cActiveCpus=%u\n", g_cRtMpNtActiveCpus, cActiveCpus),
557 VERR_INTERNAL_ERROR_3);
558 }
559 }
560 else
561 {
562 AssertFailed();
563 g_rtMpNtCpuSet = ActiveSetCopy;
564 g_cRtMpNtActiveCpus = cActiveCpus;
565 }
566 }
567 break;
568 } /* Retry loop for stable active CPU count. */
569
570#undef MY_CHECK_RETURN
571
572 /*
573 * Special IPI fun for RTMpPokeCpu.
574 *
575 * On Vista and later the DPC method doesn't seem to reliably send IPIs,
576 * so we have to use alternative methods.
577 *
578 * On AMD64 We used to use the HalSendSoftwareInterrupt API (also x86 on
579 * W10+), it looks faster and more convenient to use, however we're either
580 * using it wrong or it doesn't reliably do what we want (see @bugref{8343}).
581 *
582 * The HalRequestIpip API is thus far the only alternative to KeInsertQueueDpc
583 * for doing targetted IPIs. Trouble with this API is that it changed
584 * fundamentally in Window 7 when they added support for lots of processors.
585 *
586 * If we really think we cannot use KeInsertQueueDpc, we use the broadcast IPI
587 * API KeIpiGenericCall.
588 */
589 if ( pOsVerInfo->uMajorVer > 6
590 || (pOsVerInfo->uMajorVer == 6 && pOsVerInfo->uMinorVer > 0))
591 g_pfnrtHalRequestIpiPreW7 = NULL;
592 else
593 g_pfnrtHalRequestIpiW7Plus = NULL;
594
595 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingDpc;
596 if ( g_pfnrtHalRequestIpiW7Plus
597 && g_pfnrtKeInitializeAffinityEx
598 && g_pfnrtKeAddProcessorAffinityEx
599 && g_pfnrtKeGetProcessorIndexFromNumber)
600 {
601 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingHalReqestIpiW7Plus\n");
602 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingHalReqestIpiW7Plus;
603 }
604 else if (pOsVerInfo->uMajorVer >= 6 && g_pfnrtKeIpiGenericCall)
605 {
606 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingBroadcastIpi\n");
607 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingBroadcastIpi;
608 }
609 else
610 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingDpc\n");
611 /* else: Windows XP should send always send an IPI -> VERIFY */
612
613 return VINF_SUCCESS;
614}
615
616
617/**
618 * Called by rtR0TermNative.
619 */
620DECLHIDDEN(void) rtR0MpNtTerm(void)
621{
622 /*
623 * Deregister the processor change callback.
624 */
625 PVOID pvMpCpuChangeCallback = g_pvMpCpuChangeCallback;
626 g_pvMpCpuChangeCallback = NULL;
627 if (pvMpCpuChangeCallback)
628 {
629 AssertReturnVoid(g_pfnrtKeDeregisterProcessorChangeCallback);
630 g_pfnrtKeDeregisterProcessorChangeCallback(pvMpCpuChangeCallback);
631 }
632}
633
634
635DECLHIDDEN(int) rtR0MpNotificationNativeInit(void)
636{
637 return VINF_SUCCESS;
638}
639
640
641DECLHIDDEN(void) rtR0MpNotificationNativeTerm(void)
642{
643}
644
645
646/**
647 * Implements the NT PROCESSOR_CALLBACK_FUNCTION callback function.
648 *
649 * This maintains the g_rtMpNtCpuSet and works MP notification callbacks. When
650 * registered, it's called for each active CPU in the system, avoiding racing
651 * CPU hotplugging (as well as testing the callback).
652 *
653 * @param pvUser User context (not used).
654 * @param pChangeCtx Change context (in).
655 * @param prcOperationStatus Operation status (in/out).
656 *
657 * @remarks ASSUMES no concurrent execution of KeProcessorAddCompleteNotify
658 * notification callbacks. At least during callback registration
659 * callout, we're owning KiDynamicProcessorLock.
660 *
661 * @remarks When registering the handler, we first get KeProcessorAddStartNotify
662 * callbacks for all active CPUs, and after they all succeed we get the
663 * KeProcessorAddCompleteNotify callbacks.
664 */
665static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,
666 PNTSTATUS prcOperationStatus)
667{
668 RT_NOREF(pvUser, prcOperationStatus);
669 switch (pChangeCtx->State)
670 {
671 /*
672 * Check whether we can deal with the CPU, failing the start operation if we
673 * can't. The checks we are doing here are to avoid complicated/impossible
674 * cases in KeProcessorAddCompleteNotify. They are really just verify specs.
675 */
676 case KeProcessorAddStartNotify:
677 {
678 NTSTATUS rcNt = STATUS_SUCCESS;
679 if (pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS)
680 {
681 if (pChangeCtx->NtNumber >= g_cRtMpNtMaxCpus)
682 {
683 DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is higher than the max CPU count (%u)!\n",
684 pChangeCtx->NtNumber, g_cRtMpNtMaxCpus);
685 rcNt = STATUS_INTERNAL_ERROR;
686 }
687
688 /* The ProcessNumber field was introduced in Windows 7. */
689 PROCESSOR_NUMBER ProcNum;
690 if (g_pfnrtKeGetProcessorIndexFromNumber)
691 {
692 ProcNum = pChangeCtx->ProcNumber;
693 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
694 if (idxCpu != pChangeCtx->NtNumber)
695 {
696 DbgPrint("IPRT: KeProcessorAddStartNotify failure: g_pfnrtKeGetProcessorIndexFromNumber(%u.%u) -> %u, expected %u!\n",
697 ProcNum.Group, ProcNum.Number, idxCpu, pChangeCtx->NtNumber);
698 rcNt = STATUS_INTERNAL_ERROR;
699 }
700 }
701 else
702 {
703 ProcNum.Group = 0;
704 ProcNum.Number = pChangeCtx->NtNumber;
705 }
706
707 if ( ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups)
708 && ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers))
709 {
710 if (ProcNum.Group >= g_cRtMpNtMaxGroups)
711 {
712 DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range - max groups: %u!\n",
713 ProcNum.Group, ProcNum.Number, g_cRtMpNtMaxGroups);
714 rcNt = STATUS_INTERNAL_ERROR;
715 }
716
717 if (ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus)
718 {
719 Assert(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1);
720 if (g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == -1)
721 {
722 DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u.%u was assigned -1 as set index!\n",
723 ProcNum.Group, ProcNum.Number);
724 rcNt = STATUS_INTERNAL_ERROR;
725 }
726
727 Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID);
728 if (g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == NIL_RTCPUID)
729 {
730 DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u (%u.%u) translates to NIL_RTCPUID!\n",
731 pChangeCtx->NtNumber, ProcNum.Group, ProcNum.Number);
732 rcNt = STATUS_INTERNAL_ERROR;
733 }
734 }
735 else
736 {
737 DbgPrint("IPRT: KeProcessorAddStartNotify failure: max processors in group %u is %u, cannot add %u to it!\n",
738 ProcNum.Group, g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus, ProcNum.Group, ProcNum.Number);
739 rcNt = STATUS_INTERNAL_ERROR;
740 }
741 }
742 else
743 {
744 DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range (max %u.%u)!\n",
745 ProcNum.Group, ProcNum.Number, RT_ELEMENTS(g_aRtMpNtCpuGroups), RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers));
746 rcNt = STATUS_INTERNAL_ERROR;
747 }
748 }
749 else
750 {
751 DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is outside RTCPUSET_MAX_CPUS (%u)!\n",
752 pChangeCtx->NtNumber, RTCPUSET_MAX_CPUS);
753 rcNt = STATUS_INTERNAL_ERROR;
754 }
755 if (!NT_SUCCESS(rcNt))
756 *prcOperationStatus = rcNt;
757 break;
758 }
759
760 /*
761 * Update the globals. Since we've checked out range limits and other
762 * limitations already we just AssertBreak here.
763 */
764 case KeProcessorAddCompleteNotify:
765 {
766 /*
767 * Calc the processor number and assert conditions checked in KeProcessorAddStartNotify.
768 */
769 AssertBreak(pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS);
770 AssertBreak(pChangeCtx->NtNumber < g_cRtMpNtMaxCpus);
771 Assert(pChangeCtx->NtNumber == g_cRtMpNtActiveCpus); /* light assumption */
772 PROCESSOR_NUMBER ProcNum;
773 if (g_pfnrtKeGetProcessorIndexFromNumber)
774 {
775 ProcNum = pChangeCtx->ProcNumber;
776 AssertBreak(g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum) == pChangeCtx->NtNumber);
777 AssertBreak(ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups));
778 AssertBreak(ProcNum.Group < g_cRtMpNtMaxGroups);
779 }
780 else
781 {
782 ProcNum.Group = 0;
783 ProcNum.Number = pChangeCtx->NtNumber;
784 }
785 AssertBreak(ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers));
786 AssertBreak(ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus);
787 AssertBreak(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1);
788 AssertBreak(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID);
789
790 /*
791 * Add ourselves to the online CPU set and update the active CPU count.
792 */
793 RTCpuSetAddByIndex(&g_rtMpNtCpuSet, pChangeCtx->NtNumber);
794 ASMAtomicIncU32(&g_cRtMpNtActiveCpus);
795
796 /*
797 * Update the group info.
798 *
799 * If the index prediction failed (real hotplugging callbacks only) we
800 * have to switch it around. This is particularly annoying when we
801 * use the index as the ID.
802 */
803#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
804 RTCPUID idCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
805 RTCPUID idOld = g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber];
806 if ((idOld & ~RTMPNT_ID_F_INACTIVE) != idCpu)
807 {
808 Assert(idOld & RTMPNT_ID_F_INACTIVE);
809 int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
810 g_aRtMpNtCpuGroups[rtMpCpuIdGetGroup(idOld)].aidxCpuSetMembers[rtMpCpuIdGetGroupMember(idOld)] = idxDest;
811 g_aidRtMpNtByCpuSetIdx[idxDest] = idOld;
812 }
813 g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] = idCpu;
814#else
815 Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == pChangeCtx->NtNumber);
816 int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
817 if ((ULONG)idxDest != pChangeCtx->NtNumber)
818 {
819 bool fFound = false;
820 uint32_t idxOldGroup = g_cRtMpNtMaxGroups;
821 while (idxOldGroup-- > 0 && !fFound)
822 {
823 uint32_t idxMember = g_aRtMpNtCpuGroups[idxOldGroup].cMaxCpus;
824 while (idxMember-- > 0)
825 if (g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] == (int)pChangeCtx->NtNumber)
826 {
827 g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] = idxDest;
828 fFound = true;
829 break;
830 }
831 }
832 Assert(fFound);
833 }
834#endif
835 g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] = pChangeCtx->NtNumber;
836
837 /*
838 * Do MP notification callbacks.
839 */
840 rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, pChangeCtx->NtNumber);
841 break;
842 }
843
844 case KeProcessorAddFailureNotify:
845 /* ignore */
846 break;
847
848 default:
849 AssertMsgFailed(("State=%u\n", pChangeCtx->State));
850 }
851}
852
853
854/**
855 * Wrapper around KeQueryLogicalProcessorRelationship.
856 *
857 * @returns IPRT status code.
858 * @param ppInfo Where to return the info. Pass to RTMemFree when done.
859 */
860static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo)
861{
862 ULONG cbInfo = sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)
863 + g_cRtMpNtMaxGroups * sizeof(GROUP_RELATIONSHIP);
864 NTSTATUS rcNt;
865 do
866 {
867 SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)RTMemAlloc(cbInfo);
868 if (pInfo)
869 {
870 rcNt = g_pfnrtKeQueryLogicalProcessorRelationship(NULL /*pProcNumber*/, RelationGroup, pInfo, &cbInfo);
871 if (NT_SUCCESS(rcNt))
872 {
873 *ppInfo = pInfo;
874 return VINF_SUCCESS;
875 }
876
877 RTMemFree(pInfo);
878 pInfo = NULL;
879 }
880 else
881 rcNt = STATUS_NO_MEMORY;
882 } while (rcNt == STATUS_INFO_LENGTH_MISMATCH);
883 DbgPrint("IPRT: Fatal: KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt);
884 AssertMsgFailed(("KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt));
885 return RTErrConvertFromNtStatus(rcNt);
886}
887
888
889
890
891
892RTDECL(RTCPUID) RTMpCpuId(void)
893{
894 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
895
896#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
897 PROCESSOR_NUMBER ProcNum;
898 ProcNum.Group = 0;
899 if (g_pfnrtKeGetCurrentProcessorNumberEx)
900 {
901 ProcNum.Number = 0;
902 g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum);
903 }
904 else
905 ProcNum.Number = KeGetCurrentProcessorNumber(); /* Number is 8-bit, so we're not subject to BYTE -> WORD upgrade in WDK. */
906 return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
907
908#else
909
910 if (g_pfnrtKeGetCurrentProcessorNumberEx)
911 {
912 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL);
913 Assert(idxCpu < RTCPUSET_MAX_CPUS);
914 return idxCpu;
915 }
916
917 return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */
918#endif
919}
920
921
922RTDECL(int) RTMpCurSetIndex(void)
923{
924#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
925 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
926
927 if (g_pfnrtKeGetCurrentProcessorNumberEx)
928 {
929 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL);
930 Assert(idxCpu < RTCPUSET_MAX_CPUS);
931 return idxCpu;
932 }
933 return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */
934#else
935 return (int)RTMpCpuId();
936#endif
937}
938
939
940RTDECL(int) RTMpCurSetIndexAndId(PRTCPUID pidCpu)
941{
942#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
943 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
944
945 PROCESSOR_NUMBER ProcNum = { 0 , 0, 0 };
946 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum);
947 Assert(idxCpu < RTCPUSET_MAX_CPUS);
948 *pidCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
949 return idxCpu;
950#else
951 return *pidCpu = RTMpCpuId();
952#endif
953}
954
955
956RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu)
957{
958#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
959 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
960
961 if (idCpu != NIL_RTCPUID)
962 {
963 if (g_pfnrtKeGetProcessorIndexFromNumber)
964 {
965 PROCESSOR_NUMBER ProcNum;
966 ProcNum.Group = rtMpCpuIdGetGroup(idCpu);
967 ProcNum.Number = rtMpCpuIdGetGroupMember(idCpu);
968 ProcNum.Reserved = 0;
969 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
970 if (idxCpu != INVALID_PROCESSOR_INDEX)
971 {
972 Assert(idxCpu < g_cRtMpNtMaxCpus);
973 Assert((ULONG)g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == idxCpu);
974 return idxCpu;
975 }
976
977 /* Since NT assigned indexes as the CPUs come online, we cannot produce an ID <-> index
978 mapping for not-yet-onlined CPUS that is consistent. We just have to do our best... */
979 if ( ProcNum.Group < g_cRtMpNtMaxGroups
980 && ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus)
981 return g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
982 }
983 else if (rtMpCpuIdGetGroup(idCpu) == 0)
984 return rtMpCpuIdGetGroupMember(idCpu);
985 }
986 return -1;
987#else
988 /* 1:1 mapping, just do range checks. */
989 return idCpu < RTCPUSET_MAX_CPUS ? (int)idCpu : -1;
990#endif
991}
992
993
994RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu)
995{
996#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
997 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
998
999 if ((unsigned)iCpu < g_cRtMpNtMaxCpus)
1000 {
1001 if (g_pfnrtKeGetProcessorIndexFromNumber)
1002 {
1003 PROCESSOR_NUMBER ProcNum = { 0, 0, 0 };
1004 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(iCpu, &ProcNum);
1005 if (NT_SUCCESS(rcNt))
1006 {
1007 Assert(ProcNum.Group <= g_cRtMpNtMaxGroups);
1008 Assert( (g_aidRtMpNtByCpuSetIdx[iCpu] & ~RTMPNT_ID_F_INACTIVE)
1009 == RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number));
1010 return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
1011 }
1012 }
1013 return g_aidRtMpNtByCpuSetIdx[iCpu];
1014 }
1015 return NIL_RTCPUID;
1016#else
1017 /* 1:1 mapping, just do range checks. */
1018 return (unsigned)iCpu < RTCPUSET_MAX_CPUS ? iCpu : NIL_RTCPUID;
1019#endif
1020}
1021
1022
1023RTDECL(int) RTMpSetIndexFromCpuGroupMember(uint32_t idxGroup, uint32_t idxMember)
1024{
1025 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1026
1027 if (idxGroup < g_cRtMpNtMaxGroups)
1028 if (idxMember < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus)
1029 return g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember];
1030 return -1;
1031}
1032
1033
1034RTDECL(uint32_t) RTMpGetCpuGroupCounts(uint32_t idxGroup, uint32_t *pcActive)
1035{
1036 if (idxGroup < g_cRtMpNtMaxGroups)
1037 {
1038 if (pcActive)
1039 *pcActive = g_aRtMpNtCpuGroups[idxGroup].cActiveCpus;
1040 return g_aRtMpNtCpuGroups[idxGroup].cMaxCpus;
1041 }
1042 if (pcActive)
1043 *pcActive = 0;
1044 return 0;
1045}
1046
1047
1048RTDECL(uint32_t) RTMpGetMaxCpuGroupCount(void)
1049{
1050 return g_cRtMpNtMaxGroups;
1051}
1052
1053
1054RTDECL(RTCPUID) RTMpGetMaxCpuId(void)
1055{
1056 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1057
1058#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1059 return RTMPCPUID_FROM_GROUP_AND_NUMBER(g_cRtMpNtMaxGroups - 1, g_aRtMpNtCpuGroups[g_cRtMpNtMaxGroups - 1].cMaxCpus - 1);
1060#else
1061 /* According to MSDN the processor indexes goes from 0 to the maximum
1062 number of CPUs in the system. We've check this in initterm-r0drv-nt.cpp. */
1063 return g_cRtMpNtMaxCpus - 1;
1064#endif
1065}
1066
1067
1068RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
1069{
1070 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1071 return RTCpuSetIsMember(&g_rtMpNtCpuSet, idCpu);
1072}
1073
1074
1075RTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu)
1076{
1077 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1078
1079#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1080 if (idCpu != NIL_RTCPUID)
1081 {
1082 unsigned idxGroup = rtMpCpuIdGetGroup(idCpu);
1083 if (idxGroup < g_cRtMpNtMaxGroups)
1084 return rtMpCpuIdGetGroupMember(idCpu) < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus;
1085 }
1086 return false;
1087
1088#else
1089 /* A possible CPU ID is one with a value lower than g_cRtMpNtMaxCpus (see
1090 comment in RTMpGetMaxCpuId). */
1091 return idCpu < g_cRtMpNtMaxCpus;
1092#endif
1093}
1094
1095
1096
1097RTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet)
1098{
1099 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1100
1101 /* The set of possible CPU IDs(/indexes) are from 0 up to
1102 g_cRtMpNtMaxCpus (see comment in RTMpGetMaxCpuId). */
1103 RTCpuSetEmpty(pSet);
1104 int idxCpu = g_cRtMpNtMaxCpus;
1105 while (idxCpu-- > 0)
1106 RTCpuSetAddByIndex(pSet, idxCpu);
1107 return pSet;
1108}
1109
1110
1111RTDECL(RTCPUID) RTMpGetCount(void)
1112{
1113 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1114 return g_cRtMpNtMaxCpus;
1115}
1116
1117
1118RTDECL(PRTCPUSET) RTMpGetOnlineSet(PRTCPUSET pSet)
1119{
1120 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1121
1122 *pSet = g_rtMpNtCpuSet;
1123 return pSet;
1124}
1125
1126
1127RTDECL(RTCPUID) RTMpGetOnlineCount(void)
1128{
1129 RTCPUSET Set;
1130 RTMpGetOnlineSet(&Set);
1131 return RTCpuSetCount(&Set);
1132}
1133
1134
1135RTDECL(RTCPUID) RTMpGetOnlineCoreCount(void)
1136{
1137 /** @todo fix me */
1138 return RTMpGetOnlineCount();
1139}
1140
1141
1142
1143#if 0
1144/* Experiment with checking the undocumented KPRCB structure
1145 * 'dt nt!_kprcb 0xaddress' shows the layout
1146 */
1147typedef struct
1148{
1149 LIST_ENTRY DpcListHead;
1150 ULONG_PTR DpcLock;
1151 volatile ULONG DpcQueueDepth;
1152 ULONG DpcQueueCount;
1153} KDPC_DATA, *PKDPC_DATA;
1154
1155RTDECL(bool) RTMpIsCpuWorkPending(void)
1156{
1157 uint8_t *pkprcb;
1158 PKDPC_DATA pDpcData;
1159
1160 _asm {
1161 mov eax, fs:0x20
1162 mov pkprcb, eax
1163 }
1164 pDpcData = (PKDPC_DATA)(pkprcb + 0x19e0);
1165 if (pDpcData->DpcQueueDepth)
1166 return true;
1167
1168 pDpcData++;
1169 if (pDpcData->DpcQueueDepth)
1170 return true;
1171 return false;
1172}
1173#else
1174RTDECL(bool) RTMpIsCpuWorkPending(void)
1175{
1176 /** @todo not implemented */
1177 return false;
1178}
1179#endif
1180
1181
1182/**
1183 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1184 * the RTMpOnAll case.
1185 *
1186 * @param uUserCtx The user context argument (PRTMPARGS).
1187 */
1188static ULONG_PTR rtmpNtOnAllBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1189{
1190 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1191 /*ASMAtomicIncU32(&pArgs->cHits); - not needed */
1192 pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
1193 return 0;
1194}
1195
1196
1197/**
1198 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1199 * the RTMpOnOthers case.
1200 *
1201 * @param uUserCtx The user context argument (PRTMPARGS).
1202 */
1203static ULONG_PTR rtmpNtOnOthersBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1204{
1205 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1206 RTCPUID idCpu = RTMpCpuId();
1207 if (pArgs->idCpu != idCpu)
1208 {
1209 /*ASMAtomicIncU32(&pArgs->cHits); - not needed */
1210 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1211 }
1212 return 0;
1213}
1214
1215
1216/**
1217 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1218 * the RTMpOnPair case.
1219 *
1220 * @param uUserCtx The user context argument (PRTMPARGS).
1221 */
1222static ULONG_PTR rtmpNtOnPairBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1223{
1224 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1225 RTCPUID idCpu = RTMpCpuId();
1226 if ( pArgs->idCpu == idCpu
1227 || pArgs->idCpu2 == idCpu)
1228 {
1229 ASMAtomicIncU32(&pArgs->cHits);
1230 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1231 }
1232 return 0;
1233}
1234
1235
1236/**
1237 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1238 * the RTMpOnSpecific case.
1239 *
1240 * @param uUserCtx The user context argument (PRTMPARGS).
1241 */
1242static ULONG_PTR rtmpNtOnSpecificBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1243{
1244 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1245 RTCPUID idCpu = RTMpCpuId();
1246 if (pArgs->idCpu == idCpu)
1247 {
1248 ASMAtomicIncU32(&pArgs->cHits);
1249 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1250 }
1251 return 0;
1252}
1253
1254
1255/**
1256 * Internal worker for the RTMpOn* APIs using KeIpiGenericCall.
1257 *
1258 * @returns VINF_SUCCESS.
1259 * @param pfnWorker The callback.
1260 * @param pvUser1 User argument 1.
1261 * @param pvUser2 User argument 2.
1262 * @param pfnNativeWrapper The wrapper between the NT and IPRT callbacks.
1263 * @param idCpu First CPU to match, ultimately specific to the
1264 * pfnNativeWrapper used.
1265 * @param idCpu2 Second CPU to match, ultimately specific to the
1266 * pfnNativeWrapper used.
1267 * @param pcHits Where to return the number of this. Optional.
1268 */
1269static int rtMpCallUsingBroadcastIpi(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2,
1270 PKIPI_BROADCAST_WORKER pfnNativeWrapper, RTCPUID idCpu, RTCPUID idCpu2,
1271 uint32_t *pcHits)
1272{
1273 RTMPARGS Args;
1274 Args.pfnWorker = pfnWorker;
1275 Args.pvUser1 = pvUser1;
1276 Args.pvUser2 = pvUser2;
1277 Args.idCpu = idCpu;
1278 Args.idCpu2 = idCpu2;
1279 Args.cRefs = 0;
1280 Args.cHits = 0;
1281
1282 AssertPtr(g_pfnrtKeIpiGenericCall);
1283 g_pfnrtKeIpiGenericCall(pfnNativeWrapper, (uintptr_t)&Args);
1284 if (pcHits)
1285 *pcHits = Args.cHits;
1286 return VINF_SUCCESS;
1287}
1288
1289
1290/**
1291 * Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
1292 *
1293 * @param Dpc DPC object
1294 * @param DeferredContext Context argument specified by KeInitializeDpc
1295 * @param SystemArgument1 Argument specified by KeInsertQueueDpc
1296 * @param SystemArgument2 Argument specified by KeInsertQueueDpc
1297 */
1298static VOID rtmpNtDPCWrapper(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1299{
1300 PRTMPARGS pArgs = (PRTMPARGS)DeferredContext;
1301 RT_NOREF3(Dpc, SystemArgument1, SystemArgument2);
1302
1303 ASMAtomicIncU32(&pArgs->cHits);
1304 pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
1305
1306 /* Dereference the argument structure. */
1307 int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
1308 Assert(cRefs >= 0);
1309 if (cRefs == 0)
1310 RTMemFree(pArgs);
1311}
1312
1313
1314/**
1315 * Wrapper around KeSetTargetProcessorDpcEx / KeSetTargetProcessorDpc.
1316 *
1317 * This is shared with the timer code.
1318 *
1319 * @returns IPRT status code (errors are asserted).
1320 * @param pDpc The DPC.
1321 * @param idCpu The ID of the new target CPU.
1322 */
1323DECLHIDDEN(int) rtMpNtSetTargetProcessorDpc(KDPC *pDpc, RTCPUID idCpu)
1324{
1325 if (g_pfnrtKeSetTargetProcessorDpcEx)
1326 {
1327 /* Convert to stupid process number (bet KeSetTargetProcessorDpcEx does
1328 the reverse conversion internally). */
1329 PROCESSOR_NUMBER ProcNum;
1330 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(RTMpCpuIdToSetIndex(idCpu), &ProcNum);
1331 AssertMsgReturn(NT_SUCCESS(rcNt),
1332 ("KeGetProcessorNumberFromIndex(%u) -> %#x\n", idCpu, rcNt),
1333 RTErrConvertFromNtStatus(rcNt));
1334
1335 rcNt = g_pfnrtKeSetTargetProcessorDpcEx(pDpc, &ProcNum);
1336 AssertMsgReturn(NT_SUCCESS(rcNt),
1337 ("KeSetTargetProcessorDpcEx(,%u(%u/%u)) -> %#x\n", idCpu, ProcNum.Group, ProcNum.Number, rcNt),
1338 RTErrConvertFromNtStatus(rcNt));
1339 }
1340 else
1341 KeSetTargetProcessorDpc(pDpc, RTMpCpuIdToSetIndex(idCpu));
1342 return VINF_SUCCESS;
1343}
1344
1345
1346/**
1347 * Internal worker for the RTMpOn* APIs.
1348 *
1349 * @returns IPRT status code.
1350 * @param pfnWorker The callback.
1351 * @param pvUser1 User argument 1.
1352 * @param pvUser2 User argument 2.
1353 * @param enmCpuid What to do / is idCpu valid.
1354 * @param idCpu Used if enmCpuid is RT_NT_CPUID_SPECIFIC or
1355 * RT_NT_CPUID_PAIR, otherwise ignored.
1356 * @param idCpu2 Used if enmCpuid is RT_NT_CPUID_PAIR, otherwise ignored.
1357 * @param pcHits Where to return the number of this. Optional.
1358 */
1359static int rtMpCallUsingDpcs(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2,
1360 RT_NT_CPUID enmCpuid, RTCPUID idCpu, RTCPUID idCpu2, uint32_t *pcHits)
1361{
1362#if 0
1363 /* KeFlushQueuedDpcs must be run at IRQL PASSIVE_LEVEL according to MSDN, but the
1364 * driver verifier doesn't complain...
1365 */
1366 AssertMsg(KeGetCurrentIrql() == PASSIVE_LEVEL, ("%d != %d (PASSIVE_LEVEL)\n", KeGetCurrentIrql(), PASSIVE_LEVEL));
1367#endif
1368 /* KeFlushQueuedDpcs is not present in Windows 2000; import it dynamically so we can just fail this call. */
1369 if (!g_pfnrtNtKeFlushQueuedDpcs)
1370 return VERR_NOT_SUPPORTED;
1371
1372 /*
1373 * Make a copy of the active CPU set and figure out how many KDPCs we really need.
1374 * We must not try setup DPCs for CPUs which aren't there, because that may fail.
1375 */
1376 RTCPUSET OnlineSet = g_rtMpNtCpuSet;
1377 uint32_t cDpcsNeeded;
1378 switch (enmCpuid)
1379 {
1380 case RT_NT_CPUID_SPECIFIC:
1381 cDpcsNeeded = 1;
1382 break;
1383 case RT_NT_CPUID_PAIR:
1384 cDpcsNeeded = 2;
1385 break;
1386 default:
1387 do
1388 {
1389 cDpcsNeeded = g_cRtMpNtActiveCpus;
1390 OnlineSet = g_rtMpNtCpuSet;
1391 } while (cDpcsNeeded != g_cRtMpNtActiveCpus);
1392 break;
1393 }
1394
1395 /*
1396 * Allocate an RTMPARGS structure followed by cDpcsNeeded KDPCs
1397 * and initialize them.
1398 */
1399 PRTMPARGS pArgs = (PRTMPARGS)RTMemAllocZ(sizeof(RTMPARGS) + cDpcsNeeded * sizeof(KDPC));
1400 if (!pArgs)
1401 return VERR_NO_MEMORY;
1402
1403 pArgs->pfnWorker = pfnWorker;
1404 pArgs->pvUser1 = pvUser1;
1405 pArgs->pvUser2 = pvUser2;
1406 pArgs->idCpu = NIL_RTCPUID;
1407 pArgs->idCpu2 = NIL_RTCPUID;
1408 pArgs->cHits = 0;
1409 pArgs->cRefs = 1;
1410
1411 int rc;
1412 KDPC *paExecCpuDpcs = (KDPC *)(pArgs + 1);
1413 if (enmCpuid == RT_NT_CPUID_SPECIFIC)
1414 {
1415 KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
1416 KeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
1417 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[0], idCpu);
1418 pArgs->idCpu = idCpu;
1419 }
1420 else if (enmCpuid == RT_NT_CPUID_PAIR)
1421 {
1422 KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
1423 KeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
1424 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[0], idCpu);
1425 pArgs->idCpu = idCpu;
1426
1427 KeInitializeDpc(&paExecCpuDpcs[1], rtmpNtDPCWrapper, pArgs);
1428 KeSetImportanceDpc(&paExecCpuDpcs[1], HighImportance);
1429 if (RT_SUCCESS(rc))
1430 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[1], (int)idCpu2);
1431 pArgs->idCpu2 = idCpu2;
1432 }
1433 else
1434 {
1435 rc = VINF_SUCCESS;
1436 for (uint32_t i = 0; i < cDpcsNeeded && RT_SUCCESS(rc); i++)
1437 if (RTCpuSetIsMemberByIndex(&OnlineSet, i))
1438 {
1439 KeInitializeDpc(&paExecCpuDpcs[i], rtmpNtDPCWrapper, pArgs);
1440 KeSetImportanceDpc(&paExecCpuDpcs[i], HighImportance);
1441 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[i], RTMpCpuIdFromSetIndex(i));
1442 }
1443 }
1444 if (RT_FAILURE(rc))
1445 {
1446 RTMemFree(pArgs);
1447 return rc;
1448 }
1449
1450 /*
1451 * Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
1452 * KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL.
1453 */
1454 KIRQL oldIrql;
1455 KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
1456
1457 /*
1458 * We cannot do other than assume a 1:1 relationship between the
1459 * affinity mask and the process despite the warnings in the docs.
1460 * If someone knows a better way to get this done, please let bird know.
1461 */
1462 ASMCompilerBarrier(); /* paranoia */
1463 if (enmCpuid == RT_NT_CPUID_SPECIFIC)
1464 {
1465 ASMAtomicIncS32(&pArgs->cRefs);
1466 BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
1467 Assert(fRc); NOREF(fRc);
1468 }
1469 else if (enmCpuid == RT_NT_CPUID_PAIR)
1470 {
1471 ASMAtomicIncS32(&pArgs->cRefs);
1472 BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
1473 Assert(fRc); NOREF(fRc);
1474
1475 ASMAtomicIncS32(&pArgs->cRefs);
1476 fRc = KeInsertQueueDpc(&paExecCpuDpcs[1], 0, 0);
1477 Assert(fRc); NOREF(fRc);
1478 }
1479 else
1480 {
1481 uint32_t iSelf = RTMpCurSetIndex();
1482 for (uint32_t i = 0; i < cDpcsNeeded; i++)
1483 {
1484 if ( (i != iSelf)
1485 && RTCpuSetIsMemberByIndex(&OnlineSet, i))
1486 {
1487 ASMAtomicIncS32(&pArgs->cRefs);
1488 BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[i], 0, 0);
1489 Assert(fRc); NOREF(fRc);
1490 }
1491 }
1492 if (enmCpuid != RT_NT_CPUID_OTHERS)
1493 pfnWorker(iSelf, pvUser1, pvUser2);
1494 }
1495
1496 KeLowerIrql(oldIrql);
1497
1498 /*
1499 * Flush all DPCs and wait for completion. (can take long!)
1500 */
1501 /** @todo Consider changing this to an active wait using some atomic inc/dec
1502 * stuff (and check for the current cpu above in the specific case). */
1503 /** @todo Seems KeFlushQueuedDpcs doesn't wait for the DPCs to be completely
1504 * executed. Seen pArgs being freed while some CPU was using it before
1505 * cRefs was added. */
1506 if (g_pfnrtNtKeFlushQueuedDpcs)
1507 g_pfnrtNtKeFlushQueuedDpcs();
1508
1509 if (pcHits)
1510 *pcHits = pArgs->cHits;
1511
1512 /* Dereference the argument structure. */
1513 int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
1514 Assert(cRefs >= 0);
1515 if (cRefs == 0)
1516 RTMemFree(pArgs);
1517
1518 return VINF_SUCCESS;
1519}
1520
1521
1522RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1523{
1524 if (g_pfnrtKeIpiGenericCall)
1525 return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnAllBroadcastIpiWrapper,
1526 NIL_RTCPUID, NIL_RTCPUID, NULL);
1527 return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_ALL, NIL_RTCPUID, NIL_RTCPUID, NULL);
1528}
1529
1530
1531RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1532{
1533 if (g_pfnrtKeIpiGenericCall)
1534 return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnOthersBroadcastIpiWrapper,
1535 NIL_RTCPUID, NIL_RTCPUID, NULL);
1536 return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_OTHERS, NIL_RTCPUID, NIL_RTCPUID, NULL);
1537}
1538
1539
1540RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1541{
1542 int rc;
1543 AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
1544 AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
1545 if ((fFlags & RTMPON_F_CONCURRENT_EXEC) && !g_pfnrtKeIpiGenericCall)
1546 return VERR_NOT_SUPPORTED;
1547
1548 /*
1549 * Check that both CPUs are online before doing the broadcast call.
1550 */
1551 if ( RTMpIsCpuOnline(idCpu1)
1552 && RTMpIsCpuOnline(idCpu2))
1553 {
1554 /*
1555 * The broadcast IPI isn't quite as bad as it could have been, because
1556 * it looks like windows doesn't synchronize CPUs on the way out, they
1557 * seems to get back to normal work while the pair is still busy.
1558 */
1559 uint32_t cHits = 0;
1560 if (g_pfnrtKeIpiGenericCall)
1561 rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnPairBroadcastIpiWrapper, idCpu1, idCpu2, &cHits);
1562 else
1563 rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_PAIR, idCpu1, idCpu2, &cHits);
1564 if (RT_SUCCESS(rc))
1565 {
1566 Assert(cHits <= 2);
1567 if (cHits == 2)
1568 rc = VINF_SUCCESS;
1569 else if (cHits == 1)
1570 rc = VERR_NOT_ALL_CPUS_SHOWED;
1571 else if (cHits == 0)
1572 rc = VERR_CPU_OFFLINE;
1573 else
1574 rc = VERR_CPU_IPE_1;
1575 }
1576 }
1577 /*
1578 * A CPU must be present to be considered just offline.
1579 */
1580 else if ( RTMpIsCpuPresent(idCpu1)
1581 && RTMpIsCpuPresent(idCpu2))
1582 rc = VERR_CPU_OFFLINE;
1583 else
1584 rc = VERR_CPU_NOT_FOUND;
1585 return rc;
1586}
1587
1588
1589RTDECL(bool) RTMpOnPairIsConcurrentExecSupported(void)
1590{
1591 return g_pfnrtKeIpiGenericCall != NULL;
1592}
1593
1594
1595/**
1596 * Releases a reference to a RTMPNTONSPECIFICARGS heap allocation, freeing it
1597 * when the last reference is released.
1598 */
1599DECLINLINE(void) rtMpNtOnSpecificRelease(PRTMPNTONSPECIFICARGS pArgs)
1600{
1601 uint32_t cRefs = ASMAtomicDecU32(&pArgs->cRefs);
1602 AssertMsg(cRefs <= 1, ("cRefs=%#x\n", cRefs));
1603 if (cRefs == 0)
1604 RTMemFree(pArgs);
1605}
1606
1607
1608/**
1609 * Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
1610 *
1611 * @param Dpc DPC object
1612 * @param DeferredContext Context argument specified by KeInitializeDpc
1613 * @param SystemArgument1 Argument specified by KeInsertQueueDpc
1614 * @param SystemArgument2 Argument specified by KeInsertQueueDpc
1615 */
1616static VOID rtMpNtOnSpecificDpcWrapper(IN PKDPC Dpc, IN PVOID DeferredContext,
1617 IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1618{
1619 PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)DeferredContext;
1620 RT_NOREF3(Dpc, SystemArgument1, SystemArgument2);
1621
1622 ASMAtomicWriteBool(&pArgs->fExecuting, true);
1623
1624 pArgs->CallbackArgs.pfnWorker(RTMpCpuId(), pArgs->CallbackArgs.pvUser1, pArgs->CallbackArgs.pvUser2);
1625
1626 ASMAtomicWriteBool(&pArgs->fDone, true);
1627 KeSetEvent(&pArgs->DoneEvt, 1 /*PriorityIncrement*/, FALSE /*Wait*/);
1628
1629 rtMpNtOnSpecificRelease(pArgs);
1630}
1631
1632
1633RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1634{
1635 /*
1636 * Don't try mess with an offline CPU.
1637 */
1638 if (!RTMpIsCpuOnline(idCpu))
1639 return !RTMpIsCpuPossible(idCpu)
1640 ? VERR_CPU_NOT_FOUND
1641 : VERR_CPU_OFFLINE;
1642
1643 /*
1644 * Use the broadcast IPI routine if there are no more than two CPUs online,
1645 * or if the current IRQL is unsuitable for KeWaitForSingleObject.
1646 */
1647 int rc;
1648 uint32_t cHits = 0;
1649 if ( g_pfnrtKeIpiGenericCall
1650 && ( RTMpGetOnlineCount() <= 2
1651 || KeGetCurrentIrql() > APC_LEVEL)
1652 )
1653 {
1654 rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnSpecificBroadcastIpiWrapper,
1655 idCpu, NIL_RTCPUID, &cHits);
1656 if (RT_SUCCESS(rc))
1657 {
1658 if (cHits == 1)
1659 return VINF_SUCCESS;
1660 rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
1661 }
1662 return rc;
1663 }
1664
1665#if 0
1666 rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_SPECIFIC, idCpu, NIL_RTCPUID, &cHits);
1667 if (RT_SUCCESS(rc))
1668 {
1669 if (cHits == 1)
1670 return VINF_SUCCESS;
1671 rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
1672 }
1673 return rc;
1674
1675#else
1676 /*
1677 * Initialize the argument package and the objects within it.
1678 * The package is referenced counted to avoid unnecessary spinning to
1679 * synchronize cleanup and prevent stack corruption.
1680 */
1681 PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)RTMemAllocZ(sizeof(*pArgs));
1682 if (!pArgs)
1683 return VERR_NO_MEMORY;
1684 pArgs->cRefs = 2;
1685 pArgs->fExecuting = false;
1686 pArgs->fDone = false;
1687 pArgs->CallbackArgs.pfnWorker = pfnWorker;
1688 pArgs->CallbackArgs.pvUser1 = pvUser1;
1689 pArgs->CallbackArgs.pvUser2 = pvUser2;
1690 pArgs->CallbackArgs.idCpu = idCpu;
1691 pArgs->CallbackArgs.cHits = 0;
1692 pArgs->CallbackArgs.cRefs = 2;
1693 KeInitializeEvent(&pArgs->DoneEvt, SynchronizationEvent, FALSE /* not signalled */);
1694 KeInitializeDpc(&pArgs->Dpc, rtMpNtOnSpecificDpcWrapper, pArgs);
1695 KeSetImportanceDpc(&pArgs->Dpc, HighImportance);
1696 rc = rtMpNtSetTargetProcessorDpc(&pArgs->Dpc, idCpu);
1697 if (RT_FAILURE(rc))
1698 {
1699 RTMemFree(pArgs);
1700 return rc;
1701 }
1702
1703 /*
1704 * Disable preemption while we check the current processor and inserts the DPC.
1705 */
1706 KIRQL bOldIrql;
1707 KeRaiseIrql(DISPATCH_LEVEL, &bOldIrql);
1708 ASMCompilerBarrier(); /* paranoia */
1709
1710 if (RTMpCpuId() == idCpu)
1711 {
1712 /* Just execute the callback on the current CPU. */
1713 pfnWorker(idCpu, pvUser1, pvUser2);
1714 KeLowerIrql(bOldIrql);
1715
1716 RTMemFree(pArgs);
1717 return VINF_SUCCESS;
1718 }
1719
1720 /* Different CPU, so queue it if the CPU is still online. */
1721 if (RTMpIsCpuOnline(idCpu))
1722 {
1723 BOOLEAN fRc = KeInsertQueueDpc(&pArgs->Dpc, 0, 0);
1724 Assert(fRc); NOREF(fRc);
1725 KeLowerIrql(bOldIrql);
1726
1727 uint64_t const nsRealWaitTS = RTTimeNanoTS();
1728
1729 /*
1730 * Wait actively for a while in case the CPU/thread responds quickly.
1731 */
1732 uint32_t cLoopsLeft = 0x20000;
1733 while (cLoopsLeft-- > 0)
1734 {
1735 if (pArgs->fDone)
1736 {
1737 rtMpNtOnSpecificRelease(pArgs);
1738 return VINF_SUCCESS;
1739 }
1740 ASMNopPause();
1741 }
1742
1743 /*
1744 * It didn't respond, so wait on the event object, poking the CPU if it's slow.
1745 */
1746 LARGE_INTEGER Timeout;
1747 Timeout.QuadPart = -10000; /* 1ms */
1748 NTSTATUS rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1749 if (rcNt == STATUS_SUCCESS)
1750 {
1751 rtMpNtOnSpecificRelease(pArgs);
1752 return VINF_SUCCESS;
1753 }
1754
1755 /* If it hasn't respondend yet, maybe poke it and wait some more. */
1756 if (rcNt == STATUS_TIMEOUT)
1757 {
1758 if ( !pArgs->fExecuting
1759 && ( g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalReqestIpiW7Plus
1760 || g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalReqestIpiPreW7))
1761 RTMpPokeCpu(idCpu);
1762
1763 Timeout.QuadPart = -1280000; /* 128ms */
1764 rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1765 if (rcNt == STATUS_SUCCESS)
1766 {
1767 rtMpNtOnSpecificRelease(pArgs);
1768 return VINF_SUCCESS;
1769 }
1770 }
1771
1772 /*
1773 * Something weird is happening, try bail out.
1774 */
1775 if (KeRemoveQueueDpc(&pArgs->Dpc))
1776 {
1777 RTMemFree(pArgs); /* DPC was still queued, so we can return without further ado. */
1778 LogRel(("RTMpOnSpecific(%#x): Not processed after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1779 }
1780 else
1781 {
1782 /* DPC is running, wait a good while for it to complete. */
1783 LogRel(("RTMpOnSpecific(%#x): Still running after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1784
1785 Timeout.QuadPart = -30*1000*1000*10; /* 30 seconds */
1786 rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1787 if (rcNt != STATUS_SUCCESS)
1788 LogRel(("RTMpOnSpecific(%#x): Giving up on running worker after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1789 }
1790 rc = RTErrConvertFromNtStatus(rcNt);
1791 }
1792 else
1793 {
1794 /* CPU is offline.*/
1795 KeLowerIrql(bOldIrql);
1796 rc = !RTMpIsCpuPossible(idCpu) ? VERR_CPU_NOT_FOUND : VERR_CPU_OFFLINE;
1797 }
1798
1799 rtMpNtOnSpecificRelease(pArgs);
1800 return rc;
1801#endif
1802}
1803
1804
1805
1806
1807static VOID rtMpNtPokeCpuDummy(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1808{
1809 NOREF(Dpc);
1810 NOREF(DeferredContext);
1811 NOREF(SystemArgument1);
1812 NOREF(SystemArgument2);
1813}
1814
1815
1816/** Callback used by rtMpPokeCpuUsingBroadcastIpi. */
1817static ULONG_PTR rtMpIpiGenericCall(ULONG_PTR Argument)
1818{
1819 NOREF(Argument);
1820 return 0;
1821}
1822
1823
1824/**
1825 * RTMpPokeCpu worker that uses broadcast IPIs for doing the work.
1826 *
1827 * @returns VINF_SUCCESS
1828 * @param idCpu The CPU identifier.
1829 */
1830int rtMpPokeCpuUsingBroadcastIpi(RTCPUID idCpu)
1831{
1832 NOREF(idCpu);
1833 g_pfnrtKeIpiGenericCall(rtMpIpiGenericCall, 0);
1834 return VINF_SUCCESS;
1835}
1836
1837
1838/**
1839 * RTMpPokeCpu worker that uses the Windows 7 and later version of
1840 * HalRequestIpip to get the job done.
1841 *
1842 * @returns VINF_SUCCESS
1843 * @param idCpu The CPU identifier.
1844 */
1845int rtMpPokeCpuUsingHalReqestIpiW7Plus(RTCPUID idCpu)
1846{
1847 /* idCpu is an HAL processor index, so we can use it directly. */
1848 KAFFINITY_EX Target;
1849 g_pfnrtKeInitializeAffinityEx(&Target);
1850 g_pfnrtKeAddProcessorAffinityEx(&Target, idCpu);
1851
1852 g_pfnrtHalRequestIpiW7Plus(0, &Target);
1853 return VINF_SUCCESS;
1854}
1855
1856
1857/**
1858 * RTMpPokeCpu worker that uses the Vista and earlier version of HalRequestIpip
1859 * to get the job done.
1860 *
1861 * @returns VINF_SUCCESS
1862 * @param idCpu The CPU identifier.
1863 */
1864int rtMpPokeCpuUsingHalReqestIpiPreW7(RTCPUID idCpu)
1865{
1866 __debugbreak(); /** @todo this code needs testing!! */
1867 KAFFINITY Target = 1;
1868 Target <<= idCpu;
1869 g_pfnrtHalRequestIpiPreW7(Target);
1870 return VINF_SUCCESS;
1871}
1872
1873
1874int rtMpPokeCpuUsingDpc(RTCPUID idCpu)
1875{
1876 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1877
1878 /*
1879 * APC fallback.
1880 */
1881 static KDPC s_aPokeDpcs[RTCPUSET_MAX_CPUS] = {0};
1882 static bool s_fPokeDPCsInitialized = false;
1883
1884 if (!s_fPokeDPCsInitialized)
1885 {
1886 for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++)
1887 {
1888 KeInitializeDpc(&s_aPokeDpcs[i], rtMpNtPokeCpuDummy, NULL);
1889 KeSetImportanceDpc(&s_aPokeDpcs[i], HighImportance);
1890 int rc = rtMpNtSetTargetProcessorDpc(&s_aPokeDpcs[i], idCpu);
1891 if (RT_FAILURE(rc))
1892 return rc;
1893 }
1894
1895 s_fPokeDPCsInitialized = true;
1896 }
1897
1898 /* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
1899 KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL. */
1900 KIRQL oldIrql;
1901 KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
1902
1903 KeSetImportanceDpc(&s_aPokeDpcs[idCpu], HighImportance);
1904 KeSetTargetProcessorDpc(&s_aPokeDpcs[idCpu], (int)idCpu);
1905
1906 /* Assuming here that high importance DPCs will be delivered immediately; or at least an IPI will be sent immediately.
1907 Note! Not true on at least Vista & Windows 7 */
1908 BOOLEAN fRet = KeInsertQueueDpc(&s_aPokeDpcs[idCpu], 0, 0);
1909
1910 KeLowerIrql(oldIrql);
1911 return fRet == TRUE ? VINF_SUCCESS : VERR_ACCESS_DENIED /* already queued */;
1912}
1913
1914
1915RTDECL(int) RTMpPokeCpu(RTCPUID idCpu)
1916{
1917 if (!RTMpIsCpuOnline(idCpu))
1918 return !RTMpIsCpuPossible(idCpu)
1919 ? VERR_CPU_NOT_FOUND
1920 : VERR_CPU_OFFLINE;
1921 /* Calls rtMpPokeCpuUsingDpc, rtMpPokeCpuUsingHalReqestIpiW7Plus or rtMpPokeCpuUsingBroadcastIpi. */
1922 return g_pfnrtMpPokeCpuWorker(idCpu);
1923}
1924
1925
1926RTDECL(bool) RTMpOnAllIsConcurrentSafe(void)
1927{
1928 return false;
1929}
1930
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette