mp-r0drv-nt.cpp@ 91446

Last change on this file since 91446 was 86191, checked in by vboxsync, 4 years ago
Runtime/mp-r0drv-nt.cpp: Dynamically determine the size of the KAFFINITY_EX structure as it is not static across Windows versions (increased lately with W10 20H2) [2nd attempt]
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 76.6 KB

Line
1	/* $Id: mp-r0drv-nt.cpp 86191 2020-09-21 09:59:04Z vboxsync $ */
2	/** @file
3	* IPRT - Multiprocessor, Ring-0 Driver, NT.
4	*/
5
6	/*
7	* Copyright (C) 2008-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*********************************************************************************************************************************
29	* Header Files *
30	*********************************************************************************************************************************/
31	#include "the-nt-kernel.h"
32
33	#include <iprt/mp.h>
34	#include <iprt/cpuset.h>
35	#include <iprt/err.h>
36	#include <iprt/asm.h>
37	#include <iprt/log.h>
38	#include <iprt/mem.h>
39	#include <iprt/time.h>
40	#include "r0drv/mp-r0drv.h"
41	#include "symdb.h"
42	#include "internal-r0drv-nt.h"
43	#include "internal/mp.h"
44
45
46	/*********************************************************************************************************************************
47	* Structures and Typedefs *
48	*********************************************************************************************************************************/
49	typedef enum
50	{
51	RT_NT_CPUID_SPECIFIC,
52	RT_NT_CPUID_PAIR,
53	RT_NT_CPUID_OTHERS,
54	RT_NT_CPUID_ALL
55	} RT_NT_CPUID;
56
57
58	/**
59	* Used by the RTMpOnSpecific.
60	*/
61	typedef struct RTMPNTONSPECIFICARGS
62	{
63	/** Set if we're executing. */
64	bool volatile fExecuting;
65	/** Set when done executing. */
66	bool volatile fDone;
67	/** Number of references to this heap block. */
68	uint32_t volatile cRefs;
69	/** Event that the calling thread is waiting on. */
70	KEVENT DoneEvt;
71	/** The deferred procedure call object. */
72	KDPC Dpc;
73	/** The callback argument package. */
74	RTMPARGS CallbackArgs;
75	} RTMPNTONSPECIFICARGS;
76	/** Pointer to an argument/state structure for RTMpOnSpecific on NT. */
77	typedef RTMPNTONSPECIFICARGS *PRTMPNTONSPECIFICARGS;
78
79
80	/*********************************************************************************************************************************
81	* Defined Constants And Macros *
82	*********************************************************************************************************************************/
83	/** Inactive bit for g_aidRtMpNtByCpuSetIdx. */
84	#define RTMPNT_ID_F_INACTIVE RT_BIT_32(31)
85
86
87	/*********************************************************************************************************************************
88	* Global Variables *
89	*********************************************************************************************************************************/
90	/** Maximum number of processor groups. */
91	uint32_t g_cRtMpNtMaxGroups;
92	/** Maximum number of processors. */
93	uint32_t g_cRtMpNtMaxCpus;
94	/** Number of active processors. */
95	uint32_t volatile g_cRtMpNtActiveCpus;
96	/** The NT CPU set.
97	* KeQueryActiveProcssors() cannot be called at all IRQLs and therefore we'll
98	* have to cache it. Fortunately, NT doesn't really support taking CPUs offline,
99	* and taking them online was introduced with W2K8 where it is intended for virtual
100	* machines and not real HW. We update this, g_cRtMpNtActiveCpus and
101	* g_aidRtMpNtByCpuSetIdx from the rtR0NtMpProcessorChangeCallback.
102	*/
103	RTCPUSET g_rtMpNtCpuSet;
104
105	/** Static per group info.
106	* @remarks With 256 groups this takes up 33KB. */
107	static struct
108	{
109	/** The max CPUs in the group. */
110	uint16_t cMaxCpus;
111	/** The number of active CPUs at the time of initialization. */
112	uint16_t cActiveCpus;
113	/** CPU set indexes for each CPU in the group. */
114	int16_t aidxCpuSetMembers[64];
115	} g_aRtMpNtCpuGroups[256];
116	/** Maps CPU set indexes to RTCPUID.
117	* Inactive CPUs has bit 31 set (RTMPNT_ID_F_INACTIVE) so we can identify them
118	* and shuffle duplicates during CPU hotplugging. We assign temporary IDs to
119	* the inactive CPUs starting at g_cRtMpNtMaxCpus - 1, ASSUMING that active
120	* CPUs has IDs from 0 to g_cRtMpNtActiveCpus. */
121	RTCPUID g_aidRtMpNtByCpuSetIdx[RTCPUSET_MAX_CPUS];
122	/** The handle of the rtR0NtMpProcessorChangeCallback registration. */
123	static PVOID g_pvMpCpuChangeCallback = NULL;
124	/** Size of the KAFFINITY_EX structure.
125	* This increased from 20 to 32 bitmap words in the 2020 H2 windows 10 release
126	* (i.e. 1280 to 2048 CPUs). We expect this to increase in the future. */
127	static size_t g_cbRtMpNtKaffinityEx = RT_UOFFSETOF(KAFFINITY_EX, Bitmap)
128	+ RT_SIZEOFMEMB(KAFFINITY_EX, Bitmap[0]) * 256;
129	/** The size value of the KAFFINITY_EX structure. */
130	static uint16_t g_cRtMpNtKaffinityExEntries = 256;
131
132
133	/*********************************************************************************************************************************
134	* Internal Functions *
135	*********************************************************************************************************************************/
136	static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,
137	PNTSTATUS prcOperationStatus);
138	static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo);
139
140
141
142	/**
143	* Initalizes multiprocessor globals (called by rtR0InitNative).
144	*
145	* @returns IPRT status code.
146	* @param pOsVerInfo Version information.
147	*/
148	DECLHIDDEN(int) rtR0MpNtInit(RTNTSDBOSVER const *pOsVerInfo)
149	{
150	#define MY_CHECK_BREAK(a_Check, a_DbgPrintArgs) \
151	AssertMsgBreakStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
152	#define MY_CHECK_RETURN(a_Check, a_DbgPrintArgs, a_rcRet) \
153	AssertMsgReturnStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs, a_rcRet)
154	#define MY_CHECK(a_Check, a_DbgPrintArgs) \
155	AssertMsgStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
156
157	/*
158	* API combination checks.
159	*/
160	MY_CHECK_RETURN(!g_pfnrtKeSetTargetProcessorDpcEx \|\| g_pfnrtKeGetProcessorNumberFromIndex,
161	("IPRT: Fatal: Missing KeSetTargetProcessorDpcEx without KeGetProcessorNumberFromIndex!\n"),
162	VERR_SYMBOL_NOT_FOUND);
163
164	/*
165	* Get max number of processor groups.
166	*
167	* We may need to upadjust this number below, because windows likes to keep
168	* all options open when it comes to hotplugged CPU group assignments. A
169	* server advertising up to 64 CPUs in the ACPI table will get a result of
170	* 64 from KeQueryMaximumGroupCount. That makes sense. However, when windows
171	* server 2012 does a two processor group setup for it, the sum of the
172	* GroupInfo[*].MaximumProcessorCount members below is 128. This is probably
173	* because windows doesn't want to make decisions grouping of hotpluggable CPUs.
174	* So, we need to bump the maximum count to 128 below do deal with this as we
175	* want to have valid CPU set indexes for all potential CPUs - how could we
176	* otherwise use the RTMpGetSet() result and also RTCpuSetCount(RTMpGetSet())
177	* should equal RTMpGetCount().
178	*/
179	if (g_pfnrtKeQueryMaximumGroupCount)
180	{
181	g_cRtMpNtMaxGroups = g_pfnrtKeQueryMaximumGroupCount();
182	MY_CHECK_RETURN(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,
183	("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
184	VERR_MP_TOO_MANY_CPUS);
185	}
186	else
187	g_cRtMpNtMaxGroups = 1;
188
189	/*
190	* Get max number CPUs.
191	* This also defines the range of NT CPU indexes, RTCPUID and index into RTCPUSET.
192	*/
193	if (g_pfnrtKeQueryMaximumProcessorCountEx)
194	{
195	g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS);
196	MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
197	("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCountEx]\n",
198	g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
199	VERR_MP_TOO_MANY_CPUS);
200	}
201	else if (g_pfnrtKeQueryMaximumProcessorCount)
202	{
203	g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCount();
204	MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
205	("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCount]\n",
206	g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
207	VERR_MP_TOO_MANY_CPUS);
208	}
209	else if (g_pfnrtKeQueryActiveProcessors)
210	{
211	KAFFINITY fActiveProcessors = g_pfnrtKeQueryActiveProcessors();
212	MY_CHECK_RETURN(fActiveProcessors != 0,
213	("IPRT: Fatal: KeQueryActiveProcessors returned 0!\n"),
214	VERR_INTERNAL_ERROR_2);
215	g_cRtMpNtMaxCpus = 0;
216	do
217	{
218	g_cRtMpNtMaxCpus++;
219	fActiveProcessors >>= 1;
220	} while (fActiveProcessors);
221	}
222	else
223	g_cRtMpNtMaxCpus = KeNumberProcessors;
224
225	/*
226	* Just because we're a bit paranoid about getting something wrong wrt to the
227	* kernel interfaces, we try 16 times to get the KeQueryActiveProcessorCountEx
228	* and KeQueryLogicalProcessorRelationship information to match up.
229	*/
230	for (unsigned cTries = 0;; cTries++)
231	{
232	/*
233	* Get number of active CPUs.
234	*/
235	if (g_pfnrtKeQueryActiveProcessorCountEx)
236	{
237	g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS);
238	MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0,
239	("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCountEx]\n",
240	g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus),
241	VERR_MP_TOO_MANY_CPUS);
242	}
243	else if (g_pfnrtKeQueryActiveProcessorCount)
244	{
245	g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCount(NULL);
246	MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0,
247	("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCount]\n",
248	g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus),
249	VERR_MP_TOO_MANY_CPUS);
250	}
251	else
252	g_cRtMpNtActiveCpus = g_cRtMpNtMaxCpus;
253
254	/*
255	* Query the details for the groups to figure out which CPUs are online as
256	* well as the NT index limit.
257	*/
258	for (unsigned i = 0; i < RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx); i++)
259	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
260	g_aidRtMpNtByCpuSetIdx[i] = NIL_RTCPUID;
261	#else
262	g_aidRtMpNtByCpuSetIdx[i] = i < g_cRtMpNtMaxCpus ? i : NIL_RTCPUID;
263	#endif
264	for (unsigned idxGroup = 0; idxGroup < RT_ELEMENTS(g_aRtMpNtCpuGroups); idxGroup++)
265	{
266	g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = 0;
267	g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0;
268	for (unsigned idxMember = 0; idxMember < RT_ELEMENTS(g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers); idxMember++)
269	g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = -1;
270	}
271
272	if (g_pfnrtKeQueryLogicalProcessorRelationship)
273	{
274	MY_CHECK_RETURN(g_pfnrtKeGetProcessorIndexFromNumber,
275	("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
276	VERR_SYMBOL_NOT_FOUND);
277	MY_CHECK_RETURN(g_pfnrtKeGetProcessorNumberFromIndex,
278	("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
279	VERR_SYMBOL_NOT_FOUND);
280	MY_CHECK_RETURN(g_pfnrtKeSetTargetProcessorDpcEx,
281	("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeSetTargetProcessorDpcEx!\n"),
282	VERR_SYMBOL_NOT_FOUND);
283
284	SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = NULL;
285	int rc = rtR0NtInitQueryGroupRelations(&pInfo);
286	if (RT_FAILURE(rc))
287	return rc;
288
289	MY_CHECK(pInfo->Group.MaximumGroupCount == g_cRtMpNtMaxGroups,
290	("IPRT: Fatal: MaximumGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n",
291	pInfo->Group.MaximumGroupCount, g_cRtMpNtMaxGroups));
292	MY_CHECK(pInfo->Group.ActiveGroupCount > 0 && pInfo->Group.ActiveGroupCount <= g_cRtMpNtMaxGroups,
293	("IPRT: Fatal: ActiveGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n",
294	pInfo->Group.ActiveGroupCount, g_cRtMpNtMaxGroups));
295
296	/*
297	* First we need to recalc g_cRtMpNtMaxCpus (see above).
298	*/
299	uint32_t cMaxCpus = 0;
300	uint32_t idxGroup;
301	for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)
302	{
303	const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];
304	MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,
305	("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));
306	MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,
307	("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",
308	pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));
309	cMaxCpus += pGrpInfo->MaximumProcessorCount;
310	}
311	if (cMaxCpus > g_cRtMpNtMaxCpus && RT_SUCCESS(rc))
312	{
313	DbgPrint("IPRT: g_cRtMpNtMaxCpus=%u -> %u\n", g_cRtMpNtMaxCpus, cMaxCpus);
314	#ifndef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
315	uint32_t i = RT_MIN(cMaxCpus, RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx));
316	while (i-- > g_cRtMpNtMaxCpus)
317	g_aidRtMpNtByCpuSetIdx[i] = i;
318	#endif
319	g_cRtMpNtMaxCpus = cMaxCpus;
320	if (g_cRtMpNtMaxGroups > RTCPUSET_MAX_CPUS)
321	{
322	MY_CHECK(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,
323	("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS));
324	rc = VERR_MP_TOO_MANY_CPUS;
325	}
326	}
327
328	/*
329	* Calc online mask, partition IDs and such.
330	*
331	* Also check ASSUMPTIONS:
332	*
333	* 1. Processor indexes going from 0 and up to
334	* KeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1.
335	*
336	* 2. Currently valid processor indexes, i.e. accepted by
337	* KeGetProcessorIndexFromNumber & KeGetProcessorNumberFromIndex, goes
338	* from 0 thru KeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1.
339	*
340	* 3. PROCESSOR_GROUP_INFO::MaximumProcessorCount gives the number of
341	* relevant bits in the ActiveProcessorMask (from LSB).
342	*
343	* 4. Active processor count found in KeQueryLogicalProcessorRelationship
344	* output matches what KeQueryActiveProcessorCountEx(ALL) returns.
345	*
346	* 5. Active + inactive processor counts in same does not exceed
347	* KeQueryMaximumProcessorCountEx(ALL).
348	*
349	* Note! Processor indexes are assigned as CPUs come online and are not
350	* preallocated according to group maximums. Since CPUS are only taken
351	* online and never offlined, this means that internal CPU bitmaps are
352	* never sparse and no time is wasted scanning unused bits.
353	*
354	* Unfortunately, it means that ring-3 cannot easily guess the index
355	* assignments when hotswapping is used, and must use GIP when available.
356	*/
357	RTCpuSetEmpty(&g_rtMpNtCpuSet);
358	uint32_t cInactive = 0;
359	uint32_t cActive = 0;
360	uint32_t idxCpuMax = 0;
361	uint32_t idxCpuSetNextInactive = g_cRtMpNtMaxCpus - 1;
362	for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)
363	{
364	const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];
365	MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,
366	("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));
367	MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,
368	("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",
369	pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));
370
371	g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = pGrpInfo->MaximumProcessorCount;
372	g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = pGrpInfo->ActiveProcessorCount;
373
374	for (uint32_t idxMember = 0; idxMember < pGrpInfo->MaximumProcessorCount; idxMember++)
375	{
376	PROCESSOR_NUMBER ProcNum;
377	ProcNum.Group = (USHORT)idxGroup;
378	ProcNum.Number = (UCHAR)idxMember;
379	ProcNum.Reserved = 0;
380	ULONG idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
381	if (idxCpu != INVALID_PROCESSOR_INDEX)
382	{
383	MY_CHECK_BREAK(idxCpu < g_cRtMpNtMaxCpus && idxCpu < RTCPUSET_MAX_CPUS, /* ASSUMPTION #1 */
384	("IPRT: Fatal: idxCpu=%u >= g_cRtMpNtMaxCpus=%u (RTCPUSET_MAX_CPUS=%u)\n",
385	idxCpu, g_cRtMpNtMaxCpus, RTCPUSET_MAX_CPUS));
386	if (idxCpu > idxCpuMax)
387	idxCpuMax = idxCpu;
388	g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpu;
389	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
390	g_aidRtMpNtByCpuSetIdx[idxCpu] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember);
391	#endif
392
393	ProcNum.Group = UINT16_MAX;
394	ProcNum.Number = UINT8_MAX;
395	ProcNum.Reserved = UINT8_MAX;
396	NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(idxCpu, &ProcNum);
397	MY_CHECK_BREAK(NT_SUCCESS(rcNt),
398	("IPRT: Fatal: KeGetProcessorNumberFromIndex(%u,) -> %#x!\n", idxCpu, rcNt));
399	MY_CHECK_BREAK(ProcNum.Group == idxGroup && ProcNum.Number == idxMember,
400	("IPRT: Fatal: KeGetProcessorXxxxFromYyyy roundtrip error for %#x! Group: %u vs %u, Number: %u vs %u\n",
401	idxCpu, ProcNum.Group, idxGroup, ProcNum.Number, idxMember));
402
403	if (pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember))
404	{
405	RTCpuSetAddByIndex(&g_rtMpNtCpuSet, idxCpu);
406	cActive++;
407	}
408	else
409	cInactive++; /* (This is a little unexpected, but not important as long as things add up below.) */
410	}
411	else
412	{
413	/* Must be not present / inactive when KeGetProcessorIndexFromNumber fails. */
414	MY_CHECK_BREAK(!(pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember)),
415	("IPRT: Fatal: KeGetProcessorIndexFromNumber(%u/%u) failed but CPU is active! cMax=%u cActive=%u fActive=%p\n",
416	idxGroup, idxMember, pGrpInfo->MaximumProcessorCount, pGrpInfo->ActiveProcessorCount,
417	pGrpInfo->ActiveProcessorMask));
418	cInactive++;
419	if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus)
420	{
421	g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
422	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
423	g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember)
424	\| RTMPNT_ID_F_INACTIVE;
425	#endif
426	idxCpuSetNextInactive--;
427	}
428	}
429	}
430	}
431
432	MY_CHECK(cInactive + cActive <= g_cRtMpNtMaxCpus, /* ASSUMPTION #5 (not '==' because of inactive groups) */
433	("IPRT: Fatal: cInactive=%u + cActive=%u > g_cRtMpNtMaxCpus=%u\n", cInactive, cActive, g_cRtMpNtMaxCpus));
434
435	/* Deal with inactive groups using KeQueryMaximumProcessorCountEx or as
436	best as we can by as best we can by stipulating maximum member counts
437	from the previous group. */
438	if ( RT_SUCCESS(rc)
439	&& idxGroup < pInfo->Group.MaximumGroupCount)
440	{
441	uint16_t cInactiveLeft = g_cRtMpNtMaxCpus - (cInactive + cActive);
442	while (idxGroup < pInfo->Group.MaximumGroupCount)
443	{
444	uint32_t cMaxMembers = 0;
445	if (g_pfnrtKeQueryMaximumProcessorCountEx)
446	cMaxMembers = g_pfnrtKeQueryMaximumProcessorCountEx(idxGroup);
447	if (cMaxMembers != 0 \|\| cInactiveLeft == 0)
448	AssertStmt(cMaxMembers <= cInactiveLeft, cMaxMembers = cInactiveLeft);
449	else
450	{
451	uint16_t cGroupsLeft = pInfo->Group.MaximumGroupCount - idxGroup;
452	cMaxMembers = pInfo->Group.GroupInfo[idxGroup - 1].MaximumProcessorCount;
453	while (cMaxMembers * cGroupsLeft < cInactiveLeft)
454	cMaxMembers++;
455	if (cMaxMembers > cInactiveLeft)
456	cMaxMembers = cInactiveLeft;
457	}
458
459	g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = (uint16_t)cMaxMembers;
460	g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0;
461	for (uint16_t idxMember = 0; idxMember < cMaxMembers; idxMember++)
462	if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus)
463	{
464	g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
465	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
466	g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember)
467	\| RTMPNT_ID_F_INACTIVE;
468	#endif
469	idxCpuSetNextInactive--;
470	}
471	cInactiveLeft -= cMaxMembers;
472	idxGroup++;
473	}
474	}
475
476	/* We're done with pInfo now, free it so we can start returning when assertions fail. */
477	RTMemFree(pInfo);
478	if (RT_FAILURE(rc)) /* MY_CHECK_BREAK sets rc. */
479	return rc;
480	MY_CHECK_RETURN(cActive >= g_cRtMpNtActiveCpus,
481	("IPRT: Fatal: cActive=%u < g_cRtMpNtActiveCpus=%u - CPUs removed?\n", cActive, g_cRtMpNtActiveCpus),
482	VERR_INTERNAL_ERROR_3);
483	MY_CHECK_RETURN(idxCpuMax < cActive, /* ASSUMPTION #2 */
484	("IPRT: Fatal: idCpuMax=%u >= cActive=%u! Unexpected CPU index allocation. CPUs removed?\n",
485	idxCpuMax, cActive),
486	VERR_INTERNAL_ERROR_4);
487
488	/* Retry if CPUs were added. */
489	if ( cActive != g_cRtMpNtActiveCpus
490	&& cTries < 16)
491	continue;
492	MY_CHECK_RETURN(cActive == g_cRtMpNtActiveCpus, /* ASSUMPTION #4 */
493	("IPRT: Fatal: cActive=%u != g_cRtMpNtActiveCpus=%u\n", cActive, g_cRtMpNtActiveCpus),
494	VERR_INTERNAL_ERROR_5);
495	}
496	else
497	{
498	/* Legacy: */
499	MY_CHECK_RETURN(g_cRtMpNtMaxGroups == 1, ("IPRT: Fatal: Missing KeQueryLogicalProcessorRelationship!\n"),
500	VERR_SYMBOL_NOT_FOUND);
501
502	/** @todo Is it possible that the affinity mask returned by
503	* KeQueryActiveProcessors is sparse? */
504	if (g_pfnrtKeQueryActiveProcessors)
505	RTCpuSetFromU64(&g_rtMpNtCpuSet, g_pfnrtKeQueryActiveProcessors());
506	else if (g_cRtMpNtMaxCpus < 64)
507	RTCpuSetFromU64(&g_rtMpNtCpuSet, (UINT64_C(1) << g_cRtMpNtMaxCpus) - 1);
508	else
509	{
510	MY_CHECK_RETURN(g_cRtMpNtMaxCpus == 64, ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, expect 64 or less\n", g_cRtMpNtMaxCpus),
511	VERR_MP_TOO_MANY_CPUS);
512	RTCpuSetFromU64(&g_rtMpNtCpuSet, UINT64_MAX);
513	}
514
515	g_aRtMpNtCpuGroups[0].cMaxCpus = g_cRtMpNtMaxCpus;
516	g_aRtMpNtCpuGroups[0].cActiveCpus = g_cRtMpNtMaxCpus;
517	for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++)
518	{
519	g_aRtMpNtCpuGroups[0].aidxCpuSetMembers[i] = i;
520	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
521	g_aidRtMpNtByCpuSetIdx[i] = RTMPCPUID_FROM_GROUP_AND_NUMBER(0, i);
522	#endif
523	}
524	}
525
526	/*
527	* Register CPU hot plugging callback (it also counts active CPUs).
528	*/
529	Assert(g_pvMpCpuChangeCallback == NULL);
530	if (g_pfnrtKeRegisterProcessorChangeCallback)
531	{
532	MY_CHECK_RETURN(g_pfnrtKeDeregisterProcessorChangeCallback,
533	("IPRT: Fatal: KeRegisterProcessorChangeCallback without KeDeregisterProcessorChangeCallback!\n"),
534	VERR_SYMBOL_NOT_FOUND);
535
536	RTCPUSET const ActiveSetCopy = g_rtMpNtCpuSet;
537	RTCpuSetEmpty(&g_rtMpNtCpuSet);
538	uint32_t const cActiveCpus = g_cRtMpNtActiveCpus;
539	g_cRtMpNtActiveCpus = 0;
540
541	g_pvMpCpuChangeCallback = g_pfnrtKeRegisterProcessorChangeCallback(rtR0NtMpProcessorChangeCallback, NULL /pvUser/,
542	KE_PROCESSOR_CHANGE_ADD_EXISTING);
543	if (g_pvMpCpuChangeCallback)
544	{
545	if (cActiveCpus == g_cRtMpNtActiveCpus)
546	{ /* likely */ }
547	else
548	{
549	g_pfnrtKeDeregisterProcessorChangeCallback(g_pvMpCpuChangeCallback);
550	if (cTries < 16)
551	{
552	/* Retry if CPUs were added. */
553	MY_CHECK_RETURN(g_cRtMpNtActiveCpus >= cActiveCpus,
554	("IPRT: Fatal: g_cRtMpNtActiveCpus=%u < cActiveCpus=%u! CPUs removed?\n",
555	g_cRtMpNtActiveCpus, cActiveCpus),
556	VERR_INTERNAL_ERROR_2);
557	MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus,
558	("IPRT: Fatal: g_cRtMpNtActiveCpus=%u > g_cRtMpNtMaxCpus=%u!\n",
559	g_cRtMpNtActiveCpus, g_cRtMpNtMaxCpus),
560	VERR_INTERNAL_ERROR_2);
561	continue;
562	}
563	MY_CHECK_RETURN(0, ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u cActiveCpus=%u\n", g_cRtMpNtActiveCpus, cActiveCpus),
564	VERR_INTERNAL_ERROR_3);
565	}
566	}
567	else
568	{
569	AssertFailed();
570	g_rtMpNtCpuSet = ActiveSetCopy;
571	g_cRtMpNtActiveCpus = cActiveCpus;
572	}
573	}
574	break;
575	} /* Retry loop for stable active CPU count. */
576
577	#undef MY_CHECK_RETURN
578
579	/*
580	* Special IPI fun for RTMpPokeCpu.
581	*
582	* On Vista and later the DPC method doesn't seem to reliably send IPIs,
583	* so we have to use alternative methods.
584	*
585	* On AMD64 We used to use the HalSendSoftwareInterrupt API (also x86 on
586	* W10+), it looks faster and more convenient to use, however we're either
587	* using it wrong or it doesn't reliably do what we want (see @bugref{8343}).
588	*
589	* The HalRequestIpip API is thus far the only alternative to KeInsertQueueDpc
590	* for doing targetted IPIs. Trouble with this API is that it changed
591	* fundamentally in Window 7 when they added support for lots of processors.
592	*
593	* If we really think we cannot use KeInsertQueueDpc, we use the broadcast IPI
594	* API KeIpiGenericCall.
595	*/
596	if ( pOsVerInfo->uMajorVer > 6
597	\|\| (pOsVerInfo->uMajorVer == 6 && pOsVerInfo->uMinorVer > 0))
598	g_pfnrtHalRequestIpiPreW7 = NULL;
599	else
600	g_pfnrtHalRequestIpiW7Plus = NULL;
601
602	if ( g_pfnrtHalRequestIpiW7Plus
603	&& g_pfnrtKeInitializeAffinityEx
604	&& g_pfnrtKeAddProcessorAffinityEx
605	&& g_pfnrtKeGetProcessorIndexFromNumber)
606	{
607	/* Determine the real size of the KAFFINITY_EX structure. */
608	size_t const cbAffinity = _8K;
609	PKAFFINITY_EX pAffinity = (PKAFFINITY_EX)RTMemAllocZ(cbAffinity);
610	AssertReturn(pAffinity, VERR_NO_MEMORY);
611	size_t const cMaxEntries = (cbAffinity - RT_UOFFSETOF(KAFFINITY_EX, Bitmap[0])) / sizeof(pAffinity->Bitmap[0]);
612	g_pfnrtKeInitializeAffinityEx(pAffinity);
613	if (pAffinity->Size > 1 && pAffinity->Size <= cMaxEntries)
614	{
615	g_cRtMpNtKaffinityExEntries = pAffinity->Size;
616	g_cbRtMpNtKaffinityEx = pAffinity->Size * sizeof(pAffinity->Bitmap[0]) + RT_UOFFSETOF(KAFFINITY_EX, Bitmap[0]);
617	g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingHalRequestIpiW7Plus;
618	RTMemFree(pAffinity);
619	DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingHalRequestIpiW7Plus\n");
620	return VINF_SUCCESS;
621	}
622	DbgPrint("IPRT: RTMpPoke can't use rtMpPokeCpuUsingHalRequestIpiW7Plus! pAffinity->Size=%u\n", pAffinity->Size);
623	AssertReleaseMsg(pAffinity->Size <= cMaxEntries, ("%#x\n", pAffinity->Size)); /* stack is toast if larger (32768 CPUs). */
624	RTMemFree(pAffinity);
625	}
626
627	if (pOsVerInfo->uMajorVer >= 6 && g_pfnrtKeIpiGenericCall)
628	{
629	DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingBroadcastIpi\n");
630	g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingBroadcastIpi;
631	}
632	else if (g_pfnrtKeSetTargetProcessorDpc)
633	{
634	DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingDpc\n");
635	g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingDpc;
636	/* Windows XP should send always send an IPI -> VERIFY */
637	}
638	else
639	{
640	DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingFailureNotSupported\n");
641	Assert(pOsVerInfo->uMajorVer == 3 && pOsVerInfo->uMinorVer <= 50);
642	g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingFailureNotSupported;
643	}
644
645	return VINF_SUCCESS;
646	}
647
648
649	/**
650	* Called by rtR0TermNative.
651	*/
652	DECLHIDDEN(void) rtR0MpNtTerm(void)
653	{
654	/*
655	* Deregister the processor change callback.
656	*/
657	PVOID pvMpCpuChangeCallback = g_pvMpCpuChangeCallback;
658	g_pvMpCpuChangeCallback = NULL;
659	if (pvMpCpuChangeCallback)
660	{
661	AssertReturnVoid(g_pfnrtKeDeregisterProcessorChangeCallback);
662	g_pfnrtKeDeregisterProcessorChangeCallback(pvMpCpuChangeCallback);
663	}
664	}
665
666
667	DECLHIDDEN(int) rtR0MpNotificationNativeInit(void)
668	{
669	return VINF_SUCCESS;
670	}
671
672
673	DECLHIDDEN(void) rtR0MpNotificationNativeTerm(void)
674	{
675	}
676
677
678	/**
679	* Implements the NT PROCESSOR_CALLBACK_FUNCTION callback function.
680	*
681	* This maintains the g_rtMpNtCpuSet and works MP notification callbacks. When
682	* registered, it's called for each active CPU in the system, avoiding racing
683	* CPU hotplugging (as well as testing the callback).
684	*
685	* @param pvUser User context (not used).
686	* @param pChangeCtx Change context (in).
687	* @param prcOperationStatus Operation status (in/out).
688	*
689	* @remarks ASSUMES no concurrent execution of KeProcessorAddCompleteNotify
690	* notification callbacks. At least during callback registration
691	* callout, we're owning KiDynamicProcessorLock.
692	*
693	* @remarks When registering the handler, we first get KeProcessorAddStartNotify
694	* callbacks for all active CPUs, and after they all succeed we get the
695	* KeProcessorAddCompleteNotify callbacks.
696	*/
697	static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,
698	PNTSTATUS prcOperationStatus)
699	{
700	RT_NOREF(pvUser, prcOperationStatus);
701	switch (pChangeCtx->State)
702	{
703	/*
704	* Check whether we can deal with the CPU, failing the start operation if we
705	* can't. The checks we are doing here are to avoid complicated/impossible
706	* cases in KeProcessorAddCompleteNotify. They are really just verify specs.
707	*/
708	case KeProcessorAddStartNotify:
709	{
710	NTSTATUS rcNt = STATUS_SUCCESS;
711	if (pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS)
712	{
713	if (pChangeCtx->NtNumber >= g_cRtMpNtMaxCpus)
714	{
715	DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is higher than the max CPU count (%u)!\n",
716	pChangeCtx->NtNumber, g_cRtMpNtMaxCpus);
717	rcNt = STATUS_INTERNAL_ERROR;
718	}
719
720	/* The ProcessNumber field was introduced in Windows 7. */
721	PROCESSOR_NUMBER ProcNum;
722	if (g_pfnrtKeGetProcessorIndexFromNumber)
723	{
724	ProcNum = pChangeCtx->ProcNumber;
725	KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
726	if (idxCpu != pChangeCtx->NtNumber)
727	{
728	DbgPrint("IPRT: KeProcessorAddStartNotify failure: g_pfnrtKeGetProcessorIndexFromNumber(%u.%u) -> %u, expected %u!\n",
729	ProcNum.Group, ProcNum.Number, idxCpu, pChangeCtx->NtNumber);
730	rcNt = STATUS_INTERNAL_ERROR;
731	}
732	}
733	else
734	{
735	ProcNum.Group = 0;
736	ProcNum.Number = pChangeCtx->NtNumber;
737	}
738
739	if ( ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups)
740	&& ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers))
741	{
742	if (ProcNum.Group >= g_cRtMpNtMaxGroups)
743	{
744	DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range - max groups: %u!\n",
745	ProcNum.Group, ProcNum.Number, g_cRtMpNtMaxGroups);
746	rcNt = STATUS_INTERNAL_ERROR;
747	}
748
749	if (ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus)
750	{
751	Assert(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1);
752	if (g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == -1)
753	{
754	DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u.%u was assigned -1 as set index!\n",
755	ProcNum.Group, ProcNum.Number);
756	rcNt = STATUS_INTERNAL_ERROR;
757	}
758
759	Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID);
760	if (g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == NIL_RTCPUID)
761	{
762	DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u (%u.%u) translates to NIL_RTCPUID!\n",
763	pChangeCtx->NtNumber, ProcNum.Group, ProcNum.Number);
764	rcNt = STATUS_INTERNAL_ERROR;
765	}
766	}
767	else
768	{
769	DbgPrint("IPRT: KeProcessorAddStartNotify failure: max processors in group %u is %u, cannot add %u to it!\n",
770	ProcNum.Group, g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus, ProcNum.Group, ProcNum.Number);
771	rcNt = STATUS_INTERNAL_ERROR;
772	}
773	}
774	else
775	{
776	DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range (max %u.%u)!\n",
777	ProcNum.Group, ProcNum.Number, RT_ELEMENTS(g_aRtMpNtCpuGroups), RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers));
778	rcNt = STATUS_INTERNAL_ERROR;
779	}
780	}
781	else
782	{
783	DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is outside RTCPUSET_MAX_CPUS (%u)!\n",
784	pChangeCtx->NtNumber, RTCPUSET_MAX_CPUS);
785	rcNt = STATUS_INTERNAL_ERROR;
786	}
787	if (!NT_SUCCESS(rcNt))
788	*prcOperationStatus = rcNt;
789	break;
790	}
791
792	/*
793	* Update the globals. Since we've checked out range limits and other
794	* limitations already we just AssertBreak here.
795	*/
796	case KeProcessorAddCompleteNotify:
797	{
798	/*
799	* Calc the processor number and assert conditions checked in KeProcessorAddStartNotify.
800	*/
801	AssertBreak(pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS);
802	AssertBreak(pChangeCtx->NtNumber < g_cRtMpNtMaxCpus);
803	Assert(pChangeCtx->NtNumber == g_cRtMpNtActiveCpus); /* light assumption */
804	PROCESSOR_NUMBER ProcNum;
805	if (g_pfnrtKeGetProcessorIndexFromNumber)
806	{
807	ProcNum = pChangeCtx->ProcNumber;
808	AssertBreak(g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum) == pChangeCtx->NtNumber);
809	AssertBreak(ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups));
810	AssertBreak(ProcNum.Group < g_cRtMpNtMaxGroups);
811	}
812	else
813	{
814	ProcNum.Group = 0;
815	ProcNum.Number = pChangeCtx->NtNumber;
816	}
817	AssertBreak(ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers));
818	AssertBreak(ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus);
819	AssertBreak(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1);
820	AssertBreak(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID);
821
822	/*
823	* Add ourselves to the online CPU set and update the active CPU count.
824	*/
825	RTCpuSetAddByIndex(&g_rtMpNtCpuSet, pChangeCtx->NtNumber);
826	ASMAtomicIncU32(&g_cRtMpNtActiveCpus);
827
828	/*
829	* Update the group info.
830	*
831	* If the index prediction failed (real hotplugging callbacks only) we
832	* have to switch it around. This is particularly annoying when we
833	* use the index as the ID.
834	*/
835	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
836	RTCPUID idCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
837	RTCPUID idOld = g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber];
838	if ((idOld & ~RTMPNT_ID_F_INACTIVE) != idCpu)
839	{
840	Assert(idOld & RTMPNT_ID_F_INACTIVE);
841	int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
842	g_aRtMpNtCpuGroups[rtMpCpuIdGetGroup(idOld)].aidxCpuSetMembers[rtMpCpuIdGetGroupMember(idOld)] = idxDest;
843	g_aidRtMpNtByCpuSetIdx[idxDest] = idOld;
844	}
845	g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] = idCpu;
846	#else
847	Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == pChangeCtx->NtNumber);
848	int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
849	if ((ULONG)idxDest != pChangeCtx->NtNumber)
850	{
851	bool fFound = false;
852	uint32_t idxOldGroup = g_cRtMpNtMaxGroups;
853	while (idxOldGroup-- > 0 && !fFound)
854	{
855	uint32_t idxMember = g_aRtMpNtCpuGroups[idxOldGroup].cMaxCpus;
856	while (idxMember-- > 0)
857	if (g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] == (int)pChangeCtx->NtNumber)
858	{
859	g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] = idxDest;
860	fFound = true;
861	break;
862	}
863	}
864	Assert(fFound);
865	}
866	#endif
867	g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] = pChangeCtx->NtNumber;
868
869	/*
870	* Do MP notification callbacks.
871	*/
872	rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, pChangeCtx->NtNumber);
873	break;
874	}
875
876	case KeProcessorAddFailureNotify:
877	/* ignore */
878	break;
879
880	default:
881	AssertMsgFailed(("State=%u\n", pChangeCtx->State));
882	}
883	}
884
885
886	/**
887	* Wrapper around KeQueryLogicalProcessorRelationship.
888	*
889	* @returns IPRT status code.
890	* @param ppInfo Where to return the info. Pass to RTMemFree when done.
891	*/
892	static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo)
893	{
894	ULONG cbInfo = sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)
895	+ g_cRtMpNtMaxGroups * sizeof(GROUP_RELATIONSHIP);
896	NTSTATUS rcNt;
897	do
898	{
899	SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX pInfo = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX )RTMemAlloc(cbInfo);
900	if (pInfo)
901	{
902	rcNt = g_pfnrtKeQueryLogicalProcessorRelationship(NULL /pProcNumber/, RelationGroup, pInfo, &cbInfo);
903	if (NT_SUCCESS(rcNt))
904	{
905	*ppInfo = pInfo;
906	return VINF_SUCCESS;
907	}
908
909	RTMemFree(pInfo);
910	pInfo = NULL;
911	}
912	else
913	rcNt = STATUS_NO_MEMORY;
914	} while (rcNt == STATUS_INFO_LENGTH_MISMATCH);
915	DbgPrint("IPRT: Fatal: KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt);
916	AssertMsgFailed(("KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt));
917	return RTErrConvertFromNtStatus(rcNt);
918	}
919
920
921
922
923
924	RTDECL(RTCPUID) RTMpCpuId(void)
925	{
926	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
927
928	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
929	PROCESSOR_NUMBER ProcNum;
930	ProcNum.Group = 0;
931	if (g_pfnrtKeGetCurrentProcessorNumberEx)
932	{
933	ProcNum.Number = 0;
934	g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum);
935	}
936	else
937	ProcNum.Number = KeGetCurrentProcessorNumber(); /* Number is 8-bit, so we're not subject to BYTE -> WORD upgrade in WDK. */
938	return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
939
940	#else
941
942	if (g_pfnrtKeGetCurrentProcessorNumberEx)
943	{
944	KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL);
945	Assert(idxCpu < RTCPUSET_MAX_CPUS);
946	return idxCpu;
947	}
948
949	return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */
950	#endif
951	}
952
953
954	RTDECL(int) RTMpCurSetIndex(void)
955	{
956	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
957	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
958
959	if (g_pfnrtKeGetCurrentProcessorNumberEx)
960	{
961	KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL);
962	Assert(idxCpu < RTCPUSET_MAX_CPUS);
963	return idxCpu;
964	}
965	return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */
966	#else
967	return (int)RTMpCpuId();
968	#endif
969	}
970
971
972	RTDECL(int) RTMpCurSetIndexAndId(PRTCPUID pidCpu)
973	{
974	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
975	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
976
977	PROCESSOR_NUMBER ProcNum = { 0 , 0, 0 };
978	KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum);
979	Assert(idxCpu < RTCPUSET_MAX_CPUS);
980	*pidCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
981	return idxCpu;
982	#else
983	return *pidCpu = RTMpCpuId();
984	#endif
985	}
986
987
988	RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu)
989	{
990	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
991	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
992
993	if (idCpu != NIL_RTCPUID)
994	{
995	if (g_pfnrtKeGetProcessorIndexFromNumber)
996	{
997	PROCESSOR_NUMBER ProcNum;
998	ProcNum.Group = rtMpCpuIdGetGroup(idCpu);
999	ProcNum.Number = rtMpCpuIdGetGroupMember(idCpu);
1000	ProcNum.Reserved = 0;
1001	KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
1002	if (idxCpu != INVALID_PROCESSOR_INDEX)
1003	{
1004	Assert(idxCpu < g_cRtMpNtMaxCpus);
1005	Assert((ULONG)g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == idxCpu);
1006	return idxCpu;
1007	}
1008
1009	/* Since NT assigned indexes as the CPUs come online, we cannot produce an ID <-> index
1010	mapping for not-yet-onlined CPUS that is consistent. We just have to do our best... */
1011	if ( ProcNum.Group < g_cRtMpNtMaxGroups
1012	&& ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus)
1013	return g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
1014	}
1015	else if (rtMpCpuIdGetGroup(idCpu) == 0)
1016	return rtMpCpuIdGetGroupMember(idCpu);
1017	}
1018	return -1;
1019	#else
1020	/* 1:1 mapping, just do range checks. */
1021	return idCpu < RTCPUSET_MAX_CPUS ? (int)idCpu : -1;
1022	#endif
1023	}
1024
1025
1026	RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu)
1027	{
1028	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1029	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1030
1031	if ((unsigned)iCpu < g_cRtMpNtMaxCpus)
1032	{
1033	if (g_pfnrtKeGetProcessorIndexFromNumber)
1034	{
1035	PROCESSOR_NUMBER ProcNum = { 0, 0, 0 };
1036	NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(iCpu, &ProcNum);
1037	if (NT_SUCCESS(rcNt))
1038	{
1039	Assert(ProcNum.Group <= g_cRtMpNtMaxGroups);
1040	Assert( (g_aidRtMpNtByCpuSetIdx[iCpu] & ~RTMPNT_ID_F_INACTIVE)
1041	== RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number));
1042	return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
1043	}
1044	}
1045	return g_aidRtMpNtByCpuSetIdx[iCpu];
1046	}
1047	return NIL_RTCPUID;
1048	#else
1049	/* 1:1 mapping, just do range checks. */
1050	return (unsigned)iCpu < RTCPUSET_MAX_CPUS ? iCpu : NIL_RTCPUID;
1051	#endif
1052	}
1053
1054
1055	RTDECL(int) RTMpSetIndexFromCpuGroupMember(uint32_t idxGroup, uint32_t idxMember)
1056	{
1057	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1058
1059	if (idxGroup < g_cRtMpNtMaxGroups)
1060	if (idxMember < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus)
1061	return g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember];
1062	return -1;
1063	}
1064
1065
1066	RTDECL(uint32_t) RTMpGetCpuGroupCounts(uint32_t idxGroup, uint32_t *pcActive)
1067	{
1068	if (idxGroup < g_cRtMpNtMaxGroups)
1069	{
1070	if (pcActive)
1071	*pcActive = g_aRtMpNtCpuGroups[idxGroup].cActiveCpus;
1072	return g_aRtMpNtCpuGroups[idxGroup].cMaxCpus;
1073	}
1074	if (pcActive)
1075	*pcActive = 0;
1076	return 0;
1077	}
1078
1079
1080	RTDECL(uint32_t) RTMpGetMaxCpuGroupCount(void)
1081	{
1082	return g_cRtMpNtMaxGroups;
1083	}
1084
1085
1086	RTDECL(RTCPUID) RTMpGetMaxCpuId(void)
1087	{
1088	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1089
1090	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1091	return RTMPCPUID_FROM_GROUP_AND_NUMBER(g_cRtMpNtMaxGroups - 1, g_aRtMpNtCpuGroups[g_cRtMpNtMaxGroups - 1].cMaxCpus - 1);
1092	#else
1093	/* According to MSDN the processor indexes goes from 0 to the maximum
1094	number of CPUs in the system. We've check this in initterm-r0drv-nt.cpp. */
1095	return g_cRtMpNtMaxCpus - 1;
1096	#endif
1097	}
1098
1099
1100	RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
1101	{
1102	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1103	return RTCpuSetIsMember(&g_rtMpNtCpuSet, idCpu);
1104	}
1105
1106
1107	RTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu)
1108	{
1109	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1110
1111	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1112	if (idCpu != NIL_RTCPUID)
1113	{
1114	unsigned idxGroup = rtMpCpuIdGetGroup(idCpu);
1115	if (idxGroup < g_cRtMpNtMaxGroups)
1116	return rtMpCpuIdGetGroupMember(idCpu) < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus;
1117	}
1118	return false;
1119
1120	#else
1121	/* A possible CPU ID is one with a value lower than g_cRtMpNtMaxCpus (see
1122	comment in RTMpGetMaxCpuId). */
1123	return idCpu < g_cRtMpNtMaxCpus;
1124	#endif
1125	}
1126
1127
1128
1129	RTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet)
1130	{
1131	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1132
1133	/* The set of possible CPU IDs(/indexes) are from 0 up to
1134	g_cRtMpNtMaxCpus (see comment in RTMpGetMaxCpuId). */
1135	RTCpuSetEmpty(pSet);
1136	int idxCpu = g_cRtMpNtMaxCpus;
1137	while (idxCpu-- > 0)
1138	RTCpuSetAddByIndex(pSet, idxCpu);
1139	return pSet;
1140	}
1141
1142
1143	RTDECL(RTCPUID) RTMpGetCount(void)
1144	{
1145	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1146	return g_cRtMpNtMaxCpus;
1147	}
1148
1149
1150	RTDECL(PRTCPUSET) RTMpGetOnlineSet(PRTCPUSET pSet)
1151	{
1152	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1153
1154	*pSet = g_rtMpNtCpuSet;
1155	return pSet;
1156	}
1157
1158
1159	RTDECL(RTCPUID) RTMpGetOnlineCount(void)
1160	{
1161	RTCPUSET Set;
1162	RTMpGetOnlineSet(&Set);
1163	return RTCpuSetCount(&Set);
1164	}
1165
1166
1167	RTDECL(RTCPUID) RTMpGetOnlineCoreCount(void)
1168	{
1169	/** @todo fix me */
1170	return RTMpGetOnlineCount();
1171	}
1172
1173
1174
1175	#if 0
1176	/* Experiment with checking the undocumented KPRCB structure
1177	* 'dt nt!_kprcb 0xaddress' shows the layout
1178	*/
1179	typedef struct
1180	{
1181	LIST_ENTRY DpcListHead;
1182	ULONG_PTR DpcLock;
1183	volatile ULONG DpcQueueDepth;
1184	ULONG DpcQueueCount;
1185	} KDPC_DATA, *PKDPC_DATA;
1186
1187	RTDECL(bool) RTMpIsCpuWorkPending(void)
1188	{
1189	uint8_t *pkprcb;
1190	PKDPC_DATA pDpcData;
1191
1192	_asm {
1193	mov eax, fs:0x20
1194	mov pkprcb, eax
1195	}
1196	pDpcData = (PKDPC_DATA)(pkprcb + 0x19e0);
1197	if (pDpcData->DpcQueueDepth)
1198	return true;
1199
1200	pDpcData++;
1201	if (pDpcData->DpcQueueDepth)
1202	return true;
1203	return false;
1204	}
1205	#else
1206	RTDECL(bool) RTMpIsCpuWorkPending(void)
1207	{
1208	/** @todo not implemented */
1209	return false;
1210	}
1211	#endif
1212
1213
1214	/**
1215	* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1216	* the RTMpOnAll case.
1217	*
1218	* @param uUserCtx The user context argument (PRTMPARGS).
1219	*/
1220	static ULONG_PTR rtmpNtOnAllBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1221	{
1222	PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1223	/ASMAtomicIncU32(&pArgs->cHits); - not needed /
1224	pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
1225	return 0;
1226	}
1227
1228
1229	/**
1230	* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1231	* the RTMpOnOthers case.
1232	*
1233	* @param uUserCtx The user context argument (PRTMPARGS).
1234	*/
1235	static ULONG_PTR rtmpNtOnOthersBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1236	{
1237	PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1238	RTCPUID idCpu = RTMpCpuId();
1239	if (pArgs->idCpu != idCpu)
1240	{
1241	/ASMAtomicIncU32(&pArgs->cHits); - not needed /
1242	pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1243	}
1244	return 0;
1245	}
1246
1247
1248	/**
1249	* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1250	* the RTMpOnPair case.
1251	*
1252	* @param uUserCtx The user context argument (PRTMPARGS).
1253	*/
1254	static ULONG_PTR rtmpNtOnPairBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1255	{
1256	PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1257	RTCPUID idCpu = RTMpCpuId();
1258	if ( pArgs->idCpu == idCpu
1259	\|\| pArgs->idCpu2 == idCpu)
1260	{
1261	ASMAtomicIncU32(&pArgs->cHits);
1262	pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1263	}
1264	return 0;
1265	}
1266
1267
1268	/**
1269	* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1270	* the RTMpOnSpecific case.
1271	*
1272	* @param uUserCtx The user context argument (PRTMPARGS).
1273	*/
1274	static ULONG_PTR rtmpNtOnSpecificBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1275	{
1276	PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1277	RTCPUID idCpu = RTMpCpuId();
1278	if (pArgs->idCpu == idCpu)
1279	{
1280	ASMAtomicIncU32(&pArgs->cHits);
1281	pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1282	}
1283	return 0;
1284	}
1285
1286
1287	/**
1288	* Internal worker for the RTMpOn* APIs using KeIpiGenericCall.
1289	*
1290	* @returns VINF_SUCCESS.
1291	* @param pfnWorker The callback.
1292	* @param pvUser1 User argument 1.
1293	* @param pvUser2 User argument 2.
1294	* @param pfnNativeWrapper The wrapper between the NT and IPRT callbacks.
1295	* @param idCpu First CPU to match, ultimately specific to the
1296	* pfnNativeWrapper used.
1297	* @param idCpu2 Second CPU to match, ultimately specific to the
1298	* pfnNativeWrapper used.
1299	* @param pcHits Where to return the number of this. Optional.
1300	*/
1301	static int rtMpCallUsingBroadcastIpi(PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2,
1302	PKIPI_BROADCAST_WORKER pfnNativeWrapper, RTCPUID idCpu, RTCPUID idCpu2,
1303	uint32_t *pcHits)
1304	{
1305	RTMPARGS Args;
1306	Args.pfnWorker = pfnWorker;
1307	Args.pvUser1 = pvUser1;
1308	Args.pvUser2 = pvUser2;
1309	Args.idCpu = idCpu;
1310	Args.idCpu2 = idCpu2;
1311	Args.cRefs = 0;
1312	Args.cHits = 0;
1313
1314	AssertPtr(g_pfnrtKeIpiGenericCall);
1315	g_pfnrtKeIpiGenericCall(pfnNativeWrapper, (uintptr_t)&Args);
1316	if (pcHits)
1317	*pcHits = Args.cHits;
1318	return VINF_SUCCESS;
1319	}
1320
1321
1322	/**
1323	* Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
1324	*
1325	* @param Dpc DPC object
1326	* @param DeferredContext Context argument specified by KeInitializeDpc
1327	* @param SystemArgument1 Argument specified by KeInsertQueueDpc
1328	* @param SystemArgument2 Argument specified by KeInsertQueueDpc
1329	*/
1330	static VOID rtmpNtDPCWrapper(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1331	{
1332	PRTMPARGS pArgs = (PRTMPARGS)DeferredContext;
1333	RT_NOREF3(Dpc, SystemArgument1, SystemArgument2);
1334
1335	ASMAtomicIncU32(&pArgs->cHits);
1336	pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
1337
1338	/* Dereference the argument structure. */
1339	int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
1340	Assert(cRefs >= 0);
1341	if (cRefs == 0)
1342	RTMemFree(pArgs);
1343	}
1344
1345
1346	/**
1347	* Wrapper around KeSetTargetProcessorDpcEx / KeSetTargetProcessorDpc.
1348	*
1349	* This is shared with the timer code.
1350	*
1351	* @returns IPRT status code (errors are asserted).
1352	* @param pDpc The DPC.
1353	* @param idCpu The ID of the new target CPU.
1354	*/
1355	DECLHIDDEN(int) rtMpNtSetTargetProcessorDpc(KDPC *pDpc, RTCPUID idCpu)
1356	{
1357	if (g_pfnrtKeSetTargetProcessorDpcEx)
1358	{
1359	/* Convert to stupid process number (bet KeSetTargetProcessorDpcEx does
1360	the reverse conversion internally). */
1361	PROCESSOR_NUMBER ProcNum;
1362	NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(RTMpCpuIdToSetIndex(idCpu), &ProcNum);
1363	AssertMsgReturn(NT_SUCCESS(rcNt),
1364	("KeGetProcessorNumberFromIndex(%u) -> %#x\n", idCpu, rcNt),
1365	RTErrConvertFromNtStatus(rcNt));
1366
1367	rcNt = g_pfnrtKeSetTargetProcessorDpcEx(pDpc, &ProcNum);
1368	AssertMsgReturn(NT_SUCCESS(rcNt),
1369	("KeSetTargetProcessorDpcEx(,%u(%u/%u)) -> %#x\n", idCpu, ProcNum.Group, ProcNum.Number, rcNt),
1370	RTErrConvertFromNtStatus(rcNt));
1371	}
1372	else if (g_pfnrtKeSetTargetProcessorDpc)
1373	g_pfnrtKeSetTargetProcessorDpc(pDpc, RTMpCpuIdToSetIndex(idCpu));
1374	else
1375	return VERR_NOT_SUPPORTED;
1376	return VINF_SUCCESS;
1377	}
1378
1379
1380	/**
1381	* Internal worker for the RTMpOn* APIs.
1382	*
1383	* @returns IPRT status code.
1384	* @param pfnWorker The callback.
1385	* @param pvUser1 User argument 1.
1386	* @param pvUser2 User argument 2.
1387	* @param enmCpuid What to do / is idCpu valid.
1388	* @param idCpu Used if enmCpuid is RT_NT_CPUID_SPECIFIC or
1389	* RT_NT_CPUID_PAIR, otherwise ignored.
1390	* @param idCpu2 Used if enmCpuid is RT_NT_CPUID_PAIR, otherwise ignored.
1391	* @param pcHits Where to return the number of this. Optional.
1392	*/
1393	static int rtMpCallUsingDpcs(PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2,
1394	RT_NT_CPUID enmCpuid, RTCPUID idCpu, RTCPUID idCpu2, uint32_t *pcHits)
1395	{
1396	#if 0
1397	/* KeFlushQueuedDpcs must be run at IRQL PASSIVE_LEVEL according to MSDN, but the
1398	* driver verifier doesn't complain...
1399	*/
1400	AssertMsg(KeGetCurrentIrql() == PASSIVE_LEVEL, ("%d != %d (PASSIVE_LEVEL)\n", KeGetCurrentIrql(), PASSIVE_LEVEL));
1401	#endif
1402	/* KeFlushQueuedDpcs is not present in Windows 2000; import it dynamically so we can just fail this call. */
1403	if (!g_pfnrtNtKeFlushQueuedDpcs)
1404	return VERR_NOT_SUPPORTED;
1405
1406	/*
1407	* Make a copy of the active CPU set and figure out how many KDPCs we really need.
1408	* We must not try setup DPCs for CPUs which aren't there, because that may fail.
1409	*/
1410	RTCPUSET OnlineSet = g_rtMpNtCpuSet;
1411	uint32_t cDpcsNeeded;
1412	switch (enmCpuid)
1413	{
1414	case RT_NT_CPUID_SPECIFIC:
1415	cDpcsNeeded = 1;
1416	break;
1417	case RT_NT_CPUID_PAIR:
1418	cDpcsNeeded = 2;
1419	break;
1420	default:
1421	do
1422	{
1423	cDpcsNeeded = g_cRtMpNtActiveCpus;
1424	OnlineSet = g_rtMpNtCpuSet;
1425	} while (cDpcsNeeded != g_cRtMpNtActiveCpus);
1426	break;
1427	}
1428
1429	/*
1430	* Allocate an RTMPARGS structure followed by cDpcsNeeded KDPCs
1431	* and initialize them.
1432	*/
1433	PRTMPARGS pArgs = (PRTMPARGS)RTMemAllocZ(sizeof(RTMPARGS) + cDpcsNeeded * sizeof(KDPC));
1434	if (!pArgs)
1435	return VERR_NO_MEMORY;
1436
1437	pArgs->pfnWorker = pfnWorker;
1438	pArgs->pvUser1 = pvUser1;
1439	pArgs->pvUser2 = pvUser2;
1440	pArgs->idCpu = NIL_RTCPUID;
1441	pArgs->idCpu2 = NIL_RTCPUID;
1442	pArgs->cHits = 0;
1443	pArgs->cRefs = 1;
1444
1445	int rc;
1446	KDPC paExecCpuDpcs = (KDPC )(pArgs + 1);
1447	if (enmCpuid == RT_NT_CPUID_SPECIFIC)
1448	{
1449	KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
1450	if (g_pfnrtKeSetImportanceDpc)
1451	g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
1452	rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[0], idCpu);
1453	pArgs->idCpu = idCpu;
1454	}
1455	else if (enmCpuid == RT_NT_CPUID_PAIR)
1456	{
1457	KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
1458	if (g_pfnrtKeSetImportanceDpc)
1459	g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
1460	rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[0], idCpu);
1461	pArgs->idCpu = idCpu;
1462
1463	KeInitializeDpc(&paExecCpuDpcs[1], rtmpNtDPCWrapper, pArgs);
1464	if (g_pfnrtKeSetImportanceDpc)
1465	g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[1], HighImportance);
1466	if (RT_SUCCESS(rc))
1467	rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[1], (int)idCpu2);
1468	pArgs->idCpu2 = idCpu2;
1469	}
1470	else
1471	{
1472	rc = VINF_SUCCESS;
1473	for (uint32_t i = 0; i < cDpcsNeeded && RT_SUCCESS(rc); i++)
1474	if (RTCpuSetIsMemberByIndex(&OnlineSet, i))
1475	{
1476	KeInitializeDpc(&paExecCpuDpcs[i], rtmpNtDPCWrapper, pArgs);
1477	if (g_pfnrtKeSetImportanceDpc)
1478	g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[i], HighImportance);
1479	rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[i], RTMpCpuIdFromSetIndex(i));
1480	}
1481	}
1482	if (RT_FAILURE(rc))
1483	{
1484	RTMemFree(pArgs);
1485	return rc;
1486	}
1487
1488	/*
1489	* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
1490	* KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL.
1491	*/
1492	KIRQL oldIrql;
1493	KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
1494
1495	/*
1496	* We cannot do other than assume a 1:1 relationship between the
1497	* affinity mask and the process despite the warnings in the docs.
1498	* If someone knows a better way to get this done, please let bird know.
1499	*/
1500	ASMCompilerBarrier(); /* paranoia */
1501	if (enmCpuid == RT_NT_CPUID_SPECIFIC)
1502	{
1503	ASMAtomicIncS32(&pArgs->cRefs);
1504	BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
1505	Assert(fRc); NOREF(fRc);
1506	}
1507	else if (enmCpuid == RT_NT_CPUID_PAIR)
1508	{
1509	ASMAtomicIncS32(&pArgs->cRefs);
1510	BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
1511	Assert(fRc); NOREF(fRc);
1512
1513	ASMAtomicIncS32(&pArgs->cRefs);
1514	fRc = KeInsertQueueDpc(&paExecCpuDpcs[1], 0, 0);
1515	Assert(fRc); NOREF(fRc);
1516	}
1517	else
1518	{
1519	uint32_t iSelf = RTMpCurSetIndex();
1520	for (uint32_t i = 0; i < cDpcsNeeded; i++)
1521	{
1522	if ( (i != iSelf)
1523	&& RTCpuSetIsMemberByIndex(&OnlineSet, i))
1524	{
1525	ASMAtomicIncS32(&pArgs->cRefs);
1526	BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[i], 0, 0);
1527	Assert(fRc); NOREF(fRc);
1528	}
1529	}
1530	if (enmCpuid != RT_NT_CPUID_OTHERS)
1531	pfnWorker(iSelf, pvUser1, pvUser2);
1532	}
1533
1534	KeLowerIrql(oldIrql);
1535
1536	/*
1537	* Flush all DPCs and wait for completion. (can take long!)
1538	*/
1539	/** @todo Consider changing this to an active wait using some atomic inc/dec
1540	* stuff (and check for the current cpu above in the specific case). */
1541	/** @todo Seems KeFlushQueuedDpcs doesn't wait for the DPCs to be completely
1542	* executed. Seen pArgs being freed while some CPU was using it before
1543	* cRefs was added. */
1544	if (g_pfnrtNtKeFlushQueuedDpcs)
1545	g_pfnrtNtKeFlushQueuedDpcs();
1546
1547	if (pcHits)
1548	*pcHits = pArgs->cHits;
1549
1550	/* Dereference the argument structure. */
1551	int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
1552	Assert(cRefs >= 0);
1553	if (cRefs == 0)
1554	RTMemFree(pArgs);
1555
1556	return VINF_SUCCESS;
1557	}
1558
1559
1560	RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2)
1561	{
1562	if (g_pfnrtKeIpiGenericCall)
1563	return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnAllBroadcastIpiWrapper,
1564	NIL_RTCPUID, NIL_RTCPUID, NULL);
1565	return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_ALL, NIL_RTCPUID, NIL_RTCPUID, NULL);
1566	}
1567
1568
1569	RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2)
1570	{
1571	if (g_pfnrtKeIpiGenericCall)
1572	return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnOthersBroadcastIpiWrapper,
1573	NIL_RTCPUID, NIL_RTCPUID, NULL);
1574	return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_OTHERS, NIL_RTCPUID, NIL_RTCPUID, NULL);
1575	}
1576
1577
1578	RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2)
1579	{
1580	int rc;
1581	AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
1582	AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
1583	if ((fFlags & RTMPON_F_CONCURRENT_EXEC) && !g_pfnrtKeIpiGenericCall)
1584	return VERR_NOT_SUPPORTED;
1585
1586	/*
1587	* Check that both CPUs are online before doing the broadcast call.
1588	*/
1589	if ( RTMpIsCpuOnline(idCpu1)
1590	&& RTMpIsCpuOnline(idCpu2))
1591	{
1592	/*
1593	* The broadcast IPI isn't quite as bad as it could have been, because
1594	* it looks like windows doesn't synchronize CPUs on the way out, they
1595	* seems to get back to normal work while the pair is still busy.
1596	*/
1597	uint32_t cHits = 0;
1598	if (g_pfnrtKeIpiGenericCall)
1599	rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnPairBroadcastIpiWrapper, idCpu1, idCpu2, &cHits);
1600	else
1601	rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_PAIR, idCpu1, idCpu2, &cHits);
1602	if (RT_SUCCESS(rc))
1603	{
1604	Assert(cHits <= 2);
1605	if (cHits == 2)
1606	rc = VINF_SUCCESS;
1607	else if (cHits == 1)
1608	rc = VERR_NOT_ALL_CPUS_SHOWED;
1609	else if (cHits == 0)
1610	rc = VERR_CPU_OFFLINE;
1611	else
1612	rc = VERR_CPU_IPE_1;
1613	}
1614	}
1615	/*
1616	* A CPU must be present to be considered just offline.
1617	*/
1618	else if ( RTMpIsCpuPresent(idCpu1)
1619	&& RTMpIsCpuPresent(idCpu2))
1620	rc = VERR_CPU_OFFLINE;
1621	else
1622	rc = VERR_CPU_NOT_FOUND;
1623	return rc;
1624	}
1625
1626
1627	RTDECL(bool) RTMpOnPairIsConcurrentExecSupported(void)
1628	{
1629	return g_pfnrtKeIpiGenericCall != NULL;
1630	}
1631
1632
1633	/**
1634	* Releases a reference to a RTMPNTONSPECIFICARGS heap allocation, freeing it
1635	* when the last reference is released.
1636	*/
1637	DECLINLINE(void) rtMpNtOnSpecificRelease(PRTMPNTONSPECIFICARGS pArgs)
1638	{
1639	uint32_t cRefs = ASMAtomicDecU32(&pArgs->cRefs);
1640	AssertMsg(cRefs <= 1, ("cRefs=%#x\n", cRefs));
1641	if (cRefs == 0)
1642	RTMemFree(pArgs);
1643	}
1644
1645
1646	/**
1647	* Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
1648	*
1649	* @param Dpc DPC object
1650	* @param DeferredContext Context argument specified by KeInitializeDpc
1651	* @param SystemArgument1 Argument specified by KeInsertQueueDpc
1652	* @param SystemArgument2 Argument specified by KeInsertQueueDpc
1653	*/
1654	static VOID rtMpNtOnSpecificDpcWrapper(IN PKDPC Dpc, IN PVOID DeferredContext,
1655	IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1656	{
1657	PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)DeferredContext;
1658	RT_NOREF3(Dpc, SystemArgument1, SystemArgument2);
1659
1660	ASMAtomicWriteBool(&pArgs->fExecuting, true);
1661
1662	pArgs->CallbackArgs.pfnWorker(RTMpCpuId(), pArgs->CallbackArgs.pvUser1, pArgs->CallbackArgs.pvUser2);
1663
1664	ASMAtomicWriteBool(&pArgs->fDone, true);
1665	KeSetEvent(&pArgs->DoneEvt, 1 /PriorityIncrement/, FALSE /Wait/);
1666
1667	rtMpNtOnSpecificRelease(pArgs);
1668	}
1669
1670
1671	RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2)
1672	{
1673	/*
1674	* Don't try mess with an offline CPU.
1675	*/
1676	if (!RTMpIsCpuOnline(idCpu))
1677	return !RTMpIsCpuPossible(idCpu)
1678	? VERR_CPU_NOT_FOUND
1679	: VERR_CPU_OFFLINE;
1680
1681	/*
1682	* Use the broadcast IPI routine if there are no more than two CPUs online,
1683	* or if the current IRQL is unsuitable for KeWaitForSingleObject.
1684	*/
1685	int rc;
1686	uint32_t cHits = 0;
1687	if ( g_pfnrtKeIpiGenericCall
1688	&& ( RTMpGetOnlineCount() <= 2
1689	\|\| KeGetCurrentIrql() > APC_LEVEL)
1690	)
1691	{
1692	rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnSpecificBroadcastIpiWrapper,
1693	idCpu, NIL_RTCPUID, &cHits);
1694	if (RT_SUCCESS(rc))
1695	{
1696	if (cHits == 1)
1697	return VINF_SUCCESS;
1698	rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
1699	}
1700	return rc;
1701	}
1702
1703	#if 0
1704	rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_SPECIFIC, idCpu, NIL_RTCPUID, &cHits);
1705	if (RT_SUCCESS(rc))
1706	{
1707	if (cHits == 1)
1708	return VINF_SUCCESS;
1709	rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
1710	}
1711	return rc;
1712
1713	#else
1714	/*
1715	* Initialize the argument package and the objects within it.
1716	* The package is referenced counted to avoid unnecessary spinning to
1717	* synchronize cleanup and prevent stack corruption.
1718	*/
1719	PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)RTMemAllocZ(sizeof(*pArgs));
1720	if (!pArgs)
1721	return VERR_NO_MEMORY;
1722	pArgs->cRefs = 2;
1723	pArgs->fExecuting = false;
1724	pArgs->fDone = false;
1725	pArgs->CallbackArgs.pfnWorker = pfnWorker;
1726	pArgs->CallbackArgs.pvUser1 = pvUser1;
1727	pArgs->CallbackArgs.pvUser2 = pvUser2;
1728	pArgs->CallbackArgs.idCpu = idCpu;
1729	pArgs->CallbackArgs.cHits = 0;
1730	pArgs->CallbackArgs.cRefs = 2;
1731	KeInitializeEvent(&pArgs->DoneEvt, SynchronizationEvent, FALSE /* not signalled */);
1732	KeInitializeDpc(&pArgs->Dpc, rtMpNtOnSpecificDpcWrapper, pArgs);
1733	if (g_pfnrtKeSetImportanceDpc)
1734	g_pfnrtKeSetImportanceDpc(&pArgs->Dpc, HighImportance);
1735	rc = rtMpNtSetTargetProcessorDpc(&pArgs->Dpc, idCpu);
1736	if (RT_FAILURE(rc))
1737	{
1738	RTMemFree(pArgs);
1739	return rc;
1740	}
1741
1742	/*
1743	* Disable preemption while we check the current processor and inserts the DPC.
1744	*/
1745	KIRQL bOldIrql;
1746	KeRaiseIrql(DISPATCH_LEVEL, &bOldIrql);
1747	ASMCompilerBarrier(); /* paranoia */
1748
1749	if (RTMpCpuId() == idCpu)
1750	{
1751	/* Just execute the callback on the current CPU. */
1752	pfnWorker(idCpu, pvUser1, pvUser2);
1753	KeLowerIrql(bOldIrql);
1754
1755	RTMemFree(pArgs);
1756	return VINF_SUCCESS;
1757	}
1758
1759	/* Different CPU, so queue it if the CPU is still online. */
1760	if (RTMpIsCpuOnline(idCpu))
1761	{
1762	BOOLEAN fRc = KeInsertQueueDpc(&pArgs->Dpc, 0, 0);
1763	Assert(fRc); NOREF(fRc);
1764	KeLowerIrql(bOldIrql);
1765
1766	uint64_t const nsRealWaitTS = RTTimeNanoTS();
1767
1768	/*
1769	* Wait actively for a while in case the CPU/thread responds quickly.
1770	*/
1771	uint32_t cLoopsLeft = 0x20000;
1772	while (cLoopsLeft-- > 0)
1773	{
1774	if (pArgs->fDone)
1775	{
1776	rtMpNtOnSpecificRelease(pArgs);
1777	return VINF_SUCCESS;
1778	}
1779	ASMNopPause();
1780	}
1781
1782	/*
1783	* It didn't respond, so wait on the event object, poking the CPU if it's slow.
1784	*/
1785	LARGE_INTEGER Timeout;
1786	Timeout.QuadPart = -10000; /* 1ms */
1787	NTSTATUS rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1788	if (rcNt == STATUS_SUCCESS)
1789	{
1790	rtMpNtOnSpecificRelease(pArgs);
1791	return VINF_SUCCESS;
1792	}
1793
1794	/* If it hasn't respondend yet, maybe poke it and wait some more. */
1795	if (rcNt == STATUS_TIMEOUT)
1796	{
1797	if ( !pArgs->fExecuting
1798	&& ( g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalRequestIpiW7Plus
1799	\|\| g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalRequestIpiPreW7))
1800	RTMpPokeCpu(idCpu);
1801
1802	Timeout.QuadPart = -1280000; /* 128ms */
1803	rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1804	if (rcNt == STATUS_SUCCESS)
1805	{
1806	rtMpNtOnSpecificRelease(pArgs);
1807	return VINF_SUCCESS;
1808	}
1809	}
1810
1811	/*
1812	* Something weird is happening, try bail out.
1813	*/
1814	if (KeRemoveQueueDpc(&pArgs->Dpc))
1815	{
1816	RTMemFree(pArgs); /* DPC was still queued, so we can return without further ado. */
1817	LogRel(("RTMpOnSpecific(%#x): Not processed after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1818	}
1819	else
1820	{
1821	/* DPC is running, wait a good while for it to complete. */
1822	LogRel(("RTMpOnSpecific(%#x): Still running after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1823
1824	Timeout.QuadPart = -301000100010; / 30 seconds */
1825	rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1826	if (rcNt != STATUS_SUCCESS)
1827	LogRel(("RTMpOnSpecific(%#x): Giving up on running worker after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1828	}
1829	rc = RTErrConvertFromNtStatus(rcNt);
1830	}
1831	else
1832	{
1833	/* CPU is offline.*/
1834	KeLowerIrql(bOldIrql);
1835	rc = !RTMpIsCpuPossible(idCpu) ? VERR_CPU_NOT_FOUND : VERR_CPU_OFFLINE;
1836	}
1837
1838	rtMpNtOnSpecificRelease(pArgs);
1839	return rc;
1840	#endif
1841	}
1842
1843
1844
1845
1846	static VOID rtMpNtPokeCpuDummy(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1847	{
1848	NOREF(Dpc);
1849	NOREF(DeferredContext);
1850	NOREF(SystemArgument1);
1851	NOREF(SystemArgument2);
1852	}
1853
1854
1855	/** Callback used by rtMpPokeCpuUsingBroadcastIpi. */
1856	static ULONG_PTR rtMpIpiGenericCall(ULONG_PTR Argument)
1857	{
1858	NOREF(Argument);
1859	return 0;
1860	}
1861
1862
1863	/**
1864	* RTMpPokeCpu worker that uses broadcast IPIs for doing the work.
1865	*
1866	* @returns VINF_SUCCESS
1867	* @param idCpu The CPU identifier.
1868	*/
1869	int rtMpPokeCpuUsingBroadcastIpi(RTCPUID idCpu)
1870	{
1871	NOREF(idCpu);
1872	g_pfnrtKeIpiGenericCall(rtMpIpiGenericCall, 0);
1873	return VINF_SUCCESS;
1874	}
1875
1876
1877	/**
1878	* RTMpPokeCpu worker that uses the Windows 7 and later version of
1879	* HalRequestIpip to get the job done.
1880	*
1881	* @returns VINF_SUCCESS
1882	* @param idCpu The CPU identifier.
1883	*/
1884	int rtMpPokeCpuUsingHalRequestIpiW7Plus(RTCPUID idCpu)
1885	{
1886	/* idCpu is an HAL processor index, so we can use it directly. */
1887	PKAFFINITY_EX pTarget = (PKAFFINITY_EX)alloca(g_cbRtMpNtKaffinityEx);
1888	pTarget->Size = g_cRtMpNtKaffinityExEntries; /* (just in case KeInitializeAffinityEx starts using it) */
1889	g_pfnrtKeInitializeAffinityEx(pTarget);
1890	g_pfnrtKeAddProcessorAffinityEx(pTarget, idCpu);
1891
1892	g_pfnrtHalRequestIpiW7Plus(0, pTarget);
1893	return VINF_SUCCESS;
1894	}
1895
1896
1897	/**
1898	* RTMpPokeCpu worker that uses the Vista and earlier version of HalRequestIpip
1899	* to get the job done.
1900	*
1901	* @returns VINF_SUCCESS
1902	* @param idCpu The CPU identifier.
1903	*/
1904	int rtMpPokeCpuUsingHalRequestIpiPreW7(RTCPUID idCpu)
1905	{
1906	__debugbreak(); /** @todo this code needs testing!! */
1907	KAFFINITY Target = 1;
1908	Target <<= idCpu;
1909	g_pfnrtHalRequestIpiPreW7(Target);
1910	return VINF_SUCCESS;
1911	}
1912
1913
1914	int rtMpPokeCpuUsingFailureNotSupported(RTCPUID idCpu)
1915	{
1916	NOREF(idCpu);
1917	return VERR_NOT_SUPPORTED;
1918	}
1919
1920
1921	int rtMpPokeCpuUsingDpc(RTCPUID idCpu)
1922	{
1923	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1924
1925	/*
1926	* APC fallback.
1927	*/
1928	static KDPC s_aPokeDpcs[RTCPUSET_MAX_CPUS] = {0};
1929	static bool s_fPokeDPCsInitialized = false;
1930
1931	if (!s_fPokeDPCsInitialized)
1932	{
1933	for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++)
1934	{
1935	KeInitializeDpc(&s_aPokeDpcs[i], rtMpNtPokeCpuDummy, NULL);
1936	if (g_pfnrtKeSetImportanceDpc)
1937	g_pfnrtKeSetImportanceDpc(&s_aPokeDpcs[i], HighImportance);
1938	int rc = rtMpNtSetTargetProcessorDpc(&s_aPokeDpcs[i], idCpu);
1939	if (RT_FAILURE(rc))
1940	return rc;
1941	}
1942
1943	s_fPokeDPCsInitialized = true;
1944	}
1945
1946	/* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
1947	KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL. */
1948	KIRQL oldIrql;
1949	KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
1950
1951	if (g_pfnrtKeSetImportanceDpc)
1952	g_pfnrtKeSetImportanceDpc(&s_aPokeDpcs[idCpu], HighImportance);
1953	g_pfnrtKeSetTargetProcessorDpc(&s_aPokeDpcs[idCpu], (int)idCpu);
1954
1955	/* Assuming here that high importance DPCs will be delivered immediately; or at least an IPI will be sent immediately.
1956	Note! Not true on at least Vista & Windows 7 */
1957	BOOLEAN fRet = KeInsertQueueDpc(&s_aPokeDpcs[idCpu], 0, 0);
1958
1959	KeLowerIrql(oldIrql);
1960	return fRet == TRUE ? VINF_SUCCESS : VERR_ACCESS_DENIED /* already queued */;
1961	}
1962
1963
1964	RTDECL(int) RTMpPokeCpu(RTCPUID idCpu)
1965	{
1966	if (!RTMpIsCpuOnline(idCpu))
1967	return !RTMpIsCpuPossible(idCpu)
1968	? VERR_CPU_NOT_FOUND
1969	: VERR_CPU_OFFLINE;
1970	/* Calls rtMpPokeCpuUsingDpc, rtMpPokeCpuUsingHalRequestIpiW7Plus or rtMpPokeCpuUsingBroadcastIpi. */
1971	return g_pfnrtMpPokeCpuWorker(idCpu);
1972	}
1973
1974
1975	RTDECL(bool) RTMpOnAllIsConcurrentSafe(void)
1976	{
1977	return false;
1978	}
1979

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/nt/mp-r0drv-nt.cpp@ 91446

Download in other formats: