mp-r0drv-nt.cpp@ 86176

Last change on this file since 86176 was 86176, checked in by vboxsync, 4 years ago
Runtime/mp-r0drv-nt.cpp: Dynamically determine the size of the KAFFINITY_EX structure as it is not static across Windows versions (increased lately with W10 20H2)
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 76.5 KB

Line
1	/* $Id: mp-r0drv-nt.cpp 86176 2020-09-19 15:33:04Z vboxsync $ */
2	/** @file
3	* IPRT - Multiprocessor, Ring-0 Driver, NT.
4	*/
5
6	/*
7	* Copyright (C) 2008-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*********************************************************************************************************************************
29	* Header Files *
30	*********************************************************************************************************************************/
31	#include "the-nt-kernel.h"
32
33	#include <iprt/mp.h>
34	#include <iprt/cpuset.h>
35	#include <iprt/err.h>
36	#include <iprt/asm.h>
37	#include <iprt/log.h>
38	#include <iprt/mem.h>
39	#include <iprt/string.h>
40	#include <iprt/time.h>
41	#include "r0drv/mp-r0drv.h"
42	#include "symdb.h"
43	#include "internal-r0drv-nt.h"
44	#include "internal/mp.h"
45
46
47	/*********************************************************************************************************************************
48	* Structures and Typedefs *
49	*********************************************************************************************************************************/
50	typedef enum
51	{
52	RT_NT_CPUID_SPECIFIC,
53	RT_NT_CPUID_PAIR,
54	RT_NT_CPUID_OTHERS,
55	RT_NT_CPUID_ALL
56	} RT_NT_CPUID;
57
58
59	/**
60	* Used by the RTMpOnSpecific.
61	*/
62	typedef struct RTMPNTONSPECIFICARGS
63	{
64	/** Set if we're executing. */
65	bool volatile fExecuting;
66	/** Set when done executing. */
67	bool volatile fDone;
68	/** Number of references to this heap block. */
69	uint32_t volatile cRefs;
70	/** Event that the calling thread is waiting on. */
71	KEVENT DoneEvt;
72	/** The deferred procedure call object. */
73	KDPC Dpc;
74	/** The callback argument package. */
75	RTMPARGS CallbackArgs;
76	} RTMPNTONSPECIFICARGS;
77	/** Pointer to an argument/state structure for RTMpOnSpecific on NT. */
78	typedef RTMPNTONSPECIFICARGS *PRTMPNTONSPECIFICARGS;
79
80
81	/*********************************************************************************************************************************
82	* Defined Constants And Macros *
83	*********************************************************************************************************************************/
84	/** Inactive bit for g_aidRtMpNtByCpuSetIdx. */
85	#define RTMPNT_ID_F_INACTIVE RT_BIT_32(31)
86
87
88	/*********************************************************************************************************************************
89	* Global Variables *
90	*********************************************************************************************************************************/
91	/** Maximum number of processor groups. */
92	uint32_t g_cRtMpNtMaxGroups;
93	/** Maximum number of processors. */
94	uint32_t g_cRtMpNtMaxCpus;
95	/** Number of active processors. */
96	uint32_t volatile g_cRtMpNtActiveCpus;
97	/** The NT CPU set.
98	* KeQueryActiveProcssors() cannot be called at all IRQLs and therefore we'll
99	* have to cache it. Fortunately, NT doesn't really support taking CPUs offline,
100	* and taking them online was introduced with W2K8 where it is intended for virtual
101	* machines and not real HW. We update this, g_cRtMpNtActiveCpus and
102	* g_aidRtMpNtByCpuSetIdx from the rtR0NtMpProcessorChangeCallback.
103	*/
104	RTCPUSET g_rtMpNtCpuSet;
105
106	/** Static per group info.
107	* @remarks With 256 groups this takes up 33KB. */
108	static struct
109	{
110	/** The max CPUs in the group. */
111	uint16_t cMaxCpus;
112	/** The number of active CPUs at the time of initialization. */
113	uint16_t cActiveCpus;
114	/** CPU set indexes for each CPU in the group. */
115	int16_t aidxCpuSetMembers[64];
116	} g_aRtMpNtCpuGroups[256];
117	/** Maps CPU set indexes to RTCPUID.
118	* Inactive CPUs has bit 31 set (RTMPNT_ID_F_INACTIVE) so we can identify them
119	* and shuffle duplicates during CPU hotplugging. We assign temporary IDs to
120	* the inactive CPUs starting at g_cRtMpNtMaxCpus - 1, ASSUMING that active
121	* CPUs has IDs from 0 to g_cRtMpNtActiveCpus. */
122	RTCPUID g_aidRtMpNtByCpuSetIdx[RTCPUSET_MAX_CPUS];
123	/** The handle of the rtR0NtMpProcessorChangeCallback registration. */
124	static PVOID g_pvMpCpuChangeCallback = NULL;
125	/** Size of the KAFFINITY_EX structure.
126	* This increased from 20 to 32 bitmap words in the 2020 H2 windows 10 release
127	* (i.e. 1280 to 2048 CPUs). We expect this to increase in the future. */
128	static size_t g_cbRtMpNtKaffinityEx = RT_UOFFSETOF(KAFFINITY_EX, Bitmap)
129	+ RT_SIZEOFMEMB(KAFFINITY_EX, Bitmap[0]) * 256;
130	/** The size value of the KAFFINITY_EX structure. */
131	static uint16_t g_cRtMpNtKaffinityExEntries = 256;
132
133
134	/*********************************************************************************************************************************
135	* Internal Functions *
136	*********************************************************************************************************************************/
137	static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,
138	PNTSTATUS prcOperationStatus);
139	static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo);
140
141
142
143	/**
144	* Initalizes multiprocessor globals (called by rtR0InitNative).
145	*
146	* @returns IPRT status code.
147	* @param pOsVerInfo Version information.
148	*/
149	DECLHIDDEN(int) rtR0MpNtInit(RTNTSDBOSVER const *pOsVerInfo)
150	{
151	#define MY_CHECK_BREAK(a_Check, a_DbgPrintArgs) \
152	AssertMsgBreakStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
153	#define MY_CHECK_RETURN(a_Check, a_DbgPrintArgs, a_rcRet) \
154	AssertMsgReturnStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs, a_rcRet)
155	#define MY_CHECK(a_Check, a_DbgPrintArgs) \
156	AssertMsgStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
157
158	/*
159	* API combination checks.
160	*/
161	MY_CHECK_RETURN(!g_pfnrtKeSetTargetProcessorDpcEx \|\| g_pfnrtKeGetProcessorNumberFromIndex,
162	("IPRT: Fatal: Missing KeSetTargetProcessorDpcEx without KeGetProcessorNumberFromIndex!\n"),
163	VERR_SYMBOL_NOT_FOUND);
164
165	/*
166	* Get max number of processor groups.
167	*
168	* We may need to upadjust this number below, because windows likes to keep
169	* all options open when it comes to hotplugged CPU group assignments. A
170	* server advertising up to 64 CPUs in the ACPI table will get a result of
171	* 64 from KeQueryMaximumGroupCount. That makes sense. However, when windows
172	* server 2012 does a two processor group setup for it, the sum of the
173	* GroupInfo[*].MaximumProcessorCount members below is 128. This is probably
174	* because windows doesn't want to make decisions grouping of hotpluggable CPUs.
175	* So, we need to bump the maximum count to 128 below do deal with this as we
176	* want to have valid CPU set indexes for all potential CPUs - how could we
177	* otherwise use the RTMpGetSet() result and also RTCpuSetCount(RTMpGetSet())
178	* should equal RTMpGetCount().
179	*/
180	if (g_pfnrtKeQueryMaximumGroupCount)
181	{
182	g_cRtMpNtMaxGroups = g_pfnrtKeQueryMaximumGroupCount();
183	MY_CHECK_RETURN(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,
184	("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
185	VERR_MP_TOO_MANY_CPUS);
186	}
187	else
188	g_cRtMpNtMaxGroups = 1;
189
190	/*
191	* Get max number CPUs.
192	* This also defines the range of NT CPU indexes, RTCPUID and index into RTCPUSET.
193	*/
194	if (g_pfnrtKeQueryMaximumProcessorCountEx)
195	{
196	g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS);
197	MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
198	("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCountEx]\n",
199	g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
200	VERR_MP_TOO_MANY_CPUS);
201	}
202	else if (g_pfnrtKeQueryMaximumProcessorCount)
203	{
204	g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCount();
205	MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
206	("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCount]\n",
207	g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
208	VERR_MP_TOO_MANY_CPUS);
209	}
210	else if (g_pfnrtKeQueryActiveProcessors)
211	{
212	KAFFINITY fActiveProcessors = g_pfnrtKeQueryActiveProcessors();
213	MY_CHECK_RETURN(fActiveProcessors != 0,
214	("IPRT: Fatal: KeQueryActiveProcessors returned 0!\n"),
215	VERR_INTERNAL_ERROR_2);
216	g_cRtMpNtMaxCpus = 0;
217	do
218	{
219	g_cRtMpNtMaxCpus++;
220	fActiveProcessors >>= 1;
221	} while (fActiveProcessors);
222	}
223	else
224	g_cRtMpNtMaxCpus = KeNumberProcessors;
225
226	/*
227	* Just because we're a bit paranoid about getting something wrong wrt to the
228	* kernel interfaces, we try 16 times to get the KeQueryActiveProcessorCountEx
229	* and KeQueryLogicalProcessorRelationship information to match up.
230	*/
231	for (unsigned cTries = 0;; cTries++)
232	{
233	/*
234	* Get number of active CPUs.
235	*/
236	if (g_pfnrtKeQueryActiveProcessorCountEx)
237	{
238	g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS);
239	MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0,
240	("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCountEx]\n",
241	g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus),
242	VERR_MP_TOO_MANY_CPUS);
243	}
244	else if (g_pfnrtKeQueryActiveProcessorCount)
245	{
246	g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCount(NULL);
247	MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0,
248	("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCount]\n",
249	g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus),
250	VERR_MP_TOO_MANY_CPUS);
251	}
252	else
253	g_cRtMpNtActiveCpus = g_cRtMpNtMaxCpus;
254
255	/*
256	* Query the details for the groups to figure out which CPUs are online as
257	* well as the NT index limit.
258	*/
259	for (unsigned i = 0; i < RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx); i++)
260	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
261	g_aidRtMpNtByCpuSetIdx[i] = NIL_RTCPUID;
262	#else
263	g_aidRtMpNtByCpuSetIdx[i] = i < g_cRtMpNtMaxCpus ? i : NIL_RTCPUID;
264	#endif
265	for (unsigned idxGroup = 0; idxGroup < RT_ELEMENTS(g_aRtMpNtCpuGroups); idxGroup++)
266	{
267	g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = 0;
268	g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0;
269	for (unsigned idxMember = 0; idxMember < RT_ELEMENTS(g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers); idxMember++)
270	g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = -1;
271	}
272
273	if (g_pfnrtKeQueryLogicalProcessorRelationship)
274	{
275	MY_CHECK_RETURN(g_pfnrtKeGetProcessorIndexFromNumber,
276	("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
277	VERR_SYMBOL_NOT_FOUND);
278	MY_CHECK_RETURN(g_pfnrtKeGetProcessorNumberFromIndex,
279	("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
280	VERR_SYMBOL_NOT_FOUND);
281	MY_CHECK_RETURN(g_pfnrtKeSetTargetProcessorDpcEx,
282	("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeSetTargetProcessorDpcEx!\n"),
283	VERR_SYMBOL_NOT_FOUND);
284
285	SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = NULL;
286	int rc = rtR0NtInitQueryGroupRelations(&pInfo);
287	if (RT_FAILURE(rc))
288	return rc;
289
290	MY_CHECK(pInfo->Group.MaximumGroupCount == g_cRtMpNtMaxGroups,
291	("IPRT: Fatal: MaximumGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n",
292	pInfo->Group.MaximumGroupCount, g_cRtMpNtMaxGroups));
293	MY_CHECK(pInfo->Group.ActiveGroupCount > 0 && pInfo->Group.ActiveGroupCount <= g_cRtMpNtMaxGroups,
294	("IPRT: Fatal: ActiveGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n",
295	pInfo->Group.ActiveGroupCount, g_cRtMpNtMaxGroups));
296
297	/*
298	* First we need to recalc g_cRtMpNtMaxCpus (see above).
299	*/
300	uint32_t cMaxCpus = 0;
301	uint32_t idxGroup;
302	for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)
303	{
304	const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];
305	MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,
306	("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));
307	MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,
308	("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",
309	pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));
310	cMaxCpus += pGrpInfo->MaximumProcessorCount;
311	}
312	if (cMaxCpus > g_cRtMpNtMaxCpus && RT_SUCCESS(rc))
313	{
314	DbgPrint("IPRT: g_cRtMpNtMaxCpus=%u -> %u\n", g_cRtMpNtMaxCpus, cMaxCpus);
315	#ifndef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
316	uint32_t i = RT_MIN(cMaxCpus, RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx));
317	while (i-- > g_cRtMpNtMaxCpus)
318	g_aidRtMpNtByCpuSetIdx[i] = i;
319	#endif
320	g_cRtMpNtMaxCpus = cMaxCpus;
321	if (g_cRtMpNtMaxGroups > RTCPUSET_MAX_CPUS)
322	{
323	MY_CHECK(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,
324	("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS));
325	rc = VERR_MP_TOO_MANY_CPUS;
326	}
327	}
328
329	/*
330	* Calc online mask, partition IDs and such.
331	*
332	* Also check ASSUMPTIONS:
333	*
334	* 1. Processor indexes going from 0 and up to
335	* KeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1.
336	*
337	* 2. Currently valid processor indexes, i.e. accepted by
338	* KeGetProcessorIndexFromNumber & KeGetProcessorNumberFromIndex, goes
339	* from 0 thru KeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1.
340	*
341	* 3. PROCESSOR_GROUP_INFO::MaximumProcessorCount gives the number of
342	* relevant bits in the ActiveProcessorMask (from LSB).
343	*
344	* 4. Active processor count found in KeQueryLogicalProcessorRelationship
345	* output matches what KeQueryActiveProcessorCountEx(ALL) returns.
346	*
347	* 5. Active + inactive processor counts in same does not exceed
348	* KeQueryMaximumProcessorCountEx(ALL).
349	*
350	* Note! Processor indexes are assigned as CPUs come online and are not
351	* preallocated according to group maximums. Since CPUS are only taken
352	* online and never offlined, this means that internal CPU bitmaps are
353	* never sparse and no time is wasted scanning unused bits.
354	*
355	* Unfortunately, it means that ring-3 cannot easily guess the index
356	* assignments when hotswapping is used, and must use GIP when available.
357	*/
358	RTCpuSetEmpty(&g_rtMpNtCpuSet);
359	uint32_t cInactive = 0;
360	uint32_t cActive = 0;
361	uint32_t idxCpuMax = 0;
362	uint32_t idxCpuSetNextInactive = g_cRtMpNtMaxCpus - 1;
363	for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)
364	{
365	const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];
366	MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,
367	("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));
368	MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,
369	("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",
370	pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));
371
372	g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = pGrpInfo->MaximumProcessorCount;
373	g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = pGrpInfo->ActiveProcessorCount;
374
375	for (uint32_t idxMember = 0; idxMember < pGrpInfo->MaximumProcessorCount; idxMember++)
376	{
377	PROCESSOR_NUMBER ProcNum;
378	ProcNum.Group = (USHORT)idxGroup;
379	ProcNum.Number = (UCHAR)idxMember;
380	ProcNum.Reserved = 0;
381	ULONG idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
382	if (idxCpu != INVALID_PROCESSOR_INDEX)
383	{
384	MY_CHECK_BREAK(idxCpu < g_cRtMpNtMaxCpus && idxCpu < RTCPUSET_MAX_CPUS, /* ASSUMPTION #1 */
385	("IPRT: Fatal: idxCpu=%u >= g_cRtMpNtMaxCpus=%u (RTCPUSET_MAX_CPUS=%u)\n",
386	idxCpu, g_cRtMpNtMaxCpus, RTCPUSET_MAX_CPUS));
387	if (idxCpu > idxCpuMax)
388	idxCpuMax = idxCpu;
389	g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpu;
390	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
391	g_aidRtMpNtByCpuSetIdx[idxCpu] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember);
392	#endif
393
394	ProcNum.Group = UINT16_MAX;
395	ProcNum.Number = UINT8_MAX;
396	ProcNum.Reserved = UINT8_MAX;
397	NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(idxCpu, &ProcNum);
398	MY_CHECK_BREAK(NT_SUCCESS(rcNt),
399	("IPRT: Fatal: KeGetProcessorNumberFromIndex(%u,) -> %#x!\n", idxCpu, rcNt));
400	MY_CHECK_BREAK(ProcNum.Group == idxGroup && ProcNum.Number == idxMember,
401	("IPRT: Fatal: KeGetProcessorXxxxFromYyyy roundtrip error for %#x! Group: %u vs %u, Number: %u vs %u\n",
402	idxCpu, ProcNum.Group, idxGroup, ProcNum.Number, idxMember));
403
404	if (pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember))
405	{
406	RTCpuSetAddByIndex(&g_rtMpNtCpuSet, idxCpu);
407	cActive++;
408	}
409	else
410	cInactive++; /* (This is a little unexpected, but not important as long as things add up below.) */
411	}
412	else
413	{
414	/* Must be not present / inactive when KeGetProcessorIndexFromNumber fails. */
415	MY_CHECK_BREAK(!(pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember)),
416	("IPRT: Fatal: KeGetProcessorIndexFromNumber(%u/%u) failed but CPU is active! cMax=%u cActive=%u fActive=%p\n",
417	idxGroup, idxMember, pGrpInfo->MaximumProcessorCount, pGrpInfo->ActiveProcessorCount,
418	pGrpInfo->ActiveProcessorMask));
419	cInactive++;
420	if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus)
421	{
422	g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
423	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
424	g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember)
425	\| RTMPNT_ID_F_INACTIVE;
426	#endif
427	idxCpuSetNextInactive--;
428	}
429	}
430	}
431	}
432
433	MY_CHECK(cInactive + cActive <= g_cRtMpNtMaxCpus, /* ASSUMPTION #5 (not '==' because of inactive groups) */
434	("IPRT: Fatal: cInactive=%u + cActive=%u > g_cRtMpNtMaxCpus=%u\n", cInactive, cActive, g_cRtMpNtMaxCpus));
435
436	/* Deal with inactive groups using KeQueryMaximumProcessorCountEx or as
437	best as we can by as best we can by stipulating maximum member counts
438	from the previous group. */
439	if ( RT_SUCCESS(rc)
440	&& idxGroup < pInfo->Group.MaximumGroupCount)
441	{
442	uint16_t cInactiveLeft = g_cRtMpNtMaxCpus - (cInactive + cActive);
443	while (idxGroup < pInfo->Group.MaximumGroupCount)
444	{
445	uint32_t cMaxMembers = 0;
446	if (g_pfnrtKeQueryMaximumProcessorCountEx)
447	cMaxMembers = g_pfnrtKeQueryMaximumProcessorCountEx(idxGroup);
448	if (cMaxMembers != 0 \|\| cInactiveLeft == 0)
449	AssertStmt(cMaxMembers <= cInactiveLeft, cMaxMembers = cInactiveLeft);
450	else
451	{
452	uint16_t cGroupsLeft = pInfo->Group.MaximumGroupCount - idxGroup;
453	cMaxMembers = pInfo->Group.GroupInfo[idxGroup - 1].MaximumProcessorCount;
454	while (cMaxMembers * cGroupsLeft < cInactiveLeft)
455	cMaxMembers++;
456	if (cMaxMembers > cInactiveLeft)
457	cMaxMembers = cInactiveLeft;
458	}
459
460	g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = (uint16_t)cMaxMembers;
461	g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0;
462	for (uint16_t idxMember = 0; idxMember < cMaxMembers; idxMember++)
463	if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus)
464	{
465	g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
466	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
467	g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember)
468	\| RTMPNT_ID_F_INACTIVE;
469	#endif
470	idxCpuSetNextInactive--;
471	}
472	cInactiveLeft -= cMaxMembers;
473	idxGroup++;
474	}
475	}
476
477	/* We're done with pInfo now, free it so we can start returning when assertions fail. */
478	RTMemFree(pInfo);
479	if (RT_FAILURE(rc)) /* MY_CHECK_BREAK sets rc. */
480	return rc;
481	MY_CHECK_RETURN(cActive >= g_cRtMpNtActiveCpus,
482	("IPRT: Fatal: cActive=%u < g_cRtMpNtActiveCpus=%u - CPUs removed?\n", cActive, g_cRtMpNtActiveCpus),
483	VERR_INTERNAL_ERROR_3);
484	MY_CHECK_RETURN(idxCpuMax < cActive, /* ASSUMPTION #2 */
485	("IPRT: Fatal: idCpuMax=%u >= cActive=%u! Unexpected CPU index allocation. CPUs removed?\n",
486	idxCpuMax, cActive),
487	VERR_INTERNAL_ERROR_4);
488
489	/* Retry if CPUs were added. */
490	if ( cActive != g_cRtMpNtActiveCpus
491	&& cTries < 16)
492	continue;
493	MY_CHECK_RETURN(cActive == g_cRtMpNtActiveCpus, /* ASSUMPTION #4 */
494	("IPRT: Fatal: cActive=%u != g_cRtMpNtActiveCpus=%u\n", cActive, g_cRtMpNtActiveCpus),
495	VERR_INTERNAL_ERROR_5);
496	}
497	else
498	{
499	/* Legacy: */
500	MY_CHECK_RETURN(g_cRtMpNtMaxGroups == 1, ("IPRT: Fatal: Missing KeQueryLogicalProcessorRelationship!\n"),
501	VERR_SYMBOL_NOT_FOUND);
502
503	/** @todo Is it possible that the affinity mask returned by
504	* KeQueryActiveProcessors is sparse? */
505	if (g_pfnrtKeQueryActiveProcessors)
506	RTCpuSetFromU64(&g_rtMpNtCpuSet, g_pfnrtKeQueryActiveProcessors());
507	else if (g_cRtMpNtMaxCpus < 64)
508	RTCpuSetFromU64(&g_rtMpNtCpuSet, (UINT64_C(1) << g_cRtMpNtMaxCpus) - 1);
509	else
510	{
511	MY_CHECK_RETURN(g_cRtMpNtMaxCpus == 64, ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, expect 64 or less\n", g_cRtMpNtMaxCpus),
512	VERR_MP_TOO_MANY_CPUS);
513	RTCpuSetFromU64(&g_rtMpNtCpuSet, UINT64_MAX);
514	}
515
516	g_aRtMpNtCpuGroups[0].cMaxCpus = g_cRtMpNtMaxCpus;
517	g_aRtMpNtCpuGroups[0].cActiveCpus = g_cRtMpNtMaxCpus;
518	for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++)
519	{
520	g_aRtMpNtCpuGroups[0].aidxCpuSetMembers[i] = i;
521	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
522	g_aidRtMpNtByCpuSetIdx[i] = RTMPCPUID_FROM_GROUP_AND_NUMBER(0, i);
523	#endif
524	}
525	}
526
527	/*
528	* Register CPU hot plugging callback (it also counts active CPUs).
529	*/
530	Assert(g_pvMpCpuChangeCallback == NULL);
531	if (g_pfnrtKeRegisterProcessorChangeCallback)
532	{
533	MY_CHECK_RETURN(g_pfnrtKeDeregisterProcessorChangeCallback,
534	("IPRT: Fatal: KeRegisterProcessorChangeCallback without KeDeregisterProcessorChangeCallback!\n"),
535	VERR_SYMBOL_NOT_FOUND);
536
537	RTCPUSET const ActiveSetCopy = g_rtMpNtCpuSet;
538	RTCpuSetEmpty(&g_rtMpNtCpuSet);
539	uint32_t const cActiveCpus = g_cRtMpNtActiveCpus;
540	g_cRtMpNtActiveCpus = 0;
541
542	g_pvMpCpuChangeCallback = g_pfnrtKeRegisterProcessorChangeCallback(rtR0NtMpProcessorChangeCallback, NULL /pvUser/,
543	KE_PROCESSOR_CHANGE_ADD_EXISTING);
544	if (g_pvMpCpuChangeCallback)
545	{
546	if (cActiveCpus == g_cRtMpNtActiveCpus)
547	{ /* likely */ }
548	else
549	{
550	g_pfnrtKeDeregisterProcessorChangeCallback(g_pvMpCpuChangeCallback);
551	if (cTries < 16)
552	{
553	/* Retry if CPUs were added. */
554	MY_CHECK_RETURN(g_cRtMpNtActiveCpus >= cActiveCpus,
555	("IPRT: Fatal: g_cRtMpNtActiveCpus=%u < cActiveCpus=%u! CPUs removed?\n",
556	g_cRtMpNtActiveCpus, cActiveCpus),
557	VERR_INTERNAL_ERROR_2);
558	MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus,
559	("IPRT: Fatal: g_cRtMpNtActiveCpus=%u > g_cRtMpNtMaxCpus=%u!\n",
560	g_cRtMpNtActiveCpus, g_cRtMpNtMaxCpus),
561	VERR_INTERNAL_ERROR_2);
562	continue;
563	}
564	MY_CHECK_RETURN(0, ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u cActiveCpus=%u\n", g_cRtMpNtActiveCpus, cActiveCpus),
565	VERR_INTERNAL_ERROR_3);
566	}
567	}
568	else
569	{
570	AssertFailed();
571	g_rtMpNtCpuSet = ActiveSetCopy;
572	g_cRtMpNtActiveCpus = cActiveCpus;
573	}
574	}
575	break;
576	} /* Retry loop for stable active CPU count. */
577
578	#undef MY_CHECK_RETURN
579
580	/*
581	* Special IPI fun for RTMpPokeCpu.
582	*
583	* On Vista and later the DPC method doesn't seem to reliably send IPIs,
584	* so we have to use alternative methods.
585	*
586	* On AMD64 We used to use the HalSendSoftwareInterrupt API (also x86 on
587	* W10+), it looks faster and more convenient to use, however we're either
588	* using it wrong or it doesn't reliably do what we want (see @bugref{8343}).
589	*
590	* The HalRequestIpip API is thus far the only alternative to KeInsertQueueDpc
591	* for doing targetted IPIs. Trouble with this API is that it changed
592	* fundamentally in Window 7 when they added support for lots of processors.
593	*
594	* If we really think we cannot use KeInsertQueueDpc, we use the broadcast IPI
595	* API KeIpiGenericCall.
596	*/
597	if ( pOsVerInfo->uMajorVer > 6
598	\|\| (pOsVerInfo->uMajorVer == 6 && pOsVerInfo->uMinorVer > 0))
599	g_pfnrtHalRequestIpiPreW7 = NULL;
600	else
601	g_pfnrtHalRequestIpiW7Plus = NULL;
602
603	if ( g_pfnrtHalRequestIpiW7Plus
604	&& g_pfnrtKeInitializeAffinityEx
605	&& g_pfnrtKeAddProcessorAffinityEx
606	&& g_pfnrtKeGetProcessorIndexFromNumber)
607	{
608	/* Determine the real size of the KAFFINITY_EX structure. */
609	union
610	{
611	KAFFINITY_EX Struct;
612	uint8_t ab[_4K + 8]; /* This should suffice for determining the real size. */
613	} u;
614	RT_ZERO(u);
615	size_t const cMaxEntries = (sizeof(u) - RT_UOFFSETOF(KAFFINITY_EX, Bitmap[0])) / sizeof(u.Struct.Bitmap[0]);
616	g_pfnrtKeInitializeAffinityEx(&u.Struct);
617	if (u.Struct.Size > 1 && u.Struct.Size <= cMaxEntries)
618	{
619	g_cRtMpNtKaffinityExEntries = u.Struct.Size;
620	g_cbRtMpNtKaffinityEx = u.Struct.Size * sizeof(u.Struct.Bitmap[0]) + RT_UOFFSETOF(KAFFINITY_EX, Bitmap[0]);
621	g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingHalRequestIpiW7Plus;
622	DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingHalRequestIpiW7Plus\n");
623	return VINF_SUCCESS;
624	}
625	DbgPrint("IPRT: RTMpPoke can't use rtMpPokeCpuUsingHalRequestIpiW7Plus! u.Struct.Size=%u\n", u.Struct.Size);
626	AssertReleaseMsg(u.Struct.Size <= cMaxEntries, ("%#x\n", u.Struct.Size)); /* stack is toast if larger (32768 CPUs). */
627	}
628
629	if (pOsVerInfo->uMajorVer >= 6 && g_pfnrtKeIpiGenericCall)
630	{
631	DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingBroadcastIpi\n");
632	g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingBroadcastIpi;
633	}
634	else if (g_pfnrtKeSetTargetProcessorDpc)
635	{
636	DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingDpc\n");
637	g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingDpc;
638	/* Windows XP should send always send an IPI -> VERIFY */
639	}
640	else
641	{
642	DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingFailureNotSupported\n");
643	Assert(pOsVerInfo->uMajorVer == 3 && pOsVerInfo->uMinorVer <= 50);
644	g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingFailureNotSupported;
645	}
646
647	return VINF_SUCCESS;
648	}
649
650
651	/**
652	* Called by rtR0TermNative.
653	*/
654	DECLHIDDEN(void) rtR0MpNtTerm(void)
655	{
656	/*
657	* Deregister the processor change callback.
658	*/
659	PVOID pvMpCpuChangeCallback = g_pvMpCpuChangeCallback;
660	g_pvMpCpuChangeCallback = NULL;
661	if (pvMpCpuChangeCallback)
662	{
663	AssertReturnVoid(g_pfnrtKeDeregisterProcessorChangeCallback);
664	g_pfnrtKeDeregisterProcessorChangeCallback(pvMpCpuChangeCallback);
665	}
666	}
667
668
669	DECLHIDDEN(int) rtR0MpNotificationNativeInit(void)
670	{
671	return VINF_SUCCESS;
672	}
673
674
675	DECLHIDDEN(void) rtR0MpNotificationNativeTerm(void)
676	{
677	}
678
679
680	/**
681	* Implements the NT PROCESSOR_CALLBACK_FUNCTION callback function.
682	*
683	* This maintains the g_rtMpNtCpuSet and works MP notification callbacks. When
684	* registered, it's called for each active CPU in the system, avoiding racing
685	* CPU hotplugging (as well as testing the callback).
686	*
687	* @param pvUser User context (not used).
688	* @param pChangeCtx Change context (in).
689	* @param prcOperationStatus Operation status (in/out).
690	*
691	* @remarks ASSUMES no concurrent execution of KeProcessorAddCompleteNotify
692	* notification callbacks. At least during callback registration
693	* callout, we're owning KiDynamicProcessorLock.
694	*
695	* @remarks When registering the handler, we first get KeProcessorAddStartNotify
696	* callbacks for all active CPUs, and after they all succeed we get the
697	* KeProcessorAddCompleteNotify callbacks.
698	*/
699	static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,
700	PNTSTATUS prcOperationStatus)
701	{
702	RT_NOREF(pvUser, prcOperationStatus);
703	switch (pChangeCtx->State)
704	{
705	/*
706	* Check whether we can deal with the CPU, failing the start operation if we
707	* can't. The checks we are doing here are to avoid complicated/impossible
708	* cases in KeProcessorAddCompleteNotify. They are really just verify specs.
709	*/
710	case KeProcessorAddStartNotify:
711	{
712	NTSTATUS rcNt = STATUS_SUCCESS;
713	if (pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS)
714	{
715	if (pChangeCtx->NtNumber >= g_cRtMpNtMaxCpus)
716	{
717	DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is higher than the max CPU count (%u)!\n",
718	pChangeCtx->NtNumber, g_cRtMpNtMaxCpus);
719	rcNt = STATUS_INTERNAL_ERROR;
720	}
721
722	/* The ProcessNumber field was introduced in Windows 7. */
723	PROCESSOR_NUMBER ProcNum;
724	if (g_pfnrtKeGetProcessorIndexFromNumber)
725	{
726	ProcNum = pChangeCtx->ProcNumber;
727	KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
728	if (idxCpu != pChangeCtx->NtNumber)
729	{
730	DbgPrint("IPRT: KeProcessorAddStartNotify failure: g_pfnrtKeGetProcessorIndexFromNumber(%u.%u) -> %u, expected %u!\n",
731	ProcNum.Group, ProcNum.Number, idxCpu, pChangeCtx->NtNumber);
732	rcNt = STATUS_INTERNAL_ERROR;
733	}
734	}
735	else
736	{
737	ProcNum.Group = 0;
738	ProcNum.Number = pChangeCtx->NtNumber;
739	}
740
741	if ( ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups)
742	&& ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers))
743	{
744	if (ProcNum.Group >= g_cRtMpNtMaxGroups)
745	{
746	DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range - max groups: %u!\n",
747	ProcNum.Group, ProcNum.Number, g_cRtMpNtMaxGroups);
748	rcNt = STATUS_INTERNAL_ERROR;
749	}
750
751	if (ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus)
752	{
753	Assert(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1);
754	if (g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == -1)
755	{
756	DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u.%u was assigned -1 as set index!\n",
757	ProcNum.Group, ProcNum.Number);
758	rcNt = STATUS_INTERNAL_ERROR;
759	}
760
761	Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID);
762	if (g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == NIL_RTCPUID)
763	{
764	DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u (%u.%u) translates to NIL_RTCPUID!\n",
765	pChangeCtx->NtNumber, ProcNum.Group, ProcNum.Number);
766	rcNt = STATUS_INTERNAL_ERROR;
767	}
768	}
769	else
770	{
771	DbgPrint("IPRT: KeProcessorAddStartNotify failure: max processors in group %u is %u, cannot add %u to it!\n",
772	ProcNum.Group, g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus, ProcNum.Group, ProcNum.Number);
773	rcNt = STATUS_INTERNAL_ERROR;
774	}
775	}
776	else
777	{
778	DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range (max %u.%u)!\n",
779	ProcNum.Group, ProcNum.Number, RT_ELEMENTS(g_aRtMpNtCpuGroups), RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers));
780	rcNt = STATUS_INTERNAL_ERROR;
781	}
782	}
783	else
784	{
785	DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is outside RTCPUSET_MAX_CPUS (%u)!\n",
786	pChangeCtx->NtNumber, RTCPUSET_MAX_CPUS);
787	rcNt = STATUS_INTERNAL_ERROR;
788	}
789	if (!NT_SUCCESS(rcNt))
790	*prcOperationStatus = rcNt;
791	break;
792	}
793
794	/*
795	* Update the globals. Since we've checked out range limits and other
796	* limitations already we just AssertBreak here.
797	*/
798	case KeProcessorAddCompleteNotify:
799	{
800	/*
801	* Calc the processor number and assert conditions checked in KeProcessorAddStartNotify.
802	*/
803	AssertBreak(pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS);
804	AssertBreak(pChangeCtx->NtNumber < g_cRtMpNtMaxCpus);
805	Assert(pChangeCtx->NtNumber == g_cRtMpNtActiveCpus); /* light assumption */
806	PROCESSOR_NUMBER ProcNum;
807	if (g_pfnrtKeGetProcessorIndexFromNumber)
808	{
809	ProcNum = pChangeCtx->ProcNumber;
810	AssertBreak(g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum) == pChangeCtx->NtNumber);
811	AssertBreak(ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups));
812	AssertBreak(ProcNum.Group < g_cRtMpNtMaxGroups);
813	}
814	else
815	{
816	ProcNum.Group = 0;
817	ProcNum.Number = pChangeCtx->NtNumber;
818	}
819	AssertBreak(ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers));
820	AssertBreak(ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus);
821	AssertBreak(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1);
822	AssertBreak(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID);
823
824	/*
825	* Add ourselves to the online CPU set and update the active CPU count.
826	*/
827	RTCpuSetAddByIndex(&g_rtMpNtCpuSet, pChangeCtx->NtNumber);
828	ASMAtomicIncU32(&g_cRtMpNtActiveCpus);
829
830	/*
831	* Update the group info.
832	*
833	* If the index prediction failed (real hotplugging callbacks only) we
834	* have to switch it around. This is particularly annoying when we
835	* use the index as the ID.
836	*/
837	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
838	RTCPUID idCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
839	RTCPUID idOld = g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber];
840	if ((idOld & ~RTMPNT_ID_F_INACTIVE) != idCpu)
841	{
842	Assert(idOld & RTMPNT_ID_F_INACTIVE);
843	int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
844	g_aRtMpNtCpuGroups[rtMpCpuIdGetGroup(idOld)].aidxCpuSetMembers[rtMpCpuIdGetGroupMember(idOld)] = idxDest;
845	g_aidRtMpNtByCpuSetIdx[idxDest] = idOld;
846	}
847	g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] = idCpu;
848	#else
849	Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == pChangeCtx->NtNumber);
850	int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
851	if ((ULONG)idxDest != pChangeCtx->NtNumber)
852	{
853	bool fFound = false;
854	uint32_t idxOldGroup = g_cRtMpNtMaxGroups;
855	while (idxOldGroup-- > 0 && !fFound)
856	{
857	uint32_t idxMember = g_aRtMpNtCpuGroups[idxOldGroup].cMaxCpus;
858	while (idxMember-- > 0)
859	if (g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] == (int)pChangeCtx->NtNumber)
860	{
861	g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] = idxDest;
862	fFound = true;
863	break;
864	}
865	}
866	Assert(fFound);
867	}
868	#endif
869	g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] = pChangeCtx->NtNumber;
870
871	/*
872	* Do MP notification callbacks.
873	*/
874	rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, pChangeCtx->NtNumber);
875	break;
876	}
877
878	case KeProcessorAddFailureNotify:
879	/* ignore */
880	break;
881
882	default:
883	AssertMsgFailed(("State=%u\n", pChangeCtx->State));
884	}
885	}
886
887
888	/**
889	* Wrapper around KeQueryLogicalProcessorRelationship.
890	*
891	* @returns IPRT status code.
892	* @param ppInfo Where to return the info. Pass to RTMemFree when done.
893	*/
894	static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo)
895	{
896	ULONG cbInfo = sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)
897	+ g_cRtMpNtMaxGroups * sizeof(GROUP_RELATIONSHIP);
898	NTSTATUS rcNt;
899	do
900	{
901	SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX pInfo = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX )RTMemAlloc(cbInfo);
902	if (pInfo)
903	{
904	rcNt = g_pfnrtKeQueryLogicalProcessorRelationship(NULL /pProcNumber/, RelationGroup, pInfo, &cbInfo);
905	if (NT_SUCCESS(rcNt))
906	{
907	*ppInfo = pInfo;
908	return VINF_SUCCESS;
909	}
910
911	RTMemFree(pInfo);
912	pInfo = NULL;
913	}
914	else
915	rcNt = STATUS_NO_MEMORY;
916	} while (rcNt == STATUS_INFO_LENGTH_MISMATCH);
917	DbgPrint("IPRT: Fatal: KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt);
918	AssertMsgFailed(("KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt));
919	return RTErrConvertFromNtStatus(rcNt);
920	}
921
922
923
924
925
926	RTDECL(RTCPUID) RTMpCpuId(void)
927	{
928	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
929
930	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
931	PROCESSOR_NUMBER ProcNum;
932	ProcNum.Group = 0;
933	if (g_pfnrtKeGetCurrentProcessorNumberEx)
934	{
935	ProcNum.Number = 0;
936	g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum);
937	}
938	else
939	ProcNum.Number = KeGetCurrentProcessorNumber(); /* Number is 8-bit, so we're not subject to BYTE -> WORD upgrade in WDK. */
940	return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
941
942	#else
943
944	if (g_pfnrtKeGetCurrentProcessorNumberEx)
945	{
946	KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL);
947	Assert(idxCpu < RTCPUSET_MAX_CPUS);
948	return idxCpu;
949	}
950
951	return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */
952	#endif
953	}
954
955
956	RTDECL(int) RTMpCurSetIndex(void)
957	{
958	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
959	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
960
961	if (g_pfnrtKeGetCurrentProcessorNumberEx)
962	{
963	KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL);
964	Assert(idxCpu < RTCPUSET_MAX_CPUS);
965	return idxCpu;
966	}
967	return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */
968	#else
969	return (int)RTMpCpuId();
970	#endif
971	}
972
973
974	RTDECL(int) RTMpCurSetIndexAndId(PRTCPUID pidCpu)
975	{
976	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
977	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
978
979	PROCESSOR_NUMBER ProcNum = { 0 , 0, 0 };
980	KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum);
981	Assert(idxCpu < RTCPUSET_MAX_CPUS);
982	*pidCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
983	return idxCpu;
984	#else
985	return *pidCpu = RTMpCpuId();
986	#endif
987	}
988
989
990	RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu)
991	{
992	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
993	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
994
995	if (idCpu != NIL_RTCPUID)
996	{
997	if (g_pfnrtKeGetProcessorIndexFromNumber)
998	{
999	PROCESSOR_NUMBER ProcNum;
1000	ProcNum.Group = rtMpCpuIdGetGroup(idCpu);
1001	ProcNum.Number = rtMpCpuIdGetGroupMember(idCpu);
1002	ProcNum.Reserved = 0;
1003	KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
1004	if (idxCpu != INVALID_PROCESSOR_INDEX)
1005	{
1006	Assert(idxCpu < g_cRtMpNtMaxCpus);
1007	Assert((ULONG)g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == idxCpu);
1008	return idxCpu;
1009	}
1010
1011	/* Since NT assigned indexes as the CPUs come online, we cannot produce an ID <-> index
1012	mapping for not-yet-onlined CPUS that is consistent. We just have to do our best... */
1013	if ( ProcNum.Group < g_cRtMpNtMaxGroups
1014	&& ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus)
1015	return g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
1016	}
1017	else if (rtMpCpuIdGetGroup(idCpu) == 0)
1018	return rtMpCpuIdGetGroupMember(idCpu);
1019	}
1020	return -1;
1021	#else
1022	/* 1:1 mapping, just do range checks. */
1023	return idCpu < RTCPUSET_MAX_CPUS ? (int)idCpu : -1;
1024	#endif
1025	}
1026
1027
1028	RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu)
1029	{
1030	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1031	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1032
1033	if ((unsigned)iCpu < g_cRtMpNtMaxCpus)
1034	{
1035	if (g_pfnrtKeGetProcessorIndexFromNumber)
1036	{
1037	PROCESSOR_NUMBER ProcNum = { 0, 0, 0 };
1038	NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(iCpu, &ProcNum);
1039	if (NT_SUCCESS(rcNt))
1040	{
1041	Assert(ProcNum.Group <= g_cRtMpNtMaxGroups);
1042	Assert( (g_aidRtMpNtByCpuSetIdx[iCpu] & ~RTMPNT_ID_F_INACTIVE)
1043	== RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number));
1044	return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
1045	}
1046	}
1047	return g_aidRtMpNtByCpuSetIdx[iCpu];
1048	}
1049	return NIL_RTCPUID;
1050	#else
1051	/* 1:1 mapping, just do range checks. */
1052	return (unsigned)iCpu < RTCPUSET_MAX_CPUS ? iCpu : NIL_RTCPUID;
1053	#endif
1054	}
1055
1056
1057	RTDECL(int) RTMpSetIndexFromCpuGroupMember(uint32_t idxGroup, uint32_t idxMember)
1058	{
1059	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1060
1061	if (idxGroup < g_cRtMpNtMaxGroups)
1062	if (idxMember < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus)
1063	return g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember];
1064	return -1;
1065	}
1066
1067
1068	RTDECL(uint32_t) RTMpGetCpuGroupCounts(uint32_t idxGroup, uint32_t *pcActive)
1069	{
1070	if (idxGroup < g_cRtMpNtMaxGroups)
1071	{
1072	if (pcActive)
1073	*pcActive = g_aRtMpNtCpuGroups[idxGroup].cActiveCpus;
1074	return g_aRtMpNtCpuGroups[idxGroup].cMaxCpus;
1075	}
1076	if (pcActive)
1077	*pcActive = 0;
1078	return 0;
1079	}
1080
1081
1082	RTDECL(uint32_t) RTMpGetMaxCpuGroupCount(void)
1083	{
1084	return g_cRtMpNtMaxGroups;
1085	}
1086
1087
1088	RTDECL(RTCPUID) RTMpGetMaxCpuId(void)
1089	{
1090	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1091
1092	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1093	return RTMPCPUID_FROM_GROUP_AND_NUMBER(g_cRtMpNtMaxGroups - 1, g_aRtMpNtCpuGroups[g_cRtMpNtMaxGroups - 1].cMaxCpus - 1);
1094	#else
1095	/* According to MSDN the processor indexes goes from 0 to the maximum
1096	number of CPUs in the system. We've check this in initterm-r0drv-nt.cpp. */
1097	return g_cRtMpNtMaxCpus - 1;
1098	#endif
1099	}
1100
1101
1102	RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
1103	{
1104	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1105	return RTCpuSetIsMember(&g_rtMpNtCpuSet, idCpu);
1106	}
1107
1108
1109	RTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu)
1110	{
1111	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1112
1113	#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1114	if (idCpu != NIL_RTCPUID)
1115	{
1116	unsigned idxGroup = rtMpCpuIdGetGroup(idCpu);
1117	if (idxGroup < g_cRtMpNtMaxGroups)
1118	return rtMpCpuIdGetGroupMember(idCpu) < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus;
1119	}
1120	return false;
1121
1122	#else
1123	/* A possible CPU ID is one with a value lower than g_cRtMpNtMaxCpus (see
1124	comment in RTMpGetMaxCpuId). */
1125	return idCpu < g_cRtMpNtMaxCpus;
1126	#endif
1127	}
1128
1129
1130
1131	RTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet)
1132	{
1133	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1134
1135	/* The set of possible CPU IDs(/indexes) are from 0 up to
1136	g_cRtMpNtMaxCpus (see comment in RTMpGetMaxCpuId). */
1137	RTCpuSetEmpty(pSet);
1138	int idxCpu = g_cRtMpNtMaxCpus;
1139	while (idxCpu-- > 0)
1140	RTCpuSetAddByIndex(pSet, idxCpu);
1141	return pSet;
1142	}
1143
1144
1145	RTDECL(RTCPUID) RTMpGetCount(void)
1146	{
1147	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1148	return g_cRtMpNtMaxCpus;
1149	}
1150
1151
1152	RTDECL(PRTCPUSET) RTMpGetOnlineSet(PRTCPUSET pSet)
1153	{
1154	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1155
1156	*pSet = g_rtMpNtCpuSet;
1157	return pSet;
1158	}
1159
1160
1161	RTDECL(RTCPUID) RTMpGetOnlineCount(void)
1162	{
1163	RTCPUSET Set;
1164	RTMpGetOnlineSet(&Set);
1165	return RTCpuSetCount(&Set);
1166	}
1167
1168
1169	RTDECL(RTCPUID) RTMpGetOnlineCoreCount(void)
1170	{
1171	/** @todo fix me */
1172	return RTMpGetOnlineCount();
1173	}
1174
1175
1176
1177	#if 0
1178	/* Experiment with checking the undocumented KPRCB structure
1179	* 'dt nt!_kprcb 0xaddress' shows the layout
1180	*/
1181	typedef struct
1182	{
1183	LIST_ENTRY DpcListHead;
1184	ULONG_PTR DpcLock;
1185	volatile ULONG DpcQueueDepth;
1186	ULONG DpcQueueCount;
1187	} KDPC_DATA, *PKDPC_DATA;
1188
1189	RTDECL(bool) RTMpIsCpuWorkPending(void)
1190	{
1191	uint8_t *pkprcb;
1192	PKDPC_DATA pDpcData;
1193
1194	_asm {
1195	mov eax, fs:0x20
1196	mov pkprcb, eax
1197	}
1198	pDpcData = (PKDPC_DATA)(pkprcb + 0x19e0);
1199	if (pDpcData->DpcQueueDepth)
1200	return true;
1201
1202	pDpcData++;
1203	if (pDpcData->DpcQueueDepth)
1204	return true;
1205	return false;
1206	}
1207	#else
1208	RTDECL(bool) RTMpIsCpuWorkPending(void)
1209	{
1210	/** @todo not implemented */
1211	return false;
1212	}
1213	#endif
1214
1215
1216	/**
1217	* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1218	* the RTMpOnAll case.
1219	*
1220	* @param uUserCtx The user context argument (PRTMPARGS).
1221	*/
1222	static ULONG_PTR rtmpNtOnAllBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1223	{
1224	PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1225	/ASMAtomicIncU32(&pArgs->cHits); - not needed /
1226	pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
1227	return 0;
1228	}
1229
1230
1231	/**
1232	* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1233	* the RTMpOnOthers case.
1234	*
1235	* @param uUserCtx The user context argument (PRTMPARGS).
1236	*/
1237	static ULONG_PTR rtmpNtOnOthersBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1238	{
1239	PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1240	RTCPUID idCpu = RTMpCpuId();
1241	if (pArgs->idCpu != idCpu)
1242	{
1243	/ASMAtomicIncU32(&pArgs->cHits); - not needed /
1244	pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1245	}
1246	return 0;
1247	}
1248
1249
1250	/**
1251	* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1252	* the RTMpOnPair case.
1253	*
1254	* @param uUserCtx The user context argument (PRTMPARGS).
1255	*/
1256	static ULONG_PTR rtmpNtOnPairBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1257	{
1258	PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1259	RTCPUID idCpu = RTMpCpuId();
1260	if ( pArgs->idCpu == idCpu
1261	\|\| pArgs->idCpu2 == idCpu)
1262	{
1263	ASMAtomicIncU32(&pArgs->cHits);
1264	pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1265	}
1266	return 0;
1267	}
1268
1269
1270	/**
1271	* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1272	* the RTMpOnSpecific case.
1273	*
1274	* @param uUserCtx The user context argument (PRTMPARGS).
1275	*/
1276	static ULONG_PTR rtmpNtOnSpecificBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1277	{
1278	PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1279	RTCPUID idCpu = RTMpCpuId();
1280	if (pArgs->idCpu == idCpu)
1281	{
1282	ASMAtomicIncU32(&pArgs->cHits);
1283	pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1284	}
1285	return 0;
1286	}
1287
1288
1289	/**
1290	* Internal worker for the RTMpOn* APIs using KeIpiGenericCall.
1291	*
1292	* @returns VINF_SUCCESS.
1293	* @param pfnWorker The callback.
1294	* @param pvUser1 User argument 1.
1295	* @param pvUser2 User argument 2.
1296	* @param pfnNativeWrapper The wrapper between the NT and IPRT callbacks.
1297	* @param idCpu First CPU to match, ultimately specific to the
1298	* pfnNativeWrapper used.
1299	* @param idCpu2 Second CPU to match, ultimately specific to the
1300	* pfnNativeWrapper used.
1301	* @param pcHits Where to return the number of this. Optional.
1302	*/
1303	static int rtMpCallUsingBroadcastIpi(PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2,
1304	PKIPI_BROADCAST_WORKER pfnNativeWrapper, RTCPUID idCpu, RTCPUID idCpu2,
1305	uint32_t *pcHits)
1306	{
1307	RTMPARGS Args;
1308	Args.pfnWorker = pfnWorker;
1309	Args.pvUser1 = pvUser1;
1310	Args.pvUser2 = pvUser2;
1311	Args.idCpu = idCpu;
1312	Args.idCpu2 = idCpu2;
1313	Args.cRefs = 0;
1314	Args.cHits = 0;
1315
1316	AssertPtr(g_pfnrtKeIpiGenericCall);
1317	g_pfnrtKeIpiGenericCall(pfnNativeWrapper, (uintptr_t)&Args);
1318	if (pcHits)
1319	*pcHits = Args.cHits;
1320	return VINF_SUCCESS;
1321	}
1322
1323
1324	/**
1325	* Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
1326	*
1327	* @param Dpc DPC object
1328	* @param DeferredContext Context argument specified by KeInitializeDpc
1329	* @param SystemArgument1 Argument specified by KeInsertQueueDpc
1330	* @param SystemArgument2 Argument specified by KeInsertQueueDpc
1331	*/
1332	static VOID rtmpNtDPCWrapper(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1333	{
1334	PRTMPARGS pArgs = (PRTMPARGS)DeferredContext;
1335	RT_NOREF3(Dpc, SystemArgument1, SystemArgument2);
1336
1337	ASMAtomicIncU32(&pArgs->cHits);
1338	pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
1339
1340	/* Dereference the argument structure. */
1341	int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
1342	Assert(cRefs >= 0);
1343	if (cRefs == 0)
1344	RTMemFree(pArgs);
1345	}
1346
1347
1348	/**
1349	* Wrapper around KeSetTargetProcessorDpcEx / KeSetTargetProcessorDpc.
1350	*
1351	* This is shared with the timer code.
1352	*
1353	* @returns IPRT status code (errors are asserted).
1354	* @param pDpc The DPC.
1355	* @param idCpu The ID of the new target CPU.
1356	*/
1357	DECLHIDDEN(int) rtMpNtSetTargetProcessorDpc(KDPC *pDpc, RTCPUID idCpu)
1358	{
1359	if (g_pfnrtKeSetTargetProcessorDpcEx)
1360	{
1361	/* Convert to stupid process number (bet KeSetTargetProcessorDpcEx does
1362	the reverse conversion internally). */
1363	PROCESSOR_NUMBER ProcNum;
1364	NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(RTMpCpuIdToSetIndex(idCpu), &ProcNum);
1365	AssertMsgReturn(NT_SUCCESS(rcNt),
1366	("KeGetProcessorNumberFromIndex(%u) -> %#x\n", idCpu, rcNt),
1367	RTErrConvertFromNtStatus(rcNt));
1368
1369	rcNt = g_pfnrtKeSetTargetProcessorDpcEx(pDpc, &ProcNum);
1370	AssertMsgReturn(NT_SUCCESS(rcNt),
1371	("KeSetTargetProcessorDpcEx(,%u(%u/%u)) -> %#x\n", idCpu, ProcNum.Group, ProcNum.Number, rcNt),
1372	RTErrConvertFromNtStatus(rcNt));
1373	}
1374	else if (g_pfnrtKeSetTargetProcessorDpc)
1375	g_pfnrtKeSetTargetProcessorDpc(pDpc, RTMpCpuIdToSetIndex(idCpu));
1376	else
1377	return VERR_NOT_SUPPORTED;
1378	return VINF_SUCCESS;
1379	}
1380
1381
1382	/**
1383	* Internal worker for the RTMpOn* APIs.
1384	*
1385	* @returns IPRT status code.
1386	* @param pfnWorker The callback.
1387	* @param pvUser1 User argument 1.
1388	* @param pvUser2 User argument 2.
1389	* @param enmCpuid What to do / is idCpu valid.
1390	* @param idCpu Used if enmCpuid is RT_NT_CPUID_SPECIFIC or
1391	* RT_NT_CPUID_PAIR, otherwise ignored.
1392	* @param idCpu2 Used if enmCpuid is RT_NT_CPUID_PAIR, otherwise ignored.
1393	* @param pcHits Where to return the number of this. Optional.
1394	*/
1395	static int rtMpCallUsingDpcs(PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2,
1396	RT_NT_CPUID enmCpuid, RTCPUID idCpu, RTCPUID idCpu2, uint32_t *pcHits)
1397	{
1398	#if 0
1399	/* KeFlushQueuedDpcs must be run at IRQL PASSIVE_LEVEL according to MSDN, but the
1400	* driver verifier doesn't complain...
1401	*/
1402	AssertMsg(KeGetCurrentIrql() == PASSIVE_LEVEL, ("%d != %d (PASSIVE_LEVEL)\n", KeGetCurrentIrql(), PASSIVE_LEVEL));
1403	#endif
1404	/* KeFlushQueuedDpcs is not present in Windows 2000; import it dynamically so we can just fail this call. */
1405	if (!g_pfnrtNtKeFlushQueuedDpcs)
1406	return VERR_NOT_SUPPORTED;
1407
1408	/*
1409	* Make a copy of the active CPU set and figure out how many KDPCs we really need.
1410	* We must not try setup DPCs for CPUs which aren't there, because that may fail.
1411	*/
1412	RTCPUSET OnlineSet = g_rtMpNtCpuSet;
1413	uint32_t cDpcsNeeded;
1414	switch (enmCpuid)
1415	{
1416	case RT_NT_CPUID_SPECIFIC:
1417	cDpcsNeeded = 1;
1418	break;
1419	case RT_NT_CPUID_PAIR:
1420	cDpcsNeeded = 2;
1421	break;
1422	default:
1423	do
1424	{
1425	cDpcsNeeded = g_cRtMpNtActiveCpus;
1426	OnlineSet = g_rtMpNtCpuSet;
1427	} while (cDpcsNeeded != g_cRtMpNtActiveCpus);
1428	break;
1429	}
1430
1431	/*
1432	* Allocate an RTMPARGS structure followed by cDpcsNeeded KDPCs
1433	* and initialize them.
1434	*/
1435	PRTMPARGS pArgs = (PRTMPARGS)RTMemAllocZ(sizeof(RTMPARGS) + cDpcsNeeded * sizeof(KDPC));
1436	if (!pArgs)
1437	return VERR_NO_MEMORY;
1438
1439	pArgs->pfnWorker = pfnWorker;
1440	pArgs->pvUser1 = pvUser1;
1441	pArgs->pvUser2 = pvUser2;
1442	pArgs->idCpu = NIL_RTCPUID;
1443	pArgs->idCpu2 = NIL_RTCPUID;
1444	pArgs->cHits = 0;
1445	pArgs->cRefs = 1;
1446
1447	int rc;
1448	KDPC paExecCpuDpcs = (KDPC )(pArgs + 1);
1449	if (enmCpuid == RT_NT_CPUID_SPECIFIC)
1450	{
1451	KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
1452	if (g_pfnrtKeSetImportanceDpc)
1453	g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
1454	rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[0], idCpu);
1455	pArgs->idCpu = idCpu;
1456	}
1457	else if (enmCpuid == RT_NT_CPUID_PAIR)
1458	{
1459	KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
1460	if (g_pfnrtKeSetImportanceDpc)
1461	g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
1462	rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[0], idCpu);
1463	pArgs->idCpu = idCpu;
1464
1465	KeInitializeDpc(&paExecCpuDpcs[1], rtmpNtDPCWrapper, pArgs);
1466	if (g_pfnrtKeSetImportanceDpc)
1467	g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[1], HighImportance);
1468	if (RT_SUCCESS(rc))
1469	rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[1], (int)idCpu2);
1470	pArgs->idCpu2 = idCpu2;
1471	}
1472	else
1473	{
1474	rc = VINF_SUCCESS;
1475	for (uint32_t i = 0; i < cDpcsNeeded && RT_SUCCESS(rc); i++)
1476	if (RTCpuSetIsMemberByIndex(&OnlineSet, i))
1477	{
1478	KeInitializeDpc(&paExecCpuDpcs[i], rtmpNtDPCWrapper, pArgs);
1479	if (g_pfnrtKeSetImportanceDpc)
1480	g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[i], HighImportance);
1481	rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[i], RTMpCpuIdFromSetIndex(i));
1482	}
1483	}
1484	if (RT_FAILURE(rc))
1485	{
1486	RTMemFree(pArgs);
1487	return rc;
1488	}
1489
1490	/*
1491	* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
1492	* KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL.
1493	*/
1494	KIRQL oldIrql;
1495	KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
1496
1497	/*
1498	* We cannot do other than assume a 1:1 relationship between the
1499	* affinity mask and the process despite the warnings in the docs.
1500	* If someone knows a better way to get this done, please let bird know.
1501	*/
1502	ASMCompilerBarrier(); /* paranoia */
1503	if (enmCpuid == RT_NT_CPUID_SPECIFIC)
1504	{
1505	ASMAtomicIncS32(&pArgs->cRefs);
1506	BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
1507	Assert(fRc); NOREF(fRc);
1508	}
1509	else if (enmCpuid == RT_NT_CPUID_PAIR)
1510	{
1511	ASMAtomicIncS32(&pArgs->cRefs);
1512	BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
1513	Assert(fRc); NOREF(fRc);
1514
1515	ASMAtomicIncS32(&pArgs->cRefs);
1516	fRc = KeInsertQueueDpc(&paExecCpuDpcs[1], 0, 0);
1517	Assert(fRc); NOREF(fRc);
1518	}
1519	else
1520	{
1521	uint32_t iSelf = RTMpCurSetIndex();
1522	for (uint32_t i = 0; i < cDpcsNeeded; i++)
1523	{
1524	if ( (i != iSelf)
1525	&& RTCpuSetIsMemberByIndex(&OnlineSet, i))
1526	{
1527	ASMAtomicIncS32(&pArgs->cRefs);
1528	BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[i], 0, 0);
1529	Assert(fRc); NOREF(fRc);
1530	}
1531	}
1532	if (enmCpuid != RT_NT_CPUID_OTHERS)
1533	pfnWorker(iSelf, pvUser1, pvUser2);
1534	}
1535
1536	KeLowerIrql(oldIrql);
1537
1538	/*
1539	* Flush all DPCs and wait for completion. (can take long!)
1540	*/
1541	/** @todo Consider changing this to an active wait using some atomic inc/dec
1542	* stuff (and check for the current cpu above in the specific case). */
1543	/** @todo Seems KeFlushQueuedDpcs doesn't wait for the DPCs to be completely
1544	* executed. Seen pArgs being freed while some CPU was using it before
1545	* cRefs was added. */
1546	if (g_pfnrtNtKeFlushQueuedDpcs)
1547	g_pfnrtNtKeFlushQueuedDpcs();
1548
1549	if (pcHits)
1550	*pcHits = pArgs->cHits;
1551
1552	/* Dereference the argument structure. */
1553	int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
1554	Assert(cRefs >= 0);
1555	if (cRefs == 0)
1556	RTMemFree(pArgs);
1557
1558	return VINF_SUCCESS;
1559	}
1560
1561
1562	RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2)
1563	{
1564	if (g_pfnrtKeIpiGenericCall)
1565	return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnAllBroadcastIpiWrapper,
1566	NIL_RTCPUID, NIL_RTCPUID, NULL);
1567	return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_ALL, NIL_RTCPUID, NIL_RTCPUID, NULL);
1568	}
1569
1570
1571	RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2)
1572	{
1573	if (g_pfnrtKeIpiGenericCall)
1574	return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnOthersBroadcastIpiWrapper,
1575	NIL_RTCPUID, NIL_RTCPUID, NULL);
1576	return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_OTHERS, NIL_RTCPUID, NIL_RTCPUID, NULL);
1577	}
1578
1579
1580	RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2)
1581	{
1582	int rc;
1583	AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
1584	AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
1585	if ((fFlags & RTMPON_F_CONCURRENT_EXEC) && !g_pfnrtKeIpiGenericCall)
1586	return VERR_NOT_SUPPORTED;
1587
1588	/*
1589	* Check that both CPUs are online before doing the broadcast call.
1590	*/
1591	if ( RTMpIsCpuOnline(idCpu1)
1592	&& RTMpIsCpuOnline(idCpu2))
1593	{
1594	/*
1595	* The broadcast IPI isn't quite as bad as it could have been, because
1596	* it looks like windows doesn't synchronize CPUs on the way out, they
1597	* seems to get back to normal work while the pair is still busy.
1598	*/
1599	uint32_t cHits = 0;
1600	if (g_pfnrtKeIpiGenericCall)
1601	rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnPairBroadcastIpiWrapper, idCpu1, idCpu2, &cHits);
1602	else
1603	rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_PAIR, idCpu1, idCpu2, &cHits);
1604	if (RT_SUCCESS(rc))
1605	{
1606	Assert(cHits <= 2);
1607	if (cHits == 2)
1608	rc = VINF_SUCCESS;
1609	else if (cHits == 1)
1610	rc = VERR_NOT_ALL_CPUS_SHOWED;
1611	else if (cHits == 0)
1612	rc = VERR_CPU_OFFLINE;
1613	else
1614	rc = VERR_CPU_IPE_1;
1615	}
1616	}
1617	/*
1618	* A CPU must be present to be considered just offline.
1619	*/
1620	else if ( RTMpIsCpuPresent(idCpu1)
1621	&& RTMpIsCpuPresent(idCpu2))
1622	rc = VERR_CPU_OFFLINE;
1623	else
1624	rc = VERR_CPU_NOT_FOUND;
1625	return rc;
1626	}
1627
1628
1629	RTDECL(bool) RTMpOnPairIsConcurrentExecSupported(void)
1630	{
1631	return g_pfnrtKeIpiGenericCall != NULL;
1632	}
1633
1634
1635	/**
1636	* Releases a reference to a RTMPNTONSPECIFICARGS heap allocation, freeing it
1637	* when the last reference is released.
1638	*/
1639	DECLINLINE(void) rtMpNtOnSpecificRelease(PRTMPNTONSPECIFICARGS pArgs)
1640	{
1641	uint32_t cRefs = ASMAtomicDecU32(&pArgs->cRefs);
1642	AssertMsg(cRefs <= 1, ("cRefs=%#x\n", cRefs));
1643	if (cRefs == 0)
1644	RTMemFree(pArgs);
1645	}
1646
1647
1648	/**
1649	* Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
1650	*
1651	* @param Dpc DPC object
1652	* @param DeferredContext Context argument specified by KeInitializeDpc
1653	* @param SystemArgument1 Argument specified by KeInsertQueueDpc
1654	* @param SystemArgument2 Argument specified by KeInsertQueueDpc
1655	*/
1656	static VOID rtMpNtOnSpecificDpcWrapper(IN PKDPC Dpc, IN PVOID DeferredContext,
1657	IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1658	{
1659	PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)DeferredContext;
1660	RT_NOREF3(Dpc, SystemArgument1, SystemArgument2);
1661
1662	ASMAtomicWriteBool(&pArgs->fExecuting, true);
1663
1664	pArgs->CallbackArgs.pfnWorker(RTMpCpuId(), pArgs->CallbackArgs.pvUser1, pArgs->CallbackArgs.pvUser2);
1665
1666	ASMAtomicWriteBool(&pArgs->fDone, true);
1667	KeSetEvent(&pArgs->DoneEvt, 1 /PriorityIncrement/, FALSE /Wait/);
1668
1669	rtMpNtOnSpecificRelease(pArgs);
1670	}
1671
1672
1673	RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void pvUser1, void pvUser2)
1674	{
1675	/*
1676	* Don't try mess with an offline CPU.
1677	*/
1678	if (!RTMpIsCpuOnline(idCpu))
1679	return !RTMpIsCpuPossible(idCpu)
1680	? VERR_CPU_NOT_FOUND
1681	: VERR_CPU_OFFLINE;
1682
1683	/*
1684	* Use the broadcast IPI routine if there are no more than two CPUs online,
1685	* or if the current IRQL is unsuitable for KeWaitForSingleObject.
1686	*/
1687	int rc;
1688	uint32_t cHits = 0;
1689	if ( g_pfnrtKeIpiGenericCall
1690	&& ( RTMpGetOnlineCount() <= 2
1691	\|\| KeGetCurrentIrql() > APC_LEVEL)
1692	)
1693	{
1694	rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnSpecificBroadcastIpiWrapper,
1695	idCpu, NIL_RTCPUID, &cHits);
1696	if (RT_SUCCESS(rc))
1697	{
1698	if (cHits == 1)
1699	return VINF_SUCCESS;
1700	rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
1701	}
1702	return rc;
1703	}
1704
1705	#if 0
1706	rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_SPECIFIC, idCpu, NIL_RTCPUID, &cHits);
1707	if (RT_SUCCESS(rc))
1708	{
1709	if (cHits == 1)
1710	return VINF_SUCCESS;
1711	rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
1712	}
1713	return rc;
1714
1715	#else
1716	/*
1717	* Initialize the argument package and the objects within it.
1718	* The package is referenced counted to avoid unnecessary spinning to
1719	* synchronize cleanup and prevent stack corruption.
1720	*/
1721	PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)RTMemAllocZ(sizeof(*pArgs));
1722	if (!pArgs)
1723	return VERR_NO_MEMORY;
1724	pArgs->cRefs = 2;
1725	pArgs->fExecuting = false;
1726	pArgs->fDone = false;
1727	pArgs->CallbackArgs.pfnWorker = pfnWorker;
1728	pArgs->CallbackArgs.pvUser1 = pvUser1;
1729	pArgs->CallbackArgs.pvUser2 = pvUser2;
1730	pArgs->CallbackArgs.idCpu = idCpu;
1731	pArgs->CallbackArgs.cHits = 0;
1732	pArgs->CallbackArgs.cRefs = 2;
1733	KeInitializeEvent(&pArgs->DoneEvt, SynchronizationEvent, FALSE /* not signalled */);
1734	KeInitializeDpc(&pArgs->Dpc, rtMpNtOnSpecificDpcWrapper, pArgs);
1735	if (g_pfnrtKeSetImportanceDpc)
1736	g_pfnrtKeSetImportanceDpc(&pArgs->Dpc, HighImportance);
1737	rc = rtMpNtSetTargetProcessorDpc(&pArgs->Dpc, idCpu);
1738	if (RT_FAILURE(rc))
1739	{
1740	RTMemFree(pArgs);
1741	return rc;
1742	}
1743
1744	/*
1745	* Disable preemption while we check the current processor and inserts the DPC.
1746	*/
1747	KIRQL bOldIrql;
1748	KeRaiseIrql(DISPATCH_LEVEL, &bOldIrql);
1749	ASMCompilerBarrier(); /* paranoia */
1750
1751	if (RTMpCpuId() == idCpu)
1752	{
1753	/* Just execute the callback on the current CPU. */
1754	pfnWorker(idCpu, pvUser1, pvUser2);
1755	KeLowerIrql(bOldIrql);
1756
1757	RTMemFree(pArgs);
1758	return VINF_SUCCESS;
1759	}
1760
1761	/* Different CPU, so queue it if the CPU is still online. */
1762	if (RTMpIsCpuOnline(idCpu))
1763	{
1764	BOOLEAN fRc = KeInsertQueueDpc(&pArgs->Dpc, 0, 0);
1765	Assert(fRc); NOREF(fRc);
1766	KeLowerIrql(bOldIrql);
1767
1768	uint64_t const nsRealWaitTS = RTTimeNanoTS();
1769
1770	/*
1771	* Wait actively for a while in case the CPU/thread responds quickly.
1772	*/
1773	uint32_t cLoopsLeft = 0x20000;
1774	while (cLoopsLeft-- > 0)
1775	{
1776	if (pArgs->fDone)
1777	{
1778	rtMpNtOnSpecificRelease(pArgs);
1779	return VINF_SUCCESS;
1780	}
1781	ASMNopPause();
1782	}
1783
1784	/*
1785	* It didn't respond, so wait on the event object, poking the CPU if it's slow.
1786	*/
1787	LARGE_INTEGER Timeout;
1788	Timeout.QuadPart = -10000; /* 1ms */
1789	NTSTATUS rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1790	if (rcNt == STATUS_SUCCESS)
1791	{
1792	rtMpNtOnSpecificRelease(pArgs);
1793	return VINF_SUCCESS;
1794	}
1795
1796	/* If it hasn't respondend yet, maybe poke it and wait some more. */
1797	if (rcNt == STATUS_TIMEOUT)
1798	{
1799	if ( !pArgs->fExecuting
1800	&& ( g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalRequestIpiW7Plus
1801	\|\| g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalRequestIpiPreW7))
1802	RTMpPokeCpu(idCpu);
1803
1804	Timeout.QuadPart = -1280000; /* 128ms */
1805	rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1806	if (rcNt == STATUS_SUCCESS)
1807	{
1808	rtMpNtOnSpecificRelease(pArgs);
1809	return VINF_SUCCESS;
1810	}
1811	}
1812
1813	/*
1814	* Something weird is happening, try bail out.
1815	*/
1816	if (KeRemoveQueueDpc(&pArgs->Dpc))
1817	{
1818	RTMemFree(pArgs); /* DPC was still queued, so we can return without further ado. */
1819	LogRel(("RTMpOnSpecific(%#x): Not processed after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1820	}
1821	else
1822	{
1823	/* DPC is running, wait a good while for it to complete. */
1824	LogRel(("RTMpOnSpecific(%#x): Still running after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1825
1826	Timeout.QuadPart = -301000100010; / 30 seconds */
1827	rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1828	if (rcNt != STATUS_SUCCESS)
1829	LogRel(("RTMpOnSpecific(%#x): Giving up on running worker after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1830	}
1831	rc = RTErrConvertFromNtStatus(rcNt);
1832	}
1833	else
1834	{
1835	/* CPU is offline.*/
1836	KeLowerIrql(bOldIrql);
1837	rc = !RTMpIsCpuPossible(idCpu) ? VERR_CPU_NOT_FOUND : VERR_CPU_OFFLINE;
1838	}
1839
1840	rtMpNtOnSpecificRelease(pArgs);
1841	return rc;
1842	#endif
1843	}
1844
1845
1846
1847
1848	static VOID rtMpNtPokeCpuDummy(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1849	{
1850	NOREF(Dpc);
1851	NOREF(DeferredContext);
1852	NOREF(SystemArgument1);
1853	NOREF(SystemArgument2);
1854	}
1855
1856
1857	/** Callback used by rtMpPokeCpuUsingBroadcastIpi. */
1858	static ULONG_PTR rtMpIpiGenericCall(ULONG_PTR Argument)
1859	{
1860	NOREF(Argument);
1861	return 0;
1862	}
1863
1864
1865	/**
1866	* RTMpPokeCpu worker that uses broadcast IPIs for doing the work.
1867	*
1868	* @returns VINF_SUCCESS
1869	* @param idCpu The CPU identifier.
1870	*/
1871	int rtMpPokeCpuUsingBroadcastIpi(RTCPUID idCpu)
1872	{
1873	NOREF(idCpu);
1874	g_pfnrtKeIpiGenericCall(rtMpIpiGenericCall, 0);
1875	return VINF_SUCCESS;
1876	}
1877
1878
1879	/**
1880	* RTMpPokeCpu worker that uses the Windows 7 and later version of
1881	* HalRequestIpip to get the job done.
1882	*
1883	* @returns VINF_SUCCESS
1884	* @param idCpu The CPU identifier.
1885	*/
1886	int rtMpPokeCpuUsingHalRequestIpiW7Plus(RTCPUID idCpu)
1887	{
1888	/* idCpu is an HAL processor index, so we can use it directly. */
1889	PKAFFINITY_EX pTarget = (PKAFFINITY_EX)alloca(g_cbRtMpNtKaffinityEx);
1890	pTarget->Size = g_cRtMpNtKaffinityExEntries; /* (just in case KeInitializeAffinityEx starts using it) */
1891	g_pfnrtKeInitializeAffinityEx(pTarget);
1892	g_pfnrtKeAddProcessorAffinityEx(pTarget, idCpu);
1893
1894	g_pfnrtHalRequestIpiW7Plus(0, pTarget);
1895	return VINF_SUCCESS;
1896	}
1897
1898
1899	/**
1900	* RTMpPokeCpu worker that uses the Vista and earlier version of HalRequestIpip
1901	* to get the job done.
1902	*
1903	* @returns VINF_SUCCESS
1904	* @param idCpu The CPU identifier.
1905	*/
1906	int rtMpPokeCpuUsingHalRequestIpiPreW7(RTCPUID idCpu)
1907	{
1908	__debugbreak(); /** @todo this code needs testing!! */
1909	KAFFINITY Target = 1;
1910	Target <<= idCpu;
1911	g_pfnrtHalRequestIpiPreW7(Target);
1912	return VINF_SUCCESS;
1913	}
1914
1915
1916	int rtMpPokeCpuUsingFailureNotSupported(RTCPUID idCpu)
1917	{
1918	NOREF(idCpu);
1919	return VERR_NOT_SUPPORTED;
1920	}
1921
1922
1923	int rtMpPokeCpuUsingDpc(RTCPUID idCpu)
1924	{
1925	Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1926
1927	/*
1928	* APC fallback.
1929	*/
1930	static KDPC s_aPokeDpcs[RTCPUSET_MAX_CPUS] = {0};
1931	static bool s_fPokeDPCsInitialized = false;
1932
1933	if (!s_fPokeDPCsInitialized)
1934	{
1935	for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++)
1936	{
1937	KeInitializeDpc(&s_aPokeDpcs[i], rtMpNtPokeCpuDummy, NULL);
1938	if (g_pfnrtKeSetImportanceDpc)
1939	g_pfnrtKeSetImportanceDpc(&s_aPokeDpcs[i], HighImportance);
1940	int rc = rtMpNtSetTargetProcessorDpc(&s_aPokeDpcs[i], idCpu);
1941	if (RT_FAILURE(rc))
1942	return rc;
1943	}
1944
1945	s_fPokeDPCsInitialized = true;
1946	}
1947
1948	/* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
1949	KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL. */
1950	KIRQL oldIrql;
1951	KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
1952
1953	if (g_pfnrtKeSetImportanceDpc)
1954	g_pfnrtKeSetImportanceDpc(&s_aPokeDpcs[idCpu], HighImportance);
1955	g_pfnrtKeSetTargetProcessorDpc(&s_aPokeDpcs[idCpu], (int)idCpu);
1956
1957	/* Assuming here that high importance DPCs will be delivered immediately; or at least an IPI will be sent immediately.
1958	Note! Not true on at least Vista & Windows 7 */
1959	BOOLEAN fRet = KeInsertQueueDpc(&s_aPokeDpcs[idCpu], 0, 0);
1960
1961	KeLowerIrql(oldIrql);
1962	return fRet == TRUE ? VINF_SUCCESS : VERR_ACCESS_DENIED /* already queued */;
1963	}
1964
1965
1966	RTDECL(int) RTMpPokeCpu(RTCPUID idCpu)
1967	{
1968	if (!RTMpIsCpuOnline(idCpu))
1969	return !RTMpIsCpuPossible(idCpu)
1970	? VERR_CPU_NOT_FOUND
1971	: VERR_CPU_OFFLINE;
1972	/* Calls rtMpPokeCpuUsingDpc, rtMpPokeCpuUsingHalRequestIpiW7Plus or rtMpPokeCpuUsingBroadcastIpi. */
1973	return g_pfnrtMpPokeCpuWorker(idCpu);
1974	}
1975
1976
1977	RTDECL(bool) RTMpOnAllIsConcurrentSafe(void)
1978	{
1979	return false;
1980	}
1981

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/nt/mp-r0drv-nt.cpp@ 86176

Download in other formats: