Changeset 64281 in vbox
- Timestamp:
- Oct 15, 2016 4:46:29 PM (8 years ago)
- Location:
- trunk
- Files:
-
- 1 added
- 1 deleted
- 18 edited
- 1 copied
Legend:
- Unmodified
- Added
- Removed
-
trunk/include/VBox/sup.h
r64255 r64281 272 272 /** CPU group number (always zero, except on windows). */ 273 273 uint16_t iCpuGroup; 274 /** CPU group number (same as iCpuSet, except on windows). */274 /** CPU group member number (same as iCpuSet, except on windows). */ 275 275 uint16_t iCpuGroupMember; 276 276 /** The APIC ID of this CPU. */ … … 289 289 typedef SUPGIPCPU *PSUPGIPCPU; 290 290 291 /** 292 * CPU group information. 293 * @remarks Windows only. 294 */ 295 typedef struct SUPGIPCPUGROUP 296 { 297 /** Current number of CPUs in this group. */ 298 uint16_t volatile cMembers; 299 /** Maximum number of CPUs in the group. */ 300 uint16_t cMaxMembers; 301 /** The CPU set index of the members. This table has cMaxMembers entries. 302 * @note For various reasons, entries from cMembers and up to cMaxMembers are 303 * may change as the host OS does set dynamic assignments during CPU 304 * hotplugging. */ 305 int16_t aiCpuSetIdxs[1]; 306 } SUPGIPCPUGROUP; 307 /** Pointer to a GIP CPU group structure. */ 308 typedef SUPGIPCPUGROUP *PSUPGIPCPUGROUP; 309 /** Pointer to a const GIP CPU group structure. */ 310 typedef SUPGIPCPUGROUP const *PCSUPGIPCPUGROUP; 291 311 292 312 /** … … 358 378 /** @} */ 359 379 380 /** @def SUPGIP_MAX_CPU_GROUPS 381 * Maximum number of CPU groups. */ 382 #if RTCPUSET_MAX_CPUS >= 256 383 # define SUPGIP_MAX_CPU_GROUPS 256 384 #else 385 # define SUPGIP_MAX_CPU_GROUPS 256 386 #endif 360 387 361 388 /** … … 419 446 /** CPU set index to CPU table index. */ 420 447 uint16_t aiCpuFromCpuSetIdx[RTCPUSET_MAX_CPUS]; 421 /** Table indexed by CPU group index to get the CPU set index of the first 422 * CPU. */ 423 uint16_t aiFirstCpuSetIdxFromCpuGroup[RTCPUSET_MAX_CPUS]; 448 /** Table indexed by CPU group to containing offsets to SUPGIPCPUGROUP 449 * structures, invalid entries are set to UINT16_MAX. The offsets are relative 450 * to the start of this structure. 451 * @note Windows only. The other hosts sets all entries to UINT16_MAX! */ 452 uint16_t aoffCpuGroup[SUPGIP_MAX_CPU_GROUPS]; 424 453 425 454 /** Array of per-cpu data. … … 450 479 * Upper 16 bits is the major version. Major version is only changed with 451 480 * incompatible changes in the GIP. */ 452 #define SUPGLOBALINFOPAGE_VERSION 0x000 70000481 #define SUPGLOBALINFOPAGE_VERSION 0x00080000 453 482 454 483 /** -
trunk/include/iprt/mangling.h
r64255 r64281 1303 1303 # define RTMpGetCurFrequency RT_MANGLER(RTMpGetCurFrequency) 1304 1304 # define RTMpGetDescription RT_MANGLER(RTMpGetDescription) 1305 # define RTMpGetCpuGroupCounts RT_MANGLER(RTMpGetCpuGroupCounts) 1306 # define RTMpGetMaxCpuGroupCount RT_MANGLER(RTMpGetMaxCpuGroupCount) 1305 1307 # define RTMpGetMaxCpuId RT_MANGLER(RTMpGetMaxCpuId) 1306 1308 # define RTMpGetMaxFrequency RT_MANGLER(RTMpGetMaxFrequency) … … 1326 1328 # define RTMpOnSpecific RT_MANGLER(RTMpOnSpecific) /* r0drv */ 1327 1329 # define RTMpPokeCpu RT_MANGLER(RTMpPokeCpu) /* r0drv */ 1330 # define RTMpSetIndexFromCpuGroupMember RT_MANGLER(RTMpSetIndexFromCpuGroupMember) 1328 1331 # define RTMsgError RT_MANGLER(RTMsgError) 1329 1332 # define RTMsgErrorExit RT_MANGLER(RTMsgErrorExit) -
trunk/include/iprt/mp.h
r62473 r64281 88 88 */ 89 89 RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu); 90 91 /** 92 * Translates an NT process group member to a CPU set index. 93 * 94 * @returns CPU set index, -1 if not valid. 95 * @param idxGroup The CPU group. 96 * @param idxMember The CPU group member number. 97 * 98 * @remarks Only available on Windows. 99 */ 100 RTDECL(int) RTMpSetIndexFromCpuGroupMember(uint32_t idxGroup, uint32_t idxMember); 101 102 /** 103 * Gets the member numbers for a CPU group. 104 * 105 * @returns Maximum number of group members. 106 * @param idxGroup The CPU group. 107 * @param pcActive Where to return the number of active members. 108 * 109 * @remarks Only available on Windows. 110 */ 111 RTDECL(uint32_t) RTMpGetCpuGroupCounts(uint32_t idxGroup, uint32_t *pcActive); 112 113 /** 114 * Get the maximum number of CPU groups. 115 * 116 * @returns Maximum number of CPU groups. 117 * 118 * @remarks Only available on Windows. 119 */ 120 RTDECL(uint32_t) RTMpGetMaxCpuGroupCount(void); 90 121 91 122 /** -
trunk/include/iprt/nt/nt.h
r64255 r64281 2454 2454 typedef ULONG (NTAPI *PFNKEQUERYMAXIMUMPROCESSORCOUNTEX)(USHORT GroupNumber); 2455 2455 typedef USHORT (NTAPI *PFNKEQUERYMAXIMUMGROUPCOUNT)(VOID); 2456 typedef ULONG (NTAPI *PFNKEQUERYACTIVEPROCESSORCOUNT)(KAFFINITY *pfActiveProcessors); 2457 typedef ULONG (NTAPI *PFNKEQUERYACTIVEPROCESSORCOUNTEX)(USHORT GroupNumber); 2456 2458 typedef NTSTATUS (NTAPI *PFNKEQUERYLOGICALPROCESSORRELATIONSHIP)(PROCESSOR_NUMBER *pProcNumber, 2457 2459 LOGICAL_PROCESSOR_RELATIONSHIP RelationShipType, -
trunk/src/VBox/HostDrivers/Support/SUPDrvGip.cpp
r64260 r64281 1320 1320 pGip->aCPUs[i].iCpuGroupMember = iCpuSet; 1321 1321 #ifdef RT_OS_WINDOWS 1322 pGip->aCPUs[i].iCpuGroup = supdrvOSGipGetGroupFromCpu(pDevExt, idCpu, &pGip->aCPUs[i].iCpuGroupMember);1322 supdrvOSGipInitGroupBitsForCpu(pDevExt, pGip, &pGip->aCPUs[i]); 1323 1323 #endif 1324 1324 … … 1744 1744 * Initializes the GIP data. 1745 1745 * 1746 * @returns VBox status code. 1746 1747 * @param pDevExt Pointer to the device instance data. 1747 1748 * @param pGip Pointer to the read-write kernel mapping of the GIP. … … 1751 1752 * @param uUpdateIntervalNS The update interval in nanoseconds. 1752 1753 * @param cCpus The CPU count. 1753 */ 1754 static void supdrvGipInit(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, RTHCPHYS HCPhys, 1755 uint64_t u64NanoTS, unsigned uUpdateHz, unsigned uUpdateIntervalNS, unsigned cCpus) 1756 { 1757 size_t const cbGip = RT_ALIGN_Z(RT_OFFSETOF(SUPGLOBALINFOPAGE, aCPUs[cCpus]), PAGE_SIZE); 1758 unsigned i; 1754 * @param cbGipCpuGroups The supdrvOSGipGetGroupTableSize return value we 1755 * used when allocating the GIP structure. 1756 */ 1757 static int supdrvGipInit(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, RTHCPHYS HCPhys, 1758 uint64_t u64NanoTS, unsigned uUpdateHz, unsigned uUpdateIntervalNS, 1759 unsigned cCpus, size_t cbGipCpuGroups) 1760 { 1761 size_t const cbGip = RT_ALIGN_Z(RT_OFFSETOF(SUPGLOBALINFOPAGE, aCPUs[cCpus]) + cbGipCpuGroups, PAGE_SIZE); 1762 unsigned i; 1759 1763 #ifdef DEBUG_DARWIN_GIP 1760 1764 OSDBGPRINT(("supdrvGipInit: pGip=%p HCPhys=%lx u64NanoTS=%llu uUpdateHz=%d cCpus=%u\n", pGip, (long)HCPhys, u64NanoTS, uUpdateHz, cCpus)); … … 1794 1798 for (i = 0; i < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx); i++) 1795 1799 pGip->aiCpuFromCpuSetIdx[i] = UINT16_MAX; 1796 pGip->aiFirstCpuSetIdxFromCpuGroup[0] = 0; 1797 for (i = 1; i < RT_ELEMENTS(pGip->aiFirstCpuSetIdxFromCpuGroup); i++) 1798 pGip->aiFirstCpuSetIdxFromCpuGroup[i] = UINT16_MAX; 1799 #ifdef RT_OS_WINDOWS 1800 supdrvOSInitGipGroupTable(pDevExt, pGip); 1801 #endif 1800 for (i = 0; i < RT_ELEMENTS(pGip->aoffCpuGroup); i++) 1801 pGip->aoffCpuGroup[i] = UINT16_MAX; 1802 1802 for (i = 0; i < cCpus; i++) 1803 1803 supdrvGipInitCpu(pGip, &pGip->aCPUs[i], u64NanoTS, 0 /*uCpuHz*/); 1804 #ifdef RT_OS_WINDOWS 1805 int rc = supdrvOSInitGipGroupTable(pDevExt, pGip, cbGipCpuGroups); 1806 AssertRCReturn(rc, rc); 1807 #endif 1804 1808 1805 1809 /* … … 1809 1813 pDevExt->HCPhysGip = HCPhys; 1810 1814 pDevExt->cGipUsers = 0; 1815 1816 return VINF_SUCCESS; 1811 1817 } 1812 1818 … … 1821 1827 { 1822 1828 PSUPGLOBALINFOPAGE pGip; 1829 size_t cbGip; 1830 size_t cbGipCpuGroups; 1823 1831 RTHCPHYS HCPhysGip; 1824 1832 uint32_t u32SystemResolution; … … 1862 1870 * Allocate a contiguous set of pages with a default kernel mapping. 1863 1871 */ 1864 rc = RTR0MemObjAllocCont(&pDevExt->GipMemObj, RT_UOFFSETOF(SUPGLOBALINFOPAGE, aCPUs[cCpus]), false /*fExecutable*/); 1872 cbGipCpuGroups = supdrvOSGipGetGroupTableSize(pDevExt); 1873 cbGip = RT_UOFFSETOF(SUPGLOBALINFOPAGE, aCPUs[cCpus]) + cbGipCpuGroups; 1874 rc = RTR0MemObjAllocCont(&pDevExt->GipMemObj, cbGip, false /*fExecutable*/); 1865 1875 if (RT_FAILURE(rc)) 1866 1876 { … … 1884 1894 u32Interval += u32SystemResolution - uMod; 1885 1895 1886 supdrvGipInit(pDevExt, pGip, HCPhysGip, RTTimeSystemNanoTS(), RT_NS_1SEC / u32Interval /*=Hz*/, u32Interval, cCpus); 1887 1888 /* 1889 * Important sanity check... 1896 rc = supdrvGipInit(pDevExt, pGip, HCPhysGip, RTTimeSystemNanoTS(), RT_NS_1SEC / u32Interval /*=Hz*/, u32Interval, 1897 cCpus, cbGipCpuGroups); 1898 1899 /* 1900 * Important sanity check... (Sets rc) 1890 1901 */ 1891 1902 if (RT_UNLIKELY( pGip->enmUseTscDelta == SUPGIPUSETSCDELTA_ZERO_CLAIMED … … 1894 1905 { 1895 1906 OSDBGPRINT(("supdrvGipCreate: Host-OS/user claims the TSC-deltas are zero but we detected async. TSC! Bad.\n")); 1896 r eturnVERR_INTERNAL_ERROR_2;1907 rc = VERR_INTERNAL_ERROR_2; 1897 1908 } 1898 1909 1899 1910 /* It doesn't make sense to do TSC-delta detection on systems we detect as async. */ 1900 AssertReturn( pGip->u32Mode != SUPGIPMODE_ASYNC_TSC 1901 || pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ZERO_CLAIMED, VERR_INTERNAL_ERROR_3); 1911 AssertStmt( pGip->u32Mode != SUPGIPMODE_ASYNC_TSC 1912 || pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ZERO_CLAIMED, 1913 rc = VERR_INTERNAL_ERROR_3); 1902 1914 1903 1915 /* … … 1912 1924 * array with more reasonable values. 1913 1925 */ 1914 if (pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC)1915 {1916 rc = supdrvGipInitMeasureTscFreq(pGip, true /*fRough*/); /* cannot fail */1917 supdrvGipInitStartTimerForRefiningInvariantTscFreq(pDevExt);1918 }1919 else1920 rc = supdrvGipInitMeasureTscFreq(pGip, false /*fRough*/);1921 1926 if (RT_SUCCESS(rc)) 1922 1927 { 1923 /* 1924 * Start TSC-delta measurement thread before we start getting MP 1925 * events that will try kick it into action (includes the 1926 * RTMpOnAll/supdrvGipInitOnCpu call below). 1927 */ 1928 RTCpuSetEmpty(&pDevExt->TscDeltaCpuSet); 1929 RTCpuSetEmpty(&pDevExt->TscDeltaObtainedCpuSet); 1930 #ifdef SUPDRV_USE_TSC_DELTA_THREAD 1931 if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED) 1932 rc = supdrvTscDeltaThreadInit(pDevExt); 1933 #endif 1928 if (pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC) 1929 { 1930 rc = supdrvGipInitMeasureTscFreq(pGip, true /*fRough*/); /* cannot fail */ 1931 supdrvGipInitStartTimerForRefiningInvariantTscFreq(pDevExt); 1932 } 1933 else 1934 rc = supdrvGipInitMeasureTscFreq(pGip, false /*fRough*/); 1934 1935 if (RT_SUCCESS(rc)) 1935 1936 { 1936 rc = RTMpNotificationRegister(supdrvGipMpEvent, pDevExt); 1937 /* 1938 * Start TSC-delta measurement thread before we start getting MP 1939 * events that will try kick it into action (includes the 1940 * RTMpOnAll/supdrvGipInitOnCpu call below). 1941 */ 1942 RTCpuSetEmpty(&pDevExt->TscDeltaCpuSet); 1943 RTCpuSetEmpty(&pDevExt->TscDeltaObtainedCpuSet); 1944 #ifdef SUPDRV_USE_TSC_DELTA_THREAD 1945 if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED) 1946 rc = supdrvTscDeltaThreadInit(pDevExt); 1947 #endif 1937 1948 if (RT_SUCCESS(rc)) 1938 1949 { 1939 /* 1940 * Do GIP initialization on all online CPUs. Wake up the 1941 * TSC-delta thread afterwards. 1942 */ 1943 rc = RTMpOnAll(supdrvGipInitOnCpu, pDevExt, pGip); 1950 rc = RTMpNotificationRegister(supdrvGipMpEvent, pDevExt); 1944 1951 if (RT_SUCCESS(rc)) 1945 1952 { 1946 #ifdef SUPDRV_USE_TSC_DELTA_THREAD 1947 supdrvTscDeltaThreadStartMeasurement(pDevExt, true /* fForceAll */); 1948 #else 1949 uint16_t iCpu; 1950 if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED) 1953 /* 1954 * Do GIP initialization on all online CPUs. Wake up the 1955 * TSC-delta thread afterwards. 1956 */ 1957 rc = RTMpOnAll(supdrvGipInitOnCpu, pDevExt, pGip); 1958 if (RT_SUCCESS(rc)) 1951 1959 { 1952 /*1953 * Measure the TSC deltas now that we have MP notifications.1954 */1955 int cTries = 5;1956 do1960 #ifdef SUPDRV_USE_TSC_DELTA_THREAD 1961 supdrvTscDeltaThreadStartMeasurement(pDevExt, true /* fForceAll */); 1962 #else 1963 uint16_t iCpu; 1964 if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED) 1957 1965 { 1958 rc = supdrvTscMeasureInitialDeltas(pDevExt); 1959 if ( rc != VERR_TRY_AGAIN 1960 && rc != VERR_CPU_OFFLINE) 1961 break; 1962 } while (--cTries > 0); 1963 for (iCpu = 0; iCpu < pGip->cCpus; iCpu++) 1964 Log(("supdrvTscDeltaInit: cpu[%u] delta %lld\n", iCpu, pGip->aCPUs[iCpu].i64TSCDelta)); 1966 /* 1967 * Measure the TSC deltas now that we have MP notifications. 1968 */ 1969 int cTries = 5; 1970 do 1971 { 1972 rc = supdrvTscMeasureInitialDeltas(pDevExt); 1973 if ( rc != VERR_TRY_AGAIN 1974 && rc != VERR_CPU_OFFLINE) 1975 break; 1976 } while (--cTries > 0); 1977 for (iCpu = 0; iCpu < pGip->cCpus; iCpu++) 1978 Log(("supdrvTscDeltaInit: cpu[%u] delta %lld\n", iCpu, pGip->aCPUs[iCpu].i64TSCDelta)); 1979 } 1980 else 1981 { 1982 for (iCpu = 0; iCpu < pGip->cCpus; iCpu++) 1983 AssertMsg(!pGip->aCPUs[iCpu].i64TSCDelta, ("iCpu=%u %lld mode=%d\n", iCpu, pGip->aCPUs[iCpu].i64TSCDelta, pGip->u32Mode)); 1984 } 1985 if (RT_SUCCESS(rc)) 1986 #endif 1987 { 1988 /* 1989 * Create the timer. 1990 * If CPU_ALL isn't supported we'll have to fall back to synchronous mode. 1991 */ 1992 if (pGip->u32Mode == SUPGIPMODE_ASYNC_TSC) 1993 { 1994 rc = RTTimerCreateEx(&pDevExt->pGipTimer, u32Interval, RTTIMER_FLAGS_CPU_ALL, 1995 supdrvGipAsyncTimer, pDevExt); 1996 if (rc == VERR_NOT_SUPPORTED) 1997 { 1998 OSDBGPRINT(("supdrvGipCreate: omni timer not supported, falling back to synchronous mode\n")); 1999 pGip->u32Mode = SUPGIPMODE_SYNC_TSC; 2000 } 2001 } 2002 if (pGip->u32Mode != SUPGIPMODE_ASYNC_TSC) 2003 rc = RTTimerCreateEx(&pDevExt->pGipTimer, u32Interval, 0 /* fFlags */, 2004 supdrvGipSyncAndInvariantTimer, pDevExt); 2005 if (RT_SUCCESS(rc)) 2006 { 2007 /* 2008 * We're good. 2009 */ 2010 Log(("supdrvGipCreate: %u ns interval.\n", u32Interval)); 2011 supdrvGipReleaseHigherTimerFrequencyFromSystem(pDevExt); 2012 2013 g_pSUPGlobalInfoPage = pGip; 2014 return VINF_SUCCESS; 2015 } 2016 2017 OSDBGPRINT(("supdrvGipCreate: failed create GIP timer at %u ns interval. rc=%Rrc\n", u32Interval, rc)); 2018 Assert(!pDevExt->pGipTimer); 2019 } 1965 2020 } 1966 2021 else 1967 { 1968 for (iCpu = 0; iCpu < pGip->cCpus; iCpu++) 1969 AssertMsg(!pGip->aCPUs[iCpu].i64TSCDelta, ("iCpu=%u %lld mode=%d\n", iCpu, pGip->aCPUs[iCpu].i64TSCDelta, pGip->u32Mode)); 1970 } 1971 if (RT_SUCCESS(rc)) 1972 #endif 1973 { 1974 /* 1975 * Create the timer. 1976 * If CPU_ALL isn't supported we'll have to fall back to synchronous mode. 1977 */ 1978 if (pGip->u32Mode == SUPGIPMODE_ASYNC_TSC) 1979 { 1980 rc = RTTimerCreateEx(&pDevExt->pGipTimer, u32Interval, RTTIMER_FLAGS_CPU_ALL, 1981 supdrvGipAsyncTimer, pDevExt); 1982 if (rc == VERR_NOT_SUPPORTED) 1983 { 1984 OSDBGPRINT(("supdrvGipCreate: omni timer not supported, falling back to synchronous mode\n")); 1985 pGip->u32Mode = SUPGIPMODE_SYNC_TSC; 1986 } 1987 } 1988 if (pGip->u32Mode != SUPGIPMODE_ASYNC_TSC) 1989 rc = RTTimerCreateEx(&pDevExt->pGipTimer, u32Interval, 0 /* fFlags */, 1990 supdrvGipSyncAndInvariantTimer, pDevExt); 1991 if (RT_SUCCESS(rc)) 1992 { 1993 /* 1994 * We're good. 1995 */ 1996 Log(("supdrvGipCreate: %u ns interval.\n", u32Interval)); 1997 supdrvGipReleaseHigherTimerFrequencyFromSystem(pDevExt); 1998 1999 g_pSUPGlobalInfoPage = pGip; 2000 return VINF_SUCCESS; 2001 } 2002 2003 OSDBGPRINT(("supdrvGipCreate: failed create GIP timer at %u ns interval. rc=%Rrc\n", u32Interval, rc)); 2004 Assert(!pDevExt->pGipTimer); 2005 } 2022 OSDBGPRINT(("supdrvGipCreate: RTMpOnAll failed. rc=%Rrc\n", rc)); 2006 2023 } 2007 2024 else 2008 OSDBGPRINT(("supdrvGipCreate: RTMpOnAll failed. rc=%Rrc\n", rc));2025 OSDBGPRINT(("supdrvGipCreate: failed to register MP event notfication. rc=%Rrc\n", rc)); 2009 2026 } 2010 2027 else 2011 OSDBGPRINT(("supdrvGipCreate: failed to register MP event notfication. rc=%Rrc\n", rc));2028 OSDBGPRINT(("supdrvGipCreate: supdrvTscDeltaInit failed. rc=%Rrc\n", rc)); 2012 2029 } 2013 2030 else 2014 OSDBGPRINT(("supdrvGipCreate: supdrvTscDeltaInit failed. rc=%Rrc\n", rc)); 2015 } 2016 else 2017 OSDBGPRINT(("supdrvGipCreate: supdrvTscMeasureInitialDeltas failed. rc=%Rrc\n", rc)); 2031 OSDBGPRINT(("supdrvGipCreate: supdrvTscMeasureInitialDeltas failed. rc=%Rrc\n", rc)); 2032 } 2018 2033 2019 2034 /* Releases timer frequency increase too. */ -
trunk/src/VBox/HostDrivers/Support/SUPDrvIOC.h
r64255 r64281 215 215 * - nothing. 216 216 */ 217 #define SUPDRV_IOC_VERSION 0x002 70000217 #define SUPDRV_IOC_VERSION 0x00280000 218 218 219 219 /** SUP_IOCTL_COOKIE. */ -
trunk/src/VBox/HostDrivers/Support/SUPDrvInternal.h
r64255 r64281 801 801 802 802 /** 803 * Called during GIP initializtion to calc the CPU group table size. 804 * 805 * This is currently only implemented on windows [lazy bird]. 806 * 807 * @returns Number of bytes needed for SUPGIPCPUGROUP structures. 808 * @param pDevExt The device globals. 809 */ 810 size_t VBOXCALL supdrvOSGipGetGroupTableSize(PSUPDRVDEVEXT pDevExt); 811 812 /** 803 813 * Called during GIP initialization to set up the group table and group count. 804 814 * … … 808 818 * @param pGip The GIP which group table needs initialization. 809 819 * It's only partially initialized at this point. 810 */ 811 void VBOXCALL supdrvOSInitGipGroupTable(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip); 812 813 /** 814 * Gets the CPU group and member indexes for the given CPU ID. 820 * @param cbGipCpuGroups What supdrvOSGipGetGroupTableSize returned. 821 */ 822 int VBOXCALL supdrvOSInitGipGroupTable(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, size_t cbGipCpuGroups); 823 824 /** 825 * Initializes the group related members when a CPU is added to the GIP. 826 * 827 * This is called both during GIP initalization and during an CPU online event. 815 828 * 816 829 * This is currently only implemented on windows [lazy bird]. … … 821 834 * @param piCpuGroupMember Where to return the group member number. 822 835 */ 823 uint16_t VBOXCALL supdrvOSGipGetGroupFromCpu(PSUPDRVDEVEXT pDevExt, RTCPUID idCpu, uint16_t *piCpuGroupMember);836 void VBOXCALL supdrvOSGipInitGroupBitsForCpu(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, PSUPGIPCPU pGipCpu); 824 837 825 838 void VBOXCALL supdrvOSObjInitCreator(PSUPDRVOBJ pObj, PSUPDRVSESSION pSession); -
trunk/src/VBox/HostDrivers/Support/testcase/tstGIP-2.cpp
r64255 r64281 137 137 g_pSUPGlobalInfoPage->u32Version, 138 138 g_pSUPGlobalInfoPage->fGetGipCpu); 139 RTPrintf("tstGIP-2: cCpus=%d cPossibleCpus=%d cPossibleCpuGroups=%d cPresentCpus=%d cOnlineCpus=%d \n",139 RTPrintf("tstGIP-2: cCpus=%d cPossibleCpus=%d cPossibleCpuGroups=%d cPresentCpus=%d cOnlineCpus=%d idCpuMax=%#x\n", 140 140 g_pSUPGlobalInfoPage->cCpus, 141 141 g_pSUPGlobalInfoPage->cPossibleCpus, 142 142 g_pSUPGlobalInfoPage->cPossibleCpuGroups, 143 143 g_pSUPGlobalInfoPage->cPresentCpus, 144 g_pSUPGlobalInfoPage->cOnlineCpus); 144 g_pSUPGlobalInfoPage->cOnlineCpus, 145 g_pSUPGlobalInfoPage->idCpuMax); 145 146 RTPrintf("tstGIP-2: u32UpdateHz=%RU32 u32UpdateIntervalNS=%RU32 u64NanoTSLastUpdateHz=%RX64 u64CpuHz=%RU64 uCpuHzRef=%RU64\n", 146 147 g_pSUPGlobalInfoPage->u32UpdateHz, … … 149 150 g_pSUPGlobalInfoPage->u64CpuHz, 150 151 uCpuHzRef); 152 for (uint32_t iCpu = 0; iCpu < g_pSUPGlobalInfoPage->cCpus; iCpu++) 153 if (g_pSUPGlobalInfoPage->aCPUs[iCpu].enmState != SUPGIPCPUSTATE_INVALID) 154 { 155 SUPGIPCPU const *pGipCpu = &g_pSUPGlobalInfoPage->aCPUs[iCpu]; 156 RTPrintf("tstGIP-2: aCPU[%u]: enmState=%d iCpuSet=%u idCpu=%#010x iCpuGroup=%u iCpuGroupMember=%u idApic=%#x\n", 157 iCpu, pGipCpu->enmState, pGipCpu->iCpuSet, pGipCpu->idCpu, pGipCpu->iCpuGroup, 158 pGipCpu->iCpuGroupMember, pGipCpu->idApic); 159 } 151 160 152 161 RTPrintf(fHex -
trunk/src/VBox/HostDrivers/Support/win/SUPDrv-win.cpp
r64255 r64281 1711 1711 1712 1712 1713 void VBOXCALL supdrvOSInitGipGroupTable(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip)1713 size_t VBOXCALL supdrvOSGipGetGroupTableSize(PSUPDRVDEVEXT pDevExt) 1714 1714 { 1715 1715 NOREF(pDevExt); 1716 uint32_t cMaxCpus = RTMpGetCount(); 1717 uint32_t cGroups = RTMpGetMaxCpuGroupCount(); 1718 1719 return cGroups * RT_OFFSETOF(SUPGIPCPUGROUP, aiCpuSetIdxs) 1720 + RT_SIZEOFMEMB(SUPGIPCPUGROUP, aiCpuSetIdxs[0]) * cMaxCpus; 1721 } 1722 1723 1724 int VBOXCALL supdrvOSInitGipGroupTable(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, size_t cbGipCpuGroups) 1725 { 1726 Assert(cbGipCpuGroups > 0); NOREF(cbGipCpuGroups); NOREF(pDevExt); 1727 1728 unsigned const cGroups = RTMpGetMaxCpuGroupCount(); 1729 AssertReturn(cGroups > 0 && cGroups < RT_ELEMENTS(pGip->aoffCpuGroup), VERR_INTERNAL_ERROR_2); 1730 pGip->cPossibleCpuGroups = cGroups; 1731 1732 PSUPGIPCPUGROUP pGroup = (PSUPGIPCPUGROUP)&pGip->aCPUs[pGip->cCpus]; 1733 for (uint32_t idxGroup = 0; idxGroup < cGroups; idxGroup++) 1734 { 1735 uint32_t cActive = 0; 1736 uint32_t cMax = RTMpGetCpuGroupCounts(idxGroup, &cActive); 1737 uint32_t cbNeeded = RT_OFFSETOF(SUPGIPCPUGROUP, aiCpuSetIdxs[cMax]); 1738 AssertReturn(cbNeeded <= cbGipCpuGroups, VERR_INTERNAL_ERROR_3); 1739 AssertReturn(cActive <= cMax, VERR_INTERNAL_ERROR_4); 1740 1741 pGip->aoffCpuGroup[idxGroup] = (uint16_t)((uintptr_t)pGroup - (uintptr_t)pGip); 1742 pGroup->cMembers = cActive; 1743 pGroup->cMaxMembers = cMax; 1744 for (uint32_t idxMember = 0; idxMember < cMax; idxMember++) 1745 { 1746 pGroup->aiCpuSetIdxs[idxMember] = RTMpSetIndexFromCpuGroupMember(idxGroup, idxMember); 1747 Assert((unsigned)pGroup->aiCpuSetIdxs[idxMember] < pGip->cPossibleCpus); 1748 } 1749 1750 /* advance. */ 1751 cbGipCpuGroups -= cbNeeded; 1752 pGroup = (PSUPGIPCPUGROUP)&pGroup->aiCpuSetIdxs[cMax]; 1753 } 1754 1755 return VINF_SUCCESS; 1756 } 1757 1758 1759 void VBOXCALL supdrvOSGipInitGroupBitsForCpu(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, PSUPGIPCPU pGipCpu) 1760 { 1761 NOREF(pDevExt); 1716 1762 1717 1763 /* 1718 * T he indexes are assigned in group order (see initterm-r0drv-nt.cpp).1764 * Translate the CPU index into a group and member. 1719 1765 */ 1720 if ( g_pfnKeQueryMaximumGroupCount 1721 && g_pfnKeGetProcessorIndexFromNumber) 1722 { 1723 unsigned cGroups = g_pfnKeQueryMaximumGroupCount(); 1724 AssertStmt(cGroups > 0, cGroups = 1); 1725 AssertStmt(cGroups < RT_ELEMENTS(pGip->aiFirstCpuSetIdxFromCpuGroup), 1726 cGroups = RT_ELEMENTS(pGip->aiFirstCpuSetIdxFromCpuGroup)); 1727 pGip->cPossibleCpuGroups = cGroups; 1728 1729 KEPROCESSORINDEX idxCpuMin = 0; 1730 for (unsigned iGroup = 0; iGroup < cGroups; iGroup++) 1731 { 1732 PROCESSOR_NUMBER ProcNum; 1733 ProcNum.Group = (USHORT)iGroup; 1734 ProcNum.Number = 0; 1735 ProcNum.Reserved = 0; 1736 KEPROCESSORINDEX idxCpu = g_pfnKeGetProcessorIndexFromNumber(&ProcNum); 1737 Assert(idxCpu != INVALID_PROCESSOR_INDEX); 1738 Assert(idxCpu >= idxCpuMin); 1739 idxCpuMin = idxCpu; 1740 pGip->aiFirstCpuSetIdxFromCpuGroup[iGroup] = (uint16_t)idxCpu; 1741 } 1742 } 1743 else 1744 { 1745 Assert(!g_pfnKeQueryMaximumGroupCount); 1746 Assert(!g_pfnKeGetProcessorIndexFromNumber); 1747 1748 pGip->cPossibleCpuGroups = 1; 1749 pGip->aiFirstCpuSetIdxFromCpuGroup[0] = 0; 1750 } 1751 } 1752 1753 1754 uint16_t VBOXCALL supdrvOSGipGetGroupFromCpu(PSUPDRVDEVEXT pDevExt, RTCPUID idCpu, uint16_t *piCpuGroupMember) 1755 { 1756 NOREF(pDevExt); 1766 PROCESSOR_NUMBER ProcNum = { 0, pGipCpu->iCpuSet, 0 }; 1767 if (g_pfnKeGetProcessorNumberFromIndex) 1768 { 1769 NTSTATUS rcNt = g_pfnKeGetProcessorNumberFromIndex(pGipCpu->iCpuSet, &ProcNum); 1770 if (NT_SUCCESS(rcNt)) 1771 Assert(ProcNum.Group < g_pfnKeQueryMaximumGroupCount()); 1772 else 1773 { 1774 AssertFailed(); 1775 ProcNum.Group = 0; 1776 ProcNum.Number = pGipCpu->iCpuSet; 1777 } 1778 } 1779 pGipCpu->iCpuGroup = ProcNum.Group; 1780 pGipCpu->iCpuGroupMember = ProcNum.Number; 1757 1781 1758 1782 /* 1759 * This is just a wrapper around KeGetProcessorNumberFromIndex.1783 * Update the group info. Just do this wholesale for now (doesn't scale well). 1760 1784 */ 1761 if (g_pfnKeGetProcessorNumberFromIndex) 1762 { 1763 PROCESSOR_NUMBER ProcNum = { UINT16_MAX, UINT8_MAX, 0 }; 1764 NTSTATUS rcNt = g_pfnKeGetProcessorNumberFromIndex(idCpu, &ProcNum); 1765 if (NT_SUCCESS(rcNt)) 1766 { 1767 Assert(ProcNum.Group < g_pfnKeQueryMaximumGroupCount()); 1768 *piCpuGroupMember = ProcNum.Number; 1769 return ProcNum.Group; 1770 } 1771 1772 AssertMsgFailed(("rcNt=%#x for idCpu=%u\n", rcNt, idCpu)); 1773 } 1774 1775 *piCpuGroupMember = 0; 1776 return idCpu; 1785 for (uint32_t idxGroup = 0; idxGroup < pGip->cPossibleCpuGroups; idxGroup++) 1786 if (pGip->aoffCpuGroup[idxGroup] != UINT16_MAX) 1787 { 1788 PSUPGIPCPUGROUP pGroup = (PSUPGIPCPUGROUP)((uintptr_t)pGip + pGip->aoffCpuGroup[idxGroup]); 1789 1790 uint32_t cActive = 0; 1791 uint32_t cMax = RTMpGetCpuGroupCounts(idxGroup, &cActive); 1792 AssertStmt(cMax == pGroup->cMaxMembers, cMax = pGroup->cMaxMembers); 1793 AssertStmt(cActive <= cMax, cActive = cMax); 1794 if (pGroup->cMembers != cActive) 1795 pGroup->cMembers = cActive; 1796 1797 for (uint32_t idxMember = 0; idxMember < cMax; idxMember++) 1798 { 1799 int idxCpuSet = RTMpSetIndexFromCpuGroupMember(idxGroup, idxMember); 1800 AssertMsg((unsigned)idxCpuSet < pGip->cPossibleCpus, 1801 ("%d vs %d for %u.%u\n", idxCpuSet, pGip->cPossibleCpus, idxGroup, idxMember)); 1802 1803 if (pGroup->aiCpuSetIdxs[idxMember] != idxCpuSet) 1804 pGroup->aiCpuSetIdxs[idxMember] = idxCpuSet; 1805 } 1806 } 1777 1807 } 1778 1808 -
trunk/src/VBox/Runtime/Makefile.kmk
r64148 r64281 780 780 generic/RTSemMutexRequestDebug-generic.cpp \ 781 781 generic/RTThreadSetAffinityToCpu-generic.cpp \ 782 generic/mppresent-generic .cpp \782 generic/mppresent-generic-online.cpp \ 783 783 generic/semrw-$(if-expr defined(VBOX_WITH_LOCKLESS_SEMRW),lockless-,)generic.cpp \ 784 784 generic/uuid-generic.cpp \ … … 2247 2247 generic/RTLogWriteStdOut-stub-generic.cpp \ 2248 2248 generic/RTTimerCreate-generic.cpp \ 2249 generic/mppresent-generic .cpp \2249 generic/mppresent-generic-online.cpp \ 2250 2250 generic/RTMpGetCoreCount-generic.cpp \ 2251 2251 nt/RTErrConvertFromNtStatus.cpp \ … … 2263 2263 r0drv/nt/memuserkernel-r0drv-nt.cpp \ 2264 2264 r0drv/nt/mp-r0drv-nt.cpp \ 2265 r0drv/nt/mpnotification-r0drv-nt.cpp \2266 2265 r0drv/nt/process-r0drv-nt.cpp \ 2267 2266 r0drv/nt/RTLogWriteDebugger-r0drv-nt.cpp \ -
trunk/src/VBox/Runtime/common/misc/assert.cpp
r62477 r64281 72 72 static bool volatile g_fQuiet = false; 73 73 /** Set if assertions may panic. */ 74 static bool volatile g_fMayPanic = true;74 static bool volatile g_fMayPanic = false;//true; 75 75 76 76 -
trunk/src/VBox/Runtime/common/time/timesupref.cpp
r64255 r64281 37 37 #include <iprt/asm-math.h> 38 38 #include <iprt/asm-amd64-x86.h> 39 #include <iprt/param.h> 39 40 #include <VBox/sup.h> 40 41 #ifdef IN_RC -
trunk/src/VBox/Runtime/common/time/timesupref.h
r64255 r64281 101 101 uint16_t const iCpuSet = uAux & (RTCPUSET_MAX_CPUS - 1); 102 102 # else 103 uint16_t const iCpuSet = pGip->aiFirstCpuSetIdxFromCpuGroup[(uAux >> 8) & UINT8_MAX] + (uAux & UINT8_MAX); 103 uint16_t iCpuSet = 0; 104 uint16_t offGipCpuGroup = pGip->aoffCpuGroup[(uAux >> 8) & UINT8_MAX]; 105 if (offGipCpuGroup < pGip->cPages * PAGE_SIZE) 106 { 107 PSUPGIPCPUGROUP pGipCpuGroup = (PSUPGIPCPUGROUP)((uintptr_t)pGip + offGipCpuGroup); 108 if ( (uAux & UINT8_MAX) < pGipCpuGroup->cMaxMembers 109 && pGipCpuGroup->aiCpuSetIdxs[uAux & UINT8_MAX] != -1) 110 iCpuSet = pGipCpuGroup->aiCpuSetIdxs[uAux & UINT8_MAX]; 111 } 104 112 # endif 105 113 uint16_t const iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet]; -
trunk/src/VBox/Runtime/generic/mppresent-generic-online.cpp
r64268 r64281 1 1 /* $Id$ */ 2 2 /** @file 3 * IPRT - Multiprocessor, Stubs for the RTMp*Present* API .3 * IPRT - Multiprocessor, Stubs for the RTMp*Present* API mapping to RTMp*Online. 4 4 */ 5 5 … … 35 35 RTDECL(PRTCPUSET) RTMpGetPresentSet(PRTCPUSET pSet) 36 36 { 37 return RTMpGet Set(pSet);37 return RTMpGetOnlineSet(pSet); 38 38 } 39 39 RT_EXPORT_SYMBOL(RTMpGetPresentSet); … … 42 42 RTDECL(RTCPUID) RTMpGetPresentCount(void) 43 43 { 44 return RTMpGet Count();44 return RTMpGetOnlineCount(); 45 45 } 46 46 RT_EXPORT_SYMBOL(RTMpGetPresentCount); … … 49 49 RTDECL(RTCPUID) RTMpGetPresentCoreCount(void) 50 50 { 51 return RTMpGet CoreCount();51 return RTMpGetOnlineCoreCount(); 52 52 } 53 53 RT_EXPORT_SYMBOL(RTMpGetPresentCoreCount); … … 56 56 RTDECL(bool) RTMpIsCpuPresent(RTCPUID idCpu) 57 57 { 58 return RTMpIsCpu Possible(idCpu);58 return RTMpIsCpuOnline(idCpu); 59 59 } 60 60 RT_EXPORT_SYMBOL(RTMpIsCpuPresent); -
trunk/src/VBox/Runtime/r0drv/nt/initterm-r0drv-nt.cpp
r64267 r64281 31 31 #include "the-nt-kernel.h" 32 32 #include <iprt/asm-amd64-x86.h> 33 #include <iprt/assert.h>34 33 #include <iprt/err.h> 35 #include <iprt/mem.h>36 #include <iprt/mp.h>37 34 #include <iprt/string.h> 38 35 #include "internal/initterm.h" 39 36 #include "internal-r0drv-nt.h" 40 #include "../mp-r0drv.h"41 37 #include "symdb.h" 42 38 #include "symdbdata.h" … … 46 42 * Global Variables * 47 43 *********************************************************************************************************************************/ 48 /** The NT CPU set.49 * KeQueryActiveProcssors() cannot be called at all IRQLs and therefore we'll50 * have to cache it. Fortunately, Nt doesn't really support taking CPUs offline51 * or online. It's first with W2K8 that support for CPU hotplugging was added.52 * Once we start caring about this, we'll simply let the native MP event callback53 * and update this variable as CPUs comes online. (The code is done already.)54 */55 RTCPUSET g_rtMpNtCpuSet;56 /** Maximum number of processor groups. */57 uint32_t g_cRtMpNtMaxGroups;58 /** Maximum number of processors. */59 uint32_t g_cRtMpNtMaxCpus;60 /** The handle of the rtR0NtMpProcessorChangeCallback registration. */61 static PVOID g_pvMpCpuChangeCallback = NULL;62 63 44 /** ExSetTimerResolution, introduced in W2K. */ 64 45 PFNMYEXSETTIMERRESOLUTION g_pfnrtNtExSetTimerResolution; … … 93 74 /** KeQueryMaximumGroupCount - Introducted in Windows 7. */ 94 75 PFNKEQUERYMAXIMUMGROUPCOUNT g_pfnrtKeQueryMaximumGroupCount; 76 /** KeQueryActiveProcessorCount - Introducted in Vista and obsoleted W7. */ 77 PFNKEQUERYACTIVEPROCESSORCOUNT g_pfnrtKeQueryActiveProcessorCount; 78 /** KeQueryActiveProcessorCountEx - Introducted in Windows 7. */ 79 PFNKEQUERYACTIVEPROCESSORCOUNTEX g_pfnrtKeQueryActiveProcessorCountEx; 95 80 /** KeQueryLogicalProcessorRelationship - Introducted in Windows 7. */ 96 81 PFNKEQUERYLOGICALPROCESSORRELATIONSHIP g_pfnrtKeQueryLogicalProcessorRelationship; … … 245 230 246 231 247 /**248 * Implements the NT PROCESSOR_CALLBACK_FUNCTION callback function.249 *250 * This maintains the g_rtMpNtCpuSet and works MP notification callbacks. When251 * registered, it's called for each active CPU in the system, avoiding racing252 * CPU hotplugging (as well as testing the callback).253 *254 * @param pvUser User context (not used).255 * @param pChangeCtx Change context (in).256 * @param prcOperationStatus Operation status (in/out).257 */258 static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,259 PNTSTATUS prcOperationStatus)260 {261 RT_NOREF(pvUser, prcOperationStatus);262 switch (pChangeCtx->State)263 {264 case KeProcessorAddCompleteNotify:265 if (pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS)266 {267 RTCpuSetAddByIndex(&g_rtMpNtCpuSet, pChangeCtx->NtNumber);268 rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, pChangeCtx->NtNumber);269 }270 else271 {272 DbgPrint("rtR0NtMpProcessorChangeCallback: NtNumber=%u (%#x) is higher than RTCPUSET_MAX_CPUS (%d)\n",273 pChangeCtx->NtNumber, pChangeCtx->NtNumber, RTCPUSET_MAX_CPUS);274 AssertMsgFailed(("NtNumber=%u (%#x)\n", pChangeCtx->NtNumber, pChangeCtx->NtNumber));275 }276 break;277 278 case KeProcessorAddStartNotify:279 case KeProcessorAddFailureNotify:280 /* ignore */281 break;282 283 default:284 AssertMsgFailed(("State=%u\n", pChangeCtx->State));285 }286 }287 288 289 /**290 * Wrapper around KeQueryLogicalProcessorRelationship.291 *292 * @returns IPRT status code.293 * @param ppInfo Where to return the info. Pass to RTMemFree when done.294 */295 static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo)296 {297 ULONG cbInfo = sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)298 + g_cRtMpNtMaxGroups * sizeof(GROUP_RELATIONSHIP);299 NTSTATUS rcNt;300 do301 {302 SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)RTMemAlloc(cbInfo);303 if (pInfo)304 {305 rcNt = g_pfnrtKeQueryLogicalProcessorRelationship(NULL /*pProcNumber*/, RelationGroup, pInfo, &cbInfo);306 if (NT_SUCCESS(rcNt))307 {308 *ppInfo = pInfo;309 return VINF_SUCCESS;310 }311 312 RTMemFree(pInfo);313 pInfo = NULL;314 }315 else316 rcNt = STATUS_NO_MEMORY;317 } while (rcNt == STATUS_INFO_LENGTH_MISMATCH);318 DbgPrint("IPRT: Fatal: KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt);319 AssertMsgFailed(("KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt));320 return RTErrConvertFromNtStatus(rcNt);321 }322 323 324 /**325 * Initalizes multiprocessor globals.326 *327 * @returns IPRT status code.328 */329 static int rtR0NtInitMp(RTNTSDBOSVER const *pOsVerInfo)330 {331 #define MY_CHECK_BREAK(a_Check, a_DbgPrintArgs) \332 AssertMsgBreakStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )333 #define MY_CHECK_RETURN(a_Check, a_DbgPrintArgs, a_rcRet) \334 AssertMsgReturnStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs, a_rcRet)335 #define MY_CHECK(a_Check, a_DbgPrintArgs) \336 AssertMsgStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )337 338 /*339 * API combination checks.340 */341 MY_CHECK_RETURN(!g_pfnrtKeSetTargetProcessorDpcEx || g_pfnrtKeGetProcessorNumberFromIndex,342 ("IPRT: Fatal: Missing KeSetTargetProcessorDpcEx without KeGetProcessorNumberFromIndex!\n"),343 VERR_SYMBOL_NOT_FOUND);344 345 /*346 * Get max number of processor groups.347 */348 if (g_pfnrtKeQueryMaximumGroupCount)349 {350 g_cRtMpNtMaxGroups = g_pfnrtKeQueryMaximumGroupCount();351 MY_CHECK_RETURN(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,352 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),353 VERR_MP_TOO_MANY_CPUS);354 }355 else356 g_cRtMpNtMaxGroups = 1;357 358 /*359 * Get max number CPUs.360 * This also defines the range of NT CPU indexes, RTCPUID and index into RTCPUSET.361 */362 if (g_pfnrtKeQueryMaximumProcessorCountEx)363 {364 g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS);365 MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,366 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryMaximumProcessorCountEx]\n",367 g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),368 VERR_MP_TOO_MANY_CPUS);369 }370 else if (g_pfnrtKeQueryMaximumProcessorCount)371 {372 g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCount();373 MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,374 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryMaximumProcessorCount]\n",375 g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),376 VERR_MP_TOO_MANY_CPUS);377 }378 else if (g_pfnrtKeQueryActiveProcessors)379 {380 KAFFINITY fActiveProcessors = g_pfnrtKeQueryActiveProcessors();381 MY_CHECK_RETURN(fActiveProcessors != 0,382 ("IPRT: Fatal: KeQueryActiveProcessors returned 0!\n"),383 VERR_INTERNAL_ERROR_2);384 g_cRtMpNtMaxCpus = 0;385 do386 {387 g_cRtMpNtMaxCpus++;388 fActiveProcessors >>= 1;389 } while (fActiveProcessors);390 }391 else392 g_cRtMpNtMaxCpus = KeNumberProcessors;393 394 /*395 * Query the details for the groups to figure out which CPUs are online as396 * well as the NT index limit.397 */398 if (g_pfnrtKeQueryLogicalProcessorRelationship)399 {400 MY_CHECK_RETURN(g_pfnrtKeGetProcessorIndexFromNumber,401 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),402 VERR_SYMBOL_NOT_FOUND);403 MY_CHECK_RETURN(g_pfnrtKeGetProcessorNumberFromIndex,404 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),405 VERR_SYMBOL_NOT_FOUND);406 MY_CHECK_RETURN(g_pfnrtKeSetTargetProcessorDpcEx,407 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeSetTargetProcessorDpcEx!\n"),408 VERR_SYMBOL_NOT_FOUND);409 410 SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = NULL;411 int rc = rtR0NtInitQueryGroupRelations(&pInfo);412 if (RT_FAILURE(rc))413 return rc;414 415 AssertReturnStmt(pInfo->Group.MaximumGroupCount == g_cRtMpNtMaxGroups, RTMemFree(pInfo), VERR_INTERNAL_ERROR_3);416 417 /*418 * Calc online mask.419 *420 * Also check ASSUMPTIONS:421 * - Processor indexes going to KeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS)422 * - Processor indexes being assigned to absent hotswappable CPUs, i.e.423 * KeGetProcessorIndexFromNumber and KeGetProcessorNumberFromIndex works424 * all possible indexes. [Not yet confirmed!]425 * - Processor indexes are assigned in group order.426 * - MaximumProcessorCount specifies the highest bit in the active mask.427 * This is for confirming process IDs assigned by IPRT in ring-3.428 */429 /** @todo Test the latter on a real/virtual system. */430 RTCpuSetEmpty(&g_rtMpNtCpuSet);431 uint32_t idxCpuExpect = 0;432 for (uint32_t idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)433 {434 const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];435 MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,436 ("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));437 MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= MAXIMUM_PROC_PER_GROUP,438 ("IPRT: Fatal: ActiveProcessorCount=%u\n", pGrpInfo->ActiveProcessorCount));439 MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,440 ("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",441 pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));442 for (uint32_t idxMember = 0; idxMember < pGrpInfo->MaximumProcessorCount; idxMember++, idxCpuExpect++)443 {444 PROCESSOR_NUMBER ProcNum;445 ProcNum.Group = (USHORT)idxGroup;446 ProcNum.Number = (UCHAR)idxMember;447 ProcNum.Reserved = 0;448 ULONG idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);449 if (idxCpu != INVALID_PROCESSOR_INDEX)450 {451 MY_CHECK_BREAK(idxCpu < g_cRtMpNtMaxCpus && idxCpu < RTCPUSET_MAX_CPUS,452 ("IPRT: Fatal: idxCpu=%u >= g_cRtMpNtMaxCpu=%u (RTCPUSET_MAX_CPUS=%u)\n",453 idxCpu, g_cRtMpNtMaxCpus, RTCPUSET_MAX_CPUS));454 MY_CHECK_BREAK(idxCpu == idxCpuExpect, ("IPRT: Fatal: idxCpu=%u != idxCpuExpect=%u\n", idxCpu, idxCpuExpect));455 456 ProcNum.Group = UINT16_MAX;457 ProcNum.Number = UINT8_MAX;458 ProcNum.Reserved = UINT8_MAX;459 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(idxCpu, &ProcNum);460 MY_CHECK_BREAK(NT_SUCCESS(rcNt), ("IPRT: Fatal: KeGetProcessorNumberFromIndex(%u,) -> %#x!\n", idxCpu, rcNt));461 MY_CHECK_BREAK(ProcNum.Group == idxGroup && ProcNum.Number == idxMember,462 ("IPRT: Fatal: KeGetProcessorXxxxFromYyyy roundtrip error for %#x! Group: %u vs %u, Number: %u vs %u\n",463 idxCpu, ProcNum.Group, idxGroup, ProcNum.Number, idxMember));464 465 if (pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember))466 RTCpuSetAddByIndex(&g_rtMpNtCpuSet, idxCpu);467 }468 else469 {470 /* W2K8 server gives me a max of 64 logical CPUs, even if the system only has 12,471 causing failures here. Not yet sure how this would work with two CPU groups yet... */472 MY_CHECK_BREAK( idxMember >= pGrpInfo->ActiveProcessorCount473 && !(pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember)),474 ("IPRT: Fatal: KeGetProcessorIndexFromNumber(%u/%u) failed! cMax=%u cActive=%u\n",475 idxGroup, idxMember, pGrpInfo->MaximumProcessorCount, pGrpInfo->ActiveProcessorCount));476 }477 }478 }479 RTMemFree(pInfo);480 if (RT_FAILURE(rc)) /* MY_CHECK_BREAK sets rc. */481 return rc;482 }483 else484 {485 /* Legacy: */486 MY_CHECK_RETURN(g_cRtMpNtMaxGroups == 1, ("IPRT: Fatal: Missing KeQueryLogicalProcessorRelationship!\n"),487 VERR_SYMBOL_NOT_FOUND);488 489 if (g_pfnrtKeQueryActiveProcessors)490 RTCpuSetFromU64(&g_rtMpNtCpuSet, g_pfnrtKeQueryActiveProcessors());491 else if (g_cRtMpNtMaxCpus < 64)492 RTCpuSetFromU64(&g_rtMpNtCpuSet, (UINT64_C(1) << g_cRtMpNtMaxCpus) - 1);493 else494 {495 MY_CHECK_RETURN(g_cRtMpNtMaxCpus == 64, ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, expect 64 or less\n", g_cRtMpNtMaxCpus),496 VERR_MP_TOO_MANY_CPUS);497 RTCpuSetFromU64(&g_rtMpNtCpuSet, UINT64_MAX);498 }499 }500 501 /*502 * Register CPU hot plugging callback.503 */504 Assert(g_pvMpCpuChangeCallback == NULL);505 if (g_pfnrtKeRegisterProcessorChangeCallback)506 {507 MY_CHECK_RETURN(g_pfnrtKeDeregisterProcessorChangeCallback,508 ("IPRT: Fatal: KeRegisterProcessorChangeCallback without KeDeregisterProcessorChangeCallback!\n"),509 VERR_SYMBOL_NOT_FOUND);510 511 RTCPUSET ActiveSetCopy = g_rtMpNtCpuSet;512 RTCpuSetEmpty(&g_rtMpNtCpuSet);513 g_pvMpCpuChangeCallback = g_pfnrtKeRegisterProcessorChangeCallback(rtR0NtMpProcessorChangeCallback, NULL /*pvUser*/,514 KE_PROCESSOR_CHANGE_ADD_EXISTING);515 if (!g_pvMpCpuChangeCallback)516 {517 AssertFailed();518 g_rtMpNtCpuSet = ActiveSetCopy;519 }520 }521 522 /*523 * Special IPI fun for RTMpPokeCpu.524 *525 * On Vista and later the DPC method doesn't seem to reliably send IPIs,526 * so we have to use alternative methods.527 *528 * On AMD64 We used to use the HalSendSoftwareInterrupt API (also x86 on529 * W10+), it looks faster and more convenient to use, however we're either530 * using it wrong or it doesn't reliably do what we want (see @bugref{8343}).531 *532 * The HalRequestIpip API is thus far the only alternative to KeInsertQueueDpc533 * for doing targetted IPIs. Trouble with this API is that it changed534 * fundamentally in Window 7 when they added support for lots of processors.535 *536 * If we really think we cannot use KeInsertQueueDpc, we use the broadcast IPI537 * API KeIpiGenericCall.538 */539 if ( pOsVerInfo->uMajorVer > 6540 || (pOsVerInfo->uMajorVer == 6 && pOsVerInfo->uMinorVer > 0))541 g_pfnrtHalRequestIpiPreW7 = NULL;542 else543 g_pfnrtHalRequestIpiW7Plus = NULL;544 545 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingDpc;546 #ifndef IPRT_TARGET_NT4547 if ( g_pfnrtHalRequestIpiW7Plus548 && g_pfnrtKeInitializeAffinityEx549 && g_pfnrtKeAddProcessorAffinityEx550 && g_pfnrtKeGetProcessorIndexFromNumber)551 {552 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingHalReqestIpiW7Plus\n");553 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingHalReqestIpiW7Plus;554 }555 else if (pOsVerInfo->uMajorVer >= 6 && g_pfnrtKeIpiGenericCall)556 {557 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingBroadcastIpi\n");558 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingBroadcastIpi;559 }560 else561 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingDpc\n");562 /* else: Windows XP should send always send an IPI -> VERIFY */563 #endif564 565 return VINF_SUCCESS;566 }567 568 569 232 DECLHIDDEN(int) rtR0InitNative(void) 570 233 { … … 599 262 GET_SYSTEM_ROUTINE(KeQueryMaximumProcessorCountEx); 600 263 GET_SYSTEM_ROUTINE(KeQueryMaximumGroupCount); 264 GET_SYSTEM_ROUTINE(KeQueryActiveProcessorCount); 265 GET_SYSTEM_ROUTINE(KeQueryActiveProcessorCountEx); 601 266 GET_SYSTEM_ROUTINE(KeQueryLogicalProcessorRelationship); 602 267 GET_SYSTEM_ROUTINE(KeRegisterProcessorChangeCallback); … … 743 408 else 744 409 DbgPrint("IPRT: _KPRCB:{.QuantumEnd=%x/%d, .DpcQueueDepth=%x/%d} Kernel %u.%u %u %s\n", 745 g_offrtNtPbQuantumEnd, g_cbrtNtPbQuantumEnd, g_offrtNtPbDpcQueueDepth, 410 g_offrtNtPbQuantumEnd, g_cbrtNtPbQuantumEnd, g_offrtNtPbDpcQueueDepth, g_offrtNtPbDpcQueueDepth, 746 411 OsVerInfo.uMajorVer, OsVerInfo.uMinorVer, OsVerInfo.uBuildNo, OsVerInfo.fChecked ? "checked" : "free"); 747 412 # endif … … 752 417 * we call rtR0TermNative to do the deregistration on failure. 753 418 */ 754 int rc = rtR0 NtInitMp(&OsVerInfo);419 int rc = rtR0MpNtInit(&OsVerInfo); 755 420 if (RT_FAILURE(rc)) 756 421 { 757 422 rtR0TermNative(); 758 DbgPrint("IPRT: Fatal: rtR0 NtInitMpfailed: %d\n", rc);423 DbgPrint("IPRT: Fatal: rtR0MpNtInit failed: %d\n", rc); 759 424 return rc; 760 425 } … … 766 431 DECLHIDDEN(void) rtR0TermNative(void) 767 432 { 768 /* 769 * Deregister the processor change callback. 770 */ 771 PVOID pvMpCpuChangeCallback = g_pvMpCpuChangeCallback; 772 g_pvMpCpuChangeCallback = NULL; 773 if (pvMpCpuChangeCallback) 774 { 775 AssertReturnVoid(g_pfnrtKeDeregisterProcessorChangeCallback); 776 g_pfnrtKeDeregisterProcessorChangeCallback(pvMpCpuChangeCallback); 777 } 433 rtR0MpNtTerm(); 778 434 } 779 435 -
trunk/src/VBox/Runtime/r0drv/nt/internal-r0drv-nt.h
r64234 r64281 56 56 extern uint32_t g_cRtMpNtMaxGroups; 57 57 extern uint32_t g_cRtMpNtMaxCpus; 58 extern RTCPUID g_aidRtMpNtByCpuSetIdx[RTCPUSET_MAX_CPUS]; 58 59 59 60 extern PFNMYEXSETTIMERRESOLUTION g_pfnrtNtExSetTimerResolution; … … 74 75 extern PFNKEQUERYMAXIMUMPROCESSORCOUNTEX g_pfnrtKeQueryMaximumProcessorCountEx; 75 76 extern PFNKEQUERYMAXIMUMGROUPCOUNT g_pfnrtKeQueryMaximumGroupCount; 77 extern PFNKEQUERYACTIVEPROCESSORCOUNT g_pfnrtKeQueryActiveProcessorCount; 78 extern PFNKEQUERYACTIVEPROCESSORCOUNTEX g_pfnrtKeQueryActiveProcessorCountEx; 76 79 extern PFNKEQUERYLOGICALPROCESSORRELATIONSHIP g_pfnrtKeQueryLogicalProcessorRelationship; 77 80 extern PFNKEREGISTERPROCESSORCHANGECALLBACK g_pfnrtKeRegisterProcessorChangeCallback; … … 95 98 int __stdcall rtMpPokeCpuUsingHalReqestIpiPreW7(RTCPUID idCpu); 96 99 100 struct RTNTSDBOSVER; 101 DECLHIDDEN(int) rtR0MpNtInit(struct RTNTSDBOSVER const *pOsVerInfo); 102 DECLHIDDEN(void) rtR0MpNtTerm(void); 97 103 DECLHIDDEN(int) rtMpNtSetTargetProcessorDpc(KDPC *pDpc, RTCPUID idCpu); 98 104 -
trunk/src/VBox/Runtime/r0drv/nt/mp-r0drv-nt.cpp
r64234 r64281 36 36 #include <iprt/asm.h> 37 37 #include <iprt/log.h> 38 #include <iprt/mem.h> 38 39 #include <iprt/time.h> 39 40 #include "r0drv/mp-r0drv.h" 41 #include "symdb.h" 40 42 #include "internal-r0drv-nt.h" 43 #include "internal/mp.h" 41 44 42 45 … … 75 78 76 79 80 /********************************************************************************************************************************* 81 * Defined Constants And Macros * 82 *********************************************************************************************************************************/ 83 /** Inactive bit for g_aidRtMpNtByCpuSetIdx. */ 84 #define RTMPNT_ID_F_INACTIVE RT_BIT_32(31) 85 86 87 /********************************************************************************************************************************* 88 * Global Variables * 89 *********************************************************************************************************************************/ 90 /** Maximum number of processor groups. */ 91 uint32_t g_cRtMpNtMaxGroups; 92 /** Maximum number of processors. */ 93 uint32_t g_cRtMpNtMaxCpus; 94 /** Number of active processors. */ 95 uint32_t volatile g_cRtMpNtActiveCpus; 96 /** The NT CPU set. 97 * KeQueryActiveProcssors() cannot be called at all IRQLs and therefore we'll 98 * have to cache it. Fortunately, NT doesn't really support taking CPUs offline, 99 * and taking them online was introduced with W2K8 where it is intended for virtual 100 * machines and not real HW. We update this, g_cRtMpNtActiveCpus and 101 * g_aidRtMpNtByCpuSetIdx from the rtR0NtMpProcessorChangeCallback. 102 */ 103 RTCPUSET g_rtMpNtCpuSet; 104 105 /** Static per group info. 106 * @remarks With RTCPUSET_MAX_CPUS as 256, this takes up 33KB. */ 107 static struct 108 { 109 /** The max CPUs in the group. */ 110 uint16_t cMaxCpus; 111 /** The number of active CPUs at the time of initialization. */ 112 uint16_t cActiveCpus; 113 /** CPU set indexes for each CPU in the group. */ 114 int16_t aidxCpuSetMembers[64]; 115 } g_aRtMpNtCpuGroups[RTCPUSET_MAX_CPUS]; 116 /** Maps CPU set indexes to RTCPUID. 117 * Inactive CPUs has bit 31 set (RTMPNT_ID_F_INACTIVE) so we can identify them 118 * and shuffle duplicates during CPU hotplugging. We assign temporary IDs to 119 * the inactive CPUs starting at g_cRtMpNtMaxCpus - 1, ASSUMING that active 120 * CPUs has IDs from 0 to g_cRtMpNtActiveCpus. */ 121 RTCPUID g_aidRtMpNtByCpuSetIdx[RTCPUSET_MAX_CPUS]; 122 /** The handle of the rtR0NtMpProcessorChangeCallback registration. */ 123 static PVOID g_pvMpCpuChangeCallback = NULL; 124 125 126 /********************************************************************************************************************************* 127 * Internal Functions * 128 *********************************************************************************************************************************/ 129 static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx, 130 PNTSTATUS prcOperationStatus); 131 static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo); 132 133 134 135 /** 136 * Initalizes multiprocessor globals (called by rtR0InitNative). 137 * 138 * @returns IPRT status code. 139 * @param pOsVerInfo Version information. 140 */ 141 DECLHIDDEN(int) rtR0MpNtInit(RTNTSDBOSVER const *pOsVerInfo) 142 { 143 #define MY_CHECK_BREAK(a_Check, a_DbgPrintArgs) \ 144 AssertMsgBreakStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 ) 145 #define MY_CHECK_RETURN(a_Check, a_DbgPrintArgs, a_rcRet) \ 146 AssertMsgReturnStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs, a_rcRet) 147 #define MY_CHECK(a_Check, a_DbgPrintArgs) \ 148 AssertMsgStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 ) 149 150 /* 151 * API combination checks. 152 */ 153 MY_CHECK_RETURN(!g_pfnrtKeSetTargetProcessorDpcEx || g_pfnrtKeGetProcessorNumberFromIndex, 154 ("IPRT: Fatal: Missing KeSetTargetProcessorDpcEx without KeGetProcessorNumberFromIndex!\n"), 155 VERR_SYMBOL_NOT_FOUND); 156 157 /* 158 * Get max number of processor groups. 159 * 160 * We may need to upadjust this number below, because windows likes to keep 161 * all options open when it comes to hotplugged CPU group assignments. A 162 * server advertising up to 64 CPUs in the ACPI table will get a result of 163 * 64 from KeQueryMaximumGroupCount. That makes sense. However, when windows 164 * server 2012 does a two processor group setup for it, the sum of the 165 * GroupInfo[*].MaximumProcessorCount members below is 128. This is probably 166 * because windows doesn't want to make decisions grouping of hotpluggable CPUs. 167 * So, we need to bump the maximum count to 128 below do deal with this as we 168 * want to have valid CPU set indexes for all potential CPUs - how could we 169 * otherwise use the RTMpGetSet() result and also RTCpuSetCount(RTMpGetSet()) 170 * should equal RTMpGetCount(). 171 */ 172 if (g_pfnrtKeQueryMaximumGroupCount) 173 { 174 g_cRtMpNtMaxGroups = g_pfnrtKeQueryMaximumGroupCount(); 175 MY_CHECK_RETURN(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0, 176 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS), 177 VERR_MP_TOO_MANY_CPUS); 178 } 179 else 180 g_cRtMpNtMaxGroups = 1; 181 182 /* 183 * Get max number CPUs. 184 * This also defines the range of NT CPU indexes, RTCPUID and index into RTCPUSET. 185 */ 186 if (g_pfnrtKeQueryMaximumProcessorCountEx) 187 { 188 g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS); 189 MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0, 190 ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCountEx]\n", 191 g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS), 192 VERR_MP_TOO_MANY_CPUS); 193 } 194 else if (g_pfnrtKeQueryMaximumProcessorCount) 195 { 196 g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCount(); 197 MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0, 198 ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCount]\n", 199 g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS), 200 VERR_MP_TOO_MANY_CPUS); 201 } 202 else if (g_pfnrtKeQueryActiveProcessors) 203 { 204 KAFFINITY fActiveProcessors = g_pfnrtKeQueryActiveProcessors(); 205 MY_CHECK_RETURN(fActiveProcessors != 0, 206 ("IPRT: Fatal: KeQueryActiveProcessors returned 0!\n"), 207 VERR_INTERNAL_ERROR_2); 208 g_cRtMpNtMaxCpus = 0; 209 do 210 { 211 g_cRtMpNtMaxCpus++; 212 fActiveProcessors >>= 1; 213 } while (fActiveProcessors); 214 } 215 else 216 g_cRtMpNtMaxCpus = KeNumberProcessors; 217 218 /* 219 * Just because we're a bit paranoid about getting something wrong wrt to the 220 * kernel interfaces, we try 16 times to get the KeQueryActiveProcessorCountEx 221 * and KeQueryLogicalProcessorRelationship information to match up. 222 */ 223 for (unsigned cTries = 0;; cTries++) 224 { 225 /* 226 * Get number of active CPUs. 227 */ 228 if (g_pfnrtKeQueryActiveProcessorCountEx) 229 { 230 g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS); 231 MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0, 232 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCountEx]\n", 233 g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus), 234 VERR_MP_TOO_MANY_CPUS); 235 } 236 else if (g_pfnrtKeQueryActiveProcessorCount) 237 { 238 g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCount(NULL); 239 MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0, 240 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCount]\n", 241 g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus), 242 VERR_MP_TOO_MANY_CPUS); 243 } 244 else 245 g_cRtMpNtActiveCpus = g_cRtMpNtMaxCpus; 246 247 /* 248 * Query the details for the groups to figure out which CPUs are online as 249 * well as the NT index limit. 250 */ 251 for (unsigned i = 0; i < RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx); i++) 252 #ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 253 g_aidRtMpNtByCpuSetIdx[i] = NIL_RTCPUID; 254 #else 255 g_aidRtMpNtByCpuSetIdx[i] = i < g_cRtMpNtMaxCpus ? i : NIL_RTCPUID; 256 #endif 257 for (unsigned idxGroup = 0; idxGroup < RT_ELEMENTS(g_aRtMpNtCpuGroups); idxGroup++) 258 { 259 g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = 0; 260 g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0; 261 for (unsigned idxMember = 0; idxMember < RT_ELEMENTS(g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers); idxMember++) 262 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = -1; 263 } 264 265 if (g_pfnrtKeQueryLogicalProcessorRelationship) 266 { 267 MY_CHECK_RETURN(g_pfnrtKeGetProcessorIndexFromNumber, 268 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"), 269 VERR_SYMBOL_NOT_FOUND); 270 MY_CHECK_RETURN(g_pfnrtKeGetProcessorNumberFromIndex, 271 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"), 272 VERR_SYMBOL_NOT_FOUND); 273 MY_CHECK_RETURN(g_pfnrtKeSetTargetProcessorDpcEx, 274 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeSetTargetProcessorDpcEx!\n"), 275 VERR_SYMBOL_NOT_FOUND); 276 277 SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = NULL; 278 int rc = rtR0NtInitQueryGroupRelations(&pInfo); 279 if (RT_FAILURE(rc)) 280 return rc; 281 282 MY_CHECK(pInfo->Group.MaximumGroupCount == g_cRtMpNtMaxGroups, 283 ("IPRT: Fatal: MaximumGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n", 284 pInfo->Group.MaximumGroupCount, g_cRtMpNtMaxGroups)); 285 MY_CHECK(pInfo->Group.ActiveGroupCount > 0 && pInfo->Group.ActiveGroupCount <= g_cRtMpNtMaxGroups, 286 ("IPRT: Fatal: ActiveGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n", 287 pInfo->Group.ActiveGroupCount, g_cRtMpNtMaxGroups)); 288 289 /* 290 * First we need to recalc g_cRtMpNtMaxCpus (see above). 291 */ 292 uint32_t cMaxCpus = 0; 293 uint32_t idxGroup; 294 for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++) 295 { 296 const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup]; 297 MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP, 298 ("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount)); 299 MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount, 300 ("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n", 301 pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount)); 302 cMaxCpus += pGrpInfo->MaximumProcessorCount; 303 } 304 if (cMaxCpus > g_cRtMpNtMaxCpus && RT_SUCCESS(rc)) 305 { 306 DbgPrint("IPRT: g_cRtMpNtMaxCpus=%u -> %u\n", g_cRtMpNtMaxCpus, cMaxCpus); 307 #ifndef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 308 uint32_t i = RT_MIN(cMaxCpus, RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx)); 309 while (i-- > g_cRtMpNtMaxCpus) 310 g_aidRtMpNtByCpuSetIdx[i] = i; 311 #endif 312 g_cRtMpNtMaxCpus = cMaxCpus; 313 if (g_cRtMpNtMaxGroups > RTCPUSET_MAX_CPUS) 314 { 315 MY_CHECK(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0, 316 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS)); 317 rc = VERR_MP_TOO_MANY_CPUS; 318 } 319 } 320 321 /* 322 * Calc online mask, partition IDs and such. 323 * 324 * Also check ASSUMPTIONS: 325 * 326 * 1. Processor indexes going from 0 and up to 327 * KeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1. 328 * 329 * 2. Currently valid processor indexes, i.e. accepted by 330 * KeGetProcessorIndexFromNumber & KeGetProcessorNumberFromIndex, goes 331 * from 0 thru KeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1. 332 * 333 * 3. PROCESSOR_GROUP_INFO::MaximumProcessorCount gives the number of 334 * relevant bits in the ActiveProcessorMask (from LSB). 335 * 336 * 4. Active processor count found in KeQueryLogicalProcessorRelationship 337 * output matches what KeQueryActiveProcessorCountEx(ALL) returns. 338 * 339 * 5. Active + inactive processor counts in same does not exceed 340 * KeQueryMaximumProcessorCountEx(ALL). 341 * 342 * Note! Processor indexes are assigned as CPUs come online and are not 343 * preallocated according to group maximums. Since CPUS are only taken 344 * online and never offlined, this means that internal CPU bitmaps are 345 * never sparse and no time is wasted scanning unused bits. 346 * 347 * Unfortunately, it means that ring-3 cannot easily guess the index 348 * assignments when hotswapping is used, and must use GIP when available. 349 */ 350 RTCpuSetEmpty(&g_rtMpNtCpuSet); 351 uint32_t cInactive = 0; 352 uint32_t cActive = 0; 353 uint32_t idxCpuMax = 0; 354 uint32_t idxCpuSetNextInactive = g_cRtMpNtMaxCpus - 1; 355 for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++) 356 { 357 const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup]; 358 MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP, 359 ("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount)); 360 MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount, 361 ("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n", 362 pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount)); 363 364 g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = pGrpInfo->MaximumProcessorCount; 365 g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = pGrpInfo->ActiveProcessorCount; 366 367 for (uint32_t idxMember = 0; idxMember < pGrpInfo->MaximumProcessorCount; idxMember++) 368 { 369 PROCESSOR_NUMBER ProcNum; 370 ProcNum.Group = (USHORT)idxGroup; 371 ProcNum.Number = (UCHAR)idxMember; 372 ProcNum.Reserved = 0; 373 ULONG idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum); 374 if (idxCpu != INVALID_PROCESSOR_INDEX) 375 { 376 MY_CHECK_BREAK(idxCpu < g_cRtMpNtMaxCpus && idxCpu < RTCPUSET_MAX_CPUS, /* ASSUMPTION #1 */ 377 ("IPRT: Fatal: idxCpu=%u >= g_cRtMpNtMaxCpus=%u (RTCPUSET_MAX_CPUS=%u)\n", 378 idxCpu, g_cRtMpNtMaxCpus, RTCPUSET_MAX_CPUS)); 379 if (idxCpu > idxCpuMax) 380 idxCpuMax = idxCpu; 381 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpu; 382 #ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 383 g_aidRtMpNtByCpuSetIdx[idxCpu] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember); 384 #endif 385 386 ProcNum.Group = UINT16_MAX; 387 ProcNum.Number = UINT8_MAX; 388 ProcNum.Reserved = UINT8_MAX; 389 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(idxCpu, &ProcNum); 390 MY_CHECK_BREAK(NT_SUCCESS(rcNt), 391 ("IPRT: Fatal: KeGetProcessorNumberFromIndex(%u,) -> %#x!\n", idxCpu, rcNt)); 392 MY_CHECK_BREAK(ProcNum.Group == idxGroup && ProcNum.Number == idxMember, 393 ("IPRT: Fatal: KeGetProcessorXxxxFromYyyy roundtrip error for %#x! Group: %u vs %u, Number: %u vs %u\n", 394 idxCpu, ProcNum.Group, idxGroup, ProcNum.Number, idxMember)); 395 396 if (pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember)) 397 { 398 RTCpuSetAddByIndex(&g_rtMpNtCpuSet, idxCpu); 399 cActive++; 400 } 401 else 402 cInactive++; /* (This is a little unexpected, but not important as long as things add up below.) */ 403 } 404 else 405 { 406 /* Must be not present / inactive when KeGetProcessorIndexFromNumber fails. */ 407 MY_CHECK_BREAK(!(pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember)), 408 ("IPRT: Fatal: KeGetProcessorIndexFromNumber(%u/%u) failed but CPU is active! cMax=%u cActive=%u fActive=%p\n", 409 idxGroup, idxMember, pGrpInfo->MaximumProcessorCount, pGrpInfo->ActiveProcessorCount, 410 pGrpInfo->ActiveProcessorMask)); 411 cInactive++; 412 if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus) 413 { 414 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive; 415 #ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 416 g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember) 417 | RTMPNT_ID_F_INACTIVE; 418 #endif 419 idxCpuSetNextInactive--; 420 } 421 } 422 } 423 } 424 425 MY_CHECK(cInactive + cActive <= g_cRtMpNtMaxCpus, /* ASSUMPTION #5 (not '==' because of inactive groups) */ 426 ("IPRT: Fatal: cInactive=%u + cActive=%u > g_cRtMpNtMaxCpus=%u\n", cInactive, cActive, g_cRtMpNtMaxCpus)); 427 428 /* Deal with inactive groups using KeQueryMaximumProcessorCountEx or as 429 best as we can by as best we can by stipulating maximum member counts 430 from the previous group. */ 431 if ( RT_SUCCESS(rc) 432 && idxGroup < pInfo->Group.MaximumGroupCount) 433 { 434 uint16_t cInactiveLeft = g_cRtMpNtMaxCpus - (cInactive + cActive); 435 while (idxGroup < pInfo->Group.MaximumGroupCount) 436 { 437 uint32_t cMaxMembers = 0; 438 if (g_pfnrtKeQueryMaximumProcessorCountEx) 439 cMaxMembers = g_pfnrtKeQueryMaximumProcessorCountEx(idxGroup); 440 if (cMaxMembers != 0 || cInactiveLeft == 0) 441 AssertStmt(cMaxMembers <= cInactiveLeft, cMaxMembers = cInactiveLeft); 442 else 443 { 444 uint16_t cGroupsLeft = pInfo->Group.MaximumGroupCount - idxGroup; 445 cMaxMembers = pInfo->Group.GroupInfo[idxGroup - 1].MaximumProcessorCount; 446 while (cMaxMembers * cGroupsLeft < cInactiveLeft) 447 cMaxMembers++; 448 if (cMaxMembers > cInactiveLeft) 449 cMaxMembers = cInactiveLeft; 450 } 451 452 g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = (uint16_t)cMaxMembers; 453 g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0; 454 for (uint16_t idxMember = 0; idxMember < cMaxMembers; idxMember++) 455 if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus) 456 { 457 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive; 458 #ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 459 g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember) 460 | RTMPNT_ID_F_INACTIVE; 461 #endif 462 idxCpuSetNextInactive--; 463 } 464 cInactiveLeft -= cMaxMembers; 465 idxGroup++; 466 } 467 } 468 469 /* We're done with pInfo now, free it so we can start returning when assertions fail. */ 470 RTMemFree(pInfo); 471 if (RT_FAILURE(rc)) /* MY_CHECK_BREAK sets rc. */ 472 return rc; 473 MY_CHECK_RETURN(cActive >= g_cRtMpNtActiveCpus, 474 ("IPRT: Fatal: cActive=%u < g_cRtMpNtActiveCpus=%u - CPUs removed?\n", cActive, g_cRtMpNtActiveCpus), 475 VERR_INTERNAL_ERROR_3); 476 MY_CHECK_RETURN(idxCpuMax < cActive, /* ASSUMPTION #2 */ 477 ("IPRT: Fatal: idCpuMax=%u >= cActive=%u! Unexpected CPU index allocation. CPUs removed?\n", 478 idxCpuMax, cActive), 479 VERR_INTERNAL_ERROR_4); 480 481 /* Retry if CPUs were added. */ 482 if ( cActive != g_cRtMpNtActiveCpus 483 && cTries < 16) 484 continue; 485 MY_CHECK_RETURN(cActive == g_cRtMpNtActiveCpus, /* ASSUMPTION #4 */ 486 ("IPRT: Fatal: cActive=%u != g_cRtMpNtActiveCpus=%u\n", cActive, g_cRtMpNtActiveCpus), 487 VERR_INTERNAL_ERROR_5); 488 } 489 else 490 { 491 /* Legacy: */ 492 MY_CHECK_RETURN(g_cRtMpNtMaxGroups == 1, ("IPRT: Fatal: Missing KeQueryLogicalProcessorRelationship!\n"), 493 VERR_SYMBOL_NOT_FOUND); 494 495 /** @todo Is it possible that the affinity mask returned by 496 * KeQueryActiveProcessors is sparse? */ 497 if (g_pfnrtKeQueryActiveProcessors) 498 RTCpuSetFromU64(&g_rtMpNtCpuSet, g_pfnrtKeQueryActiveProcessors()); 499 else if (g_cRtMpNtMaxCpus < 64) 500 RTCpuSetFromU64(&g_rtMpNtCpuSet, (UINT64_C(1) << g_cRtMpNtMaxCpus) - 1); 501 else 502 { 503 MY_CHECK_RETURN(g_cRtMpNtMaxCpus == 64, ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, expect 64 or less\n", g_cRtMpNtMaxCpus), 504 VERR_MP_TOO_MANY_CPUS); 505 RTCpuSetFromU64(&g_rtMpNtCpuSet, UINT64_MAX); 506 } 507 508 g_aRtMpNtCpuGroups[0].cMaxCpus = g_cRtMpNtMaxCpus; 509 g_aRtMpNtCpuGroups[0].cActiveCpus = g_cRtMpNtMaxCpus; 510 for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++) 511 { 512 g_aRtMpNtCpuGroups[0].aidxCpuSetMembers[i] = i; 513 #ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 514 g_aidRtMpNtByCpuSetIdx[i] = RTMPCPUID_FROM_GROUP_AND_NUMBER(0, i); 515 #endif 516 } 517 } 518 519 /* 520 * Register CPU hot plugging callback (it also counts active CPUs). 521 */ 522 Assert(g_pvMpCpuChangeCallback == NULL); 523 if (g_pfnrtKeRegisterProcessorChangeCallback) 524 { 525 MY_CHECK_RETURN(g_pfnrtKeDeregisterProcessorChangeCallback, 526 ("IPRT: Fatal: KeRegisterProcessorChangeCallback without KeDeregisterProcessorChangeCallback!\n"), 527 VERR_SYMBOL_NOT_FOUND); 528 529 RTCPUSET const ActiveSetCopy = g_rtMpNtCpuSet; 530 RTCpuSetEmpty(&g_rtMpNtCpuSet); 531 uint32_t const cActiveCpus = g_cRtMpNtActiveCpus; 532 g_cRtMpNtActiveCpus = 0; 533 534 g_pvMpCpuChangeCallback = g_pfnrtKeRegisterProcessorChangeCallback(rtR0NtMpProcessorChangeCallback, NULL /*pvUser*/, 535 KE_PROCESSOR_CHANGE_ADD_EXISTING); 536 if (g_pvMpCpuChangeCallback) 537 { 538 if (cActiveCpus == g_cRtMpNtActiveCpus) 539 { /* likely */ } 540 else 541 { 542 g_pfnrtKeDeregisterProcessorChangeCallback(g_pvMpCpuChangeCallback); 543 if (cTries < 16) 544 { 545 /* Retry if CPUs were added. */ 546 MY_CHECK_RETURN(g_cRtMpNtActiveCpus >= cActiveCpus, 547 ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u < cActiveCpus=%u! CPUs removed?\n", 548 g_cRtMpNtActiveCpus, cActiveCpus), 549 VERR_INTERNAL_ERROR_2); 550 MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus, 551 ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u > g_cRtMpNtMaxCpus=%u!\n", 552 g_cRtMpNtActiveCpus, g_cRtMpNtMaxCpus), 553 VERR_INTERNAL_ERROR_2); 554 continue; 555 } 556 MY_CHECK_RETURN(0, ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u cActiveCpus=%u\n", g_cRtMpNtActiveCpus, cActiveCpus), 557 VERR_INTERNAL_ERROR_3); 558 } 559 } 560 else 561 { 562 AssertFailed(); 563 g_rtMpNtCpuSet = ActiveSetCopy; 564 g_cRtMpNtActiveCpus = cActiveCpus; 565 } 566 } 567 break; 568 } /* Retry loop for stable active CPU count. */ 569 570 #undef MY_CHECK_RETURN 571 572 /* 573 * Special IPI fun for RTMpPokeCpu. 574 * 575 * On Vista and later the DPC method doesn't seem to reliably send IPIs, 576 * so we have to use alternative methods. 577 * 578 * On AMD64 We used to use the HalSendSoftwareInterrupt API (also x86 on 579 * W10+), it looks faster and more convenient to use, however we're either 580 * using it wrong or it doesn't reliably do what we want (see @bugref{8343}). 581 * 582 * The HalRequestIpip API is thus far the only alternative to KeInsertQueueDpc 583 * for doing targetted IPIs. Trouble with this API is that it changed 584 * fundamentally in Window 7 when they added support for lots of processors. 585 * 586 * If we really think we cannot use KeInsertQueueDpc, we use the broadcast IPI 587 * API KeIpiGenericCall. 588 */ 589 if ( pOsVerInfo->uMajorVer > 6 590 || (pOsVerInfo->uMajorVer == 6 && pOsVerInfo->uMinorVer > 0)) 591 g_pfnrtHalRequestIpiPreW7 = NULL; 592 else 593 g_pfnrtHalRequestIpiW7Plus = NULL; 594 595 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingDpc; 596 #ifndef IPRT_TARGET_NT4 597 if ( g_pfnrtHalRequestIpiW7Plus 598 && g_pfnrtKeInitializeAffinityEx 599 && g_pfnrtKeAddProcessorAffinityEx 600 && g_pfnrtKeGetProcessorIndexFromNumber) 601 { 602 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingHalReqestIpiW7Plus\n"); 603 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingHalReqestIpiW7Plus; 604 } 605 else if (pOsVerInfo->uMajorVer >= 6 && g_pfnrtKeIpiGenericCall) 606 { 607 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingBroadcastIpi\n"); 608 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingBroadcastIpi; 609 } 610 else 611 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingDpc\n"); 612 /* else: Windows XP should send always send an IPI -> VERIFY */ 613 #endif 614 615 return VINF_SUCCESS; 616 } 617 618 619 /** 620 * Called by rtR0TermNative. 621 */ 622 DECLHIDDEN(void) rtR0MpNtTerm(void) 623 { 624 /* 625 * Deregister the processor change callback. 626 */ 627 PVOID pvMpCpuChangeCallback = g_pvMpCpuChangeCallback; 628 g_pvMpCpuChangeCallback = NULL; 629 if (pvMpCpuChangeCallback) 630 { 631 AssertReturnVoid(g_pfnrtKeDeregisterProcessorChangeCallback); 632 g_pfnrtKeDeregisterProcessorChangeCallback(pvMpCpuChangeCallback); 633 } 634 } 635 636 637 DECLHIDDEN(int) rtR0MpNotificationNativeInit(void) 638 { 639 return VINF_SUCCESS; 640 } 641 642 643 DECLHIDDEN(void) rtR0MpNotificationNativeTerm(void) 644 { 645 } 646 647 648 /** 649 * Implements the NT PROCESSOR_CALLBACK_FUNCTION callback function. 650 * 651 * This maintains the g_rtMpNtCpuSet and works MP notification callbacks. When 652 * registered, it's called for each active CPU in the system, avoiding racing 653 * CPU hotplugging (as well as testing the callback). 654 * 655 * @param pvUser User context (not used). 656 * @param pChangeCtx Change context (in). 657 * @param prcOperationStatus Operation status (in/out). 658 * 659 * @remarks ASSUMES no concurrent execution of KeProcessorAddCompleteNotify 660 * notification callbacks. At least during callback registration 661 * callout, we're owning KiDynamicProcessorLock. 662 * 663 * @remarks When registering the handler, we first get KeProcessorAddStartNotify 664 * callbacks for all active CPUs, and after they all succeed we get the 665 * KeProcessorAddCompleteNotify callbacks. 666 */ 667 static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx, 668 PNTSTATUS prcOperationStatus) 669 { 670 RT_NOREF(pvUser, prcOperationStatus); 671 switch (pChangeCtx->State) 672 { 673 /* 674 * Check whether we can deal with the CPU, failing the start operation if we 675 * can't. The checks we are doing here are to avoid complicated/impossible 676 * cases in KeProcessorAddCompleteNotify. They are really just verify specs. 677 */ 678 case KeProcessorAddStartNotify: 679 { 680 NTSTATUS rcNt = STATUS_SUCCESS; 681 if (pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS) 682 { 683 if (pChangeCtx->NtNumber >= g_cRtMpNtMaxCpus) 684 { 685 DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is higher than the max CPU count (%u)!\n", 686 pChangeCtx->NtNumber, g_cRtMpNtMaxCpus); 687 rcNt = STATUS_INTERNAL_ERROR; 688 } 689 690 /* The ProcessNumber field was introduced in Windows 7. */ 691 PROCESSOR_NUMBER ProcNum; 692 if (g_pfnrtKeGetProcessorIndexFromNumber) 693 { 694 ProcNum = pChangeCtx->ProcNumber; 695 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum); 696 if (idxCpu != pChangeCtx->NtNumber) 697 { 698 DbgPrint("IPRT: KeProcessorAddStartNotify failure: g_pfnrtKeGetProcessorIndexFromNumber(%u.%u) -> %u, expected %u!\n", 699 ProcNum.Group, ProcNum.Number, idxCpu, pChangeCtx->NtNumber); 700 rcNt = STATUS_INTERNAL_ERROR; 701 } 702 } 703 else 704 { 705 ProcNum.Group = 0; 706 ProcNum.Number = pChangeCtx->NtNumber; 707 } 708 709 if ( ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups) 710 && ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers)) 711 { 712 if (ProcNum.Group >= g_cRtMpNtMaxGroups) 713 { 714 DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range - max groups: %u!\n", 715 ProcNum.Group, ProcNum.Number, g_cRtMpNtMaxGroups); 716 rcNt = STATUS_INTERNAL_ERROR; 717 } 718 719 if (ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus) 720 { 721 Assert(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1); 722 if (g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == -1) 723 { 724 DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u.%u was assigned -1 as set index!\n", 725 ProcNum.Group, ProcNum.Number); 726 rcNt = STATUS_INTERNAL_ERROR; 727 } 728 729 Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID); 730 if (g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == NIL_RTCPUID) 731 { 732 DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u (%u.%u) translates to NIL_RTCPUID!\n", 733 pChangeCtx->NtNumber, ProcNum.Group, ProcNum.Number); 734 rcNt = STATUS_INTERNAL_ERROR; 735 } 736 } 737 else 738 { 739 DbgPrint("IPRT: KeProcessorAddStartNotify failure: max processors in group %u is %u, cannot add %u to it!\n", 740 ProcNum.Group, g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus, ProcNum.Group, ProcNum.Number); 741 rcNt = STATUS_INTERNAL_ERROR; 742 } 743 } 744 else 745 { 746 DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range (max %u.%u)!\n", 747 ProcNum.Group, ProcNum.Number, RT_ELEMENTS(g_aRtMpNtCpuGroups), RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers)); 748 rcNt = STATUS_INTERNAL_ERROR; 749 } 750 } 751 else 752 { 753 DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is outside RTCPUSET_MAX_CPUS (%u)!\n", 754 pChangeCtx->NtNumber, RTCPUSET_MAX_CPUS); 755 rcNt = STATUS_INTERNAL_ERROR; 756 } 757 if (!NT_SUCCESS(rcNt)) 758 *prcOperationStatus = rcNt; 759 break; 760 } 761 762 /* 763 * Update the globals. Since we've checked out range limits and other 764 * limitations already we just AssertBreak here. 765 */ 766 case KeProcessorAddCompleteNotify: 767 { 768 /* 769 * Calc the processor number and assert conditions checked in KeProcessorAddStartNotify. 770 */ 771 AssertBreak(pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS); 772 AssertBreak(pChangeCtx->NtNumber < g_cRtMpNtMaxCpus); 773 Assert(pChangeCtx->NtNumber == g_cRtMpNtActiveCpus); /* light assumption */ 774 PROCESSOR_NUMBER ProcNum; 775 if (g_pfnrtKeGetProcessorIndexFromNumber) 776 { 777 ProcNum = pChangeCtx->ProcNumber; 778 AssertBreak(g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum) == pChangeCtx->NtNumber); 779 AssertBreak(ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups)); 780 AssertBreak(ProcNum.Group < g_cRtMpNtMaxGroups); 781 } 782 else 783 { 784 ProcNum.Group = 0; 785 ProcNum.Number = pChangeCtx->NtNumber; 786 } 787 AssertBreak(ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers)); 788 AssertBreak(ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus); 789 AssertBreak(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1); 790 AssertBreak(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID); 791 792 /* 793 * Add ourselves to the online CPU set and update the active CPU count. 794 */ 795 RTCpuSetAddByIndex(&g_rtMpNtCpuSet, pChangeCtx->NtNumber); 796 ASMAtomicIncU32(&g_cRtMpNtActiveCpus); 797 798 /* 799 * Update the group info. 800 * 801 * If the index prediction failed (real hotplugging callbacks only) we 802 * have to switch it around. This is particularly annoying when we 803 * use the index as the ID. 804 */ 805 #ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 806 RTCPUID idCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number); 807 RTCPUID idOld = g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber]; 808 if ((idOld & ~RTMPNT_ID_F_INACTIVE) != idCpu) 809 { 810 Assert(idOld & RTMPNT_ID_F_INACTIVE); 811 int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number]; 812 g_aRtMpNtCpuGroups[rtMpCpuIdGetGroup(idOld)].aidxCpuSetMembers[rtMpCpuIdGetGroupMember(idOld)] = idxDest; 813 g_aidRtMpNtByCpuSetIdx[idxDest] = idOld; 814 } 815 g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] = idCpu; 816 #else 817 Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == pChangeCtx->NtNumber); 818 int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number]; 819 if ((ULONG)idxDest != pChangeCtx->NtNumber) 820 { 821 bool fFound = false; 822 uint32_t idxOldGroup = g_cRtMpNtMaxGroups; 823 while (idxOldGroup-- > 0 && !fFound) 824 { 825 uint32_t idxMember = g_aRtMpNtCpuGroups[idxOldGroup].cMaxCpus; 826 while (idxMember-- > 0) 827 if (g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] == (int)pChangeCtx->NtNumber) 828 { 829 g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] = idxDest; 830 fFound = true; 831 break; 832 } 833 } 834 Assert(fFound); 835 } 836 #endif 837 g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] = pChangeCtx->NtNumber; 838 839 /* 840 * Do MP notification callbacks. 841 */ 842 rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, pChangeCtx->NtNumber); 843 break; 844 } 845 846 case KeProcessorAddFailureNotify: 847 /* ignore */ 848 break; 849 850 default: 851 AssertMsgFailed(("State=%u\n", pChangeCtx->State)); 852 } 853 } 854 855 856 /** 857 * Wrapper around KeQueryLogicalProcessorRelationship. 858 * 859 * @returns IPRT status code. 860 * @param ppInfo Where to return the info. Pass to RTMemFree when done. 861 */ 862 static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo) 863 { 864 ULONG cbInfo = sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) 865 + g_cRtMpNtMaxGroups * sizeof(GROUP_RELATIONSHIP); 866 NTSTATUS rcNt; 867 do 868 { 869 SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)RTMemAlloc(cbInfo); 870 if (pInfo) 871 { 872 rcNt = g_pfnrtKeQueryLogicalProcessorRelationship(NULL /*pProcNumber*/, RelationGroup, pInfo, &cbInfo); 873 if (NT_SUCCESS(rcNt)) 874 { 875 *ppInfo = pInfo; 876 return VINF_SUCCESS; 877 } 878 879 RTMemFree(pInfo); 880 pInfo = NULL; 881 } 882 else 883 rcNt = STATUS_NO_MEMORY; 884 } while (rcNt == STATUS_INFO_LENGTH_MISMATCH); 885 DbgPrint("IPRT: Fatal: KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt); 886 AssertMsgFailed(("KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt)); 887 return RTErrConvertFromNtStatus(rcNt); 888 } 889 890 891 892 77 893 78 894 RTDECL(RTCPUID) RTMpCpuId(void) 79 895 { 80 896 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ 897 898 #ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 899 PROCESSOR_NUMBER ProcNum; 900 ProcNum.Group = 0; 901 if (g_pfnrtKeGetCurrentProcessorNumberEx) 902 { 903 ProcNum.Number = 0; 904 g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum); 905 } 906 else 907 ProcNum.Number = KeGetCurrentProcessorNumber(); /* Number is 8-bit, so we're not subject to BYTE -> WORD upgrade in WDK. */ 908 return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number); 909 910 #else 81 911 82 912 if (g_pfnrtKeGetCurrentProcessorNumberEx) … … 87 917 } 88 918 89 /* WDK upgrade warning: PCR->Number changed from BYTE to WORD. */90 return KeGetCurrentProcessorNumber(); 919 return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */ 920 #endif 91 921 } 92 922 … … 94 924 RTDECL(int) RTMpCurSetIndex(void) 95 925 { 926 #ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 927 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ 928 929 if (g_pfnrtKeGetCurrentProcessorNumberEx) 930 { 931 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL); 932 Assert(idxCpu < RTCPUSET_MAX_CPUS); 933 return idxCpu; 934 } 935 return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */ 936 #else 96 937 return (int)RTMpCpuId(); 938 #endif 97 939 } 98 940 … … 100 942 RTDECL(int) RTMpCurSetIndexAndId(PRTCPUID pidCpu) 101 943 { 944 #ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 945 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ 946 947 PROCESSOR_NUMBER ProcNum = { 0 , 0, 0 }; 948 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum); 949 Assert(idxCpu < RTCPUSET_MAX_CPUS); 950 *pidCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number); 951 return idxCpu; 952 #else 102 953 return *pidCpu = RTMpCpuId(); 954 #endif 103 955 } 104 956 … … 106 958 RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu) 107 959 { 960 #ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 961 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ 962 963 if (idCpu != NIL_RTCPUID) 964 { 965 if (g_pfnrtKeGetProcessorIndexFromNumber) 966 { 967 PROCESSOR_NUMBER ProcNum; 968 ProcNum.Group = rtMpCpuIdGetGroup(idCpu); 969 ProcNum.Number = rtMpCpuIdGetGroupMember(idCpu); 970 ProcNum.Reserved = 0; 971 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum); 972 if (idxCpu != INVALID_PROCESSOR_INDEX) 973 { 974 Assert(idxCpu < g_cRtMpNtMaxCpus); 975 Assert((ULONG)g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == idxCpu); 976 return idxCpu; 977 } 978 979 /* Since NT assigned indexes as the CPUs come online, we cannot produce an ID <-> index 980 mapping for not-yet-onlined CPUS that is consistent. We just have to do our best... */ 981 if ( ProcNum.Group < g_cRtMpNtMaxGroups 982 && ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus) 983 return g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number]; 984 } 985 else if (rtMpCpuIdGetGroup(idCpu) == 0) 986 return rtMpCpuIdGetGroupMember(idCpu); 987 } 988 return -1; 989 #else 108 990 /* 1:1 mapping, just do range checks. */ 109 991 return idCpu < RTCPUSET_MAX_CPUS ? (int)idCpu : -1; 992 #endif 110 993 } 111 994 … … 113 996 RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu) 114 997 { 998 #ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 999 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ 1000 1001 if ((unsigned)iCpu < g_cRtMpNtMaxCpus) 1002 { 1003 if (g_pfnrtKeGetProcessorIndexFromNumber) 1004 { 1005 PROCESSOR_NUMBER ProcNum = { 0, 0, 0 }; 1006 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(iCpu, &ProcNum); 1007 if (NT_SUCCESS(rcNt)) 1008 { 1009 Assert(ProcNum.Group <= g_cRtMpNtMaxGroups); 1010 Assert( (g_aidRtMpNtByCpuSetIdx[iCpu] & ~RTMPNT_ID_F_INACTIVE) 1011 == RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number)); 1012 return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number); 1013 } 1014 } 1015 return g_aidRtMpNtByCpuSetIdx[iCpu]; 1016 } 1017 return NIL_RTCPUID; 1018 #else 115 1019 /* 1:1 mapping, just do range checks. */ 116 1020 return (unsigned)iCpu < RTCPUSET_MAX_CPUS ? iCpu : NIL_RTCPUID; 1021 #endif 1022 } 1023 1024 1025 RTDECL(int) RTMpSetIndexFromCpuGroupMember(uint32_t idxGroup, uint32_t idxMember) 1026 { 1027 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ 1028 1029 if (idxGroup < g_cRtMpNtMaxGroups) 1030 if (idxMember < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus) 1031 return g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember]; 1032 return -1; 1033 } 1034 1035 1036 RTDECL(uint32_t) RTMpGetCpuGroupCounts(uint32_t idxGroup, uint32_t *pcActive) 1037 { 1038 if (idxGroup < g_cRtMpNtMaxGroups) 1039 { 1040 if (pcActive) 1041 *pcActive = g_aRtMpNtCpuGroups[idxGroup].cActiveCpus; 1042 return g_aRtMpNtCpuGroups[idxGroup].cMaxCpus; 1043 } 1044 if (pcActive) 1045 *pcActive = 0; 1046 return 0; 1047 } 1048 1049 1050 RTDECL(uint32_t) RTMpGetMaxCpuGroupCount(void) 1051 { 1052 return g_cRtMpNtMaxGroups; 117 1053 } 118 1054 … … 122 1058 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ 123 1059 1060 #ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 1061 return RTMPCPUID_FROM_GROUP_AND_NUMBER(g_cRtMpNtMaxGroups - 1, g_aRtMpNtCpuGroups[g_cRtMpNtMaxGroups - 1].cMaxCpus - 1); 1062 #else 124 1063 /* According to MSDN the processor indexes goes from 0 to the maximum 125 1064 number of CPUs in the system. We've check this in initterm-r0drv-nt.cpp. */ 126 1065 return g_cRtMpNtMaxCpus - 1; 1066 #endif 127 1067 } 128 1068 … … 131 1071 { 132 1072 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ 133 return idCpu < RTCPUSET_MAX_CPUS 134 && RTCpuSetIsMember(&g_rtMpNtCpuSet, idCpu); 1073 return RTCpuSetIsMember(&g_rtMpNtCpuSet, idCpu); 135 1074 } 136 1075 … … 140 1079 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ 141 1080 1081 #ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 1082 if (idCpu != NIL_RTCPUID) 1083 { 1084 unsigned idxGroup = rtMpCpuIdGetGroup(idCpu); 1085 if (idxGroup < g_cRtMpNtMaxGroups) 1086 return rtMpCpuIdGetGroupMember(idCpu) < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus; 1087 } 1088 return false; 1089 1090 #else 142 1091 /* A possible CPU ID is one with a value lower than g_cRtMpNtMaxCpus (see 143 1092 comment in RTMpGetMaxCpuId). */ 144 1093 return idCpu < g_cRtMpNtMaxCpus; 1094 #endif 145 1095 } 146 1096 … … 183 1133 return RTCpuSetCount(&Set); 184 1134 } 1135 1136 1137 RTDECL(RTCPUID) RTMpGetOnlineCoreCount(void) 1138 { 1139 /** @todo fix me */ 1140 return RTMpGetOnlineCount(); 1141 } 1142 185 1143 186 1144 … … 372 1330 the reverse conversion internally). */ 373 1331 PROCESSOR_NUMBER ProcNum; 374 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex( idCpu, &ProcNum);1332 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(RTMpCpuIdToSetIndex(idCpu), &ProcNum); 375 1333 AssertMsgReturn(NT_SUCCESS(rcNt), 376 1334 ("KeGetProcessorNumberFromIndex(%u) -> %#x\n", idCpu, rcNt), … … 383 1341 } 384 1342 else 385 KeSetTargetProcessorDpc(pDpc, (int)idCpu);1343 KeSetTargetProcessorDpc(pDpc, RTMpCpuIdToSetIndex(idCpu)); 386 1344 return VINF_SUCCESS; 387 1345 } … … 410 1368 411 1369 #else /* !IPRT_TARGET_NT4 */ 412 PRTMPARGS pArgs;413 KDPC *paExecCpuDpcs;414 415 1370 # if 0 416 1371 /* KeFlushQueuedDpcs must be run at IRQL PASSIVE_LEVEL according to MSDN, but the … … 419 1374 AssertMsg(KeGetCurrentIrql() == PASSIVE_LEVEL, ("%d != %d (PASSIVE_LEVEL)\n", KeGetCurrentIrql(), PASSIVE_LEVEL)); 420 1375 # endif 421 422 KAFFINITY Mask = KeQueryActiveProcessors();423 424 1376 /* KeFlushQueuedDpcs is not present in Windows 2000; import it dynamically so we can just fail this call. */ 425 1377 if (!g_pfnrtNtKeFlushQueuedDpcs) 426 1378 return VERR_NOT_SUPPORTED; 427 1379 428 pArgs = (PRTMPARGS)ExAllocatePoolWithTag(NonPagedPool, g_cRtMpNtMaxCpus * sizeof(KDPC) + sizeof(RTMPARGS), (ULONG)'RTMp'); 1380 /* 1381 * Make a copy of the active CPU set and figure out how many KDPCs we really need. 1382 * We must not try setup DPCs for CPUs which aren't there, because that may fail. 1383 */ 1384 RTCPUSET OnlineSet = g_rtMpNtCpuSet; 1385 uint32_t cDpcsNeeded; 1386 switch (enmCpuid) 1387 { 1388 case RT_NT_CPUID_SPECIFIC: 1389 cDpcsNeeded = 1; 1390 break; 1391 case RT_NT_CPUID_PAIR: 1392 cDpcsNeeded = 2; 1393 break; 1394 default: 1395 do 1396 { 1397 cDpcsNeeded = g_cRtMpNtActiveCpus; 1398 OnlineSet = g_rtMpNtCpuSet; 1399 } while (cDpcsNeeded != g_cRtMpNtActiveCpus); 1400 break; 1401 } 1402 1403 /* 1404 * Allocate an RTMPARGS structure followed by cDpcsNeeded KDPCs 1405 * and initialize them. 1406 */ 1407 PRTMPARGS pArgs = (PRTMPARGS)ExAllocatePoolWithTag(NonPagedPool, sizeof(RTMPARGS) + cDpcsNeeded * sizeof(KDPC), (ULONG)'RTMp'); 429 1408 if (!pArgs) 430 1409 return VERR_NO_MEMORY; … … 438 1417 pArgs->cRefs = 1; 439 1418 440 paExecCpuDpcs = (KDPC *)(pArgs + 1);441 442 1419 int rc; 1420 KDPC *paExecCpuDpcs = (KDPC *)(pArgs + 1); 443 1421 if (enmCpuid == RT_NT_CPUID_SPECIFIC) 444 1422 { … … 464 1442 { 465 1443 rc = VINF_SUCCESS; 466 for (unsigned i = 0; i < g_cRtMpNtMaxCpus && RT_SUCCESS(rc); i++) 467 { 468 KeInitializeDpc(&paExecCpuDpcs[i], rtmpNtDPCWrapper, pArgs); 469 KeSetImportanceDpc(&paExecCpuDpcs[i], HighImportance); 470 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[i], i); 471 } 1444 for (uint32_t i = 0; i < cDpcsNeeded && RT_SUCCESS(rc); i++) 1445 if (RTCpuSetIsMemberByIndex(&OnlineSet, i)) 1446 { 1447 KeInitializeDpc(&paExecCpuDpcs[i], rtmpNtDPCWrapper, pArgs); 1448 KeSetImportanceDpc(&paExecCpuDpcs[i], HighImportance); 1449 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[i], RTMpCpuIdFromSetIndex(i)); 1450 } 472 1451 } 473 1452 if (RT_FAILURE(rc)) … … 477 1456 } 478 1457 479 /* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu. 1458 /* 1459 * Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu. 480 1460 * KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL. 481 1461 */ … … 507 1487 else 508 1488 { 509 unsigned iSelf = RTMpCpuId(); 510 511 for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++) 512 { 513 if ( (i != iSelf) 514 && (Mask & RT_BIT_64(i))) 1489 uint32_t iSelf = RTMpCurSetIndex(); 1490 for (uint32_t i = 0; i < cDpcsNeeded; i++) 1491 { 1492 if ( (i != iSelf) 1493 && RTCpuSetIsMemberByIndex(&OnlineSet, i)) 515 1494 { 516 1495 ASMAtomicIncS32(&pArgs->cRefs); … … 525 1504 KeLowerIrql(oldIrql); 526 1505 527 /* Flush all DPCs and wait for completion. (can take long!) */ 1506 /* 1507 * Flush all DPCs and wait for completion. (can take long!) 1508 */ 528 1509 /** @todo Consider changing this to an active wait using some atomic inc/dec 529 1510 * stuff (and check for the current cpu above in the specific case). */ -
trunk/src/VBox/Runtime/r3/win/mp-win.cpp
r64234 r64281 40 40 #include <iprt/mem.h> 41 41 #include <iprt/once.h> 42 #include <iprt/param.h> 42 43 #if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64) 43 44 # include <iprt/asm-amd64-x86.h> 44 45 #endif 46 #if defined(VBOX) && !defined(IN_GUEST) 47 # include <VBox/sup.h> 48 # define IPRT_WITH_GIP_MP_INFO 49 #else 50 # undef IPRT_WITH_GIP_MP_INFO 51 #endif 45 52 46 53 #include "internal-r3-win.h" 47 48 54 49 55 … … 51 57 * Defined Constants And Macros * 52 58 *********************************************************************************************************************************/ 59 /** @def RTMPWIN_UPDATE_GIP_GLOBAL 60 * Does lazy (re-)initialization using information provieded by GIP. */ 61 #ifdef IPRT_WITH_GIP_MP_INFO 62 # define RTMPWIN_UPDATE_GIP_GLOBAL() \ 63 do { RTMPWIN_UPDATE_GIP_GLOBALS_AND_GET_PGIP(); } while (0) 64 #else 65 # define RTMPWIN_UPDATE_GIP_GLOBAL() do { } while (0) 66 #endif 67 68 /** @def RTMPWIN_UPDATE_GIP_GLOBALS_AND_GET_PGIP 69 * Does lazy (re-)initialization using information provieded by GIP and 70 * declare and initalize a pGip local variable. */ 71 #ifdef IPRT_WITH_GIP_MP_INFO 72 #define RTMPWIN_UPDATE_GIP_GLOBALS_AND_GET_PGIP() \ 73 PSUPGLOBALINFOPAGE pGip = g_pSUPGlobalInfoPage; \ 74 if (pGip) \ 75 { \ 76 if ( pGip->u32Magic == SUPGLOBALINFOPAGE_MAGIC \ 77 && RTOnce(&g_MpInitOnceGip, rtMpWinInitOnceGip, NULL) == VINF_SUCCESS) \ 78 { \ 79 if (g_cRtMpWinActiveCpus >= pGip->cOnlineCpus) \ 80 { /* likely */ } \ 81 else \ 82 rtMpWinRefreshGip(); \ 83 } \ 84 else \ 85 pGip = NULL; \ 86 } else do { } while (0) 87 #else 88 # define RTMPWIN_UPDATE_GIP_GLOBALS_AND_GET_PGIP() do { } while (0) 89 #endif 90 91 92 /********************************************************************************************************************************* 93 * Global Variables * 94 *********************************************************************************************************************************/ 53 95 /** Initialize once. */ 54 96 static RTONCE g_MpInitOnce = RTONCE_INITIALIZER; 55 //static decltype(GetMaximumProcessorCount) *g_pfnGetMaximumProcessorCount; 97 #ifdef IPRT_WITH_GIP_MP_INFO 98 /** Initialize once using GIP. */ 99 static RTONCE g_MpInitOnceGip = RTONCE_INITIALIZER; 100 #endif 101 102 static decltype(GetMaximumProcessorCount) *g_pfnGetMaximumProcessorCount; 103 //static decltype(GetActiveProcessorCount) *g_pfnGetActiveProcessorCount; 56 104 static decltype(GetCurrentProcessorNumber) *g_pfnGetCurrentProcessorNumber; 57 105 static decltype(GetCurrentProcessorNumberEx) *g_pfnGetCurrentProcessorNumberEx; … … 60 108 61 109 62 /*********************************************************************************************************************************63 * Global Variables *64 *********************************************************************************************************************************/65 110 /** The required buffer size for getting group relations. */ 66 111 static uint32_t g_cbRtMpWinGrpRelBuf; … … 71 116 /** The max number of groups. */ 72 117 static uint32_t g_cRtMpWinMaxCpuGroups; 73 /** Static per group info. */ 118 /** The number of active CPUs the last time we checked. */ 119 static uint32_t volatile g_cRtMpWinActiveCpus; 120 /** Static per group info. 121 * @remarks With RTCPUSET_MAX_CPUS as 256, this takes up 33KB. 122 * @sa g_aRtMpNtCpuGroups */ 74 123 static struct 75 124 { 76 /** The CPU ID (and CPU set index) of the first CPU in the group. */77 uint16_t idFirstCpu;78 125 /** The max CPUs in the group. */ 79 126 uint16_t cMaxCpus; 80 } g_aRtMpWinCpuGroups[RTCPUSET_MAX_CPUS]; 127 /** The number of active CPUs at the time of initialization. */ 128 uint16_t cActiveCpus; 129 /** CPU set indexes for each CPU in the group. */ 130 int16_t aidxCpuSetMembers[64]; 131 } g_aRtMpWinCpuGroups[RTCPUSET_MAX_CPUS]; 132 /** Maps CPU set indexes to RTCPUID. 133 * @sa g_aidRtMpNtByCpuSetIdx */ 134 RTCPUID g_aidRtMpWinByCpuSetIdx[RTCPUSET_MAX_CPUS]; 81 135 82 136 83 137 /** 84 * @callback_method_impl{FNRTONCE, Resolves dynamic imports.} 138 * @callback_method_impl{FNRTONCE, 139 * Resolves dynamic imports and initializes globals.} 85 140 */ 86 141 static DECLCALLBACK(int32_t) rtMpWinInitOnce(void *pvUser) … … 98 153 RT_CONCAT(g_pfn,a_FnName) = (decltype(a_FnName) *)GetProcAddress(g_hModKernel32, #a_FnName); \ 99 154 } while (0) 100 //RESOLVE_API("kernel32.dll", GetMaximumProcessorCount); /* Calls GetLogicalProcessorInformationEx/RelationGroup in W10. */ 155 RESOLVE_API("kernel32.dll", GetMaximumProcessorCount); 156 //RESOLVE_API("kernel32.dll", GetActiveProcessorCount); - slow :/ 101 157 RESOLVE_API("kernel32.dll", GetCurrentProcessorNumber); 102 158 RESOLVE_API("kernel32.dll", GetCurrentProcessorNumberEx); … … 105 161 106 162 /* 107 * Query group information, partitioning CPU IDs and CPU set 108 * indexes (they are the same). 163 * Reset globals. 164 */ 165 for (unsigned i = 0; i < RT_ELEMENTS(g_aidRtMpWinByCpuSetIdx); i++) 166 g_aidRtMpWinByCpuSetIdx[i] = NIL_RTCPUID; 167 for (unsigned idxGroup = 0; idxGroup < RT_ELEMENTS(g_aRtMpWinCpuGroups); idxGroup++) 168 { 169 g_aRtMpWinCpuGroups[idxGroup].cMaxCpus = 0; 170 g_aRtMpWinCpuGroups[idxGroup].cActiveCpus = 0; 171 for (unsigned idxMember = 0; idxMember < RT_ELEMENTS(g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers); idxMember++) 172 g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = -1; 173 } 174 175 /* 176 * Query group information, partitioning CPU IDs and CPU set indexes. 109 177 * 110 178 * We ASSUME the the GroupInfo index is the same as the group number. 111 179 * 112 * We ASSUME there are no inactive groups, because otherwise it will 113 * be difficult to tell how many possible CPUs we can have and do a 114 * reasonable CPU ID/index partitioning. [probably bad assumption] 180 * We CANNOT ASSUME that the kernel CPU indexes are assigned in any given 181 * way, though they usually are in group order by active processor. So, 182 * we do that to avoid trouble. We must use information provided thru GIP 183 * if we want the kernel CPU set indexes. Even there, the inactive CPUs 184 * wont have sensible indexes. Sigh. 115 185 * 116 * We ASSUME that the kernel processor indexes are assigned in group order, 117 * which we match here with our own ID+index assignments. This claim is 118 * verified by initterm-r0drv-nt.cpp. 186 * We try to assign IDs to inactive CPUs in the same manner as mp-r0drv-nt.cpp 119 187 * 120 * Note! We will die if there are too many processors!188 * Note! We will die (AssertFatal) if there are too many processors! 121 189 */ 122 190 union … … 146 214 AssertFatal(uBuf.Info.Group.MaximumGroupCount >= uBuf.Info.Group.ActiveGroupCount); 147 215 216 g_cRtMpWinMaxCpuGroups = uBuf.Info.Group.MaximumGroupCount; 217 218 /* Count max cpus (see mp-r0drv0-nt.cpp) why we don't use GetMaximumProcessorCount(ALL). */ 219 uint32_t idxGroup; 220 g_cRtMpWinMaxCpus = 0; 221 for (idxGroup = 0; idxGroup < uBuf.Info.Group.ActiveGroupCount; idxGroup++) 222 g_cRtMpWinMaxCpus += uBuf.Info.Group.GroupInfo[idxGroup].MaximumProcessorCount; 148 223 149 224 /* Process the active groups. */ 150 g_cRtMpWinMaxCpuGroups = uBuf.Info.Group.MaximumGroupCount; 151 uint16_t idxCpu = 0; 152 uint32_t idxGroup = 0; 153 for (; idxGroup < uBuf.Info.Group.ActiveGroupCount; idxGroup++) 154 { 155 g_aRtMpWinCpuGroups[idxGroup].idFirstCpu = idxCpu; 156 g_aRtMpWinCpuGroups[idxGroup].cMaxCpus = uBuf.Info.Group.GroupInfo[idxGroup].MaximumProcessorCount; 157 idxCpu += uBuf.Info.Group.GroupInfo[idxGroup].MaximumProcessorCount; 158 } 225 uint32_t cActive = 0; 226 uint32_t cInactive = 0; 227 uint32_t idxCpu = 0; 228 uint32_t idxCpuSetNextInactive = g_cRtMpWinMaxCpus - 1; 229 for (idxGroup = 0; idxGroup < uBuf.Info.Group.ActiveGroupCount; idxGroup++) 230 { 231 PROCESSOR_GROUP_INFO const *pGroupInfo = &uBuf.Info.Group.GroupInfo[idxGroup]; 232 g_aRtMpWinCpuGroups[idxGroup].cMaxCpus = pGroupInfo->MaximumProcessorCount; 233 g_aRtMpWinCpuGroups[idxGroup].cActiveCpus = pGroupInfo->ActiveProcessorCount; 234 for (uint32_t idxMember = 0; idxMember < pGroupInfo->MaximumProcessorCount; idxMember++) 235 { 236 if (pGroupInfo->ActiveProcessorMask & RT_BIT_64(idxMember)) 237 { 238 g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpu; 239 g_aidRtMpWinByCpuSetIdx[idxCpu] = idxCpu; 240 idxCpu++; 241 cActive++; 242 } 243 else 244 { 245 if (idxCpuSetNextInactive >= idxCpu) 246 { 247 g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive; 248 g_aidRtMpWinByCpuSetIdx[idxCpuSetNextInactive] = idxCpuSetNextInactive; 249 idxCpuSetNextInactive--; 250 } 251 cInactive++; 252 } 253 } 254 } 255 g_cRtMpWinActiveCpus = cActive; 256 Assert(cActive + cInactive <= g_cRtMpWinMaxCpus); 257 Assert(idxCpu <= idxCpuSetNextInactive + 1); 258 Assert(idxCpu <= g_cRtMpWinMaxCpus); 159 259 160 260 /* Just in case the 2nd assumption doesn't hold true and there are inactive groups. */ 161 261 for (; idxGroup < uBuf.Info.Group.MaximumGroupCount; idxGroup++) 162 262 { 163 g_aRtMpWinCpuGroups[idxGroup].idFirstCpu = idxCpu; 164 g_aRtMpWinCpuGroups[idxGroup].cMaxCpus = RT_MAX(MAXIMUM_PROC_PER_GROUP, 64); 165 idxCpu += RT_MAX(MAXIMUM_PROC_PER_GROUP, 64); 166 } 167 168 g_cRtMpWinMaxCpus = idxCpu; 263 DWORD cMaxMembers = g_pfnGetMaximumProcessorCount(idxGroup); 264 g_aRtMpWinCpuGroups[idxGroup].cMaxCpus = cMaxMembers; 265 g_aRtMpWinCpuGroups[idxGroup].cActiveCpus = 0; 266 for (uint32_t idxMember = 0; idxMember < cMaxMembers; idxMember++) 267 { 268 if (idxCpuSetNextInactive >= idxCpu) 269 { 270 g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive; 271 g_aidRtMpWinByCpuSetIdx[idxCpuSetNextInactive] = idxCpuSetNextInactive; 272 idxCpuSetNextInactive--; 273 } 274 cInactive++; 275 } 276 } 277 Assert(cActive + cInactive <= g_cRtMpWinMaxCpus); 278 Assert(idxCpu <= idxCpuSetNextInactive + 1); 169 279 } 170 280 else … … 172 282 /* Legacy: */ 173 283 GetSystemInfo(&uBuf.SysInfo); 284 g_cRtMpWinMaxCpuGroups = 1; 174 285 g_cRtMpWinMaxCpus = uBuf.SysInfo.dwNumberOfProcessors; 175 g_cRtMpWinMaxCpuGroups = 1;176 g_aRtMpWinCpuGroups[0].idFirstCpu = 0;177 286 g_aRtMpWinCpuGroups[0].cMaxCpus = uBuf.SysInfo.dwNumberOfProcessors; 287 g_aRtMpWinCpuGroups[0].cActiveCpus = uBuf.SysInfo.dwNumberOfProcessors; 288 289 for (uint32_t idxMember = 0; idxMember < uBuf.SysInfo.dwNumberOfProcessors; idxMember++) 290 { 291 g_aRtMpWinCpuGroups[0].aidxCpuSetMembers[idxMember] = idxMember; 292 g_aidRtMpWinByCpuSetIdx[idxMember] = idxMember; 293 } 178 294 } 179 295 … … 268 384 269 385 386 #ifdef IPRT_WITH_GIP_MP_INFO 387 /** 388 * @callback_method_impl{FNRTONCE, Updates globals with information from GIP.} 389 */ 390 static DECLCALLBACK(int32_t) rtMpWinInitOnceGip(void *pvUser) 391 { 392 RT_NOREF(pvUser); 393 RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL); 394 395 PSUPGLOBALINFOPAGE pGip = g_pSUPGlobalInfoPage; 396 if ( pGip 397 && pGip->u32Magic == SUPGLOBALINFOPAGE_MAGIC) 398 { 399 /* 400 * Update globals. 401 */ 402 if (g_cRtMpWinMaxCpus != pGip->cPossibleCpus) 403 g_cRtMpWinMaxCpus = pGip->cPossibleCpus; 404 if (g_cRtMpWinActiveCpus != pGip->cOnlineCpus) 405 g_cRtMpWinActiveCpus = pGip->cOnlineCpus; 406 Assert(g_cRtMpWinMaxCpuGroups == pGip->cPossibleCpuGroups); 407 if (g_cRtMpWinMaxCpuGroups != pGip->cPossibleCpuGroups) 408 { 409 g_cRtMpWinMaxCpuGroups = pGip->cPossibleCpuGroups; 410 g_cbRtMpWinGrpRelBuf = sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) 411 + (g_cRtMpWinMaxCpuGroups + 2) * sizeof(PROCESSOR_GROUP_INFO); 412 } 413 414 /* 415 * Update CPU set IDs. 416 */ 417 for (unsigned i = g_cRtMpWinMaxCpus; i < RT_ELEMENTS(g_aidRtMpWinByCpuSetIdx); i++) 418 g_aidRtMpWinByCpuSetIdx[i] = NIL_RTCPUID; 419 420 unsigned const cbGip = pGip->cPages * PAGE_SIZE; 421 for (uint32_t idxGroup = 0; idxGroup < g_cRtMpWinMaxCpus; idxGroup++) 422 { 423 uint32_t idxMember; 424 unsigned offCpuGroup = pGip->aoffCpuGroup[idxGroup]; 425 if (offCpuGroup < cbGip) 426 { 427 PSUPGIPCPUGROUP pGipCpuGrp = (PSUPGIPCPUGROUP)((uintptr_t)pGip + offCpuGroup); 428 uint32_t cMaxMembers = pGipCpuGrp->cMaxMembers; 429 AssertStmt(cMaxMembers < RT_ELEMENTS(g_aRtMpWinCpuGroups[0].aidxCpuSetMembers), 430 cMaxMembers = RT_ELEMENTS(g_aRtMpWinCpuGroups[0].aidxCpuSetMembers)); 431 g_aRtMpWinCpuGroups[idxGroup].cMaxCpus = cMaxMembers; 432 g_aRtMpWinCpuGroups[idxGroup].cActiveCpus = RT_MIN(pGipCpuGrp->cMembers, cMaxMembers); 433 434 for (idxMember = 0; idxMember < cMaxMembers; idxMember++) 435 { 436 int16_t idxSet = pGipCpuGrp->aiCpuSetIdxs[idxMember]; 437 g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxSet; 438 if ((unsigned)idxSet < RT_ELEMENTS(g_aidRtMpWinByCpuSetIdx)) 439 # ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 440 g_aidRtMpWinByCpuSetIdx[idxSet] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember); 441 # else 442 g_aidRtMpWinByCpuSetIdx[idxSet] = idxSet; 443 # endif 444 } 445 } 446 else 447 idxMember = 0; 448 for (; idxMember < RT_ELEMENTS(g_aRtMpWinCpuGroups[0].aidxCpuSetMembers); idxMember++) 449 g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = -1; 450 } 451 } 452 453 return VINF_SUCCESS; 454 } 455 456 457 /** 458 * Refreshes globals from GIP after one or more CPUs were added. 459 * 460 * There are potential races here. We might race other threads and we may race 461 * more CPUs being added. 462 */ 463 static void rtMpWinRefreshGip(void) 464 { 465 PSUPGLOBALINFOPAGE pGip = g_pSUPGlobalInfoPage; 466 if ( pGip 467 && pGip->u32Magic == SUPGLOBALINFOPAGE_MAGIC) 468 { 469 /* 470 * Since CPUs cannot be removed, we only have to update the IDs and 471 * indexes of CPUs that we think are inactive and the group member counts. 472 */ 473 for (;;) 474 { 475 unsigned const cbGip = pGip->cPages * PAGE_SIZE; 476 uint32_t const cGipActiveCpus = pGip->cOnlineCpus; 477 uint32_t const cMyActiveCpus = ASMAtomicReadU32(&g_cRtMpWinActiveCpus); 478 ASMCompilerBarrier(); 479 480 for (uint32_t idxGroup = 0; idxGroup < g_cRtMpWinMaxCpus; idxGroup++) 481 { 482 unsigned offCpuGroup = pGip->aoffCpuGroup[idxGroup]; 483 if (offCpuGroup < cbGip) 484 { 485 PSUPGIPCPUGROUP pGipCpuGrp = (PSUPGIPCPUGROUP)((uintptr_t)pGip + offCpuGroup); 486 uint32_t cMaxMembers = pGipCpuGrp->cMaxMembers; 487 AssertStmt(cMaxMembers < RT_ELEMENTS(g_aRtMpWinCpuGroups[0].aidxCpuSetMembers), 488 cMaxMembers = RT_ELEMENTS(g_aRtMpWinCpuGroups[0].aidxCpuSetMembers)); 489 for (uint32_t idxMember = g_aRtMpWinCpuGroups[idxGroup].cActiveCpus; idxMember < cMaxMembers; idxMember++) 490 { 491 int16_t idxSet = pGipCpuGrp->aiCpuSetIdxs[idxMember]; 492 g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxSet; 493 if ((unsigned)idxSet < RT_ELEMENTS(g_aidRtMpWinByCpuSetIdx)) 494 # ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 495 g_aidRtMpWinByCpuSetIdx[idxSet] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember); 496 # else 497 g_aidRtMpWinByCpuSetIdx[idxSet] = idxSet; 498 # endif 499 } 500 g_aRtMpWinCpuGroups[idxGroup].cMaxCpus = RT_MIN(pGipCpuGrp->cMembers, cMaxMembers); 501 g_aRtMpWinCpuGroups[idxGroup].cActiveCpus = RT_MIN(pGipCpuGrp->cMembers, cMaxMembers); 502 } 503 else 504 Assert(g_aRtMpWinCpuGroups[idxGroup].cActiveCpus == 0); 505 } 506 507 ASMCompilerBarrier(); 508 if (cGipActiveCpus == pGip->cOnlineCpus) 509 if (ASMAtomicCmpXchgU32(&g_cRtMpWinActiveCpus, cGipActiveCpus, cMyActiveCpus)) 510 break; 511 } 512 } 513 } 514 515 #endif /* IPRT_WITH_GIP_MP_INFO */ 516 517 518 /* 519 * Conversion between CPU ID and set index. 520 */ 521 522 RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu) 523 { 524 RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL); 525 RTMPWIN_UPDATE_GIP_GLOBAL(); 526 527 #ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 528 if (idCpu != NIL_RTCPUID) 529 return RTMpSetIndexFromCpuGroupMember(rtMpCpuIdGetGroup(idCpu), rtMpCpuIdGetGroupMember(idCpu)); 530 return -1; 531 532 #else 533 /* 1:1 mapping, just do range checking. */ 534 return idCpu < g_cRtMpWinMaxCpus ? idCpu : -1; 535 #endif 536 } 537 538 539 RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu) 540 { 541 RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL); 542 RTMPWIN_UPDATE_GIP_GLOBAL(); 543 544 if ((unsigned)iCpu < RT_ELEMENTS(g_aidRtMpWinByCpuSetIdx)) 545 { 546 RTCPUID idCpu = g_aidRtMpWinByCpuSetIdx[iCpu]; 547 548 #if defined(IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER) && defined(RT_STRICT) 549 /* Check the correctness of the mapping table. */ 550 RTCPUID idCpuGip = NIL_RTCPUID; 551 if ( pGip 552 && (unsigned)iCpu < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)) 553 { 554 unsigned idxSupCpu = pGip->aiCpuFromCpuSetIdx[idxGuess]; 555 if (idxSupCpu < pGip->cCpus) 556 if (pGip->aCPUs[idxSupCpu].enmState != SUPGIPCPUSTATE_INVALID) 557 idCpuGip = pGip->aCPUs[idxSupCpu].idCpu; 558 } 559 AssertMsg(idCpu == idCpuGip, ("table:%#x gip:%#x\n", idCpu, idCpuGip)); 560 #endif 561 562 return idCpu; 563 } 564 return NIL_RTCPUID; 565 } 566 567 568 RTDECL(int) RTMpSetIndexFromCpuGroupMember(uint32_t idxGroup, uint32_t idxMember) 569 { 570 if (idxGroup < g_cRtMpWinMaxCpuGroups) 571 if (idxMember < g_aRtMpWinCpuGroups[idxGroup].cMaxCpus) 572 return g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember]; 573 return -1; 574 } 575 576 577 RTDECL(uint32_t) RTMpGetCpuGroupCounts(uint32_t idxGroup, uint32_t *pcActive) 578 { 579 if (idxGroup < g_cRtMpWinMaxCpuGroups) 580 { 581 if (pcActive) 582 *pcActive = g_aRtMpWinCpuGroups[idxGroup].cActiveCpus; 583 return g_aRtMpWinCpuGroups[idxGroup].cMaxCpus; 584 } 585 if (pcActive) 586 *pcActive = 0; 587 return 0; 588 } 589 590 591 RTDECL(uint32_t) RTMpGetMaxCpuGroupCount(void) 592 { 593 return g_cRtMpWinMaxCpuGroups; 594 } 595 596 597 598 /* 599 * Get current CPU. 600 */ 601 270 602 RTDECL(RTCPUID) RTMpCpuId(void) 271 603 { 272 604 RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL); 273 605 #ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 606 RTMPWIN_UPDATE_GIP_GLOBAL(); 607 #endif 608 609 PROCESSOR_NUMBER ProcNum; 610 ProcNum.Group = 0; 611 ProcNum.Number = 0xff; 274 612 if (g_pfnGetCurrentProcessorNumberEx) 275 {276 PROCESSOR_NUMBER ProcNum;277 613 g_pfnGetCurrentProcessorNumberEx(&ProcNum); 278 Assert(ProcNum.Group < g_cRtMpWinMaxCpuGroups); 279 Assert(ProcNum.Number < g_aRtMpWinCpuGroups[ProcNum.Group].cMaxCpus); 280 return g_aRtMpWinCpuGroups[ProcNum.Group].idFirstCpu + ProcNum.Number; 281 } 282 283 if (g_pfnGetCurrentProcessorNumber) 284 { 285 /* Should be safe wrt processor numbering, I hope... Only affects W2k3 and Vista. */ 286 Assert(g_cRtMpWinMaxCpuGroups == 1); 287 return g_pfnGetCurrentProcessorNumber(); 288 } 289 290 /* The API was introduced with W2K3 according to MSDN. */ 614 else if (g_pfnGetCurrentProcessorNumber) 615 { 616 DWORD iCpu = g_pfnGetCurrentProcessorNumber(); 617 Assert(iCpu < g_cRtMpWinMaxCpus); 618 ProcNum.Number = iCpu; 619 } 620 else 621 { 291 622 #if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64) 292 returnASMGetApicId();623 ProcNum.Number = ASMGetApicId(); 293 624 #else 294 625 # error "Not ported to this architecture." 295 return NIL_RTAPICID; 296 #endif 297 } 298 299 300 RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu) 626 return NIL_RTCPUID; 627 #endif 628 } 629 630 #ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 631 return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number); 632 #else 633 return RTMpSetIndexFromCpuGroupMember(ProcNum.Group, ProcNum.Number); 634 #endif 635 } 636 637 638 /* 639 * Possible CPUs and cores. 640 */ 641 642 RTDECL(RTCPUID) RTMpGetMaxCpuId(void) 301 643 { 302 644 RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL); 303 304 /* 1:1 mapping, just do range checking. */ 305 return idCpu < g_cRtMpWinMaxCpus ? idCpu : -1; 306 } 307 308 309 RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu) 645 #ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER 646 return RTMPCPUID_FROM_GROUP_AND_NUMBER(g_cRtMpWinMaxCpuGroups - 1, 647 g_aRtMpWinCpuGroups[g_cRtMpWinMaxCpuGroups - 1].cMaxCpus - 1); 648 #else 649 return g_cRtMpWinMaxCpus - 1; 650 #endif 651 } 652 653 654 RTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu) 310 655 { 311 656 RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL); 312 313 /* 1:1 mapping, just do range checking. */ 314 return (unsigned)iCpu < g_cRtMpWinMaxCpus ? iCpu : NIL_RTCPUID; 315 } 316 317 318 RTDECL(RTCPUID) RTMpGetMaxCpuId(void) 319 { 320 RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL); 321 return g_cRtMpWinMaxCpus - 1; 322 } 323 324 325 RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu) 326 { 327 RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL); 328 RTCPUSET Set; 329 return RTCpuSetIsMember(RTMpGetOnlineSet(&Set), idCpu); 330 } 331 332 333 RTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu) 334 { 335 RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL); 657 RTMPWIN_UPDATE_GIP_GLOBAL(); 658 336 659 /* Any CPU between 0 and g_cRtMpWinMaxCpus are possible. */ 337 660 return idCpu < g_cRtMpWinMaxCpus; … … 341 664 RTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet) 342 665 { 343 RTCPUID i dCpu = RTMpGetCount();666 RTCPUID iCpu = RTMpGetCount(); 344 667 RTCpuSetEmpty(pSet); 345 while (i dCpu-- > 0)346 RTCpuSetAdd (pSet, idCpu);668 while (iCpu-- > 0) 669 RTCpuSetAddByIndex(pSet, iCpu); 347 670 return pSet; 348 671 } … … 352 675 { 353 676 RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL); 677 354 678 return g_cRtMpWinMaxCpus; 355 679 } … … 359 683 { 360 684 RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL); 685 361 686 return g_cRtMpWinMaxCpuCores; 362 687 } 363 688 364 689 690 /* 691 * Online CPUs and cores. 692 */ 693 365 694 RTDECL(PRTCPUSET) RTMpGetOnlineSet(PRTCPUSET pSet) 366 695 { 367 696 RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL); 697 698 #ifdef IPRT_WITH_GIP_MP_INFO 699 RTMPWIN_UPDATE_GIP_GLOBALS_AND_GET_PGIP(); 700 if (pGip) 701 { 702 *pSet = pGip->OnlineCpuSet; 703 return pSet; 704 } 705 #endif 368 706 369 707 if (g_pfnGetLogicalProcessorInformationEx) … … 376 714 * active processor mask width. 377 715 */ 716 /** @todo this is not correct for WOW64 */ 378 717 DWORD cbInfo = g_cbRtMpWinGrpRelBuf; 379 718 SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)alloca(cbInfo); … … 397 736 uint32_t cMembersLeft = pInfo->Group.GroupInfo[idxGroup].ActiveProcessorCount; 398 737 #endif 399 int const idxFirst = g_aRtMpWinCpuGroups[idxGroup].idFirstCpu;400 738 int const cMembers = g_aRtMpWinCpuGroups[idxGroup].cMaxCpus; 401 739 for (int idxMember = 0; idxMember < cMembers; idxMember++) … … 406 744 cMembersLeft--; 407 745 #endif 408 RTCpuSetAddByIndex(pSet, idxFirst + idxMember);746 RTCpuSetAddByIndex(pSet, g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember]); 409 747 fActive >>= 1; 410 748 if (!fActive) … … 434 772 435 773 774 RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu) 775 { 776 RTCPUSET Set; 777 return RTCpuSetIsMember(RTMpGetOnlineSet(&Set), idCpu); 778 } 779 780 436 781 RTDECL(RTCPUID) RTMpGetOnlineCount(void) 437 782 { 783 #ifdef IPRT_WITH_GIP_MP_INFO 784 RTMPWIN_UPDATE_GIP_GLOBALS_AND_GET_PGIP(); 785 if (pGip) 786 return pGip->cOnlineCpus; 787 #endif 788 438 789 RTCPUSET Set; 439 790 RTMpGetOnlineSet(&Set); -
trunk/src/VBox/Runtime/testcase/tstRTMp-1.cpp
r62477 r64281 34 34 #include <iprt/string.h> 35 35 #include <iprt/test.h> 36 37 38 39 int main() 36 #ifdef VBOX 37 # include <VBox/sup.h> 38 #endif 39 40 41 42 int main(int argc, char **argv) 40 43 { 41 44 RTTEST hTest; … … 44 47 return rcExit; 45 48 RTTestBanner(hTest); 49 50 NOREF(argc); NOREF(argv); 51 #ifdef VBOX 52 if (argc > 1) 53 SUPR3Init(NULL); 54 #endif 46 55 47 56 /*
Note:
See TracChangeset
for help on using the changeset viewer.