VirtualBox

Changeset 64281 in vbox


Ignore:
Timestamp:
Oct 15, 2016 4:46:29 PM (8 years ago)
Author:
vboxsync
Message:

IPRT,SUP: Major vboxdrv and GIP version change; more flexible processor group handling on Windows.

Location:
trunk
Files:
1 added
1 deleted
18 edited
1 copied

Legend:

Unmodified
Added
Removed
  • trunk/include/VBox/sup.h

    r64255 r64281  
    272272    /** CPU group number (always zero, except on windows). */
    273273    uint16_t                iCpuGroup;
    274     /** CPU group number (same as iCpuSet, except on windows). */
     274    /** CPU group member number (same as iCpuSet, except on windows). */
    275275    uint16_t                iCpuGroupMember;
    276276    /** The APIC ID of this CPU. */
     
    289289typedef SUPGIPCPU *PSUPGIPCPU;
    290290
     291/**
     292 * CPU group information.
     293 * @remarks Windows only.
     294 */
     295typedef struct SUPGIPCPUGROUP
     296{
     297    /** Current number of CPUs in this group. */
     298    uint16_t volatile       cMembers;
     299    /** Maximum number of CPUs in the group. */
     300    uint16_t                cMaxMembers;
     301    /** The CPU set index of the members. This table has cMaxMembers entries.
     302     * @note For various reasons, entries from cMembers and up to cMaxMembers are
     303     *       may change as the host OS does set dynamic assignments during CPU
     304     *       hotplugging. */
     305    int16_t                 aiCpuSetIdxs[1];
     306} SUPGIPCPUGROUP;
     307/** Pointer to a GIP CPU group structure. */
     308typedef SUPGIPCPUGROUP *PSUPGIPCPUGROUP;
     309/** Pointer to a const GIP CPU group structure. */
     310typedef SUPGIPCPUGROUP const *PCSUPGIPCPUGROUP;
    291311
    292312/**
     
    358378/** @} */
    359379
     380/** @def SUPGIP_MAX_CPU_GROUPS
     381 * Maximum number of CPU groups.  */
     382#if RTCPUSET_MAX_CPUS >= 256
     383# define SUPGIP_MAX_CPU_GROUPS 256
     384#else
     385# define SUPGIP_MAX_CPU_GROUPS 256
     386#endif
    360387
    361388/**
     
    419446    /** CPU set index to CPU table index. */
    420447    uint16_t            aiCpuFromCpuSetIdx[RTCPUSET_MAX_CPUS];
    421     /** Table indexed by CPU group index to get the CPU set index of the first
    422      *  CPU. */
    423     uint16_t            aiFirstCpuSetIdxFromCpuGroup[RTCPUSET_MAX_CPUS];
     448    /** Table indexed by CPU group to containing offsets to SUPGIPCPUGROUP
     449     * structures, invalid entries are set to UINT16_MAX.  The offsets are relative
     450     * to the start of this structure.
     451     * @note Windows only. The other hosts sets all entries to UINT16_MAX! */
     452    uint16_t            aoffCpuGroup[SUPGIP_MAX_CPU_GROUPS];
    424453
    425454    /** Array of per-cpu data.
     
    450479 * Upper 16 bits is the major version. Major version is only changed with
    451480 * incompatible changes in the GIP. */
    452 #define SUPGLOBALINFOPAGE_VERSION   0x00070000
     481#define SUPGLOBALINFOPAGE_VERSION   0x00080000
    453482
    454483/**
  • trunk/include/iprt/mangling.h

    r64255 r64281  
    13031303# define RTMpGetCurFrequency                            RT_MANGLER(RTMpGetCurFrequency)
    13041304# define RTMpGetDescription                             RT_MANGLER(RTMpGetDescription)
     1305# define RTMpGetCpuGroupCounts                          RT_MANGLER(RTMpGetCpuGroupCounts)
     1306# define RTMpGetMaxCpuGroupCount                        RT_MANGLER(RTMpGetMaxCpuGroupCount)
    13051307# define RTMpGetMaxCpuId                                RT_MANGLER(RTMpGetMaxCpuId)
    13061308# define RTMpGetMaxFrequency                            RT_MANGLER(RTMpGetMaxFrequency)
     
    13261328# define RTMpOnSpecific                                 RT_MANGLER(RTMpOnSpecific)             /* r0drv */
    13271329# define RTMpPokeCpu                                    RT_MANGLER(RTMpPokeCpu)                /* r0drv */
     1330# define RTMpSetIndexFromCpuGroupMember                 RT_MANGLER(RTMpSetIndexFromCpuGroupMember)
    13281331# define RTMsgError                                     RT_MANGLER(RTMsgError)
    13291332# define RTMsgErrorExit                                 RT_MANGLER(RTMsgErrorExit)
  • trunk/include/iprt/mp.h

    r62473 r64281  
    8888 */
    8989RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu);
     90
     91/**
     92 * Translates an NT process group member to a CPU set index.
     93 *
     94 * @returns CPU set index, -1 if not valid.
     95 * @param   idxGroup        The CPU group.
     96 * @param   idxMember       The CPU group member number.
     97 *
     98 * @remarks Only available on Windows.
     99 */
     100RTDECL(int) RTMpSetIndexFromCpuGroupMember(uint32_t idxGroup, uint32_t idxMember);
     101
     102/**
     103 * Gets the member numbers for a CPU group.
     104 *
     105 * @returns Maximum number of group members.
     106 * @param   idxGroup        The CPU group.
     107 * @param   pcActive        Where to return the number of active members.
     108 *
     109 * @remarks Only available on Windows.
     110 */
     111RTDECL(uint32_t) RTMpGetCpuGroupCounts(uint32_t idxGroup, uint32_t *pcActive);
     112
     113/**
     114 * Get the maximum number of CPU groups.
     115 *
     116 * @returns Maximum number of CPU groups.
     117 *
     118 * @remarks Only available on Windows.
     119 */
     120RTDECL(uint32_t) RTMpGetMaxCpuGroupCount(void);
    90121
    91122/**
  • trunk/include/iprt/nt/nt.h

    r64255 r64281  
    24542454typedef  ULONG   (NTAPI *PFNKEQUERYMAXIMUMPROCESSORCOUNTEX)(USHORT GroupNumber);
    24552455typedef  USHORT  (NTAPI *PFNKEQUERYMAXIMUMGROUPCOUNT)(VOID);
     2456typedef  ULONG   (NTAPI *PFNKEQUERYACTIVEPROCESSORCOUNT)(KAFFINITY *pfActiveProcessors);
     2457typedef  ULONG   (NTAPI *PFNKEQUERYACTIVEPROCESSORCOUNTEX)(USHORT GroupNumber);
    24562458typedef  NTSTATUS (NTAPI *PFNKEQUERYLOGICALPROCESSORRELATIONSHIP)(PROCESSOR_NUMBER *pProcNumber,
    24572459                                                                  LOGICAL_PROCESSOR_RELATIONSHIP RelationShipType,
  • trunk/src/VBox/HostDrivers/Support/SUPDrvGip.cpp

    r64260 r64281  
    13201320    pGip->aCPUs[i].iCpuGroupMember = iCpuSet;
    13211321#ifdef RT_OS_WINDOWS
    1322     pGip->aCPUs[i].iCpuGroup = supdrvOSGipGetGroupFromCpu(pDevExt, idCpu, &pGip->aCPUs[i].iCpuGroupMember);
     1322    supdrvOSGipInitGroupBitsForCpu(pDevExt, pGip, &pGip->aCPUs[i]);
    13231323#endif
    13241324
     
    17441744 * Initializes the GIP data.
    17451745 *
     1746 * @returns VBox status code.
    17461747 * @param   pDevExt             Pointer to the device instance data.
    17471748 * @param   pGip                Pointer to the read-write kernel mapping of the GIP.
     
    17511752 * @param   uUpdateIntervalNS   The update interval in nanoseconds.
    17521753 * @param   cCpus               The CPU count.
    1753  */
    1754 static void supdrvGipInit(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, RTHCPHYS HCPhys,
    1755                           uint64_t u64NanoTS, unsigned uUpdateHz, unsigned uUpdateIntervalNS, unsigned cCpus)
    1756 {
    1757     size_t const    cbGip = RT_ALIGN_Z(RT_OFFSETOF(SUPGLOBALINFOPAGE, aCPUs[cCpus]), PAGE_SIZE);
    1758     unsigned        i;
     1754 * @param   cbGipCpuGroups      The supdrvOSGipGetGroupTableSize return value we
     1755 *                              used when allocating the GIP structure.
     1756 */
     1757static int supdrvGipInit(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, RTHCPHYS HCPhys,
     1758                         uint64_t u64NanoTS, unsigned uUpdateHz, unsigned uUpdateIntervalNS,
     1759                         unsigned cCpus, size_t cbGipCpuGroups)
     1760{
     1761    size_t const cbGip = RT_ALIGN_Z(RT_OFFSETOF(SUPGLOBALINFOPAGE, aCPUs[cCpus]) + cbGipCpuGroups, PAGE_SIZE);
     1762    unsigned i;
    17591763#ifdef DEBUG_DARWIN_GIP
    17601764    OSDBGPRINT(("supdrvGipInit: pGip=%p HCPhys=%lx u64NanoTS=%llu uUpdateHz=%d cCpus=%u\n", pGip, (long)HCPhys, u64NanoTS, uUpdateHz, cCpus));
     
    17941798    for (i = 0; i < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx); i++)
    17951799        pGip->aiCpuFromCpuSetIdx[i] = UINT16_MAX;
    1796     pGip->aiFirstCpuSetIdxFromCpuGroup[0] = 0;
    1797     for (i = 1; i < RT_ELEMENTS(pGip->aiFirstCpuSetIdxFromCpuGroup); i++)
    1798         pGip->aiFirstCpuSetIdxFromCpuGroup[i] = UINT16_MAX;
    1799 #ifdef RT_OS_WINDOWS
    1800     supdrvOSInitGipGroupTable(pDevExt, pGip);
    1801 #endif
     1800    for (i = 0; i < RT_ELEMENTS(pGip->aoffCpuGroup); i++)
     1801        pGip->aoffCpuGroup[i] = UINT16_MAX;
    18021802    for (i = 0; i < cCpus; i++)
    18031803        supdrvGipInitCpu(pGip, &pGip->aCPUs[i], u64NanoTS, 0 /*uCpuHz*/);
     1804#ifdef RT_OS_WINDOWS
     1805    int rc = supdrvOSInitGipGroupTable(pDevExt, pGip, cbGipCpuGroups);
     1806    AssertRCReturn(rc, rc);
     1807#endif
    18041808
    18051809    /*
     
    18091813    pDevExt->HCPhysGip = HCPhys;
    18101814    pDevExt->cGipUsers = 0;
     1815
     1816    return VINF_SUCCESS;
    18111817}
    18121818
     
    18211827{
    18221828    PSUPGLOBALINFOPAGE  pGip;
     1829    size_t              cbGip;
     1830    size_t              cbGipCpuGroups;
    18231831    RTHCPHYS            HCPhysGip;
    18241832    uint32_t            u32SystemResolution;
     
    18621870     * Allocate a contiguous set of pages with a default kernel mapping.
    18631871     */
    1864     rc = RTR0MemObjAllocCont(&pDevExt->GipMemObj, RT_UOFFSETOF(SUPGLOBALINFOPAGE, aCPUs[cCpus]), false /*fExecutable*/);
     1872    cbGipCpuGroups = supdrvOSGipGetGroupTableSize(pDevExt);
     1873    cbGip = RT_UOFFSETOF(SUPGLOBALINFOPAGE, aCPUs[cCpus]) + cbGipCpuGroups;
     1874    rc = RTR0MemObjAllocCont(&pDevExt->GipMemObj, cbGip, false /*fExecutable*/);
    18651875    if (RT_FAILURE(rc))
    18661876    {
     
    18841894        u32Interval += u32SystemResolution - uMod;
    18851895
    1886     supdrvGipInit(pDevExt, pGip, HCPhysGip, RTTimeSystemNanoTS(), RT_NS_1SEC / u32Interval /*=Hz*/, u32Interval, cCpus);
    1887 
    1888     /*
    1889      * Important sanity check...
     1896    rc = supdrvGipInit(pDevExt, pGip, HCPhysGip, RTTimeSystemNanoTS(), RT_NS_1SEC / u32Interval /*=Hz*/, u32Interval,
     1897                       cCpus, cbGipCpuGroups);
     1898
     1899    /*
     1900     * Important sanity check...  (Sets rc)
    18901901     */
    18911902    if (RT_UNLIKELY(   pGip->enmUseTscDelta == SUPGIPUSETSCDELTA_ZERO_CLAIMED
     
    18941905    {
    18951906        OSDBGPRINT(("supdrvGipCreate: Host-OS/user claims the TSC-deltas are zero but we detected async. TSC! Bad.\n"));
    1896         return VERR_INTERNAL_ERROR_2;
     1907        rc = VERR_INTERNAL_ERROR_2;
    18971908    }
    18981909
    18991910    /* It doesn't make sense to do TSC-delta detection on systems we detect as async. */
    1900     AssertReturn(   pGip->u32Mode != SUPGIPMODE_ASYNC_TSC
    1901                  || pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ZERO_CLAIMED, VERR_INTERNAL_ERROR_3);
     1911    AssertStmt(   pGip->u32Mode != SUPGIPMODE_ASYNC_TSC
     1912               || pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ZERO_CLAIMED,
     1913               rc = VERR_INTERNAL_ERROR_3);
    19021914
    19031915    /*
     
    19121924     * array with more reasonable values.
    19131925     */
    1914     if (pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC)
    1915     {
    1916         rc = supdrvGipInitMeasureTscFreq(pGip, true /*fRough*/); /* cannot fail */
    1917         supdrvGipInitStartTimerForRefiningInvariantTscFreq(pDevExt);
    1918     }
    1919     else
    1920         rc = supdrvGipInitMeasureTscFreq(pGip, false /*fRough*/);
    19211926    if (RT_SUCCESS(rc))
    19221927    {
    1923         /*
    1924          * Start TSC-delta measurement thread before we start getting MP
    1925          * events that will try kick it into action (includes the
    1926          * RTMpOnAll/supdrvGipInitOnCpu call below).
    1927          */
    1928         RTCpuSetEmpty(&pDevExt->TscDeltaCpuSet);
    1929         RTCpuSetEmpty(&pDevExt->TscDeltaObtainedCpuSet);
    1930 #ifdef SUPDRV_USE_TSC_DELTA_THREAD
    1931         if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
    1932             rc = supdrvTscDeltaThreadInit(pDevExt);
    1933 #endif
     1928        if (pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC)
     1929        {
     1930            rc = supdrvGipInitMeasureTscFreq(pGip, true /*fRough*/); /* cannot fail */
     1931            supdrvGipInitStartTimerForRefiningInvariantTscFreq(pDevExt);
     1932        }
     1933        else
     1934            rc = supdrvGipInitMeasureTscFreq(pGip, false /*fRough*/);
    19341935        if (RT_SUCCESS(rc))
    19351936        {
    1936             rc = RTMpNotificationRegister(supdrvGipMpEvent, pDevExt);
     1937            /*
     1938             * Start TSC-delta measurement thread before we start getting MP
     1939             * events that will try kick it into action (includes the
     1940             * RTMpOnAll/supdrvGipInitOnCpu call below).
     1941             */
     1942            RTCpuSetEmpty(&pDevExt->TscDeltaCpuSet);
     1943            RTCpuSetEmpty(&pDevExt->TscDeltaObtainedCpuSet);
     1944    #ifdef SUPDRV_USE_TSC_DELTA_THREAD
     1945            if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
     1946                rc = supdrvTscDeltaThreadInit(pDevExt);
     1947    #endif
    19371948            if (RT_SUCCESS(rc))
    19381949            {
    1939                 /*
    1940                  * Do GIP initialization on all online CPUs.  Wake up the
    1941                  * TSC-delta thread afterwards.
    1942                  */
    1943                 rc = RTMpOnAll(supdrvGipInitOnCpu, pDevExt, pGip);
     1950                rc = RTMpNotificationRegister(supdrvGipMpEvent, pDevExt);
    19441951                if (RT_SUCCESS(rc))
    19451952                {
    1946 #ifdef SUPDRV_USE_TSC_DELTA_THREAD
    1947                     supdrvTscDeltaThreadStartMeasurement(pDevExt, true /* fForceAll */);
    1948 #else
    1949                     uint16_t iCpu;
    1950                     if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
     1953                    /*
     1954                     * Do GIP initialization on all online CPUs.  Wake up the
     1955                     * TSC-delta thread afterwards.
     1956                     */
     1957                    rc = RTMpOnAll(supdrvGipInitOnCpu, pDevExt, pGip);
     1958                    if (RT_SUCCESS(rc))
    19511959                    {
    1952                         /*
    1953                          * Measure the TSC deltas now that we have MP notifications.
    1954                          */
    1955                         int cTries = 5;
    1956                         do
     1960    #ifdef SUPDRV_USE_TSC_DELTA_THREAD
     1961                        supdrvTscDeltaThreadStartMeasurement(pDevExt, true /* fForceAll */);
     1962    #else
     1963                        uint16_t iCpu;
     1964                        if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
    19571965                        {
    1958                             rc = supdrvTscMeasureInitialDeltas(pDevExt);
    1959                             if (   rc != VERR_TRY_AGAIN
    1960                                 && rc != VERR_CPU_OFFLINE)
    1961                                 break;
    1962                         } while (--cTries > 0);
    1963                         for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
    1964                             Log(("supdrvTscDeltaInit: cpu[%u] delta %lld\n", iCpu, pGip->aCPUs[iCpu].i64TSCDelta));
     1966                            /*
     1967                             * Measure the TSC deltas now that we have MP notifications.
     1968                             */
     1969                            int cTries = 5;
     1970                            do
     1971                            {
     1972                                rc = supdrvTscMeasureInitialDeltas(pDevExt);
     1973                                if (   rc != VERR_TRY_AGAIN
     1974                                    && rc != VERR_CPU_OFFLINE)
     1975                                    break;
     1976                            } while (--cTries > 0);
     1977                            for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
     1978                                Log(("supdrvTscDeltaInit: cpu[%u] delta %lld\n", iCpu, pGip->aCPUs[iCpu].i64TSCDelta));
     1979                        }
     1980                        else
     1981                        {
     1982                            for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
     1983                                AssertMsg(!pGip->aCPUs[iCpu].i64TSCDelta, ("iCpu=%u %lld mode=%d\n", iCpu, pGip->aCPUs[iCpu].i64TSCDelta, pGip->u32Mode));
     1984                        }
     1985                        if (RT_SUCCESS(rc))
     1986    #endif
     1987                        {
     1988                            /*
     1989                             * Create the timer.
     1990                             * If CPU_ALL isn't supported we'll have to fall back to synchronous mode.
     1991                             */
     1992                            if (pGip->u32Mode == SUPGIPMODE_ASYNC_TSC)
     1993                            {
     1994                                rc = RTTimerCreateEx(&pDevExt->pGipTimer, u32Interval, RTTIMER_FLAGS_CPU_ALL,
     1995                                                     supdrvGipAsyncTimer, pDevExt);
     1996                                if (rc == VERR_NOT_SUPPORTED)
     1997                                {
     1998                                    OSDBGPRINT(("supdrvGipCreate: omni timer not supported, falling back to synchronous mode\n"));
     1999                                    pGip->u32Mode = SUPGIPMODE_SYNC_TSC;
     2000                                }
     2001                            }
     2002                            if (pGip->u32Mode != SUPGIPMODE_ASYNC_TSC)
     2003                                rc = RTTimerCreateEx(&pDevExt->pGipTimer, u32Interval, 0 /* fFlags */,
     2004                                                     supdrvGipSyncAndInvariantTimer, pDevExt);
     2005                            if (RT_SUCCESS(rc))
     2006                            {
     2007                                /*
     2008                                 * We're good.
     2009                                 */
     2010                                Log(("supdrvGipCreate: %u ns interval.\n", u32Interval));
     2011                                supdrvGipReleaseHigherTimerFrequencyFromSystem(pDevExt);
     2012
     2013                                g_pSUPGlobalInfoPage = pGip;
     2014                                return VINF_SUCCESS;
     2015                            }
     2016
     2017                            OSDBGPRINT(("supdrvGipCreate: failed create GIP timer at %u ns interval. rc=%Rrc\n", u32Interval, rc));
     2018                            Assert(!pDevExt->pGipTimer);
     2019                        }
    19652020                    }
    19662021                    else
    1967                     {
    1968                         for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
    1969                             AssertMsg(!pGip->aCPUs[iCpu].i64TSCDelta, ("iCpu=%u %lld mode=%d\n", iCpu, pGip->aCPUs[iCpu].i64TSCDelta, pGip->u32Mode));
    1970                     }
    1971                     if (RT_SUCCESS(rc))
    1972 #endif
    1973                     {
    1974                         /*
    1975                          * Create the timer.
    1976                          * If CPU_ALL isn't supported we'll have to fall back to synchronous mode.
    1977                          */
    1978                         if (pGip->u32Mode == SUPGIPMODE_ASYNC_TSC)
    1979                         {
    1980                             rc = RTTimerCreateEx(&pDevExt->pGipTimer, u32Interval, RTTIMER_FLAGS_CPU_ALL,
    1981                                                  supdrvGipAsyncTimer, pDevExt);
    1982                             if (rc == VERR_NOT_SUPPORTED)
    1983                             {
    1984                                 OSDBGPRINT(("supdrvGipCreate: omni timer not supported, falling back to synchronous mode\n"));
    1985                                 pGip->u32Mode = SUPGIPMODE_SYNC_TSC;
    1986                             }
    1987                         }
    1988                         if (pGip->u32Mode != SUPGIPMODE_ASYNC_TSC)
    1989                             rc = RTTimerCreateEx(&pDevExt->pGipTimer, u32Interval, 0 /* fFlags */,
    1990                                                  supdrvGipSyncAndInvariantTimer, pDevExt);
    1991                         if (RT_SUCCESS(rc))
    1992                         {
    1993                             /*
    1994                              * We're good.
    1995                              */
    1996                             Log(("supdrvGipCreate: %u ns interval.\n", u32Interval));
    1997                             supdrvGipReleaseHigherTimerFrequencyFromSystem(pDevExt);
    1998 
    1999                             g_pSUPGlobalInfoPage = pGip;
    2000                             return VINF_SUCCESS;
    2001                         }
    2002 
    2003                         OSDBGPRINT(("supdrvGipCreate: failed create GIP timer at %u ns interval. rc=%Rrc\n", u32Interval, rc));
    2004                         Assert(!pDevExt->pGipTimer);
    2005                     }
     2022                        OSDBGPRINT(("supdrvGipCreate: RTMpOnAll failed. rc=%Rrc\n", rc));
    20062023                }
    20072024                else
    2008                     OSDBGPRINT(("supdrvGipCreate: RTMpOnAll failed. rc=%Rrc\n", rc));
     2025                    OSDBGPRINT(("supdrvGipCreate: failed to register MP event notfication. rc=%Rrc\n", rc));
    20092026            }
    20102027            else
    2011                 OSDBGPRINT(("supdrvGipCreate: failed to register MP event notfication. rc=%Rrc\n", rc));
     2028                OSDBGPRINT(("supdrvGipCreate: supdrvTscDeltaInit failed. rc=%Rrc\n", rc));
    20122029        }
    20132030        else
    2014             OSDBGPRINT(("supdrvGipCreate: supdrvTscDeltaInit failed. rc=%Rrc\n", rc));
    2015     }
    2016     else
    2017         OSDBGPRINT(("supdrvGipCreate: supdrvTscMeasureInitialDeltas failed. rc=%Rrc\n", rc));
     2031            OSDBGPRINT(("supdrvGipCreate: supdrvTscMeasureInitialDeltas failed. rc=%Rrc\n", rc));
     2032    }
    20182033
    20192034    /* Releases timer frequency increase too. */
  • trunk/src/VBox/HostDrivers/Support/SUPDrvIOC.h

    r64255 r64281  
    215215 *          - nothing.
    216216 */
    217 #define SUPDRV_IOC_VERSION                              0x00270000
     217#define SUPDRV_IOC_VERSION                              0x00280000
    218218
    219219/** SUP_IOCTL_COOKIE. */
  • trunk/src/VBox/HostDrivers/Support/SUPDrvInternal.h

    r64255 r64281  
    801801
    802802/**
     803 * Called during GIP initializtion to calc the CPU group table size.
     804 *
     805 * This is currently only implemented on windows [lazy bird].
     806 *
     807 * @returns Number of bytes needed for SUPGIPCPUGROUP structures.
     808 * @param   pDevExt             The device globals.
     809 */
     810size_t VBOXCALL supdrvOSGipGetGroupTableSize(PSUPDRVDEVEXT pDevExt);
     811
     812/**
    803813 * Called during GIP initialization to set up the group table and group count.
    804814 *
     
    808818 * @param   pGip                The GIP which group table needs initialization.
    809819 *                              It's only partially initialized at this point.
    810  */
    811 void VBOXCALL   supdrvOSInitGipGroupTable(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip);
    812 
    813 /**
    814  * Gets the CPU group and member indexes for the given CPU ID.
     820 * @param   cbGipCpuGroups      What supdrvOSGipGetGroupTableSize returned.
     821 */
     822int VBOXCALL    supdrvOSInitGipGroupTable(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, size_t cbGipCpuGroups);
     823
     824/**
     825 * Initializes the group related members when a CPU is added to the GIP.
     826 *
     827 * This is called both during GIP initalization and during an CPU online event.
    815828 *
    816829 * This is currently only implemented on windows [lazy bird].
     
    821834 * @param   piCpuGroupMember    Where to return the group member number.
    822835 */
    823 uint16_t VBOXCALL supdrvOSGipGetGroupFromCpu(PSUPDRVDEVEXT pDevExt, RTCPUID idCpu, uint16_t *piCpuGroupMember);
     836void VBOXCALL supdrvOSGipInitGroupBitsForCpu(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, PSUPGIPCPU pGipCpu);
    824837
    825838void VBOXCALL   supdrvOSObjInitCreator(PSUPDRVOBJ pObj, PSUPDRVSESSION pSession);
  • trunk/src/VBox/HostDrivers/Support/testcase/tstGIP-2.cpp

    r64255 r64281  
    137137                     g_pSUPGlobalInfoPage->u32Version,
    138138                     g_pSUPGlobalInfoPage->fGetGipCpu);
    139             RTPrintf("tstGIP-2: cCpus=%d  cPossibleCpus=%d cPossibleCpuGroups=%d cPresentCpus=%d cOnlineCpus=%d\n",
     139            RTPrintf("tstGIP-2: cCpus=%d  cPossibleCpus=%d cPossibleCpuGroups=%d cPresentCpus=%d cOnlineCpus=%d idCpuMax=%#x\n",
    140140                     g_pSUPGlobalInfoPage->cCpus,
    141141                     g_pSUPGlobalInfoPage->cPossibleCpus,
    142142                     g_pSUPGlobalInfoPage->cPossibleCpuGroups,
    143143                     g_pSUPGlobalInfoPage->cPresentCpus,
    144                      g_pSUPGlobalInfoPage->cOnlineCpus);
     144                     g_pSUPGlobalInfoPage->cOnlineCpus,
     145                     g_pSUPGlobalInfoPage->idCpuMax);
    145146            RTPrintf("tstGIP-2: u32UpdateHz=%RU32  u32UpdateIntervalNS=%RU32  u64NanoTSLastUpdateHz=%RX64  u64CpuHz=%RU64  uCpuHzRef=%RU64\n",
    146147                     g_pSUPGlobalInfoPage->u32UpdateHz,
     
    149150                     g_pSUPGlobalInfoPage->u64CpuHz,
    150151                     uCpuHzRef);
     152            for (uint32_t iCpu = 0; iCpu < g_pSUPGlobalInfoPage->cCpus; iCpu++)
     153                if (g_pSUPGlobalInfoPage->aCPUs[iCpu].enmState != SUPGIPCPUSTATE_INVALID)
     154                {
     155                    SUPGIPCPU const *pGipCpu = &g_pSUPGlobalInfoPage->aCPUs[iCpu];
     156                    RTPrintf("tstGIP-2: aCPU[%u]: enmState=%d iCpuSet=%u idCpu=%#010x iCpuGroup=%u iCpuGroupMember=%u idApic=%#x\n",
     157                             iCpu, pGipCpu->enmState, pGipCpu->iCpuSet, pGipCpu->idCpu, pGipCpu->iCpuGroup,
     158                             pGipCpu->iCpuGroupMember, pGipCpu->idApic);
     159                }
    151160
    152161            RTPrintf(fHex
  • trunk/src/VBox/HostDrivers/Support/win/SUPDrv-win.cpp

    r64255 r64281  
    17111711
    17121712
    1713 void VBOXCALL supdrvOSInitGipGroupTable(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip)
     1713size_t VBOXCALL supdrvOSGipGetGroupTableSize(PSUPDRVDEVEXT pDevExt)
    17141714{
    17151715    NOREF(pDevExt);
     1716    uint32_t cMaxCpus = RTMpGetCount();
     1717    uint32_t cGroups  = RTMpGetMaxCpuGroupCount();
     1718
     1719    return cGroups * RT_OFFSETOF(SUPGIPCPUGROUP, aiCpuSetIdxs)
     1720         + RT_SIZEOFMEMB(SUPGIPCPUGROUP, aiCpuSetIdxs[0]) * cMaxCpus;
     1721}
     1722
     1723
     1724int VBOXCALL supdrvOSInitGipGroupTable(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, size_t cbGipCpuGroups)
     1725{
     1726    Assert(cbGipCpuGroups > 0); NOREF(cbGipCpuGroups); NOREF(pDevExt);
     1727
     1728    unsigned const  cGroups = RTMpGetMaxCpuGroupCount();
     1729    AssertReturn(cGroups > 0 && cGroups < RT_ELEMENTS(pGip->aoffCpuGroup), VERR_INTERNAL_ERROR_2);
     1730    pGip->cPossibleCpuGroups = cGroups;
     1731
     1732    PSUPGIPCPUGROUP pGroup = (PSUPGIPCPUGROUP)&pGip->aCPUs[pGip->cCpus];
     1733    for (uint32_t idxGroup = 0; idxGroup < cGroups; idxGroup++)
     1734    {
     1735        uint32_t cActive  = 0;
     1736        uint32_t cMax     = RTMpGetCpuGroupCounts(idxGroup, &cActive);
     1737        uint32_t cbNeeded = RT_OFFSETOF(SUPGIPCPUGROUP, aiCpuSetIdxs[cMax]);
     1738        AssertReturn(cbNeeded <= cbGipCpuGroups, VERR_INTERNAL_ERROR_3);
     1739        AssertReturn(cActive <= cMax, VERR_INTERNAL_ERROR_4);
     1740
     1741        pGip->aoffCpuGroup[idxGroup] = (uint16_t)((uintptr_t)pGroup - (uintptr_t)pGip);
     1742        pGroup->cMembers    = cActive;
     1743        pGroup->cMaxMembers = cMax;
     1744        for (uint32_t idxMember = 0; idxMember < cMax; idxMember++)
     1745        {
     1746            pGroup->aiCpuSetIdxs[idxMember] = RTMpSetIndexFromCpuGroupMember(idxGroup, idxMember);
     1747            Assert((unsigned)pGroup->aiCpuSetIdxs[idxMember] < pGip->cPossibleCpus);
     1748        }
     1749
     1750        /* advance. */
     1751        cbGipCpuGroups -= cbNeeded;
     1752        pGroup = (PSUPGIPCPUGROUP)&pGroup->aiCpuSetIdxs[cMax];
     1753    }
     1754
     1755    return VINF_SUCCESS;
     1756}
     1757
     1758
     1759void VBOXCALL supdrvOSGipInitGroupBitsForCpu(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, PSUPGIPCPU pGipCpu)
     1760{
     1761    NOREF(pDevExt);
    17161762
    17171763    /*
    1718      * The indexes are assigned in group order (see initterm-r0drv-nt.cpp).
     1764     * Translate the CPU index into a group and member.
    17191765     */
    1720     if (   g_pfnKeQueryMaximumGroupCount
    1721         && g_pfnKeGetProcessorIndexFromNumber)
    1722     {
    1723         unsigned cGroups = g_pfnKeQueryMaximumGroupCount();
    1724         AssertStmt(cGroups > 0, cGroups = 1);
    1725         AssertStmt(cGroups < RT_ELEMENTS(pGip->aiFirstCpuSetIdxFromCpuGroup),
    1726                    cGroups = RT_ELEMENTS(pGip->aiFirstCpuSetIdxFromCpuGroup));
    1727         pGip->cPossibleCpuGroups = cGroups;
    1728 
    1729         KEPROCESSORINDEX idxCpuMin = 0;
    1730         for (unsigned iGroup = 0; iGroup < cGroups; iGroup++)
    1731         {
    1732             PROCESSOR_NUMBER ProcNum;
    1733             ProcNum.Group    = (USHORT)iGroup;
    1734             ProcNum.Number   = 0;
    1735             ProcNum.Reserved = 0;
    1736             KEPROCESSORINDEX idxCpu = g_pfnKeGetProcessorIndexFromNumber(&ProcNum);
    1737             Assert(idxCpu != INVALID_PROCESSOR_INDEX);
    1738             Assert(idxCpu >= idxCpuMin);
    1739             idxCpuMin = idxCpu;
    1740             pGip->aiFirstCpuSetIdxFromCpuGroup[iGroup] = (uint16_t)idxCpu;
    1741         }
    1742     }
    1743     else
    1744     {
    1745         Assert(!g_pfnKeQueryMaximumGroupCount);
    1746         Assert(!g_pfnKeGetProcessorIndexFromNumber);
    1747 
    1748         pGip->cPossibleCpuGroups              = 1;
    1749         pGip->aiFirstCpuSetIdxFromCpuGroup[0] = 0;
    1750     }
    1751 }
    1752 
    1753 
    1754 uint16_t VBOXCALL supdrvOSGipGetGroupFromCpu(PSUPDRVDEVEXT pDevExt, RTCPUID idCpu, uint16_t *piCpuGroupMember)
    1755 {
    1756     NOREF(pDevExt);
     1766    PROCESSOR_NUMBER ProcNum = { 0, pGipCpu->iCpuSet, 0 };
     1767    if (g_pfnKeGetProcessorNumberFromIndex)
     1768    {
     1769        NTSTATUS rcNt = g_pfnKeGetProcessorNumberFromIndex(pGipCpu->iCpuSet, &ProcNum);
     1770        if (NT_SUCCESS(rcNt))
     1771            Assert(ProcNum.Group < g_pfnKeQueryMaximumGroupCount());
     1772        else
     1773        {
     1774            AssertFailed();
     1775            ProcNum.Group  = 0;
     1776            ProcNum.Number = pGipCpu->iCpuSet;
     1777        }
     1778    }
     1779    pGipCpu->iCpuGroup       = ProcNum.Group;
     1780    pGipCpu->iCpuGroupMember = ProcNum.Number;
    17571781
    17581782    /*
    1759      * This is just a wrapper around KeGetProcessorNumberFromIndex.
     1783     * Update the group info.  Just do this wholesale for now (doesn't scale well).
    17601784     */
    1761     if (g_pfnKeGetProcessorNumberFromIndex)
    1762     {
    1763         PROCESSOR_NUMBER ProcNum = { UINT16_MAX, UINT8_MAX, 0 };
    1764         NTSTATUS rcNt = g_pfnKeGetProcessorNumberFromIndex(idCpu, &ProcNum);
    1765         if (NT_SUCCESS(rcNt))
    1766         {
    1767             Assert(ProcNum.Group < g_pfnKeQueryMaximumGroupCount());
    1768             *piCpuGroupMember = ProcNum.Number;
    1769             return ProcNum.Group;
    1770         }
    1771 
    1772         AssertMsgFailed(("rcNt=%#x for idCpu=%u\n", rcNt, idCpu));
    1773     }
    1774 
    1775     *piCpuGroupMember = 0;
    1776     return idCpu;
     1785    for (uint32_t idxGroup = 0; idxGroup < pGip->cPossibleCpuGroups; idxGroup++)
     1786        if (pGip->aoffCpuGroup[idxGroup] != UINT16_MAX)
     1787        {
     1788            PSUPGIPCPUGROUP pGroup = (PSUPGIPCPUGROUP)((uintptr_t)pGip + pGip->aoffCpuGroup[idxGroup]);
     1789
     1790            uint32_t cActive  = 0;
     1791            uint32_t cMax     = RTMpGetCpuGroupCounts(idxGroup, &cActive);
     1792            AssertStmt(cMax == pGroup->cMaxMembers, cMax = pGroup->cMaxMembers);
     1793            AssertStmt(cActive <= cMax, cActive = cMax);
     1794            if (pGroup->cMembers != cActive)
     1795                pGroup->cMembers = cActive;
     1796
     1797            for (uint32_t idxMember = 0; idxMember < cMax; idxMember++)
     1798            {
     1799                int idxCpuSet = RTMpSetIndexFromCpuGroupMember(idxGroup, idxMember);
     1800                AssertMsg((unsigned)idxCpuSet < pGip->cPossibleCpus,
     1801                          ("%d vs %d for %u.%u\n", idxCpuSet, pGip->cPossibleCpus, idxGroup, idxMember));
     1802
     1803                if (pGroup->aiCpuSetIdxs[idxMember] != idxCpuSet)
     1804                    pGroup->aiCpuSetIdxs[idxMember] = idxCpuSet;
     1805            }
     1806        }
    17771807}
    17781808
  • trunk/src/VBox/Runtime/Makefile.kmk

    r64148 r64281  
    780780        generic/RTSemMutexRequestDebug-generic.cpp \
    781781        generic/RTThreadSetAffinityToCpu-generic.cpp \
    782         generic/mppresent-generic.cpp \
     782        generic/mppresent-generic-online.cpp \
    783783        generic/semrw-$(if-expr defined(VBOX_WITH_LOCKLESS_SEMRW),lockless-,)generic.cpp \
    784784        generic/uuid-generic.cpp \
     
    22472247        generic/RTLogWriteStdOut-stub-generic.cpp \
    22482248        generic/RTTimerCreate-generic.cpp \
    2249         generic/mppresent-generic.cpp \
     2249        generic/mppresent-generic-online.cpp \
    22502250        generic/RTMpGetCoreCount-generic.cpp \
    22512251        nt/RTErrConvertFromNtStatus.cpp \
     
    22632263        r0drv/nt/memuserkernel-r0drv-nt.cpp \
    22642264        r0drv/nt/mp-r0drv-nt.cpp \
    2265         r0drv/nt/mpnotification-r0drv-nt.cpp \
    22662265        r0drv/nt/process-r0drv-nt.cpp \
    22672266        r0drv/nt/RTLogWriteDebugger-r0drv-nt.cpp \
  • trunk/src/VBox/Runtime/common/misc/assert.cpp

    r62477 r64281  
    7272static bool volatile                g_fQuiet = false;
    7373/** Set if assertions may panic. */
    74 static bool volatile                g_fMayPanic = true;
     74static bool volatile                g_fMayPanic = false;//true;
    7575
    7676
  • trunk/src/VBox/Runtime/common/time/timesupref.cpp

    r64255 r64281  
    3737#include <iprt/asm-math.h>
    3838#include <iprt/asm-amd64-x86.h>
     39#include <iprt/param.h>
    3940#include <VBox/sup.h>
    4041#ifdef IN_RC
  • trunk/src/VBox/Runtime/common/time/timesupref.h

    r64255 r64281  
    101101            uint16_t const  iCpuSet  = uAux & (RTCPUSET_MAX_CPUS - 1);
    102102#  else
    103             uint16_t const  iCpuSet  = pGip->aiFirstCpuSetIdxFromCpuGroup[(uAux >> 8) & UINT8_MAX] + (uAux & UINT8_MAX);
     103            uint16_t        iCpuSet = 0;
     104            uint16_t        offGipCpuGroup = pGip->aoffCpuGroup[(uAux >> 8) & UINT8_MAX];
     105            if (offGipCpuGroup < pGip->cPages * PAGE_SIZE)
     106            {
     107                PSUPGIPCPUGROUP pGipCpuGroup = (PSUPGIPCPUGROUP)((uintptr_t)pGip + offGipCpuGroup);
     108                if (   (uAux & UINT8_MAX) < pGipCpuGroup->cMaxMembers
     109                    && pGipCpuGroup->aiCpuSetIdxs[uAux & UINT8_MAX] != -1)
     110                    iCpuSet = pGipCpuGroup->aiCpuSetIdxs[uAux & UINT8_MAX];
     111            }
    104112#  endif
    105113            uint16_t const  iGipCpu  = pGip->aiCpuFromCpuSetIdx[iCpuSet];
  • trunk/src/VBox/Runtime/generic/mppresent-generic-online.cpp

    r64268 r64281  
    11/* $Id$ */
    22/** @file
    3  * IPRT - Multiprocessor, Stubs for the RTMp*Present* API.
     3 * IPRT - Multiprocessor, Stubs for the RTMp*Present* API mapping to RTMp*Online.
    44 */
    55
     
    3535RTDECL(PRTCPUSET) RTMpGetPresentSet(PRTCPUSET pSet)
    3636{
    37     return RTMpGetSet(pSet);
     37    return RTMpGetOnlineSet(pSet);
    3838}
    3939RT_EXPORT_SYMBOL(RTMpGetPresentSet);
     
    4242RTDECL(RTCPUID) RTMpGetPresentCount(void)
    4343{
    44     return RTMpGetCount();
     44    return RTMpGetOnlineCount();
    4545}
    4646RT_EXPORT_SYMBOL(RTMpGetPresentCount);
     
    4949RTDECL(RTCPUID) RTMpGetPresentCoreCount(void)
    5050{
    51     return RTMpGetCoreCount();
     51    return RTMpGetOnlineCoreCount();
    5252}
    5353RT_EXPORT_SYMBOL(RTMpGetPresentCoreCount);
     
    5656RTDECL(bool) RTMpIsCpuPresent(RTCPUID idCpu)
    5757{
    58     return RTMpIsCpuPossible(idCpu);
     58    return RTMpIsCpuOnline(idCpu);
    5959}
    6060RT_EXPORT_SYMBOL(RTMpIsCpuPresent);
  • trunk/src/VBox/Runtime/r0drv/nt/initterm-r0drv-nt.cpp

    r64267 r64281  
    3131#include "the-nt-kernel.h"
    3232#include <iprt/asm-amd64-x86.h>
    33 #include <iprt/assert.h>
    3433#include <iprt/err.h>
    35 #include <iprt/mem.h>
    36 #include <iprt/mp.h>
    3734#include <iprt/string.h>
    3835#include "internal/initterm.h"
    3936#include "internal-r0drv-nt.h"
    40 #include "../mp-r0drv.h"
    4137#include "symdb.h"
    4238#include "symdbdata.h"
     
    4642*   Global Variables                                                                                                             *
    4743*********************************************************************************************************************************/
    48 /** The NT CPU set.
    49  * KeQueryActiveProcssors() cannot be called at all IRQLs and therefore we'll
    50  * have to cache it. Fortunately, Nt doesn't really support taking CPUs offline
    51  * or online. It's first with W2K8 that support for CPU hotplugging was added.
    52  * Once we start caring about this, we'll simply let the native MP event callback
    53  * and update this variable as CPUs comes online. (The code is done already.)
    54  */
    55 RTCPUSET                                g_rtMpNtCpuSet;
    56 /** Maximum number of processor groups. */
    57 uint32_t                                g_cRtMpNtMaxGroups;
    58 /** Maximum number of processors. */
    59 uint32_t                                g_cRtMpNtMaxCpus;
    60 /** The handle of the rtR0NtMpProcessorChangeCallback registration. */
    61 static PVOID                            g_pvMpCpuChangeCallback = NULL;
    62 
    6344/** ExSetTimerResolution, introduced in W2K. */
    6445PFNMYEXSETTIMERRESOLUTION               g_pfnrtNtExSetTimerResolution;
     
    9374/** KeQueryMaximumGroupCount - Introducted in Windows 7. */
    9475PFNKEQUERYMAXIMUMGROUPCOUNT             g_pfnrtKeQueryMaximumGroupCount;
     76/** KeQueryActiveProcessorCount   - Introducted in Vista and obsoleted W7. */
     77PFNKEQUERYACTIVEPROCESSORCOUNT          g_pfnrtKeQueryActiveProcessorCount;
     78/** KeQueryActiveProcessorCountEx - Introducted in Windows 7. */
     79PFNKEQUERYACTIVEPROCESSORCOUNTEX        g_pfnrtKeQueryActiveProcessorCountEx;
    9580/** KeQueryLogicalProcessorRelationship - Introducted in Windows 7. */
    9681PFNKEQUERYLOGICALPROCESSORRELATIONSHIP  g_pfnrtKeQueryLogicalProcessorRelationship;
     
    245230
    246231
    247 /**
    248  * Implements the NT PROCESSOR_CALLBACK_FUNCTION callback function.
    249  *
    250  * This maintains the g_rtMpNtCpuSet and works MP notification callbacks.  When
    251  * registered, it's called for each active CPU in the system, avoiding racing
    252  * CPU hotplugging (as well as testing the callback).
    253  *
    254  * @param   pvUser              User context (not used).
    255  * @param   pChangeCtx          Change context (in).
    256  * @param   prcOperationStatus  Operation status (in/out).
    257  */
    258 static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,
    259                                                       PNTSTATUS prcOperationStatus)
    260 {
    261     RT_NOREF(pvUser, prcOperationStatus);
    262     switch (pChangeCtx->State)
    263     {
    264         case KeProcessorAddCompleteNotify:
    265             if (pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS)
    266             {
    267                 RTCpuSetAddByIndex(&g_rtMpNtCpuSet, pChangeCtx->NtNumber);
    268                 rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, pChangeCtx->NtNumber);
    269             }
    270             else
    271             {
    272                 DbgPrint("rtR0NtMpProcessorChangeCallback: NtNumber=%u (%#x) is higher than RTCPUSET_MAX_CPUS (%d)\n",
    273                          pChangeCtx->NtNumber, pChangeCtx->NtNumber, RTCPUSET_MAX_CPUS);
    274                 AssertMsgFailed(("NtNumber=%u (%#x)\n", pChangeCtx->NtNumber, pChangeCtx->NtNumber));
    275             }
    276             break;
    277 
    278         case KeProcessorAddStartNotify:
    279         case KeProcessorAddFailureNotify:
    280             /* ignore */
    281             break;
    282 
    283         default:
    284             AssertMsgFailed(("State=%u\n", pChangeCtx->State));
    285     }
    286 }
    287 
    288 
    289 /**
    290  * Wrapper around KeQueryLogicalProcessorRelationship.
    291  *
    292  * @returns IPRT status code.
    293  * @param   ppInfo  Where to return the info. Pass to RTMemFree when done.
    294  */
    295 static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo)
    296 {
    297     ULONG    cbInfo = sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)
    298                     + g_cRtMpNtMaxGroups * sizeof(GROUP_RELATIONSHIP);
    299     NTSTATUS rcNt;
    300     do
    301     {
    302         SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)RTMemAlloc(cbInfo);
    303         if (pInfo)
    304         {
    305             rcNt = g_pfnrtKeQueryLogicalProcessorRelationship(NULL /*pProcNumber*/, RelationGroup, pInfo, &cbInfo);
    306             if (NT_SUCCESS(rcNt))
    307             {
    308                 *ppInfo = pInfo;
    309                 return VINF_SUCCESS;
    310             }
    311 
    312             RTMemFree(pInfo);
    313             pInfo = NULL;
    314         }
    315         else
    316             rcNt = STATUS_NO_MEMORY;
    317     } while (rcNt == STATUS_INFO_LENGTH_MISMATCH);
    318     DbgPrint("IPRT: Fatal: KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt);
    319     AssertMsgFailed(("KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt));
    320     return RTErrConvertFromNtStatus(rcNt);
    321 }
    322 
    323 
    324 /**
    325  * Initalizes multiprocessor globals.
    326  *
    327  * @returns IPRT status code.
    328  */
    329 static int rtR0NtInitMp(RTNTSDBOSVER const *pOsVerInfo)
    330 {
    331 #define MY_CHECK_BREAK(a_Check, a_DbgPrintArgs) \
    332         AssertMsgBreakStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
    333 #define MY_CHECK_RETURN(a_Check, a_DbgPrintArgs, a_rcRet) \
    334         AssertMsgReturnStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs, a_rcRet)
    335 #define MY_CHECK(a_Check, a_DbgPrintArgs) \
    336         AssertMsgStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
    337 
    338     /*
    339      * API combination checks.
    340      */
    341     MY_CHECK_RETURN(!g_pfnrtKeSetTargetProcessorDpcEx || g_pfnrtKeGetProcessorNumberFromIndex,
    342                     ("IPRT: Fatal: Missing KeSetTargetProcessorDpcEx without KeGetProcessorNumberFromIndex!\n"),
    343                     VERR_SYMBOL_NOT_FOUND);
    344 
    345     /*
    346      * Get max number of processor groups.
    347      */
    348     if (g_pfnrtKeQueryMaximumGroupCount)
    349     {
    350         g_cRtMpNtMaxGroups = g_pfnrtKeQueryMaximumGroupCount();
    351         MY_CHECK_RETURN(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,
    352                         ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
    353                         VERR_MP_TOO_MANY_CPUS);
    354     }
    355     else
    356         g_cRtMpNtMaxGroups = 1;
    357 
    358     /*
    359      * Get max number CPUs.
    360      * This also defines the range of NT CPU indexes, RTCPUID and index into RTCPUSET.
    361      */
    362     if (g_pfnrtKeQueryMaximumProcessorCountEx)
    363     {
    364         g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS);
    365         MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
    366                         ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryMaximumProcessorCountEx]\n",
    367                          g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
    368                         VERR_MP_TOO_MANY_CPUS);
    369     }
    370     else if (g_pfnrtKeQueryMaximumProcessorCount)
    371     {
    372         g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCount();
    373         MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
    374                         ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryMaximumProcessorCount]\n",
    375                          g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
    376                         VERR_MP_TOO_MANY_CPUS);
    377     }
    378     else if (g_pfnrtKeQueryActiveProcessors)
    379     {
    380         KAFFINITY fActiveProcessors = g_pfnrtKeQueryActiveProcessors();
    381         MY_CHECK_RETURN(fActiveProcessors != 0,
    382                         ("IPRT: Fatal: KeQueryActiveProcessors returned 0!\n"),
    383                         VERR_INTERNAL_ERROR_2);
    384         g_cRtMpNtMaxCpus = 0;
    385         do
    386         {
    387             g_cRtMpNtMaxCpus++;
    388             fActiveProcessors >>= 1;
    389         } while (fActiveProcessors);
    390     }
    391     else
    392         g_cRtMpNtMaxCpus = KeNumberProcessors;
    393 
    394     /*
    395      * Query the details for the groups to figure out which CPUs are online as
    396      * well as the NT index limit.
    397      */
    398     if (g_pfnrtKeQueryLogicalProcessorRelationship)
    399     {
    400         MY_CHECK_RETURN(g_pfnrtKeGetProcessorIndexFromNumber,
    401                         ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
    402                         VERR_SYMBOL_NOT_FOUND);
    403         MY_CHECK_RETURN(g_pfnrtKeGetProcessorNumberFromIndex,
    404                         ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
    405                         VERR_SYMBOL_NOT_FOUND);
    406         MY_CHECK_RETURN(g_pfnrtKeSetTargetProcessorDpcEx,
    407                         ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeSetTargetProcessorDpcEx!\n"),
    408                         VERR_SYMBOL_NOT_FOUND);
    409 
    410         SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = NULL;
    411         int rc = rtR0NtInitQueryGroupRelations(&pInfo);
    412         if (RT_FAILURE(rc))
    413             return rc;
    414 
    415         AssertReturnStmt(pInfo->Group.MaximumGroupCount == g_cRtMpNtMaxGroups, RTMemFree(pInfo), VERR_INTERNAL_ERROR_3);
    416 
    417         /*
    418          * Calc online mask.
    419          *
    420          * Also check ASSUMPTIONS:
    421          *      - Processor indexes going to KeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS)
    422          *      - Processor indexes being assigned to absent hotswappable CPUs, i.e.
    423          *        KeGetProcessorIndexFromNumber and KeGetProcessorNumberFromIndex works
    424          *        all possible indexes. [Not yet confirmed!]
    425          *      - Processor indexes are assigned in group order.
    426          *      - MaximumProcessorCount specifies the highest bit in the active mask.
    427          *        This is for confirming process IDs assigned by IPRT in ring-3.
    428          */
    429         /** @todo Test the latter on a real/virtual system. */
    430         RTCpuSetEmpty(&g_rtMpNtCpuSet);
    431         uint32_t idxCpuExpect = 0;
    432         for (uint32_t idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)
    433         {
    434             const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];
    435             MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,
    436                            ("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));
    437             MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= MAXIMUM_PROC_PER_GROUP,
    438                            ("IPRT: Fatal: ActiveProcessorCount=%u\n", pGrpInfo->ActiveProcessorCount));
    439             MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,
    440                            ("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",
    441                             pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));
    442             for (uint32_t idxMember = 0; idxMember < pGrpInfo->MaximumProcessorCount; idxMember++, idxCpuExpect++)
    443             {
    444                 PROCESSOR_NUMBER ProcNum;
    445                 ProcNum.Group    = (USHORT)idxGroup;
    446                 ProcNum.Number   = (UCHAR)idxMember;
    447                 ProcNum.Reserved = 0;
    448                 ULONG idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
    449                 if (idxCpu != INVALID_PROCESSOR_INDEX)
    450                 {
    451                     MY_CHECK_BREAK(idxCpu < g_cRtMpNtMaxCpus && idxCpu < RTCPUSET_MAX_CPUS,
    452                                    ("IPRT: Fatal: idxCpu=%u >= g_cRtMpNtMaxCpu=%u (RTCPUSET_MAX_CPUS=%u)\n",
    453                                     idxCpu, g_cRtMpNtMaxCpus, RTCPUSET_MAX_CPUS));
    454                     MY_CHECK_BREAK(idxCpu == idxCpuExpect, ("IPRT: Fatal: idxCpu=%u != idxCpuExpect=%u\n", idxCpu, idxCpuExpect));
    455 
    456                     ProcNum.Group    = UINT16_MAX;
    457                     ProcNum.Number   = UINT8_MAX;
    458                     ProcNum.Reserved = UINT8_MAX;
    459                     NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(idxCpu, &ProcNum);
    460                     MY_CHECK_BREAK(NT_SUCCESS(rcNt), ("IPRT: Fatal: KeGetProcessorNumberFromIndex(%u,) -> %#x!\n", idxCpu, rcNt));
    461                     MY_CHECK_BREAK(ProcNum.Group == idxGroup && ProcNum.Number == idxMember,
    462                                    ("IPRT: Fatal: KeGetProcessorXxxxFromYyyy roundtrip error for %#x! Group: %u vs %u, Number: %u vs %u\n",
    463                                     idxCpu, ProcNum.Group, idxGroup, ProcNum.Number, idxMember));
    464 
    465                     if (pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember))
    466                         RTCpuSetAddByIndex(&g_rtMpNtCpuSet, idxCpu);
    467                 }
    468                 else
    469                 {
    470                     /* W2K8 server gives me a max of 64 logical CPUs, even if the system only has 12,
    471                        causing failures here.  Not yet sure how this would work with two CPU groups yet... */
    472                     MY_CHECK_BREAK(   idxMember >= pGrpInfo->ActiveProcessorCount
    473                                    && !(pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember)),
    474                                    ("IPRT: Fatal: KeGetProcessorIndexFromNumber(%u/%u) failed! cMax=%u cActive=%u\n",
    475                                     idxGroup, idxMember, pGrpInfo->MaximumProcessorCount, pGrpInfo->ActiveProcessorCount));
    476                 }
    477             }
    478         }
    479         RTMemFree(pInfo);
    480         if (RT_FAILURE(rc)) /* MY_CHECK_BREAK sets rc. */
    481             return rc;
    482     }
    483     else
    484     {
    485         /* Legacy: */
    486         MY_CHECK_RETURN(g_cRtMpNtMaxGroups == 1, ("IPRT: Fatal: Missing KeQueryLogicalProcessorRelationship!\n"),
    487                         VERR_SYMBOL_NOT_FOUND);
    488 
    489         if (g_pfnrtKeQueryActiveProcessors)
    490             RTCpuSetFromU64(&g_rtMpNtCpuSet, g_pfnrtKeQueryActiveProcessors());
    491         else if (g_cRtMpNtMaxCpus < 64)
    492             RTCpuSetFromU64(&g_rtMpNtCpuSet, (UINT64_C(1) << g_cRtMpNtMaxCpus) - 1);
    493         else
    494         {
    495             MY_CHECK_RETURN(g_cRtMpNtMaxCpus == 64, ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, expect 64 or less\n", g_cRtMpNtMaxCpus),
    496                             VERR_MP_TOO_MANY_CPUS);
    497             RTCpuSetFromU64(&g_rtMpNtCpuSet, UINT64_MAX);
    498         }
    499     }
    500 
    501     /*
    502      * Register CPU hot plugging callback.
    503      */
    504     Assert(g_pvMpCpuChangeCallback == NULL);
    505     if (g_pfnrtKeRegisterProcessorChangeCallback)
    506     {
    507         MY_CHECK_RETURN(g_pfnrtKeDeregisterProcessorChangeCallback,
    508                         ("IPRT: Fatal: KeRegisterProcessorChangeCallback without KeDeregisterProcessorChangeCallback!\n"),
    509                         VERR_SYMBOL_NOT_FOUND);
    510 
    511         RTCPUSET ActiveSetCopy = g_rtMpNtCpuSet;
    512         RTCpuSetEmpty(&g_rtMpNtCpuSet);
    513         g_pvMpCpuChangeCallback = g_pfnrtKeRegisterProcessorChangeCallback(rtR0NtMpProcessorChangeCallback, NULL /*pvUser*/,
    514                                                                            KE_PROCESSOR_CHANGE_ADD_EXISTING);
    515         if (!g_pvMpCpuChangeCallback) 
    516         {
    517             AssertFailed();
    518             g_rtMpNtCpuSet = ActiveSetCopy;
    519         }
    520     }
    521 
    522     /*
    523      * Special IPI fun for RTMpPokeCpu.
    524      *
    525      * On Vista and later the DPC method doesn't seem to reliably send IPIs,
    526      * so we have to use alternative methods.
    527      *
    528      * On AMD64 We used to use the HalSendSoftwareInterrupt API (also x86 on
    529      * W10+), it looks faster and more convenient to use, however we're either
    530      * using it wrong or it doesn't reliably do what we want (see @bugref{8343}).
    531      *
    532      * The HalRequestIpip API is thus far the only alternative to KeInsertQueueDpc
    533      * for doing targetted IPIs.  Trouble with this API is that it changed
    534      * fundamentally in Window 7 when they added support for lots of processors.
    535      *
    536      * If we really think we cannot use KeInsertQueueDpc, we use the broadcast IPI
    537      * API KeIpiGenericCall.
    538      */
    539     if (   pOsVerInfo->uMajorVer > 6
    540         || (pOsVerInfo->uMajorVer == 6 && pOsVerInfo->uMinorVer > 0))
    541         g_pfnrtHalRequestIpiPreW7 = NULL;
    542     else
    543         g_pfnrtHalRequestIpiW7Plus = NULL;
    544 
    545     g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingDpc;
    546 #ifndef IPRT_TARGET_NT4
    547     if (   g_pfnrtHalRequestIpiW7Plus
    548         && g_pfnrtKeInitializeAffinityEx
    549         && g_pfnrtKeAddProcessorAffinityEx
    550         && g_pfnrtKeGetProcessorIndexFromNumber)
    551     {
    552         DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingHalReqestIpiW7Plus\n");
    553         g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingHalReqestIpiW7Plus;
    554     }
    555     else if (pOsVerInfo->uMajorVer >= 6 && g_pfnrtKeIpiGenericCall)
    556     {
    557         DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingBroadcastIpi\n");
    558         g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingBroadcastIpi;
    559     }
    560     else
    561         DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingDpc\n");
    562     /* else: Windows XP should send always send an IPI -> VERIFY */
    563 #endif
    564 
    565     return VINF_SUCCESS;
    566 }
    567 
    568 
    569232DECLHIDDEN(int) rtR0InitNative(void)
    570233{
     
    599262    GET_SYSTEM_ROUTINE(KeQueryMaximumProcessorCountEx);
    600263    GET_SYSTEM_ROUTINE(KeQueryMaximumGroupCount);
     264    GET_SYSTEM_ROUTINE(KeQueryActiveProcessorCount);
     265    GET_SYSTEM_ROUTINE(KeQueryActiveProcessorCountEx);
    601266    GET_SYSTEM_ROUTINE(KeQueryLogicalProcessorRelationship);
    602267    GET_SYSTEM_ROUTINE(KeRegisterProcessorChangeCallback);
     
    743408    else
    744409        DbgPrint("IPRT: _KPRCB:{.QuantumEnd=%x/%d, .DpcQueueDepth=%x/%d} Kernel %u.%u %u %s\n",
    745                  g_offrtNtPbQuantumEnd, g_cbrtNtPbQuantumEnd, g_offrtNtPbDpcQueueDepth,
     410                 g_offrtNtPbQuantumEnd, g_cbrtNtPbQuantumEnd, g_offrtNtPbDpcQueueDepth, g_offrtNtPbDpcQueueDepth,
    746411                 OsVerInfo.uMajorVer, OsVerInfo.uMinorVer, OsVerInfo.uBuildNo, OsVerInfo.fChecked ? "checked" : "free");
    747412# endif
     
    752417     * we call rtR0TermNative to do the deregistration on failure.
    753418     */
    754     int rc = rtR0NtInitMp(&OsVerInfo);
     419    int rc = rtR0MpNtInit(&OsVerInfo);
    755420    if (RT_FAILURE(rc))
    756421    {
    757422        rtR0TermNative();
    758         DbgPrint("IPRT: Fatal: rtR0NtInitMp failed: %d\n", rc);
     423        DbgPrint("IPRT: Fatal: rtR0MpNtInit failed: %d\n", rc);
    759424        return rc;
    760425    }
     
    766431DECLHIDDEN(void) rtR0TermNative(void)
    767432{
    768     /*
    769      * Deregister the processor change callback.
    770      */
    771     PVOID pvMpCpuChangeCallback = g_pvMpCpuChangeCallback;
    772     g_pvMpCpuChangeCallback = NULL;
    773     if (pvMpCpuChangeCallback)
    774     {
    775         AssertReturnVoid(g_pfnrtKeDeregisterProcessorChangeCallback);
    776         g_pfnrtKeDeregisterProcessorChangeCallback(pvMpCpuChangeCallback);
    777     }
     433    rtR0MpNtTerm();
    778434}
    779435
  • trunk/src/VBox/Runtime/r0drv/nt/internal-r0drv-nt.h

    r64234 r64281  
    5656extern uint32_t                                g_cRtMpNtMaxGroups;
    5757extern uint32_t                                g_cRtMpNtMaxCpus;
     58extern RTCPUID                                 g_aidRtMpNtByCpuSetIdx[RTCPUSET_MAX_CPUS];
    5859
    5960extern PFNMYEXSETTIMERRESOLUTION               g_pfnrtNtExSetTimerResolution;
     
    7475extern PFNKEQUERYMAXIMUMPROCESSORCOUNTEX       g_pfnrtKeQueryMaximumProcessorCountEx;
    7576extern PFNKEQUERYMAXIMUMGROUPCOUNT             g_pfnrtKeQueryMaximumGroupCount;
     77extern PFNKEQUERYACTIVEPROCESSORCOUNT          g_pfnrtKeQueryActiveProcessorCount;
     78extern PFNKEQUERYACTIVEPROCESSORCOUNTEX        g_pfnrtKeQueryActiveProcessorCountEx;
    7679extern PFNKEQUERYLOGICALPROCESSORRELATIONSHIP  g_pfnrtKeQueryLogicalProcessorRelationship;
    7780extern PFNKEREGISTERPROCESSORCHANGECALLBACK    g_pfnrtKeRegisterProcessorChangeCallback;
     
    9598int __stdcall rtMpPokeCpuUsingHalReqestIpiPreW7(RTCPUID idCpu);
    9699
     100struct RTNTSDBOSVER;
     101DECLHIDDEN(int)  rtR0MpNtInit(struct RTNTSDBOSVER const *pOsVerInfo);
     102DECLHIDDEN(void) rtR0MpNtTerm(void);
    97103DECLHIDDEN(int) rtMpNtSetTargetProcessorDpc(KDPC *pDpc, RTCPUID idCpu);
    98104
  • trunk/src/VBox/Runtime/r0drv/nt/mp-r0drv-nt.cpp

    r64234 r64281  
    3636#include <iprt/asm.h>
    3737#include <iprt/log.h>
     38#include <iprt/mem.h>
    3839#include <iprt/time.h>
    3940#include "r0drv/mp-r0drv.h"
     41#include "symdb.h"
    4042#include "internal-r0drv-nt.h"
     43#include "internal/mp.h"
    4144
    4245
     
    7578
    7679
     80/*********************************************************************************************************************************
     81*   Defined Constants And Macros                                                                                                 *
     82*********************************************************************************************************************************/
     83/** Inactive bit for g_aidRtMpNtByCpuSetIdx. */
     84#define RTMPNT_ID_F_INACTIVE    RT_BIT_32(31)
     85
     86
     87/*********************************************************************************************************************************
     88*   Global Variables                                                                                                             *
     89*********************************************************************************************************************************/
     90/** Maximum number of processor groups. */
     91uint32_t                                g_cRtMpNtMaxGroups;
     92/** Maximum number of processors. */
     93uint32_t                                g_cRtMpNtMaxCpus;
     94/** Number of active processors. */
     95uint32_t volatile                       g_cRtMpNtActiveCpus;
     96/** The NT CPU set.
     97 * KeQueryActiveProcssors() cannot be called at all IRQLs and therefore we'll
     98 * have to cache it.  Fortunately, NT doesn't really support taking CPUs offline,
     99 * and taking them online was introduced with W2K8 where it is intended for virtual
     100 * machines and not real HW.  We update this, g_cRtMpNtActiveCpus and
     101 * g_aidRtMpNtByCpuSetIdx from the rtR0NtMpProcessorChangeCallback.
     102 */
     103RTCPUSET                                g_rtMpNtCpuSet;
     104
     105/** Static per group info.
     106 * @remarks  With RTCPUSET_MAX_CPUS as 256, this takes up 33KB. */
     107static struct
     108{
     109    /** The max CPUs in the group. */
     110    uint16_t    cMaxCpus;
     111    /** The number of active CPUs at the time of initialization. */
     112    uint16_t    cActiveCpus;
     113    /** CPU set indexes for each CPU in the group. */
     114    int16_t     aidxCpuSetMembers[64];
     115}                                       g_aRtMpNtCpuGroups[RTCPUSET_MAX_CPUS];
     116/** Maps CPU set indexes to RTCPUID.
     117 * Inactive CPUs has bit 31 set (RTMPNT_ID_F_INACTIVE) so we can identify them
     118 * and shuffle duplicates during CPU hotplugging.  We assign temporary IDs to
     119 * the inactive CPUs starting at g_cRtMpNtMaxCpus - 1, ASSUMING that active
     120 * CPUs has IDs from 0 to g_cRtMpNtActiveCpus. */
     121RTCPUID                                 g_aidRtMpNtByCpuSetIdx[RTCPUSET_MAX_CPUS];
     122/** The handle of the rtR0NtMpProcessorChangeCallback registration. */
     123static PVOID                            g_pvMpCpuChangeCallback = NULL;
     124
     125
     126/*********************************************************************************************************************************
     127*   Internal Functions                                                                                                           *
     128*********************************************************************************************************************************/
     129static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,
     130                                                      PNTSTATUS prcOperationStatus);
     131static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo);
     132
     133
     134
     135/**
     136 * Initalizes multiprocessor globals (called by rtR0InitNative).
     137 *
     138 * @returns IPRT status code.
     139 * @param   pOsVerInfo          Version information.
     140 */
     141DECLHIDDEN(int) rtR0MpNtInit(RTNTSDBOSVER const *pOsVerInfo)
     142{
     143#define MY_CHECK_BREAK(a_Check, a_DbgPrintArgs) \
     144        AssertMsgBreakStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
     145#define MY_CHECK_RETURN(a_Check, a_DbgPrintArgs, a_rcRet) \
     146        AssertMsgReturnStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs, a_rcRet)
     147#define MY_CHECK(a_Check, a_DbgPrintArgs) \
     148        AssertMsgStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
     149
     150    /*
     151     * API combination checks.
     152     */
     153    MY_CHECK_RETURN(!g_pfnrtKeSetTargetProcessorDpcEx || g_pfnrtKeGetProcessorNumberFromIndex,
     154                    ("IPRT: Fatal: Missing KeSetTargetProcessorDpcEx without KeGetProcessorNumberFromIndex!\n"),
     155                    VERR_SYMBOL_NOT_FOUND);
     156
     157    /*
     158     * Get max number of processor groups.
     159     *
     160     * We may need to upadjust this number below, because windows likes to keep
     161     * all options open when it comes to hotplugged CPU group assignments.  A
     162     * server advertising up to 64 CPUs in the ACPI table will get a result of
     163     * 64 from KeQueryMaximumGroupCount.  That makes sense.  However, when windows
     164     * server 2012 does a two processor group setup for it, the sum of the
     165     * GroupInfo[*].MaximumProcessorCount members below is 128.  This is probably
     166     * because windows doesn't want to make decisions grouping of hotpluggable CPUs.
     167     * So, we need to bump the maximum count to 128 below do deal with this as we
     168     * want to have valid CPU set indexes for all potential CPUs - how could we
     169     * otherwise use the RTMpGetSet() result and also RTCpuSetCount(RTMpGetSet())
     170     * should equal RTMpGetCount().
     171     */
     172    if (g_pfnrtKeQueryMaximumGroupCount)
     173    {
     174        g_cRtMpNtMaxGroups = g_pfnrtKeQueryMaximumGroupCount();
     175        MY_CHECK_RETURN(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,
     176                        ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
     177                        VERR_MP_TOO_MANY_CPUS);
     178    }
     179    else
     180        g_cRtMpNtMaxGroups = 1;
     181
     182    /*
     183     * Get max number CPUs.
     184     * This also defines the range of NT CPU indexes, RTCPUID and index into RTCPUSET.
     185     */
     186    if (g_pfnrtKeQueryMaximumProcessorCountEx)
     187    {
     188        g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS);
     189        MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
     190                        ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCountEx]\n",
     191                         g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
     192                        VERR_MP_TOO_MANY_CPUS);
     193    }
     194    else if (g_pfnrtKeQueryMaximumProcessorCount)
     195    {
     196        g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCount();
     197        MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
     198                        ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCount]\n",
     199                         g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
     200                        VERR_MP_TOO_MANY_CPUS);
     201    }
     202    else if (g_pfnrtKeQueryActiveProcessors)
     203    {
     204        KAFFINITY fActiveProcessors = g_pfnrtKeQueryActiveProcessors();
     205        MY_CHECK_RETURN(fActiveProcessors != 0,
     206                        ("IPRT: Fatal: KeQueryActiveProcessors returned 0!\n"),
     207                        VERR_INTERNAL_ERROR_2);
     208        g_cRtMpNtMaxCpus = 0;
     209        do
     210        {
     211            g_cRtMpNtMaxCpus++;
     212            fActiveProcessors >>= 1;
     213        } while (fActiveProcessors);
     214    }
     215    else
     216        g_cRtMpNtMaxCpus = KeNumberProcessors;
     217
     218    /*
     219     * Just because we're a bit paranoid about getting something wrong wrt to the
     220     * kernel interfaces, we try 16 times to get the KeQueryActiveProcessorCountEx
     221     * and KeQueryLogicalProcessorRelationship information to match up.
     222     */
     223    for (unsigned cTries = 0;; cTries++)
     224    {
     225        /*
     226         * Get number of active CPUs.
     227         */
     228        if (g_pfnrtKeQueryActiveProcessorCountEx)
     229        {
     230            g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS);
     231            MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0,
     232                            ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCountEx]\n",
     233                             g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus),
     234                            VERR_MP_TOO_MANY_CPUS);
     235        }
     236        else if (g_pfnrtKeQueryActiveProcessorCount)
     237        {
     238            g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCount(NULL);
     239            MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0,
     240                            ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCount]\n",
     241                             g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus),
     242                            VERR_MP_TOO_MANY_CPUS);
     243        }
     244        else
     245            g_cRtMpNtActiveCpus = g_cRtMpNtMaxCpus;
     246
     247        /*
     248         * Query the details for the groups to figure out which CPUs are online as
     249         * well as the NT index limit.
     250         */
     251        for (unsigned i = 0; i < RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx); i++)
     252#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     253            g_aidRtMpNtByCpuSetIdx[i] = NIL_RTCPUID;
     254#else
     255            g_aidRtMpNtByCpuSetIdx[i] = i < g_cRtMpNtMaxCpus ? i : NIL_RTCPUID;
     256#endif
     257        for (unsigned idxGroup = 0; idxGroup < RT_ELEMENTS(g_aRtMpNtCpuGroups); idxGroup++)
     258        {
     259            g_aRtMpNtCpuGroups[idxGroup].cMaxCpus    = 0;
     260            g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0;
     261            for (unsigned idxMember = 0; idxMember < RT_ELEMENTS(g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers); idxMember++)
     262                g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = -1;
     263        }
     264
     265        if (g_pfnrtKeQueryLogicalProcessorRelationship)
     266        {
     267            MY_CHECK_RETURN(g_pfnrtKeGetProcessorIndexFromNumber,
     268                            ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
     269                            VERR_SYMBOL_NOT_FOUND);
     270            MY_CHECK_RETURN(g_pfnrtKeGetProcessorNumberFromIndex,
     271                            ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
     272                            VERR_SYMBOL_NOT_FOUND);
     273            MY_CHECK_RETURN(g_pfnrtKeSetTargetProcessorDpcEx,
     274                            ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeSetTargetProcessorDpcEx!\n"),
     275                            VERR_SYMBOL_NOT_FOUND);
     276
     277            SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = NULL;
     278            int rc = rtR0NtInitQueryGroupRelations(&pInfo);
     279            if (RT_FAILURE(rc))
     280                return rc;
     281
     282            MY_CHECK(pInfo->Group.MaximumGroupCount == g_cRtMpNtMaxGroups,
     283                     ("IPRT: Fatal: MaximumGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n",
     284                      pInfo->Group.MaximumGroupCount, g_cRtMpNtMaxGroups));
     285            MY_CHECK(pInfo->Group.ActiveGroupCount > 0 && pInfo->Group.ActiveGroupCount <= g_cRtMpNtMaxGroups,
     286                     ("IPRT: Fatal: ActiveGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n",
     287                      pInfo->Group.ActiveGroupCount, g_cRtMpNtMaxGroups));
     288
     289            /*
     290             * First we need to recalc g_cRtMpNtMaxCpus (see above).
     291             */
     292            uint32_t cMaxCpus = 0;
     293            uint32_t idxGroup;
     294            for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)
     295            {
     296                const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];
     297                MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,
     298                               ("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));
     299                MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,
     300                               ("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",
     301                                pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));
     302                cMaxCpus += pGrpInfo->MaximumProcessorCount;
     303            }
     304            if (cMaxCpus > g_cRtMpNtMaxCpus && RT_SUCCESS(rc))
     305            {
     306                DbgPrint("IPRT: g_cRtMpNtMaxCpus=%u -> %u\n", g_cRtMpNtMaxCpus, cMaxCpus);
     307#ifndef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     308                uint32_t i = RT_MIN(cMaxCpus, RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx));
     309                while (i-- > g_cRtMpNtMaxCpus)
     310                    g_aidRtMpNtByCpuSetIdx[i] = i;
     311#endif
     312                g_cRtMpNtMaxCpus = cMaxCpus;
     313                if (g_cRtMpNtMaxGroups > RTCPUSET_MAX_CPUS)
     314                {
     315                    MY_CHECK(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,
     316                             ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS));
     317                    rc = VERR_MP_TOO_MANY_CPUS;
     318                }
     319            }
     320
     321            /*
     322             * Calc online mask, partition IDs and such.
     323             *
     324             * Also check ASSUMPTIONS:
     325             *
     326             *      1. Processor indexes going from 0 and up to
     327             *         KeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1.
     328             *
     329             *      2. Currently valid processor indexes, i.e. accepted by
     330             *         KeGetProcessorIndexFromNumber & KeGetProcessorNumberFromIndex, goes
     331             *         from 0 thru KeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1.
     332             *
     333             *      3. PROCESSOR_GROUP_INFO::MaximumProcessorCount gives the number of
     334             *         relevant bits in the ActiveProcessorMask (from LSB).
     335             *
     336             *      4. Active processor count found in KeQueryLogicalProcessorRelationship
     337             *         output matches what KeQueryActiveProcessorCountEx(ALL) returns.
     338             *
     339             *      5. Active + inactive processor counts in same does not exceed
     340             *         KeQueryMaximumProcessorCountEx(ALL).
     341             *
     342             * Note! Processor indexes are assigned as CPUs come online and are not
     343             *       preallocated according to group maximums.  Since CPUS are only taken
     344             *       online and never offlined, this means that internal CPU bitmaps are
     345             *       never sparse and no time is wasted scanning unused bits.
     346             *
     347             *       Unfortunately, it means that ring-3 cannot easily guess the index
     348             *       assignments when hotswapping is used, and must use GIP when available.
     349             */
     350            RTCpuSetEmpty(&g_rtMpNtCpuSet);
     351            uint32_t cInactive = 0;
     352            uint32_t cActive   = 0;
     353            uint32_t idxCpuMax = 0;
     354            uint32_t idxCpuSetNextInactive = g_cRtMpNtMaxCpus - 1;
     355            for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)
     356            {
     357                const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];
     358                MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,
     359                               ("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));
     360                MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,
     361                               ("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",
     362                                pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));
     363
     364                g_aRtMpNtCpuGroups[idxGroup].cMaxCpus    = pGrpInfo->MaximumProcessorCount;
     365                g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = pGrpInfo->ActiveProcessorCount;
     366
     367                for (uint32_t idxMember = 0; idxMember < pGrpInfo->MaximumProcessorCount; idxMember++)
     368                {
     369                    PROCESSOR_NUMBER ProcNum;
     370                    ProcNum.Group    = (USHORT)idxGroup;
     371                    ProcNum.Number   = (UCHAR)idxMember;
     372                    ProcNum.Reserved = 0;
     373                    ULONG idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
     374                    if (idxCpu != INVALID_PROCESSOR_INDEX)
     375                    {
     376                        MY_CHECK_BREAK(idxCpu < g_cRtMpNtMaxCpus && idxCpu < RTCPUSET_MAX_CPUS, /* ASSUMPTION #1 */
     377                                       ("IPRT: Fatal: idxCpu=%u >= g_cRtMpNtMaxCpus=%u (RTCPUSET_MAX_CPUS=%u)\n",
     378                                        idxCpu, g_cRtMpNtMaxCpus, RTCPUSET_MAX_CPUS));
     379                        if (idxCpu > idxCpuMax)
     380                            idxCpuMax = idxCpu;
     381                        g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpu;
     382#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     383                        g_aidRtMpNtByCpuSetIdx[idxCpu] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember);
     384#endif
     385
     386                        ProcNum.Group    = UINT16_MAX;
     387                        ProcNum.Number   = UINT8_MAX;
     388                        ProcNum.Reserved = UINT8_MAX;
     389                        NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(idxCpu, &ProcNum);
     390                        MY_CHECK_BREAK(NT_SUCCESS(rcNt),
     391                                       ("IPRT: Fatal: KeGetProcessorNumberFromIndex(%u,) -> %#x!\n", idxCpu, rcNt));
     392                        MY_CHECK_BREAK(ProcNum.Group == idxGroup && ProcNum.Number == idxMember,
     393                                       ("IPRT: Fatal: KeGetProcessorXxxxFromYyyy roundtrip error for %#x! Group: %u vs %u, Number: %u vs %u\n",
     394                                        idxCpu, ProcNum.Group, idxGroup, ProcNum.Number, idxMember));
     395
     396                        if (pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember))
     397                        {
     398                            RTCpuSetAddByIndex(&g_rtMpNtCpuSet, idxCpu);
     399                            cActive++;
     400                        }
     401                        else
     402                            cInactive++; /* (This is a little unexpected, but not important as long as things add up below.) */
     403                    }
     404                    else
     405                    {
     406                        /* Must be not present / inactive when KeGetProcessorIndexFromNumber fails. */
     407                        MY_CHECK_BREAK(!(pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember)),
     408                                       ("IPRT: Fatal: KeGetProcessorIndexFromNumber(%u/%u) failed but CPU is active! cMax=%u cActive=%u fActive=%p\n",
     409                                        idxGroup, idxMember, pGrpInfo->MaximumProcessorCount, pGrpInfo->ActiveProcessorCount,
     410                                        pGrpInfo->ActiveProcessorMask));
     411                        cInactive++;
     412                        if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus)
     413                        {
     414                            g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
     415#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     416                            g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember)
     417                                                                          | RTMPNT_ID_F_INACTIVE;
     418#endif
     419                            idxCpuSetNextInactive--;
     420                        }
     421                    }
     422                }
     423            }
     424
     425            MY_CHECK(cInactive + cActive <= g_cRtMpNtMaxCpus, /* ASSUMPTION #5 (not '==' because of inactive groups) */
     426                     ("IPRT: Fatal: cInactive=%u + cActive=%u > g_cRtMpNtMaxCpus=%u\n", cInactive, cActive, g_cRtMpNtMaxCpus));
     427
     428            /* Deal with inactive groups using KeQueryMaximumProcessorCountEx or as
     429               best as we can by as best we can by stipulating maximum member counts
     430               from the previous group. */
     431            if (   RT_SUCCESS(rc)
     432                && idxGroup < pInfo->Group.MaximumGroupCount)
     433            {
     434                uint16_t cInactiveLeft = g_cRtMpNtMaxCpus - (cInactive + cActive);
     435                while (idxGroup < pInfo->Group.MaximumGroupCount)
     436                {
     437                    uint32_t cMaxMembers = 0;
     438                    if (g_pfnrtKeQueryMaximumProcessorCountEx)
     439                        cMaxMembers = g_pfnrtKeQueryMaximumProcessorCountEx(idxGroup);
     440                    if (cMaxMembers != 0 || cInactiveLeft == 0)
     441                        AssertStmt(cMaxMembers <= cInactiveLeft, cMaxMembers = cInactiveLeft);
     442                    else
     443                    {
     444                        uint16_t cGroupsLeft = pInfo->Group.MaximumGroupCount - idxGroup;
     445                        cMaxMembers = pInfo->Group.GroupInfo[idxGroup - 1].MaximumProcessorCount;
     446                        while (cMaxMembers * cGroupsLeft < cInactiveLeft)
     447                            cMaxMembers++;
     448                        if (cMaxMembers > cInactiveLeft)
     449                            cMaxMembers = cInactiveLeft;
     450                    }
     451
     452                    g_aRtMpNtCpuGroups[idxGroup].cMaxCpus    = (uint16_t)cMaxMembers;
     453                    g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0;
     454                    for (uint16_t idxMember = 0; idxMember < cMaxMembers; idxMember++)
     455                        if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus)
     456                        {
     457                            g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
     458#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     459                            g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember)
     460                                                                          | RTMPNT_ID_F_INACTIVE;
     461#endif
     462                            idxCpuSetNextInactive--;
     463                        }
     464                    cInactiveLeft -= cMaxMembers;
     465                    idxGroup++;
     466                }
     467            }
     468
     469            /* We're done with pInfo now, free it so we can start returning when assertions fail. */
     470            RTMemFree(pInfo);
     471            if (RT_FAILURE(rc)) /* MY_CHECK_BREAK sets rc. */
     472                return rc;
     473            MY_CHECK_RETURN(cActive >= g_cRtMpNtActiveCpus,
     474                            ("IPRT: Fatal: cActive=%u < g_cRtMpNtActiveCpus=%u - CPUs removed?\n", cActive, g_cRtMpNtActiveCpus),
     475                            VERR_INTERNAL_ERROR_3);
     476            MY_CHECK_RETURN(idxCpuMax < cActive, /* ASSUMPTION #2 */
     477                            ("IPRT: Fatal: idCpuMax=%u >= cActive=%u! Unexpected CPU index allocation. CPUs removed?\n",
     478                             idxCpuMax, cActive),
     479                            VERR_INTERNAL_ERROR_4);
     480
     481            /* Retry if CPUs were added. */
     482            if (   cActive != g_cRtMpNtActiveCpus
     483                && cTries < 16)
     484                continue;
     485            MY_CHECK_RETURN(cActive == g_cRtMpNtActiveCpus, /* ASSUMPTION #4 */
     486                            ("IPRT: Fatal: cActive=%u != g_cRtMpNtActiveCpus=%u\n", cActive, g_cRtMpNtActiveCpus),
     487                            VERR_INTERNAL_ERROR_5);
     488        }
     489        else
     490        {
     491            /* Legacy: */
     492            MY_CHECK_RETURN(g_cRtMpNtMaxGroups == 1, ("IPRT: Fatal: Missing KeQueryLogicalProcessorRelationship!\n"),
     493                            VERR_SYMBOL_NOT_FOUND);
     494
     495            /** @todo Is it possible that the affinity mask returned by
     496             *        KeQueryActiveProcessors is sparse? */
     497            if (g_pfnrtKeQueryActiveProcessors)
     498                RTCpuSetFromU64(&g_rtMpNtCpuSet, g_pfnrtKeQueryActiveProcessors());
     499            else if (g_cRtMpNtMaxCpus < 64)
     500                RTCpuSetFromU64(&g_rtMpNtCpuSet, (UINT64_C(1) << g_cRtMpNtMaxCpus) - 1);
     501            else
     502            {
     503                MY_CHECK_RETURN(g_cRtMpNtMaxCpus == 64, ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, expect 64 or less\n", g_cRtMpNtMaxCpus),
     504                                VERR_MP_TOO_MANY_CPUS);
     505                RTCpuSetFromU64(&g_rtMpNtCpuSet, UINT64_MAX);
     506            }
     507
     508            g_aRtMpNtCpuGroups[0].cMaxCpus    = g_cRtMpNtMaxCpus;
     509            g_aRtMpNtCpuGroups[0].cActiveCpus = g_cRtMpNtMaxCpus;
     510            for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++)
     511            {
     512                g_aRtMpNtCpuGroups[0].aidxCpuSetMembers[i] = i;
     513#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     514                g_aidRtMpNtByCpuSetIdx[i] = RTMPCPUID_FROM_GROUP_AND_NUMBER(0, i);
     515#endif
     516            }
     517        }
     518
     519        /*
     520         * Register CPU hot plugging callback (it also counts active CPUs).
     521         */
     522        Assert(g_pvMpCpuChangeCallback == NULL);
     523        if (g_pfnrtKeRegisterProcessorChangeCallback)
     524        {
     525            MY_CHECK_RETURN(g_pfnrtKeDeregisterProcessorChangeCallback,
     526                            ("IPRT: Fatal: KeRegisterProcessorChangeCallback without KeDeregisterProcessorChangeCallback!\n"),
     527                            VERR_SYMBOL_NOT_FOUND);
     528
     529            RTCPUSET const ActiveSetCopy = g_rtMpNtCpuSet;
     530            RTCpuSetEmpty(&g_rtMpNtCpuSet);
     531            uint32_t const cActiveCpus   = g_cRtMpNtActiveCpus;
     532            g_cRtMpNtActiveCpus = 0;
     533
     534            g_pvMpCpuChangeCallback = g_pfnrtKeRegisterProcessorChangeCallback(rtR0NtMpProcessorChangeCallback, NULL /*pvUser*/,
     535                                                                               KE_PROCESSOR_CHANGE_ADD_EXISTING);
     536            if (g_pvMpCpuChangeCallback)
     537            {
     538                if (cActiveCpus == g_cRtMpNtActiveCpus)
     539                { /* likely */ }
     540                else
     541                {
     542                    g_pfnrtKeDeregisterProcessorChangeCallback(g_pvMpCpuChangeCallback);
     543                    if (cTries < 16)
     544                    {
     545                        /* Retry if CPUs were added. */
     546                        MY_CHECK_RETURN(g_cRtMpNtActiveCpus >= cActiveCpus,
     547                                        ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u < cActiveCpus=%u! CPUs removed?\n",
     548                                         g_cRtMpNtActiveCpus, cActiveCpus),
     549                                        VERR_INTERNAL_ERROR_2);
     550                        MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus,
     551                                        ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u > g_cRtMpNtMaxCpus=%u!\n",
     552                                         g_cRtMpNtActiveCpus, g_cRtMpNtMaxCpus),
     553                                        VERR_INTERNAL_ERROR_2);
     554                        continue;
     555                    }
     556                    MY_CHECK_RETURN(0, ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u cActiveCpus=%u\n", g_cRtMpNtActiveCpus, cActiveCpus),
     557                                    VERR_INTERNAL_ERROR_3);
     558                }
     559            }
     560            else
     561            {
     562                AssertFailed();
     563                g_rtMpNtCpuSet      = ActiveSetCopy;
     564                g_cRtMpNtActiveCpus = cActiveCpus;
     565            }
     566        }
     567        break;
     568    } /* Retry loop for stable active CPU count. */
     569
     570#undef MY_CHECK_RETURN
     571
     572    /*
     573     * Special IPI fun for RTMpPokeCpu.
     574     *
     575     * On Vista and later the DPC method doesn't seem to reliably send IPIs,
     576     * so we have to use alternative methods.
     577     *
     578     * On AMD64 We used to use the HalSendSoftwareInterrupt API (also x86 on
     579     * W10+), it looks faster and more convenient to use, however we're either
     580     * using it wrong or it doesn't reliably do what we want (see @bugref{8343}).
     581     *
     582     * The HalRequestIpip API is thus far the only alternative to KeInsertQueueDpc
     583     * for doing targetted IPIs.  Trouble with this API is that it changed
     584     * fundamentally in Window 7 when they added support for lots of processors.
     585     *
     586     * If we really think we cannot use KeInsertQueueDpc, we use the broadcast IPI
     587     * API KeIpiGenericCall.
     588     */
     589    if (   pOsVerInfo->uMajorVer > 6
     590        || (pOsVerInfo->uMajorVer == 6 && pOsVerInfo->uMinorVer > 0))
     591        g_pfnrtHalRequestIpiPreW7 = NULL;
     592    else
     593        g_pfnrtHalRequestIpiW7Plus = NULL;
     594
     595    g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingDpc;
     596#ifndef IPRT_TARGET_NT4
     597    if (   g_pfnrtHalRequestIpiW7Plus
     598        && g_pfnrtKeInitializeAffinityEx
     599        && g_pfnrtKeAddProcessorAffinityEx
     600        && g_pfnrtKeGetProcessorIndexFromNumber)
     601    {
     602        DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingHalReqestIpiW7Plus\n");
     603        g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingHalReqestIpiW7Plus;
     604    }
     605    else if (pOsVerInfo->uMajorVer >= 6 && g_pfnrtKeIpiGenericCall)
     606    {
     607        DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingBroadcastIpi\n");
     608        g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingBroadcastIpi;
     609    }
     610    else
     611        DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingDpc\n");
     612    /* else: Windows XP should send always send an IPI -> VERIFY */
     613#endif
     614
     615    return VINF_SUCCESS;
     616}
     617
     618
     619/**
     620 * Called by rtR0TermNative.
     621 */
     622DECLHIDDEN(void) rtR0MpNtTerm(void)
     623{
     624    /*
     625     * Deregister the processor change callback.
     626     */
     627    PVOID pvMpCpuChangeCallback = g_pvMpCpuChangeCallback;
     628    g_pvMpCpuChangeCallback = NULL;
     629    if (pvMpCpuChangeCallback)
     630    {
     631        AssertReturnVoid(g_pfnrtKeDeregisterProcessorChangeCallback);
     632        g_pfnrtKeDeregisterProcessorChangeCallback(pvMpCpuChangeCallback);
     633    }
     634}
     635
     636
     637DECLHIDDEN(int) rtR0MpNotificationNativeInit(void)
     638{
     639    return VINF_SUCCESS;
     640}
     641
     642
     643DECLHIDDEN(void) rtR0MpNotificationNativeTerm(void)
     644{
     645}
     646
     647
     648/**
     649 * Implements the NT PROCESSOR_CALLBACK_FUNCTION callback function.
     650 *
     651 * This maintains the g_rtMpNtCpuSet and works MP notification callbacks.  When
     652 * registered, it's called for each active CPU in the system, avoiding racing
     653 * CPU hotplugging (as well as testing the callback).
     654 *
     655 * @param   pvUser              User context (not used).
     656 * @param   pChangeCtx          Change context (in).
     657 * @param   prcOperationStatus  Operation status (in/out).
     658 *
     659 * @remarks ASSUMES no concurrent execution of KeProcessorAddCompleteNotify
     660 *          notification callbacks.  At least during callback registration
     661 *          callout, we're owning KiDynamicProcessorLock.
     662 *
     663 * @remarks When registering the handler, we first get KeProcessorAddStartNotify
     664 *          callbacks for all active CPUs, and after they all succeed we get the
     665 *          KeProcessorAddCompleteNotify callbacks.
     666 */
     667static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,
     668                                                      PNTSTATUS prcOperationStatus)
     669{
     670    RT_NOREF(pvUser, prcOperationStatus);
     671    switch (pChangeCtx->State)
     672    {
     673        /*
     674         * Check whether we can deal with the CPU, failing the start operation if we
     675         * can't.  The checks we are doing here are to avoid complicated/impossible
     676         * cases in KeProcessorAddCompleteNotify.  They are really just verify specs.
     677         */
     678        case KeProcessorAddStartNotify:
     679        {
     680            NTSTATUS rcNt = STATUS_SUCCESS;
     681            if (pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS)
     682            {
     683                if (pChangeCtx->NtNumber >= g_cRtMpNtMaxCpus)
     684                {
     685                    DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is higher than the max CPU count (%u)!\n",
     686                             pChangeCtx->NtNumber, g_cRtMpNtMaxCpus);
     687                    rcNt = STATUS_INTERNAL_ERROR;
     688                }
     689
     690                /* The ProcessNumber field was introduced in Windows 7. */
     691                PROCESSOR_NUMBER ProcNum;
     692                if (g_pfnrtKeGetProcessorIndexFromNumber)
     693                {
     694                    ProcNum = pChangeCtx->ProcNumber;
     695                    KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
     696                    if (idxCpu != pChangeCtx->NtNumber)
     697                    {
     698                        DbgPrint("IPRT: KeProcessorAddStartNotify failure: g_pfnrtKeGetProcessorIndexFromNumber(%u.%u) -> %u, expected %u!\n",
     699                                 ProcNum.Group, ProcNum.Number, idxCpu, pChangeCtx->NtNumber);
     700                        rcNt = STATUS_INTERNAL_ERROR;
     701                    }
     702                }
     703                else
     704                {
     705                    ProcNum.Group  = 0;
     706                    ProcNum.Number = pChangeCtx->NtNumber;
     707                }
     708
     709                if (   ProcNum.Group  < RT_ELEMENTS(g_aRtMpNtCpuGroups)
     710                    && ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers))
     711                {
     712                    if (ProcNum.Group >= g_cRtMpNtMaxGroups)
     713                    {
     714                        DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range - max groups: %u!\n",
     715                                 ProcNum.Group, ProcNum.Number, g_cRtMpNtMaxGroups);
     716                        rcNt = STATUS_INTERNAL_ERROR;
     717                    }
     718
     719                    if (ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus)
     720                    {
     721                        Assert(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1);
     722                        if (g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == -1)
     723                        {
     724                            DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u.%u was assigned -1 as set index!\n",
     725                                     ProcNum.Group, ProcNum.Number);
     726                            rcNt = STATUS_INTERNAL_ERROR;
     727                        }
     728
     729                        Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID);
     730                        if (g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == NIL_RTCPUID)
     731                        {
     732                            DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u (%u.%u) translates to NIL_RTCPUID!\n",
     733                                     pChangeCtx->NtNumber, ProcNum.Group, ProcNum.Number);
     734                            rcNt = STATUS_INTERNAL_ERROR;
     735                        }
     736                    }
     737                    else
     738                    {
     739                        DbgPrint("IPRT: KeProcessorAddStartNotify failure: max processors in group %u is %u, cannot add %u to it!\n",
     740                                 ProcNum.Group, g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus, ProcNum.Group, ProcNum.Number);
     741                        rcNt = STATUS_INTERNAL_ERROR;
     742                    }
     743                }
     744                else
     745                {
     746                    DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range (max %u.%u)!\n",
     747                             ProcNum.Group, ProcNum.Number, RT_ELEMENTS(g_aRtMpNtCpuGroups), RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers));
     748                    rcNt = STATUS_INTERNAL_ERROR;
     749                }
     750            }
     751            else
     752            {
     753                DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is outside RTCPUSET_MAX_CPUS (%u)!\n",
     754                         pChangeCtx->NtNumber, RTCPUSET_MAX_CPUS);
     755                rcNt = STATUS_INTERNAL_ERROR;
     756            }
     757            if (!NT_SUCCESS(rcNt))
     758                *prcOperationStatus = rcNt;
     759            break;
     760        }
     761
     762        /*
     763         * Update the globals.  Since we've checked out range limits and other
     764         * limitations already we just AssertBreak here.
     765         */
     766        case KeProcessorAddCompleteNotify:
     767        {
     768            /*
     769             * Calc the processor number and assert conditions checked in KeProcessorAddStartNotify.
     770             */
     771            AssertBreak(pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS);
     772            AssertBreak(pChangeCtx->NtNumber < g_cRtMpNtMaxCpus);
     773            Assert(pChangeCtx->NtNumber == g_cRtMpNtActiveCpus); /* light assumption */
     774            PROCESSOR_NUMBER ProcNum;
     775            if (g_pfnrtKeGetProcessorIndexFromNumber)
     776            {
     777                ProcNum = pChangeCtx->ProcNumber;
     778                AssertBreak(g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum) == pChangeCtx->NtNumber);
     779                AssertBreak(ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups));
     780                AssertBreak(ProcNum.Group < g_cRtMpNtMaxGroups);
     781            }
     782            else
     783            {
     784                ProcNum.Group  = 0;
     785                ProcNum.Number = pChangeCtx->NtNumber;
     786            }
     787            AssertBreak(ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers));
     788            AssertBreak(ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus);
     789            AssertBreak(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1);
     790            AssertBreak(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID);
     791
     792            /*
     793             * Add ourselves to the online CPU set and update the active CPU count.
     794             */
     795            RTCpuSetAddByIndex(&g_rtMpNtCpuSet, pChangeCtx->NtNumber);
     796            ASMAtomicIncU32(&g_cRtMpNtActiveCpus);
     797
     798            /*
     799             * Update the group info.
     800             *
     801             * If the index prediction failed (real hotplugging callbacks only) we
     802             * have to switch it around.  This is particularly annoying when we
     803             * use the index as the ID.
     804             */
     805#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     806            RTCPUID idCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
     807            RTCPUID idOld = g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber];
     808            if ((idOld & ~RTMPNT_ID_F_INACTIVE) != idCpu)
     809            {
     810                Assert(idOld & RTMPNT_ID_F_INACTIVE);
     811                int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
     812                g_aRtMpNtCpuGroups[rtMpCpuIdGetGroup(idOld)].aidxCpuSetMembers[rtMpCpuIdGetGroupMember(idOld)] = idxDest;
     813                g_aidRtMpNtByCpuSetIdx[idxDest] = idOld;
     814            }
     815            g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] = idCpu;
     816#else
     817            Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == pChangeCtx->NtNumber);
     818            int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
     819            if ((ULONG)idxDest != pChangeCtx->NtNumber)
     820            {
     821                bool     fFound = false;
     822                uint32_t idxOldGroup = g_cRtMpNtMaxGroups;
     823                while (idxOldGroup-- > 0 && !fFound)
     824                {
     825                    uint32_t idxMember = g_aRtMpNtCpuGroups[idxOldGroup].cMaxCpus;
     826                    while (idxMember-- > 0)
     827                        if (g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] == (int)pChangeCtx->NtNumber)
     828                        {
     829                            g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] = idxDest;
     830                            fFound = true;
     831                            break;
     832                        }
     833                }
     834                Assert(fFound);
     835            }
     836#endif
     837            g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] = pChangeCtx->NtNumber;
     838
     839            /*
     840             * Do MP notification callbacks.
     841             */
     842            rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, pChangeCtx->NtNumber);
     843            break;
     844        }
     845
     846        case KeProcessorAddFailureNotify:
     847            /* ignore */
     848            break;
     849
     850        default:
     851            AssertMsgFailed(("State=%u\n", pChangeCtx->State));
     852    }
     853}
     854
     855
     856/**
     857 * Wrapper around KeQueryLogicalProcessorRelationship.
     858 *
     859 * @returns IPRT status code.
     860 * @param   ppInfo  Where to return the info. Pass to RTMemFree when done.
     861 */
     862static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo)
     863{
     864    ULONG    cbInfo = sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)
     865                    + g_cRtMpNtMaxGroups * sizeof(GROUP_RELATIONSHIP);
     866    NTSTATUS rcNt;
     867    do
     868    {
     869        SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)RTMemAlloc(cbInfo);
     870        if (pInfo)
     871        {
     872            rcNt = g_pfnrtKeQueryLogicalProcessorRelationship(NULL /*pProcNumber*/, RelationGroup, pInfo, &cbInfo);
     873            if (NT_SUCCESS(rcNt))
     874            {
     875                *ppInfo = pInfo;
     876                return VINF_SUCCESS;
     877            }
     878
     879            RTMemFree(pInfo);
     880            pInfo = NULL;
     881        }
     882        else
     883            rcNt = STATUS_NO_MEMORY;
     884    } while (rcNt == STATUS_INFO_LENGTH_MISMATCH);
     885    DbgPrint("IPRT: Fatal: KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt);
     886    AssertMsgFailed(("KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt));
     887    return RTErrConvertFromNtStatus(rcNt);
     888}
     889
     890
     891
     892
    77893
    78894RTDECL(RTCPUID) RTMpCpuId(void)
    79895{
    80896    Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
     897
     898#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     899    PROCESSOR_NUMBER ProcNum;
     900    ProcNum.Group = 0;
     901    if (g_pfnrtKeGetCurrentProcessorNumberEx)
     902    {
     903        ProcNum.Number = 0;
     904        g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum);
     905    }
     906    else
     907        ProcNum.Number = KeGetCurrentProcessorNumber(); /* Number is 8-bit, so we're not subject to BYTE -> WORD upgrade in WDK.  */
     908    return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
     909
     910#else
    81911
    82912    if (g_pfnrtKeGetCurrentProcessorNumberEx)
     
    87917    }
    88918
    89     /* WDK upgrade warning: PCR->Number changed from BYTE to WORD. */
    90     return KeGetCurrentProcessorNumber();
     919    return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */
     920#endif
    91921}
    92922
     
    94924RTDECL(int) RTMpCurSetIndex(void)
    95925{
     926#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     927    Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
     928
     929    if (g_pfnrtKeGetCurrentProcessorNumberEx)
     930    {
     931        KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL);
     932        Assert(idxCpu < RTCPUSET_MAX_CPUS);
     933        return idxCpu;
     934    }
     935    return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */
     936#else
    96937    return (int)RTMpCpuId();
     938#endif
    97939}
    98940
     
    100942RTDECL(int) RTMpCurSetIndexAndId(PRTCPUID pidCpu)
    101943{
     944#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     945    Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
     946
     947    PROCESSOR_NUMBER ProcNum = { 0 , 0,  0 };
     948    KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum);
     949    Assert(idxCpu < RTCPUSET_MAX_CPUS);
     950    *pidCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
     951    return idxCpu;
     952#else
    102953    return *pidCpu = RTMpCpuId();
     954#endif
    103955}
    104956
     
    106958RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu)
    107959{
     960#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     961    Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
     962
     963    if (idCpu != NIL_RTCPUID)
     964    {
     965        if (g_pfnrtKeGetProcessorIndexFromNumber)
     966        {
     967            PROCESSOR_NUMBER ProcNum;
     968            ProcNum.Group    = rtMpCpuIdGetGroup(idCpu);
     969            ProcNum.Number   = rtMpCpuIdGetGroupMember(idCpu);
     970            ProcNum.Reserved = 0;
     971            KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
     972            if (idxCpu != INVALID_PROCESSOR_INDEX)
     973            {
     974                Assert(idxCpu < g_cRtMpNtMaxCpus);
     975                Assert((ULONG)g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == idxCpu);
     976                return idxCpu;
     977            }
     978
     979            /* Since NT assigned indexes as the CPUs come online, we cannot produce an ID <-> index
     980               mapping for not-yet-onlined CPUS that is consistent.  We just have to do our best... */
     981            if (   ProcNum.Group < g_cRtMpNtMaxGroups
     982                && ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus)
     983                return g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
     984        }
     985        else if (rtMpCpuIdGetGroup(idCpu) == 0)
     986            return rtMpCpuIdGetGroupMember(idCpu);
     987    }
     988    return -1;
     989#else
    108990    /* 1:1 mapping, just do range checks. */
    109991    return idCpu < RTCPUSET_MAX_CPUS ? (int)idCpu : -1;
     992#endif
    110993}
    111994
     
    113996RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu)
    114997{
     998#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     999    Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
     1000
     1001    if ((unsigned)iCpu < g_cRtMpNtMaxCpus)
     1002    {
     1003        if (g_pfnrtKeGetProcessorIndexFromNumber)
     1004        {
     1005            PROCESSOR_NUMBER ProcNum = { 0, 0, 0 };
     1006            NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(iCpu, &ProcNum);
     1007            if (NT_SUCCESS(rcNt))
     1008            {
     1009                Assert(ProcNum.Group <= g_cRtMpNtMaxGroups);
     1010                Assert(   (g_aidRtMpNtByCpuSetIdx[iCpu] & ~RTMPNT_ID_F_INACTIVE)
     1011                       == RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number));
     1012                return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
     1013            }
     1014        }
     1015        return g_aidRtMpNtByCpuSetIdx[iCpu];
     1016    }
     1017    return NIL_RTCPUID;
     1018#else
    1151019    /* 1:1 mapping, just do range checks. */
    1161020    return (unsigned)iCpu < RTCPUSET_MAX_CPUS ? iCpu : NIL_RTCPUID;
     1021#endif
     1022}
     1023
     1024
     1025RTDECL(int) RTMpSetIndexFromCpuGroupMember(uint32_t idxGroup, uint32_t idxMember)
     1026{
     1027    Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
     1028
     1029    if (idxGroup < g_cRtMpNtMaxGroups)
     1030        if (idxMember < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus)
     1031            return g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember];
     1032    return -1;
     1033}
     1034
     1035
     1036RTDECL(uint32_t) RTMpGetCpuGroupCounts(uint32_t idxGroup, uint32_t *pcActive)
     1037{
     1038    if (idxGroup < g_cRtMpNtMaxGroups)
     1039    {
     1040        if (pcActive)
     1041            *pcActive = g_aRtMpNtCpuGroups[idxGroup].cActiveCpus;
     1042        return g_aRtMpNtCpuGroups[idxGroup].cMaxCpus;
     1043    }
     1044    if (pcActive)
     1045        *pcActive = 0;
     1046    return 0;
     1047}
     1048
     1049
     1050RTDECL(uint32_t) RTMpGetMaxCpuGroupCount(void)
     1051{
     1052    return g_cRtMpNtMaxGroups;
    1171053}
    1181054
     
    1221058    Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
    1231059
     1060#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     1061    return RTMPCPUID_FROM_GROUP_AND_NUMBER(g_cRtMpNtMaxGroups - 1, g_aRtMpNtCpuGroups[g_cRtMpNtMaxGroups - 1].cMaxCpus - 1);
     1062#else
    1241063    /* According to MSDN the processor indexes goes from 0 to the maximum
    1251064       number of CPUs in the system.  We've check this in initterm-r0drv-nt.cpp. */
    1261065    return g_cRtMpNtMaxCpus - 1;
     1066#endif
    1271067}
    1281068
     
    1311071{
    1321072    Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
    133     return idCpu < RTCPUSET_MAX_CPUS
    134         && RTCpuSetIsMember(&g_rtMpNtCpuSet, idCpu);
     1073    return RTCpuSetIsMember(&g_rtMpNtCpuSet, idCpu);
    1351074}
    1361075
     
    1401079    Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
    1411080
     1081#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     1082    if (idCpu != NIL_RTCPUID)
     1083    {
     1084        unsigned idxGroup = rtMpCpuIdGetGroup(idCpu);
     1085        if (idxGroup < g_cRtMpNtMaxGroups)
     1086            return rtMpCpuIdGetGroupMember(idCpu) < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus;
     1087    }
     1088    return false;
     1089
     1090#else
    1421091    /* A possible CPU ID is one with a value lower than g_cRtMpNtMaxCpus (see
    1431092       comment in RTMpGetMaxCpuId). */
    1441093    return idCpu < g_cRtMpNtMaxCpus;
     1094#endif
    1451095}
    1461096
     
    1831133    return RTCpuSetCount(&Set);
    1841134}
     1135
     1136
     1137RTDECL(RTCPUID) RTMpGetOnlineCoreCount(void)
     1138{
     1139    /** @todo fix me */
     1140    return RTMpGetOnlineCount();
     1141}
     1142
    1851143
    1861144
     
    3721330           the reverse conversion internally). */
    3731331        PROCESSOR_NUMBER ProcNum;
    374         NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(idCpu, &ProcNum);
     1332        NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(RTMpCpuIdToSetIndex(idCpu), &ProcNum);
    3751333        AssertMsgReturn(NT_SUCCESS(rcNt),
    3761334                        ("KeGetProcessorNumberFromIndex(%u) -> %#x\n", idCpu, rcNt),
     
    3831341    }
    3841342    else
    385         KeSetTargetProcessorDpc(pDpc, (int)idCpu);
     1343        KeSetTargetProcessorDpc(pDpc, RTMpCpuIdToSetIndex(idCpu));
    3861344    return VINF_SUCCESS;
    3871345}
     
    4101368
    4111369#else  /* !IPRT_TARGET_NT4 */
    412     PRTMPARGS pArgs;
    413     KDPC     *paExecCpuDpcs;
    414 
    4151370# if 0
    4161371    /* KeFlushQueuedDpcs must be run at IRQL PASSIVE_LEVEL according to MSDN, but the
     
    4191374    AssertMsg(KeGetCurrentIrql() == PASSIVE_LEVEL, ("%d != %d (PASSIVE_LEVEL)\n", KeGetCurrentIrql(), PASSIVE_LEVEL));
    4201375# endif
    421 
    422     KAFFINITY Mask = KeQueryActiveProcessors();
    423 
    4241376    /* KeFlushQueuedDpcs is not present in Windows 2000; import it dynamically so we can just fail this call. */
    4251377    if (!g_pfnrtNtKeFlushQueuedDpcs)
    4261378        return VERR_NOT_SUPPORTED;
    4271379
    428     pArgs = (PRTMPARGS)ExAllocatePoolWithTag(NonPagedPool, g_cRtMpNtMaxCpus * sizeof(KDPC) + sizeof(RTMPARGS), (ULONG)'RTMp');
     1380    /*
     1381     * Make a copy of the active CPU set and figure out how many KDPCs we really need.
     1382     * We must not try setup DPCs for CPUs which aren't there, because that may fail.
     1383     */
     1384    RTCPUSET  OnlineSet = g_rtMpNtCpuSet;
     1385    uint32_t  cDpcsNeeded;
     1386    switch (enmCpuid)
     1387    {
     1388        case RT_NT_CPUID_SPECIFIC:
     1389            cDpcsNeeded = 1;
     1390            break;
     1391        case RT_NT_CPUID_PAIR:
     1392            cDpcsNeeded = 2;
     1393            break;
     1394        default:
     1395            do
     1396            {
     1397                cDpcsNeeded = g_cRtMpNtActiveCpus;
     1398                OnlineSet   = g_rtMpNtCpuSet;
     1399            } while (cDpcsNeeded != g_cRtMpNtActiveCpus);
     1400            break;
     1401    }
     1402
     1403    /*
     1404     * Allocate an RTMPARGS structure followed by cDpcsNeeded KDPCs
     1405     * and initialize them.
     1406     */
     1407    PRTMPARGS pArgs = (PRTMPARGS)ExAllocatePoolWithTag(NonPagedPool, sizeof(RTMPARGS) + cDpcsNeeded * sizeof(KDPC), (ULONG)'RTMp');
    4291408    if (!pArgs)
    4301409        return VERR_NO_MEMORY;
     
    4381417    pArgs->cRefs     = 1;
    4391418
    440     paExecCpuDpcs = (KDPC *)(pArgs + 1);
    441 
    4421419    int rc;
     1420    KDPC *paExecCpuDpcs = (KDPC *)(pArgs + 1);
    4431421    if (enmCpuid == RT_NT_CPUID_SPECIFIC)
    4441422    {
     
    4641442    {
    4651443        rc = VINF_SUCCESS;
    466         for (unsigned i = 0; i < g_cRtMpNtMaxCpus && RT_SUCCESS(rc); i++)
    467         {
    468             KeInitializeDpc(&paExecCpuDpcs[i], rtmpNtDPCWrapper, pArgs);
    469             KeSetImportanceDpc(&paExecCpuDpcs[i], HighImportance);
    470             rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[i], i);
    471         }
     1444        for (uint32_t i = 0; i < cDpcsNeeded && RT_SUCCESS(rc); i++)
     1445            if (RTCpuSetIsMemberByIndex(&OnlineSet, i))
     1446            {
     1447                KeInitializeDpc(&paExecCpuDpcs[i], rtmpNtDPCWrapper, pArgs);
     1448                KeSetImportanceDpc(&paExecCpuDpcs[i], HighImportance);
     1449                rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[i], RTMpCpuIdFromSetIndex(i));
     1450            }
    4721451    }
    4731452    if (RT_FAILURE(rc))
     
    4771456    }
    4781457
    479     /* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
     1458    /*
     1459     * Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
    4801460     * KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL.
    4811461     */
     
    5071487    else
    5081488    {
    509         unsigned iSelf = RTMpCpuId();
    510 
    511         for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++)
    512         {
    513             if (    (i != iSelf)
    514                 &&  (Mask & RT_BIT_64(i)))
     1489        uint32_t iSelf = RTMpCurSetIndex();
     1490        for (uint32_t i = 0; i < cDpcsNeeded; i++)
     1491        {
     1492            if (   (i != iSelf)
     1493                && RTCpuSetIsMemberByIndex(&OnlineSet, i))
    5151494            {
    5161495                ASMAtomicIncS32(&pArgs->cRefs);
     
    5251504    KeLowerIrql(oldIrql);
    5261505
    527     /* Flush all DPCs and wait for completion. (can take long!) */
     1506    /*
     1507     * Flush all DPCs and wait for completion. (can take long!)
     1508     */
    5281509    /** @todo Consider changing this to an active wait using some atomic inc/dec
    5291510     *  stuff (and check for the current cpu above in the specific case). */
  • trunk/src/VBox/Runtime/r3/win/mp-win.cpp

    r64234 r64281  
    4040#include <iprt/mem.h>
    4141#include <iprt/once.h>
     42#include <iprt/param.h>
    4243#if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)
    4344# include <iprt/asm-amd64-x86.h>
    4445#endif
     46#if defined(VBOX) && !defined(IN_GUEST)
     47# include <VBox/sup.h>
     48# define IPRT_WITH_GIP_MP_INFO
     49#else
     50# undef  IPRT_WITH_GIP_MP_INFO
     51#endif
    4552
    4653#include "internal-r3-win.h"
    47 
    4854
    4955
     
    5157*   Defined Constants And Macros                                                                                                 *
    5258*********************************************************************************************************************************/
     59/** @def RTMPWIN_UPDATE_GIP_GLOBAL
     60 * Does lazy (re-)initialization using information provieded by GIP. */
     61#ifdef IPRT_WITH_GIP_MP_INFO
     62# define RTMPWIN_UPDATE_GIP_GLOBAL() \
     63    do { RTMPWIN_UPDATE_GIP_GLOBALS_AND_GET_PGIP(); } while (0)
     64#else
     65# define RTMPWIN_UPDATE_GIP_GLOBAL() do { } while (0)
     66#endif
     67
     68/** @def RTMPWIN_UPDATE_GIP_GLOBALS_AND_GET_PGIP
     69 * Does lazy (re-)initialization using information provieded by GIP and
     70 * declare and initalize a pGip local variable. */
     71#ifdef IPRT_WITH_GIP_MP_INFO
     72#define RTMPWIN_UPDATE_GIP_GLOBALS_AND_GET_PGIP() \
     73    PSUPGLOBALINFOPAGE pGip = g_pSUPGlobalInfoPage; \
     74    if (pGip) \
     75    { \
     76        if (   pGip->u32Magic == SUPGLOBALINFOPAGE_MAGIC \
     77            && RTOnce(&g_MpInitOnceGip, rtMpWinInitOnceGip, NULL) == VINF_SUCCESS) \
     78        { \
     79            if (g_cRtMpWinActiveCpus >= pGip->cOnlineCpus) \
     80            { /* likely */ } \
     81            else \
     82                rtMpWinRefreshGip(); \
     83        } \
     84        else \
     85            pGip = NULL; \
     86    } else do { } while (0)
     87#else
     88# define RTMPWIN_UPDATE_GIP_GLOBALS_AND_GET_PGIP() do { } while (0)
     89#endif
     90
     91
     92/*********************************************************************************************************************************
     93*   Global Variables                                                                                                             *
     94*********************************************************************************************************************************/
    5395/** Initialize once. */
    5496static RTONCE                                       g_MpInitOnce = RTONCE_INITIALIZER;
    55 //static decltype(GetMaximumProcessorCount)          *g_pfnGetMaximumProcessorCount;
     97#ifdef IPRT_WITH_GIP_MP_INFO
     98/** Initialize once using GIP. */
     99static RTONCE                                       g_MpInitOnceGip = RTONCE_INITIALIZER;
     100#endif
     101
     102static decltype(GetMaximumProcessorCount)          *g_pfnGetMaximumProcessorCount;
     103//static decltype(GetActiveProcessorCount)           *g_pfnGetActiveProcessorCount;
    56104static decltype(GetCurrentProcessorNumber)         *g_pfnGetCurrentProcessorNumber;
    57105static decltype(GetCurrentProcessorNumberEx)       *g_pfnGetCurrentProcessorNumberEx;
     
    60108
    61109
    62 /*********************************************************************************************************************************
    63 *   Global Variables                                                                                                             *
    64 *********************************************************************************************************************************/
    65110/** The required buffer size for getting group relations. */
    66111static uint32_t     g_cbRtMpWinGrpRelBuf;
     
    71116/** The max number of groups. */
    72117static uint32_t     g_cRtMpWinMaxCpuGroups;
    73 /** Static per group info. */
     118/** The number of active CPUs the last time we checked. */
     119static uint32_t volatile g_cRtMpWinActiveCpus;
     120/** Static per group info.
     121 * @remarks  With RTCPUSET_MAX_CPUS as 256, this takes up 33KB.
     122 * @sa g_aRtMpNtCpuGroups */
    74123static struct
    75124{
    76     /** The CPU ID (and CPU set index) of the first CPU in the group. */
    77     uint16_t    idFirstCpu;
    78125    /** The max CPUs in the group. */
    79126    uint16_t    cMaxCpus;
    80 } g_aRtMpWinCpuGroups[RTCPUSET_MAX_CPUS];
     127    /** The number of active CPUs at the time of initialization. */
     128    uint16_t    cActiveCpus;
     129    /** CPU set indexes for each CPU in the group. */
     130    int16_t     aidxCpuSetMembers[64];
     131}                   g_aRtMpWinCpuGroups[RTCPUSET_MAX_CPUS];
     132/** Maps CPU set indexes to RTCPUID.
     133 * @sa g_aidRtMpNtByCpuSetIdx  */
     134RTCPUID             g_aidRtMpWinByCpuSetIdx[RTCPUSET_MAX_CPUS];
    81135
    82136
    83137/**
    84  * @callback_method_impl{FNRTONCE, Resolves dynamic imports.}
     138 * @callback_method_impl{FNRTONCE,
     139 *      Resolves dynamic imports and initializes globals.}
    85140 */
    86141static DECLCALLBACK(int32_t) rtMpWinInitOnce(void *pvUser)
     
    98153            RT_CONCAT(g_pfn,a_FnName) = (decltype(a_FnName) *)GetProcAddress(g_hModKernel32, #a_FnName); \
    99154        } while (0)
    100     //RESOLVE_API("kernel32.dll", GetMaximumProcessorCount); /* Calls GetLogicalProcessorInformationEx/RelationGroup in W10. */
     155    RESOLVE_API("kernel32.dll", GetMaximumProcessorCount);
     156    //RESOLVE_API("kernel32.dll", GetActiveProcessorCount); - slow :/
    101157    RESOLVE_API("kernel32.dll", GetCurrentProcessorNumber);
    102158    RESOLVE_API("kernel32.dll", GetCurrentProcessorNumberEx);
     
    105161
    106162    /*
    107      * Query group information, partitioning CPU IDs and CPU set
    108      * indexes (they are the same).
     163     * Reset globals.
     164     */
     165    for (unsigned i = 0; i < RT_ELEMENTS(g_aidRtMpWinByCpuSetIdx); i++)
     166        g_aidRtMpWinByCpuSetIdx[i] = NIL_RTCPUID;
     167    for (unsigned idxGroup = 0; idxGroup < RT_ELEMENTS(g_aRtMpWinCpuGroups); idxGroup++)
     168    {
     169        g_aRtMpWinCpuGroups[idxGroup].cMaxCpus    = 0;
     170        g_aRtMpWinCpuGroups[idxGroup].cActiveCpus = 0;
     171        for (unsigned idxMember = 0; idxMember < RT_ELEMENTS(g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers); idxMember++)
     172            g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = -1;
     173    }
     174
     175    /*
     176     * Query group information, partitioning CPU IDs and CPU set indexes.
    109177     *
    110178     * We ASSUME the the GroupInfo index is the same as the group number.
    111179     *
    112      * We ASSUME there are no inactive groups, because otherwise it will
    113      * be difficult to tell how many possible CPUs we can have and do a
    114      * reasonable CPU ID/index partitioning. [probably bad assumption]
     180     * We CANNOT ASSUME that the kernel CPU indexes are assigned in any given
     181     * way, though they usually are in group order by active processor.  So,
     182     * we do that to avoid trouble.  We must use information provided thru GIP
     183     * if we want the kernel CPU set indexes.  Even there, the inactive CPUs
     184     * wont have sensible indexes.  Sigh.
    115185     *
    116      * We ASSUME that the kernel processor indexes are assigned in group order,
    117      * which we match here with our own ID+index assignments.  This claim is
    118      * verified by initterm-r0drv-nt.cpp.
     186     * We try to assign IDs to inactive CPUs in the same manner as mp-r0drv-nt.cpp
    119187     *
    120      * Note! We will die if there are too many processors!
     188     * Note! We will die (AssertFatal) if there are too many processors!
    121189     */
    122190    union
     
    146214        AssertFatal(uBuf.Info.Group.MaximumGroupCount >= uBuf.Info.Group.ActiveGroupCount);
    147215
     216        g_cRtMpWinMaxCpuGroups = uBuf.Info.Group.MaximumGroupCount;
     217
     218        /* Count max cpus (see mp-r0drv0-nt.cpp) why we don't use GetMaximumProcessorCount(ALL). */
     219        uint32_t idxGroup;
     220        g_cRtMpWinMaxCpus = 0;
     221        for (idxGroup = 0; idxGroup < uBuf.Info.Group.ActiveGroupCount; idxGroup++)
     222            g_cRtMpWinMaxCpus += uBuf.Info.Group.GroupInfo[idxGroup].MaximumProcessorCount;
    148223
    149224        /* Process the active groups. */
    150         g_cRtMpWinMaxCpuGroups = uBuf.Info.Group.MaximumGroupCount;
    151         uint16_t idxCpu        = 0;
    152         uint32_t idxGroup      = 0;
    153         for (; idxGroup < uBuf.Info.Group.ActiveGroupCount; idxGroup++)
    154         {
    155             g_aRtMpWinCpuGroups[idxGroup].idFirstCpu = idxCpu;
    156             g_aRtMpWinCpuGroups[idxGroup].cMaxCpus   = uBuf.Info.Group.GroupInfo[idxGroup].MaximumProcessorCount;
    157             idxCpu += uBuf.Info.Group.GroupInfo[idxGroup].MaximumProcessorCount;
    158         }
     225        uint32_t cActive   = 0;
     226        uint32_t cInactive = 0;
     227        uint32_t idxCpu    = 0;
     228        uint32_t idxCpuSetNextInactive = g_cRtMpWinMaxCpus - 1;
     229        for (idxGroup = 0; idxGroup < uBuf.Info.Group.ActiveGroupCount; idxGroup++)
     230        {
     231            PROCESSOR_GROUP_INFO const *pGroupInfo = &uBuf.Info.Group.GroupInfo[idxGroup];
     232            g_aRtMpWinCpuGroups[idxGroup].cMaxCpus    = pGroupInfo->MaximumProcessorCount;
     233            g_aRtMpWinCpuGroups[idxGroup].cActiveCpus = pGroupInfo->ActiveProcessorCount;
     234            for (uint32_t idxMember = 0; idxMember < pGroupInfo->MaximumProcessorCount; idxMember++)
     235            {
     236                if (pGroupInfo->ActiveProcessorMask & RT_BIT_64(idxMember))
     237                {
     238                    g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpu;
     239                    g_aidRtMpWinByCpuSetIdx[idxCpu] = idxCpu;
     240                    idxCpu++;
     241                    cActive++;
     242                }
     243                else
     244                {
     245                    if (idxCpuSetNextInactive >= idxCpu)
     246                    {
     247                        g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
     248                        g_aidRtMpWinByCpuSetIdx[idxCpuSetNextInactive] = idxCpuSetNextInactive;
     249                        idxCpuSetNextInactive--;
     250                    }
     251                    cInactive++;
     252                }
     253            }
     254        }
     255        g_cRtMpWinActiveCpus = cActive;
     256        Assert(cActive + cInactive <= g_cRtMpWinMaxCpus);
     257        Assert(idxCpu <= idxCpuSetNextInactive + 1);
     258        Assert(idxCpu <= g_cRtMpWinMaxCpus);
    159259
    160260        /* Just in case the 2nd assumption doesn't hold true and there are inactive groups. */
    161261        for (; idxGroup < uBuf.Info.Group.MaximumGroupCount; idxGroup++)
    162262        {
    163             g_aRtMpWinCpuGroups[idxGroup].idFirstCpu = idxCpu;
    164             g_aRtMpWinCpuGroups[idxGroup].cMaxCpus   = RT_MAX(MAXIMUM_PROC_PER_GROUP, 64);
    165             idxCpu += RT_MAX(MAXIMUM_PROC_PER_GROUP, 64);
    166         }
    167 
    168         g_cRtMpWinMaxCpus = idxCpu;
     263            DWORD cMaxMembers = g_pfnGetMaximumProcessorCount(idxGroup);
     264            g_aRtMpWinCpuGroups[idxGroup].cMaxCpus    = cMaxMembers;
     265            g_aRtMpWinCpuGroups[idxGroup].cActiveCpus = 0;
     266            for (uint32_t idxMember = 0; idxMember < cMaxMembers; idxMember++)
     267            {
     268                if (idxCpuSetNextInactive >= idxCpu)
     269                {
     270                    g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
     271                    g_aidRtMpWinByCpuSetIdx[idxCpuSetNextInactive] = idxCpuSetNextInactive;
     272                    idxCpuSetNextInactive--;
     273                }
     274                cInactive++;
     275            }
     276        }
     277        Assert(cActive + cInactive <= g_cRtMpWinMaxCpus);
     278        Assert(idxCpu <= idxCpuSetNextInactive + 1);
    169279    }
    170280    else
     
    172282        /* Legacy: */
    173283        GetSystemInfo(&uBuf.SysInfo);
     284        g_cRtMpWinMaxCpuGroups              = 1;
    174285        g_cRtMpWinMaxCpus                   = uBuf.SysInfo.dwNumberOfProcessors;
    175         g_cRtMpWinMaxCpuGroups              = 1;
    176         g_aRtMpWinCpuGroups[0].idFirstCpu   = 0;
    177286        g_aRtMpWinCpuGroups[0].cMaxCpus     = uBuf.SysInfo.dwNumberOfProcessors;
     287        g_aRtMpWinCpuGroups[0].cActiveCpus  = uBuf.SysInfo.dwNumberOfProcessors;
     288
     289        for (uint32_t idxMember = 0; idxMember < uBuf.SysInfo.dwNumberOfProcessors; idxMember++)
     290        {
     291            g_aRtMpWinCpuGroups[0].aidxCpuSetMembers[idxMember] = idxMember;
     292            g_aidRtMpWinByCpuSetIdx[idxMember] = idxMember;
     293        }
    178294    }
    179295
     
    268384
    269385
     386#ifdef IPRT_WITH_GIP_MP_INFO
     387/**
     388 * @callback_method_impl{FNRTONCE, Updates globals with information from GIP.}
     389 */
     390static DECLCALLBACK(int32_t) rtMpWinInitOnceGip(void *pvUser)
     391{
     392    RT_NOREF(pvUser);
     393    RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL);
     394
     395    PSUPGLOBALINFOPAGE pGip = g_pSUPGlobalInfoPage;
     396    if (   pGip
     397        && pGip->u32Magic == SUPGLOBALINFOPAGE_MAGIC)
     398    {
     399        /*
     400         * Update globals.
     401         */
     402        if (g_cRtMpWinMaxCpus != pGip->cPossibleCpus)
     403            g_cRtMpWinMaxCpus = pGip->cPossibleCpus;
     404        if (g_cRtMpWinActiveCpus != pGip->cOnlineCpus)
     405            g_cRtMpWinActiveCpus = pGip->cOnlineCpus;
     406        Assert(g_cRtMpWinMaxCpuGroups == pGip->cPossibleCpuGroups);
     407        if (g_cRtMpWinMaxCpuGroups != pGip->cPossibleCpuGroups)
     408        {
     409            g_cRtMpWinMaxCpuGroups = pGip->cPossibleCpuGroups;
     410            g_cbRtMpWinGrpRelBuf   = sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)
     411                                   + (g_cRtMpWinMaxCpuGroups + 2) * sizeof(PROCESSOR_GROUP_INFO);
     412        }
     413
     414        /*
     415         * Update CPU set IDs.
     416         */
     417        for (unsigned i = g_cRtMpWinMaxCpus; i < RT_ELEMENTS(g_aidRtMpWinByCpuSetIdx); i++)
     418            g_aidRtMpWinByCpuSetIdx[i] = NIL_RTCPUID;
     419
     420        unsigned const cbGip = pGip->cPages * PAGE_SIZE;
     421        for (uint32_t idxGroup = 0; idxGroup < g_cRtMpWinMaxCpus; idxGroup++)
     422        {
     423            uint32_t idxMember;
     424            unsigned offCpuGroup = pGip->aoffCpuGroup[idxGroup];
     425            if (offCpuGroup < cbGip)
     426            {
     427                PSUPGIPCPUGROUP pGipCpuGrp  = (PSUPGIPCPUGROUP)((uintptr_t)pGip + offCpuGroup);
     428                uint32_t        cMaxMembers = pGipCpuGrp->cMaxMembers;
     429                AssertStmt(cMaxMembers < RT_ELEMENTS(g_aRtMpWinCpuGroups[0].aidxCpuSetMembers),
     430                           cMaxMembers = RT_ELEMENTS(g_aRtMpWinCpuGroups[0].aidxCpuSetMembers));
     431                g_aRtMpWinCpuGroups[idxGroup].cMaxCpus     = cMaxMembers;
     432                g_aRtMpWinCpuGroups[idxGroup].cActiveCpus  = RT_MIN(pGipCpuGrp->cMembers, cMaxMembers);
     433
     434                for (idxMember = 0; idxMember < cMaxMembers; idxMember++)
     435                {
     436                    int16_t idxSet = pGipCpuGrp->aiCpuSetIdxs[idxMember];
     437                    g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxSet;
     438                    if ((unsigned)idxSet < RT_ELEMENTS(g_aidRtMpWinByCpuSetIdx))
     439# ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     440                        g_aidRtMpWinByCpuSetIdx[idxSet] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember);
     441# else
     442                        g_aidRtMpWinByCpuSetIdx[idxSet] = idxSet;
     443# endif
     444                }
     445            }
     446            else
     447                idxMember = 0;
     448            for (; idxMember < RT_ELEMENTS(g_aRtMpWinCpuGroups[0].aidxCpuSetMembers); idxMember++)
     449                g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = -1;
     450        }
     451    }
     452
     453    return VINF_SUCCESS;
     454}
     455
     456
     457/**
     458 * Refreshes globals from GIP after one or more CPUs were added.
     459 *
     460 * There are potential races here.  We might race other threads and we may race
     461 * more CPUs being added.
     462 */
     463static void rtMpWinRefreshGip(void)
     464{
     465    PSUPGLOBALINFOPAGE pGip = g_pSUPGlobalInfoPage;
     466    if (   pGip
     467        && pGip->u32Magic == SUPGLOBALINFOPAGE_MAGIC)
     468    {
     469        /*
     470         * Since CPUs cannot be removed, we only have to update the IDs and
     471         * indexes of CPUs that we think are inactive and the group member counts.
     472         */
     473        for (;;)
     474        {
     475            unsigned const cbGip          = pGip->cPages * PAGE_SIZE;
     476            uint32_t const cGipActiveCpus = pGip->cOnlineCpus;
     477            uint32_t const cMyActiveCpus  = ASMAtomicReadU32(&g_cRtMpWinActiveCpus);
     478            ASMCompilerBarrier();
     479
     480            for (uint32_t idxGroup = 0; idxGroup < g_cRtMpWinMaxCpus; idxGroup++)
     481            {
     482                unsigned offCpuGroup = pGip->aoffCpuGroup[idxGroup];
     483                if (offCpuGroup < cbGip)
     484                {
     485                    PSUPGIPCPUGROUP pGipCpuGrp  = (PSUPGIPCPUGROUP)((uintptr_t)pGip + offCpuGroup);
     486                    uint32_t        cMaxMembers = pGipCpuGrp->cMaxMembers;
     487                    AssertStmt(cMaxMembers < RT_ELEMENTS(g_aRtMpWinCpuGroups[0].aidxCpuSetMembers),
     488                               cMaxMembers = RT_ELEMENTS(g_aRtMpWinCpuGroups[0].aidxCpuSetMembers));
     489                    for (uint32_t idxMember = g_aRtMpWinCpuGroups[idxGroup].cActiveCpus; idxMember < cMaxMembers; idxMember++)
     490                    {
     491                        int16_t idxSet = pGipCpuGrp->aiCpuSetIdxs[idxMember];
     492                        g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxSet;
     493                        if ((unsigned)idxSet < RT_ELEMENTS(g_aidRtMpWinByCpuSetIdx))
     494# ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     495                            g_aidRtMpWinByCpuSetIdx[idxSet] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember);
     496# else
     497                            g_aidRtMpWinByCpuSetIdx[idxSet] = idxSet;
     498# endif
     499                    }
     500                    g_aRtMpWinCpuGroups[idxGroup].cMaxCpus    = RT_MIN(pGipCpuGrp->cMembers, cMaxMembers);
     501                    g_aRtMpWinCpuGroups[idxGroup].cActiveCpus = RT_MIN(pGipCpuGrp->cMembers, cMaxMembers);
     502                }
     503                else
     504                    Assert(g_aRtMpWinCpuGroups[idxGroup].cActiveCpus == 0);
     505            }
     506
     507            ASMCompilerBarrier();
     508            if (cGipActiveCpus == pGip->cOnlineCpus)
     509                if (ASMAtomicCmpXchgU32(&g_cRtMpWinActiveCpus, cGipActiveCpus, cMyActiveCpus))
     510                    break;
     511        }
     512    }
     513}
     514
     515#endif /* IPRT_WITH_GIP_MP_INFO */
     516
     517
     518/*
     519 * Conversion between CPU ID and set index.
     520 */
     521
     522RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu)
     523{
     524    RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL);
     525    RTMPWIN_UPDATE_GIP_GLOBAL();
     526
     527#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     528    if (idCpu != NIL_RTCPUID)
     529        return RTMpSetIndexFromCpuGroupMember(rtMpCpuIdGetGroup(idCpu), rtMpCpuIdGetGroupMember(idCpu));
     530    return -1;
     531
     532#else
     533    /* 1:1 mapping, just do range checking. */
     534    return idCpu < g_cRtMpWinMaxCpus ? idCpu : -1;
     535#endif
     536}
     537
     538
     539RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu)
     540{
     541    RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL);
     542    RTMPWIN_UPDATE_GIP_GLOBAL();
     543
     544    if ((unsigned)iCpu < RT_ELEMENTS(g_aidRtMpWinByCpuSetIdx))
     545    {
     546        RTCPUID idCpu = g_aidRtMpWinByCpuSetIdx[iCpu];
     547
     548#if defined(IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER) && defined(RT_STRICT)
     549        /* Check the correctness of the mapping table. */
     550        RTCPUID idCpuGip = NIL_RTCPUID;
     551        if (   pGip
     552            && (unsigned)iCpu < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx))
     553        {
     554            unsigned idxSupCpu = pGip->aiCpuFromCpuSetIdx[idxGuess];
     555            if (idxSupCpu < pGip->cCpus)
     556                if (pGip->aCPUs[idxSupCpu].enmState != SUPGIPCPUSTATE_INVALID)
     557                    idCpuGip = pGip->aCPUs[idxSupCpu].idCpu;
     558        }
     559        AssertMsg(idCpu == idCpuGip, ("table:%#x  gip:%#x\n", idCpu, idCpuGip));
     560#endif
     561
     562        return idCpu;
     563    }
     564    return NIL_RTCPUID;
     565}
     566
     567
     568RTDECL(int) RTMpSetIndexFromCpuGroupMember(uint32_t idxGroup, uint32_t idxMember)
     569{
     570    if (idxGroup < g_cRtMpWinMaxCpuGroups)
     571        if (idxMember < g_aRtMpWinCpuGroups[idxGroup].cMaxCpus)
     572            return g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember];
     573    return -1;
     574}
     575
     576
     577RTDECL(uint32_t) RTMpGetCpuGroupCounts(uint32_t idxGroup, uint32_t *pcActive)
     578{
     579    if (idxGroup < g_cRtMpWinMaxCpuGroups)
     580    {
     581        if (pcActive)
     582            *pcActive = g_aRtMpWinCpuGroups[idxGroup].cActiveCpus;
     583        return g_aRtMpWinCpuGroups[idxGroup].cMaxCpus;
     584    }
     585    if (pcActive)
     586        *pcActive = 0;
     587    return 0;
     588}
     589
     590
     591RTDECL(uint32_t) RTMpGetMaxCpuGroupCount(void)
     592{
     593    return g_cRtMpWinMaxCpuGroups;
     594}
     595
     596
     597
     598/*
     599 * Get current CPU.
     600 */
     601
    270602RTDECL(RTCPUID) RTMpCpuId(void)
    271603{
    272604    RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL);
    273 
     605#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     606    RTMPWIN_UPDATE_GIP_GLOBAL();
     607#endif
     608
     609    PROCESSOR_NUMBER ProcNum;
     610    ProcNum.Group = 0;
     611    ProcNum.Number = 0xff;
    274612    if (g_pfnGetCurrentProcessorNumberEx)
    275     {
    276         PROCESSOR_NUMBER ProcNum;
    277613        g_pfnGetCurrentProcessorNumberEx(&ProcNum);
    278         Assert(ProcNum.Group < g_cRtMpWinMaxCpuGroups);
    279         Assert(ProcNum.Number < g_aRtMpWinCpuGroups[ProcNum.Group].cMaxCpus);
    280         return g_aRtMpWinCpuGroups[ProcNum.Group].idFirstCpu + ProcNum.Number;
    281     }
    282 
    283     if (g_pfnGetCurrentProcessorNumber)
    284     {
    285         /* Should be safe wrt processor numbering, I hope... Only affects W2k3 and Vista. */
    286         Assert(g_cRtMpWinMaxCpuGroups == 1);
    287         return g_pfnGetCurrentProcessorNumber();
    288     }
    289 
    290     /* The API was introduced with W2K3 according to MSDN. */
     614    else if (g_pfnGetCurrentProcessorNumber)
     615    {
     616        DWORD iCpu = g_pfnGetCurrentProcessorNumber();
     617        Assert(iCpu < g_cRtMpWinMaxCpus);
     618        ProcNum.Number = iCpu;
     619    }
     620    else
     621    {
    291622#if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)
    292     return ASMGetApicId();
     623        ProcNum.Number = ASMGetApicId();
    293624#else
    294625# error "Not ported to this architecture."
    295     return NIL_RTAPICID;
    296 #endif
    297 }
    298 
    299 
    300 RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu)
     626        return NIL_RTCPUID;
     627#endif
     628    }
     629
     630#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     631    return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
     632#else
     633    return RTMpSetIndexFromCpuGroupMember(ProcNum.Group, ProcNum.Number);
     634#endif
     635}
     636
     637
     638/*
     639 * Possible CPUs and cores.
     640 */
     641
     642RTDECL(RTCPUID) RTMpGetMaxCpuId(void)
    301643{
    302644    RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL);
    303 
    304     /* 1:1 mapping, just do range checking. */
    305     return idCpu < g_cRtMpWinMaxCpus ? idCpu : -1;
    306 }
    307 
    308 
    309 RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu)
     645#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
     646    return RTMPCPUID_FROM_GROUP_AND_NUMBER(g_cRtMpWinMaxCpuGroups - 1,
     647                                           g_aRtMpWinCpuGroups[g_cRtMpWinMaxCpuGroups - 1].cMaxCpus - 1);
     648#else
     649    return g_cRtMpWinMaxCpus - 1;
     650#endif
     651}
     652
     653
     654RTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu)
    310655{
    311656    RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL);
    312 
    313     /* 1:1 mapping, just do range checking. */
    314     return (unsigned)iCpu < g_cRtMpWinMaxCpus ? iCpu : NIL_RTCPUID;
    315 }
    316 
    317 
    318 RTDECL(RTCPUID) RTMpGetMaxCpuId(void)
    319 {
    320     RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL);
    321     return g_cRtMpWinMaxCpus - 1;
    322 }
    323 
    324 
    325 RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
    326 {
    327     RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL);
    328     RTCPUSET Set;
    329     return RTCpuSetIsMember(RTMpGetOnlineSet(&Set), idCpu);
    330 }
    331 
    332 
    333 RTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu)
    334 {
    335     RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL);
     657    RTMPWIN_UPDATE_GIP_GLOBAL();
     658
    336659    /* Any CPU between 0 and g_cRtMpWinMaxCpus are possible. */
    337660    return idCpu < g_cRtMpWinMaxCpus;
     
    341664RTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet)
    342665{
    343     RTCPUID idCpu = RTMpGetCount();
     666    RTCPUID iCpu = RTMpGetCount();
    344667    RTCpuSetEmpty(pSet);
    345     while (idCpu-- > 0)
    346         RTCpuSetAdd(pSet, idCpu);
     668    while (iCpu-- > 0)
     669        RTCpuSetAddByIndex(pSet, iCpu);
    347670    return pSet;
    348671}
     
    352675{
    353676    RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL);
     677
    354678    return g_cRtMpWinMaxCpus;
    355679}
     
    359683{
    360684    RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL);
     685
    361686    return g_cRtMpWinMaxCpuCores;
    362687}
    363688
    364689
     690/*
     691 * Online CPUs and cores.
     692 */
     693
    365694RTDECL(PRTCPUSET) RTMpGetOnlineSet(PRTCPUSET pSet)
    366695{
    367696    RTOnce(&g_MpInitOnce, rtMpWinInitOnce, NULL);
     697
     698#ifdef IPRT_WITH_GIP_MP_INFO
     699    RTMPWIN_UPDATE_GIP_GLOBALS_AND_GET_PGIP();
     700    if (pGip)
     701    {
     702        *pSet = pGip->OnlineCpuSet;
     703        return pSet;
     704    }
     705#endif
    368706
    369707    if (g_pfnGetLogicalProcessorInformationEx)
     
    376714         * active processor mask width.
    377715         */
     716        /** @todo this is not correct for WOW64   */
    378717        DWORD                                    cbInfo = g_cbRtMpWinGrpRelBuf;
    379718        SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)alloca(cbInfo);
     
    397736                uint32_t    cMembersLeft = pInfo->Group.GroupInfo[idxGroup].ActiveProcessorCount;
    398737#endif
    399                 int const   idxFirst  = g_aRtMpWinCpuGroups[idxGroup].idFirstCpu;
    400738                int const   cMembers  = g_aRtMpWinCpuGroups[idxGroup].cMaxCpus;
    401739                for (int idxMember = 0; idxMember < cMembers; idxMember++)
     
    406744                        cMembersLeft--;
    407745#endif
    408                         RTCpuSetAddByIndex(pSet, idxFirst + idxMember);
     746                        RTCpuSetAddByIndex(pSet, g_aRtMpWinCpuGroups[idxGroup].aidxCpuSetMembers[idxMember]);
    409747                        fActive >>= 1;
    410748                        if (!fActive)
     
    434772
    435773
     774RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
     775{
     776    RTCPUSET Set;
     777    return RTCpuSetIsMember(RTMpGetOnlineSet(&Set), idCpu);
     778}
     779
     780
    436781RTDECL(RTCPUID) RTMpGetOnlineCount(void)
    437782{
     783#ifdef IPRT_WITH_GIP_MP_INFO
     784    RTMPWIN_UPDATE_GIP_GLOBALS_AND_GET_PGIP();
     785    if (pGip)
     786        return pGip->cOnlineCpus;
     787#endif
     788
    438789    RTCPUSET Set;
    439790    RTMpGetOnlineSet(&Set);
  • trunk/src/VBox/Runtime/testcase/tstRTMp-1.cpp

    r62477 r64281  
    3434#include <iprt/string.h>
    3535#include <iprt/test.h>
    36 
    37 
    38 
    39 int main()
     36#ifdef VBOX
     37# include <VBox/sup.h>
     38#endif
     39
     40
     41
     42int main(int argc, char **argv)
    4043{
    4144    RTTEST hTest;
     
    4447        return rcExit;
    4548    RTTestBanner(hTest);
     49
     50    NOREF(argc); NOREF(argv);
     51#ifdef VBOX
     52    if (argc > 1)
     53        SUPR3Init(NULL);
     54#endif
    4655
    4756    /*
Note: See TracChangeset for help on using the changeset viewer.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette