Changeset 37062 in vbox for trunk

Timestamp:

May 13, 2011 10:18:29 AM (14 years ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

71699

Message:

VMM: Support for online/offlining of CPUs.

Location:

trunk/src/VBox

Files:

• HostDrivers/Support/SUPDrv.c (modified) (10 diffs)
• Runtime/r0drv/solaris/vbi/mp-r0drv-solaris.c (modified) (2 diffs)
• Runtime/r0drv/solaris/vbi/mpnotification-r0drv-solaris.c (modified) (2 diffs)
• VMM/VMMR0/HWACCMR0.cpp (modified) (10 diffs)

trunk/src/VBox/HostDrivers/Support/SUPDrv.c

@@ -322 +322 @@
     { "RTMpIsCpuOnline",                        (void *)RTMpIsCpuOnline },
     { "RTMpIsCpuWorkPending",                   (void *)RTMpIsCpuWorkPending },
+    { "RTMpNotificationRegister",               (void *)RTMpNotificationRegister },
+    { "RTMpNotificationDeregister",             (void *)RTMpNotificationDeregister },
     { "RTMpOnAll",                              (void *)RTMpOnAll },
     { "RTMpOnOthers",                           (void *)RTMpOnOthers },
@@ -4990 +4992 @@
 
 /**
+ * Helper for finding the CPU index from the CPU Id.
+ *
+ * @param    pGip               The GIP.
+ * @param    idCpu              The CPU ID.
+ *
+ * @returns Index of the CPU in the cache set.
+ */
+static inline uint32_t supdrvGipCpuIndexFromCpuId(PSUPGLOBALINFOPAGE pGip, RTCPUID idCpu)
+{
+    uint32_t i = 0;
+
+    /*
+     * Find our entry, or allocate one if not found.
+     * ASSUMES that CPU IDs are constant.
+     */
+    for (i = 0; i < pGip->cCpus; i++)
+        if (pGip->aCPUs[i].idCpu == idCpu)
+            break;
+
+    if (i >= pGip->cCpus)
+        for (i = 0; i < pGip->cCpus; i++)
+        {
+            bool fRc;
+            ASMAtomicCmpXchgSize(&pGip->aCPUs[i].idCpu, idCpu, NIL_RTCPUID, fRc);
+            if (fRc)
+                break;
+        }
+
+    AssertRelease(i < pGip->cCpus);
+    return i;
+}
+
+
+/**
  * The calling CPU should be accounted as online, update GIP accordingly.
  *
@@ -4999 +5035 @@
 static void supdrvGipMpEventOnline(PSUPGLOBALINFOPAGE pGip, RTCPUID idCpu)
 {
-    int         iCpuSet;
-    uint8_t     idApic;
-    uint32_t    i;
-
-    Assert(idCpu == RTMpCpuId());
+    int         iCpuSet = 0;
+    uint16_t    idApic = UINT16_MAX;
+    uint32_t    i = 0;
+
+    AssertRelease(idCpu == RTMpCpuId());
     Assert(pGip->cPossibleCpus == RTMpGetCount());
 
@@ -5019 +5055 @@
     }
 
-    /*
-     * Find our entry, or allocate one if not found.
-     * ASSUMES that CPU IDs are constant.
-     */
-    for (i = 0; i < pGip->cCpus; i++)
-        if (pGip->aCPUs[i].idCpu == idCpu)
-            break;
-
-    if (i >= pGip->cCpus)
-        for (i = 0; i < pGip->cCpus; i++)
-        {
-            bool fRc;
-            ASMAtomicCmpXchgSize(&pGip->aCPUs[i].idCpu, idCpu, NIL_RTCPUID, fRc);
-            if (fRc)
-                break;
-        }
-
-    AssertReturnVoid(i < pGip->cCpus);
+    i = supdrvGipCpuIndexFromCpuId(pGip, idCpu);
 
     /*
@@ -5045 +5064 @@
     ASMAtomicUoWriteS16(&pGip->aCPUs[i].iCpuSet, (int16_t)iCpuSet);
     ASMAtomicUoWriteSize(&pGip->aCPUs[i].idCpu,  idCpu);
-    ASMAtomicWriteSize(&pGip->aCPUs[i].enmState, SUPGIPCPUSTATE_ONLINE);
 
     /*
@@ -5052 +5070 @@
     ASMAtomicWriteU16(&pGip->aiCpuFromApicId[idApic],     i);
     ASMAtomicWriteU16(&pGip->aiCpuFromCpuSetIdx[iCpuSet], i);
+
+    ASMAtomicWriteSize(&pGip->aCPUs[i].enmState, SUPGIPCPUSTATE_ONLINE);
 }
 
@@ -5076 +5096 @@
 
     Assert(RTCpuSetIsMemberByIndex(&pGip->PossibleCpuSet, iCpuSet));
+    RTCpuSetDelByIndex(&pGip->OnlineCpuSet, iCpuSet);
     ASMAtomicWriteSize(&pGip->aCPUs[i].enmState, SUPGIPCPUSTATE_OFFLINE);
-    RTCpuSetDelByIndex(&pGip->OnlineCpuSet, iCpuSet);
 }
 
@@ -5095 +5115 @@
     PSUPDRVDEVEXT       pDevExt = (PSUPDRVDEVEXT)pvUser;
     PSUPGLOBALINFOPAGE  pGip    = pDevExt->pGip;
+
+    AssertRelease(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
 
     /*
@@ -5359 +5381 @@
         pGip->aCPUs[i].idCpu             = NIL_RTCPUID;
         pGip->aCPUs[i].iCpuSet           = -1;
-        pGip->aCPUs[i].idApic            = UINT8_MAX;
+        pGip->aCPUs[i].idApic            = UINT16_MAX;
 
         /*
@@ -5618 +5640 @@
                                   RTCPUID idCpu, uint8_t idApic, uint64_t iTick)
 {
+    /*
+     * Avoid a potential race when a CPU online notification doesn't fire on the onlined CPU
+     * but the tick creeps in before the event notification is run.
+     */
+    if (iTick == 1)
+    {
+        uint32_t i = supdrvGipCpuIndexFromCpuId(pGip, idCpu);
+        if (pGip->aCPUs[i].enmState == SUPGIPCPUSTATE_OFFLINE)
+            supdrvGipMpEventOnline(pGip, idCpu);
+    }
+
     unsigned iCpu = pGip->aiCpuFromApicId[idApic];
 

trunk/src/VBox/Runtime/r0drv/solaris/vbi/mp-r0drv-solaris.c

@@ -174 +174 @@
     PRTMPARGS pArgs = (PRTMPARGS)(uArg);
 
+    /*
+     * Solaris CPU cross calls execute on offline CPUs too. Check our CPU cache
+     * set and ignore if it's offline.
+     */
+    if (!RTMpIsCpuOnline(RTMpCpuId()))
+        return 0;
+
     pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
 
@@ -279 +286 @@
         return VERR_CPU_NOT_FOUND;
 
+    if (RT_UNLIKELY(!RTMpIsCpuOnline(idCpu)))
+        return RTMpIsCpuPresent(idCpu) ? VERR_CPU_OFFLINE : VERR_CPU_NOT_FOUND;
+
     Args.pfnWorker = pfnWorker;
     Args.pvUser1 = pvUser1;

trunk/src/VBox/Runtime/r0drv/solaris/vbi/mpnotification-r0drv-solaris.c

@@ -34 +34 @@
 #include <iprt/mp.h>
 #include <iprt/cpuset.h>
+#include <iprt/string.h>
 #include "r0drv/mp-r0drv.h"
 
@@ -49 +50 @@
 
 
-static void rtMpNotificationSolarisCallback(void *pvUser, int iCpu, int online)
+static void rtMpNotificationSolarisOnCurrentCpu(void *pvArgs, void *uIgnored1, void *uIgnored2)
 {
-    NOREF(pvUser);
+    NOREF(uIgnored1);
+    NOREF(uIgnored2);
 
-    /* ASSUMES iCpu == RTCPUID */
+    PRTMPARGS pArgs = (PRTMPARGS)(pvArgs);
+    AssertRelease(pArgs && pArgs->idCpu == RTMpCpuId());
+    Assert(pArgs->pvUser2);
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+    int online = *(int *)pArgs->pvUser2;
     if (online)
     {
-        RTCpuSetAdd(&g_rtMpSolarisCpuSet, iCpu);
-        rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, iCpu);
+        RTCpuSetAdd(&g_rtMpSolarisCpuSet, pArgs->idCpu);
+        rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, pArgs->idCpu);
     }
     else
     {
-        RTCpuSetDel(&g_rtMpSolarisCpuSet, iCpu);
-        rtMpNotificationDoCallbacks(RTMPEVENT_OFFLINE, iCpu);
+        RTCpuSetDel(&g_rtMpSolarisCpuSet, pArgs->idCpu);
+        rtMpNotificationDoCallbacks(RTMPEVENT_OFFLINE, pArgs->idCpu);
     }
+}
+
+
+static void rtMpNotificationSolarisCallback(void *pvUser, int iCpu, int online)
+{
+    vbi_preempt_disable();
+
+    RTMPARGS Args;
+    RT_ZERO(Args);
+    Args.pvUser1 = pvUser;
+    Args.pvUser2 = &online;
+    Args.idCpu   = iCpu;
+
+    /*
+     * If we're not on the target CPU, schedule (synchronous) the event notification callback
+     * to run on the target CPU i.e. the one pertaining to the MP event.
+     */
+    bool fRunningOnTargetCpu = iCpu == RTMpCpuId();      /* ASSUMES iCpu == RTCPUID */
+    if (fRunningOnTargetCpu)
+        rtMpNotificationSolarisOnCurrentCpu(&Args, NULL /* pvIgnored1 */, NULL /* pvIgnored2 */);
+    else
+        vbi_execute_on_one(rtMpNotificationSolarisOnCurrentCpu, &Args, iCpu);
+
+    vbi_preempt_enable();
 }
 

trunk/src/VBox/VMM/VMMR0/HWACCMR0.cpp

@@ -51 +51 @@
 static bool               hwaccmR0IsSubjectToVmxPreemptionTimerErratum(void);
 static DECLCALLBACK(void) hwaccmR0PowerCallback(RTPOWEREVENT enmEvent, void *pvUser);
+static DECLCALLBACK(void) hwaccmR0CpuCallback(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvData);
 
 /*******************************************************************************
@@ -498 +499 @@
     if (!HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx)
     {
+        rc = RTMpNotificationRegister(hwaccmR0CpuCallback, 0);
+        AssertRC(rc);
+
         rc = RTPowerNotificationRegister(hwaccmR0PowerCallback, 0);
         AssertRC(rc);
@@ -575 +579 @@
         if (!HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx)
         {
+            /* Doesn't really matter if this fails. */
+            rc = RTMpNotificationDeregister(hwaccmR0CpuCallback, 0);
+            AssertRC(rc);
             rc = RTPowerNotificationDeregister(hwaccmR0PowerCallback, 0);
             AssertRC(rc);
@@ -732 +739 @@
                 Assert(!HWACCMR0Globals.aCpuInfo[i].pMemObj);
 
-                /** @todo this is rather dangerous if cpus can be taken offline; we don't care for now */
-                if (RTMpIsCpuOnline(i))
+                if (RTMpIsCpuPossible(RTMpCpuId()))
                 {
                     rc = RTR0MemObjAllocCont(&HWACCMR0Globals.aCpuInfo[i].pMemObj, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
@@ -748 +754 @@
 #endif
                 }
+                HWACCMR0Globals.aCpuInfo[i].fConfigured = false;
             }
             if (HWACCMR0Globals.fGlobalInit)
@@ -798 +805 @@
     if (!pCpu->pMemObj)
     {
+        LogRel(("HWACCMR0: hwaccmR0EnableCpu failed idCpu=%d.\n", idCpu));
         AssertFailed();
         return VERR_INTERNAL_ERROR;
@@ -883 +891 @@
     hwaccmR0FirstRcSetStatus(pFirstRc, hwaccmR0DisableCpu(idCpu));
 }
+
+
+/**
+ * Callback function invoked when a cpu goes online or offline.
+ *
+ * @param   enmEvent            The Mp event.
+ * @param   idCpu               The identifier for the CPU the function is called on.
+ * @param   pvData              Opaque data (PVM pointer).
+ */
+static DECLCALLBACK(void) hwaccmR0CpuCallback(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvData)
+{
+    /*
+     * We only care about uninitializing a CPU that is going offline. When a
+     * CPU comes online, the initialization is done lazily in HWACCMR0Enter().
+     */
+    AssertRelease(idCpu == RTMpCpuId());
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    switch (enmEvent)
+    {
+        case RTMPEVENT_OFFLINE:
+        {
+            int rc = hwaccmR0DisableCpu(idCpu);
+            AssertRC(rc);
+            break;
+        }
+
+        default:
+            break;
+    }
+}
+
 
 /**
@@ -1129 +1168 @@
  *
  * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   pVCpu      VMCPUD id.
+ * @param   pVM        The VM to operate on.
+ * @param   pVCpu      VMCPU handle.
  */
 VMMR0DECL(int) HWACCMR0Enter(PVM pVM, PVMCPU pVCpu)
@@ -1163 +1202 @@
         pVM->hwaccm.s.u64RegisterMask = UINT64_C(0xFFFFFFFF);
 
-    /* Enable VT-x or AMD-V if local init is required. */
-    if (!HWACCMR0Globals.fGlobalInit)
+    /* Enable VT-x or AMD-V if local init is required, or enable if it's a freshly onlined CPU. */
+    if (   !pCpu->fConfigured
+        || !HWACCMR0Globals.fGlobalInit)
     {
         rc = hwaccmR0EnableCpu(pVM, idCpu);
@@ -1198 +1238 @@
  *
  * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   pVCpu      VMCPUD id.
+ * @param   pVM        The VM to operate on.
+ * @param   pVCpu      VMCPU handle.
  */
 VMMR0DECL(int) HWACCMR0Leave(PVM pVM, PVMCPU pVCpu)