Changeset 88484 in vbox for trunk/src

Timestamp:

Apr 13, 2021 5:50:44 AM (4 years ago)

Author:

vboxsync

Message:

Intel IOMMU: bugref:9967 Main: Use the same I/O APIC PCI address reservation as we do on AMD, as it keeps things simpler than moving PCI devices/slots around again.
Also for now don't opt out of x2APIC (seems Linux VMs are configured with x2APIC by default, so let's try get things working with x2APIC).

Location:

trunk/src/VBox

Files:

• Devices/Bus/DevIommuIntel.h (modified) (1 diff)
• Devices/PC/DevACPI.cpp (modified) (7 diffs)
• Main/src-client/BusAssignmentManager.cpp (modified) (1 diff)
• Main/src-client/ConsoleImpl2.cpp (modified) (7 diffs)

trunk/src/VBox/Devices/Bus/DevIommuIntel.h

@@ -31 +31 @@
 #define DMAR_PCI_REVISION_ID                        0x01
 
-/** Feature/capability flags exposed to the guest (x2APIC Opt Out until we get
- *  regular APIC setup working). */
-#define DMAR_ACPI_DMAR_FLAGS                        (ACPI_DMAR_F_INTR_REMAP | ACPI_DMAR_F_X2APIC_OPT_OUT)
+/** Feature/capability flags exposed to the guest. */
+#define DMAR_ACPI_DMAR_FLAGS                        ACPI_DMAR_F_INTR_REMAP
 
 /** The MMIO base address of the DMAR unit (taken from real hardware). */

trunk/src/VBox/Devices/PC/DevACPI.cpp

@@ -866 +866 @@
     ACPIDMAR            Dmar;
     ACPIDRHD            Drhd;
-    /* ACPIDMARDEVSCOPE    DevScope; */
+    ACPIDMARDEVSCOPE    DevScopeIoApic;
 } ACPITBLVTD;
 #endif  /* VBOX_WITH_IOMMU_INTEL */
@@ -3385 +3385 @@
 
     /* DRHD. */
-    VtdTable.Drhd.cbLength     = sizeof(ACPIDRHD) /* + sizeof(VtdTable.DevScope) */;
+    VtdTable.Drhd.cbLength     = sizeof(ACPIDRHD);
     VtdTable.Drhd.fFlags       = ACPI_DRHD_F_INCLUDE_PCI_ALL;
     VtdTable.Drhd.uRegBaseAddr = DMAR_MMIO_BASE_PHYSADDR;
+
+    /* Device Scopes: I/O APIC. */
+    if (pThis->u8UseIOApic)
+    {
+        uint8_t const uIoApicBus = 0;
+        uint8_t const uIoApicDev = RT_HI_U16(pThis->u32SbIoApicPciAddress);
+        uint8_t const uIoApicFn  = RT_LO_U16(pThis->u32SbIoApicPciAddress);
+
+        VtdTable.DevScopeIoApic.uType          = ACPIDMARDEVSCOPE_TYPE_IOAPIC;
+        VtdTable.DevScopeIoApic.cbLength       = sizeof(ACPIDMARDEVSCOPE);
+        VtdTable.DevScopeIoApic.idEnum         = pThis->cCpus;   /* The I/O APIC ID, see u8IOApicId in acpiR3SetupMadt(). */
+        VtdTable.DevScopeIoApic.uStartBusNum   = uIoApicBus;
+        VtdTable.DevScopeIoApic.Path.uDevice   = uIoApicDev;
+        VtdTable.DevScopeIoApic.Path.uFunction = uIoApicFn;
+
+        VtdTable.Drhd.cbLength += sizeof(VtdTable.DevScopeIoApic);
+    }
 
     /* Finally, compute checksum. */
@@ -3538 +3555 @@
 #ifdef VBOX_WITH_IOMMU_INTEL
     if (pThis->fUseIommuIntel)
-        iIommu = cAddr++;      /* IOMMU (AMD) */
+        iIommu = cAddr++;      /* IOMMU (Intel) */
 #endif
 
@@ -3555 +3572 @@
     Assert(cAddr < RT_ELEMENTS(aGCPhysXsdt));
 
-    cbRsdt += cAddr*sizeof(uint32_t);  /* each entry: 32 bits phys. address. */
-    cbXsdt += cAddr*sizeof(uint64_t);  /* each entry: 64 bits phys. address. */
+    cbRsdt += cAddr * sizeof(uint32_t);  /* each entry: 32 bits phys. address. */
+    cbXsdt += cAddr * sizeof(uint64_t);  /* each entry: 64 bits phys. address. */
 
     /*
@@ -4243 +4260 @@
     {
         /* Query IOMMU AMD address (IOMA). */
-        rc = pHlp->pfnCFGMQueryU32Def(pCfg, "IommuPciAddress", &pThis->u32IommuPciAddress, 0);
+        rc = pHlp->pfnCFGMQueryU32(pCfg, "IommuPciAddress", &pThis->u32IommuPciAddress);
         if (RT_FAILURE(rc))
             return PDMDEV_SET_ERROR(pDevIns, rc, N_("Configuration error: Failed to read \"IommuPciAddress\""));
 
         /* Query southbridge I/O APIC address (required when an AMD IOMMU is configured). */
-        rc = pHlp->pfnCFGMQueryU32Def(pCfg, "SbIoApicPciAddress", &pThis->u32SbIoApicPciAddress, 0);
+        rc = pHlp->pfnCFGMQueryU32(pCfg, "SbIoApicPciAddress", &pThis->u32SbIoApicPciAddress);
         if (RT_FAILURE(rc))
             return PDMDEV_SET_ERROR(pDevIns, rc, N_("Configuration error: Failed to read \"SbIoApicAddress\""));
@@ -4258 +4275 @@
             LogRel(("ACPI: Warning! AMD IOMMU assigned the PCI host bridge address.\n"));
 
-        /* Warn if the SB IOAPIC is not at the required address if an AMD IOMMU is configured. */
+        /* Warn if the IOAPIC is not at the expected address. */
         if (pThis->u32SbIoApicPciAddress != RT_MAKE_U32(VBOX_PCI_FN_SB_IOAPIC, VBOX_PCI_DEV_SB_IOAPIC))
         {
-            /** @todo Maybe make this a VM startup failure later. */
-            LogRel(("ACPI: Warning! Southbridge I/O APIC not at %#x:%#x:%#x when an AMD IOMMU is present.\n",
+            LogRel(("ACPI: Southbridge I/O APIC not at %#x:%#x:%#x when an AMD IOMMU is present.\n",
                     VBOX_PCI_BUS_SB_IOAPIC, VBOX_PCI_DEV_SB_IOAPIC, VBOX_PCI_FN_SB_IOAPIC));
+            return PDMDEV_SET_ERROR(pDevIns, VERR_MISMATCH, N_("Configuration error: \"SbIoApicAddress\" mismatch"));
         }
     }
@@ -4277 +4294 @@
     {
         /* Query IOMMU Intel address. */
-        rc = pHlp->pfnCFGMQueryU32Def(pCfg, "IommuPciAddress", &pThis->u32IommuPciAddress, 0);
+        rc = pHlp->pfnCFGMQueryU32(pCfg, "IommuPciAddress", &pThis->u32IommuPciAddress);
         if (RT_FAILURE(rc))
             return PDMDEV_SET_ERROR(pDevIns, rc, N_("Configuration error: Failed to read \"IommuPciAddress\""));
+
+        /* Get the reserved I/O APIC PCI address (required when an Intel IOMMU is configured). */
+        rc = pHlp->pfnCFGMQueryU32(pCfg, "SbIoApicPciAddress", &pThis->u32SbIoApicPciAddress);
+        if (RT_FAILURE(rc))
+            return PDMDEV_SET_ERROR(pDevIns, rc, N_("Configuration error: Failed to read \"SbIoApicAddress\""));
+
+        /* Warn if the IOAPIC is not at the expected address. */
+        if (pThis->u32SbIoApicPciAddress != RT_MAKE_U32(VBOX_PCI_FN_SB_IOAPIC, VBOX_PCI_DEV_SB_IOAPIC))
+        {
+            LogRel(("ACPI: Southbridge I/O APIC not at %#x:%#x:%#x when an Intel IOMMU is present.\n",
+                    VBOX_PCI_BUS_SB_IOAPIC, VBOX_PCI_DEV_SB_IOAPIC, VBOX_PCI_FN_SB_IOAPIC));
+            return PDMDEV_SET_ERROR(pDevIns, VERR_MISMATCH, N_("Configuration error: \"SbIoApicAddress\" mismatch"));
+        }
     }
 #endif

trunk/src/VBox/Main/src-client/BusAssignmentManager.cpp

@@ -255 +255 @@
  * Intel IOMMU.
  * The VT-d misc, address remapping, system management device is
- * located at BDF 00:5:0 on real hardware so we mimick the same.
- * LSI logic remains at 0:20:0.
+ * located at BDF 0:5:0 on real hardware but we use 0:1:0 since that
+ * slot isn't used for anything else.
+ *
+ * While we could place the I/O APIC anywhere, we keep it consistent
+ * with the AMD IOMMU and we assign the LSI Logic controller to
+ * device number 23 (and I/O APIC at device 20).
  */
 static const DeviceAssignmentRule g_aIch9IommuIntelRules[] =
 {
     /* Intel IOMMU. */
-    {"iommu-intel",   0,  5,  0, 0},
+    {"iommu-intel",   0,  1,  0, 0},
+    /* Intel IOMMU: Reserved for I/O APIC. */
+    {"sb-ioapic",     0, 20,  0, 0},
 
     /* Storage controller */
-    {"lsilogic",      0, 20,  0, 1},
+    {"lsilogic",      0, 23,  0, 1},
     { NULL,          -1, -1, -1, 0}
 };

trunk/src/VBox/Main/src-client/ConsoleImpl2.cpp

@@ -831 +831 @@
     hrc = pMachine->COMGETTER(ParavirtDebug)(strParavirtDebug.asOutParam());                H();
 
+    BOOL fIOAPIC;
+    hrc = biosSettings->COMGETTER(IOAPICEnabled)(&fIOAPIC);                                 H();
+
     ChipsetType_T chipsetType;
     hrc = pMachine->COMGETTER(ChipsetType)(&chipsetType);                                   H();
@@ -870 +873 @@
 #endif
     }
+
     if (iommuType == IommuType_Intel)
     {
@@ -877 +881 @@
 #endif
     }
-    if (   (iommuType == IommuType_AMD || iommuType == IommuType_Intel)
-        && chipsetType != ChipsetType_ICH9)
+
+    if (   iommuType == IommuType_AMD
+        || iommuType == IommuType_Intel)
+    {
+        if (chipsetType != ChipsetType_ICH9)
             return VMR3SetError(pUVM, VERR_INVALID_PARAMETER, RT_SRC_POS,
                                 N_("IOMMU uses MSIs which requires the ICH9 chipset implementation."));
+        if (!fIOAPIC)
+            return VMR3SetError(pUVM, VERR_INVALID_PARAMETER, RT_SRC_POS,
+                            N_("IOMMU requires an I/O APIC for remapping interrupts."));
+    }
 #else
     IommuType_T const iommuType = IommuType_None;
@@ -896 +907 @@
     hrc = pMachine->COMGETTER(OSTypeId)(osTypeId.asOutParam());                             H();
     LogRel(("Guest OS type: '%s'\n", Utf8Str(osTypeId).c_str()));
-
-    BOOL fIOAPIC;
-    hrc = biosSettings->COMGETTER(IOAPICEnabled)(&fIOAPIC);                                 H();
 
     APICMode_T apicMode;
@@ -1607 +1615 @@
                 InsertConfigNode(pInst,    "Config", &pCfg);
                 hrc = pBusMgr->assignPCIDevice("iommu-intel", pInst);                       H();
+
+                /*
+                 * Reserve a specific PCI address for the I/O APIC when using
+                 * an Intel IOMMU. For convenience we use the same address as
+                 * we do on AMD, see @bugref{9967#c13}.
+                 */
+                PCIBusAddress PCIAddr = PCIBusAddress(VBOX_PCI_BUS_SB_IOAPIC, VBOX_PCI_DEV_SB_IOAPIC, VBOX_PCI_FN_SB_IOAPIC);
+                hrc = pBusMgr->assignPCIDevice("sb-ioapic", NULL /* pCfg */, PCIAddr, true /*fGuestAddressRequired*/);  H();
             }
         }
@@ -3384 +3400 @@
                     if (pBusMgr->findPCIAddress("sb-ioapic", 0, Address))
                     {
-                        uint32_t u32SbIoapicAddress = (Address.miDevice << 16) | Address.miFn;
+                        uint32_t const u32SbIoapicAddress = (Address.miDevice << 16) | Address.miFn;
                         InsertConfigInteger(pCfg, "SbIoApicPciAddress", u32SbIoapicAddress);
                     }
                     else
-                        LogRel(("IOMMU: AMD IOMMU is enabled, but southbridge I/O APIC is not assigned a PCI address!\n"));
+                        return VMR3SetError(pUVM, VERR_INVALID_PARAMETER, RT_SRC_POS,
+                                            N_("AMD IOMMU is enabled, but the I/O APIC is not assigned a PCI address!"));
                 }
             }
@@ -3399 +3416 @@
                     InsertConfigInteger(pCfg, "IommuIntelEnabled", true);
                     InsertConfigInteger(pCfg, "IommuPciAddress", u32IommuAddress);
+                    if (pBusMgr->findPCIAddress("sb-ioapic", 0, Address))
+                    {
+                        uint32_t const u32SbIoapicAddress = (Address.miDevice << 16) | Address.miFn;
+                        InsertConfigInteger(pCfg, "SbIoApicPciAddress", u32SbIoapicAddress);
+                    }
+                    else
+                        return VMR3SetError(pUVM, VERR_INVALID_PARAMETER, RT_SRC_POS,
+                                            N_("Intel IOMMU is enabled, but the I/O APIC is not assigned a PCI address!"));
                 }
             }