conf: Support multiple device-pluggable smmuv3 IOMMUs

Add support for parsing multiple IOMMU devices from
the VM definition when "smmuv3" is the IOMMU model.

Signed-off-by: Nathan Chen <nathanc@nvidia.com>
Signed-off-by: Ján Tomko <jtomko@redhat.com>
Reviewed-by: Ján Tomko <jtomko@redhat.com>

src/conf/domain_conf.c

@@ -4141,7 +4141,9 @@ void virDomainDefFree(virDomainDef *def)
         virDomainCryptoDefFree(def->cryptos[i]);
     g_free(def->cryptos);
 
-    virDomainIOMMUDefFree(def->iommu);
+    for (i = 0; i < def->niommus; i++)
+        virDomainIOMMUDefFree(def->iommus[i]);
+    g_free(def->iommus);
 
     virDomainPstoreDefFree(def->pstore);
 
@@ -5013,9 +5015,9 @@ virDomainDeviceInfoIterateFlags(virDomainDef *def,
     }
 
     device.type = VIR_DOMAIN_DEVICE_IOMMU;
-    if (def->iommu) {
-        device.data.iommu = def->iommu;
-        if ((rc = cb(def, &device, &def->iommu->info, opaque)) != 0)
+    for (i = 0; i < def->niommus; i++) {
+        device.data.iommu = def->iommus[i];
+        if ((rc = cb(def, &device, &def->iommus[i]->info, opaque)) != 0)
             return rc;
     }
 
@@ -16540,6 +16542,42 @@ virDomainInputDefFind(const virDomainDef *def,
 }
 
 
+bool
+virDomainIOMMUDefEquals(const virDomainIOMMUDef *a,
+                        const virDomainIOMMUDef *b)
+{
+    if (a->model != b->model ||
+        a->intremap != b->intremap ||
+        a->caching_mode != b->caching_mode ||
+        a->eim != b->eim ||
+        a->iotlb != b->iotlb ||
+        a->aw_bits != b->aw_bits ||
+        a->dma_translation != b->dma_translation)
+        return false;
+
+    if (a->info.type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE &&
+        !virDomainDeviceInfoAddressIsEqual(&a->info, &b->info))
+        return false;
+
+    return true;
+}
+
+
+ssize_t
+virDomainIOMMUDefFind(const virDomainDef *def,
+                      const virDomainIOMMUDef *iommu)
+{
+    size_t i;
+
+    for (i = 0; i < def->niommus; i++) {
+        if (virDomainIOMMUDefEquals(iommu, def->iommus[i]))
+            return i;
+    }
+
+    return -1;
+}
+
+
 bool
 virDomainVsockDefEquals(const virDomainVsockDef *a,
                         const virDomainVsockDef *b)
@@ -20209,19 +20247,22 @@ virDomainDefParseXML(xmlXPathContextPtr ctxt,
     }
     VIR_FREE(nodes);
 
+    /* Parsing iommu device definitions */
     if ((n = virXPathNodeSet("./devices/iommu", ctxt, &nodes)) < 0)
         return NULL;
 
-    if (n > 1) {
-        virReportError(VIR_ERR_XML_ERROR, "%s",
-                       _("only a single IOMMU device is supported"));
-        return NULL;
-    }
+    if (n > 0)
+        def->iommus = g_new0(virDomainIOMMUDef *, n);
 
-    if (n > 0) {
-        if (!(def->iommu = virDomainIOMMUDefParseXML(xmlopt, nodes[0],
-                                                     ctxt, flags)))
+    for (i = 0; i < n; i++) {
+        virDomainIOMMUDef *iommu;
+
+        iommu = virDomainIOMMUDefParseXML(xmlopt, nodes[i], ctxt, flags);
+
+        if (!iommu)
             return NULL;
+
+        def->iommus[def->niommus++] = iommu;
     }
     VIR_FREE(nodes);
 
@@ -22700,15 +22741,17 @@ virDomainDefCheckABIStabilityFlags(virDomainDef *src,
             goto error;
     }
 
-    if (!!src->iommu != !!dst->iommu) {
-        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
-                       _("Target domain IOMMU device count does not match source"));
+    if (src->niommus != dst->niommus) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                       _("Target domain IOMMU device count %1$zu does not match source %2$zu"),
+                       dst->niommus, src->niommus);
         goto error;
     }
 
-    if (src->iommu &&
-        !virDomainIOMMUDefCheckABIStability(src->iommu, dst->iommu))
-        goto error;
+    for (i = 0; i < src->niommus; i++) {
+        if (!virDomainIOMMUDefCheckABIStability(src->iommus[i], dst->iommus[i]))
+            goto error;
+    }
 
     if (!!src->vsock != !!dst->vsock) {
         virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
@@ -29568,8 +29611,9 @@ virDomainDefFormatInternalSetRootName(virDomainDef *def,
     for (n = 0; n < def->ncryptos; n++) {
         virDomainCryptoDefFormat(buf, def->cryptos[n], flags);
     }
-    if (def->iommu)
-        virDomainIOMMUDefFormat(buf, def->iommu);
+
+    for (n = 0; n < def->niommus; n++)
+        virDomainIOMMUDefFormat(buf, def->iommus[n]);
 
     if (def->vsock)
         virDomainVsockDefFormat(buf, def->vsock);

src/conf/domain_conf.h

@@ -3304,6 +3304,9 @@ struct _virDomainDef {
     size_t nwatchdogs;
     virDomainWatchdogDef **watchdogs;
 
+    size_t niommus;
+    virDomainIOMMUDef **iommus;
+
     /* At maximum 2 TPMs on the domain if a TPM Proxy is present. */
     size_t ntpms;
     virDomainTPMDef **tpms;
@@ -3313,7 +3316,6 @@ struct _virDomainDef {
     virDomainNVRAMDef *nvram;
     virCPUDef *cpu;
     virDomainRedirFilterDef *redirfilter;
-    virDomainIOMMUDef *iommu;
     virDomainVsockDef *vsock;
     virDomainPstoreDef *pstore;
 
@@ -4321,6 +4323,11 @@ virDomainShmemDef *virDomainShmemDefRemove(virDomainDef *def, size_t idx)
 ssize_t virDomainInputDefFind(const virDomainDef *def,
                               const virDomainInputDef *input)
     ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) G_GNUC_WARN_UNUSED_RESULT;
+bool virDomainIOMMUDefEquals(const virDomainIOMMUDef *a,
+                             const virDomainIOMMUDef *b);
+ssize_t virDomainIOMMUDefFind(const virDomainDef *def,
+                              const virDomainIOMMUDef *iommu)
+    ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) G_GNUC_WARN_UNUSED_RESULT;
 bool virDomainVsockDefEquals(const virDomainVsockDef *a,
                              const virDomainVsockDef *b)
     ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) G_GNUC_WARN_UNUSED_RESULT;

src/conf/domain_validate.c

@@ -1844,21 +1844,28 @@ virDomainDefCputuneValidate(const virDomainDef *def)
 static int
 virDomainDefIOMMUValidate(const virDomainDef *def)
 {
-    if (!def->iommu)
-        return 0;
+    size_t i;
 
-    if (def->iommu->intremap == VIR_TRISTATE_SWITCH_ON &&
-        def->features[VIR_DOMAIN_FEATURE_IOAPIC] != VIR_DOMAIN_IOAPIC_QEMU) {
-        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
-                       _("IOMMU interrupt remapping requires split I/O APIC (ioapic driver='qemu')"));
-        return -1;
-    }
+    for (i = 0; i < def->niommus; i++) {
+        virDomainIOMMUDef *iommu = def->iommus[i];
+        if (def->niommus > 1 && iommu->model != VIR_DOMAIN_IOMMU_MODEL_SMMUV3) {
+            virReportError(VIR_ERR_XML_ERROR, "%s",
+                           _("IOMMU model smmuv3 must be specified for multiple IOMMU definitions"));
+        }
 
-    if (def->iommu->eim == VIR_TRISTATE_SWITCH_ON &&
-        def->iommu->intremap != VIR_TRISTATE_SWITCH_ON) {
-        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
-                       _("IOMMU eim requires interrupt remapping to be enabled"));
-        return -1;
+        if (iommu->intremap == VIR_TRISTATE_SWITCH_ON &&
+            def->features[VIR_DOMAIN_FEATURE_IOAPIC] != VIR_DOMAIN_IOAPIC_QEMU) {
+            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                           _("IOMMU interrupt remapping requires split I/O APIC (ioapic driver='qemu')"));
+            return -1;
+        }
+
+        if (iommu->eim == VIR_TRISTATE_SWITCH_ON &&
+            iommu->intremap != VIR_TRISTATE_SWITCH_ON) {
+            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                           _("IOMMU eim requires interrupt remapping to be enabled"));
+            return -1;
+        }
     }
 
     return 0;

src/conf/schemas/domaincommon.rng

@@ -6969,9 +6969,9 @@
         <zeroOrMore>
           <ref name="panic"/>
         </zeroOrMore>
-        <optional>
+        <zeroOrMore>
           <ref name="iommu"/>
-        </optional>
+        </zeroOrMore>
         <optional>
           <ref name="vsock"/>
         </optional>

src/libvirt_private.syms

@@ -495,6 +495,8 @@ virDomainInputSourceGrabToggleTypeToString;
 virDomainInputSourceGrabTypeFromString;
 virDomainInputSourceGrabTypeToString;
 virDomainInputTypeToString;
+virDomainIOMMUDefEquals;
+virDomainIOMMUDefFind;
 virDomainIOMMUDefFree;
 virDomainIOMMUDefNew;
 virDomainIOMMUModelTypeFromString;

src/qemu/qemu_alias.c

@@ -650,10 +650,14 @@ qemuAssignDeviceVsockAlias(virDomainVsockDef *vsock)
 
 
 static void
-qemuAssignDeviceIOMMUAlias(virDomainIOMMUDef *iommu)
+qemuAssignDeviceIOMMUAlias(virDomainDef *def,
+                           virDomainIOMMUDef **iommu)
 {
-    if (!iommu->info.alias)
-        iommu->info.alias = g_strdup("iommu0");
+    size_t i;
+    for (i = 0; i < def->niommus; i++) {
+        if (!iommu[i]->info.alias)
+            iommu[i]->info.alias = g_strdup_printf("iommu%zu", i);
+    }
 }
 
 
@@ -769,8 +773,9 @@ qemuAssignDeviceAliases(virDomainDef *def)
     if (def->vsock) {
         qemuAssignDeviceVsockAlias(def->vsock);
     }
-    if (def->iommu)
-        qemuAssignDeviceIOMMUAlias(def->iommu);
+    if (def->niommus > 0) {
+        qemuAssignDeviceIOMMUAlias(def, def->iommus);
+    }
     for (i = 0; i < def->ncryptos; i++) {
         qemuAssignDeviceCryptoAlias(def, def->cryptos[i]);
     }

src/qemu/qemu_command.c

@@ -6244,84 +6244,83 @@ qemuBuildIOMMUCommandLine(virCommand *cmd,
                           const virDomainDef *def,
                           virQEMUCaps *qemuCaps)
 {
-    g_autoptr(virJSONValue) props = NULL;
-    g_autoptr(virJSONValue) wrapperProps = NULL;
-    const virDomainIOMMUDef *iommu = def->iommu;
+    size_t i;
 
-    if (!iommu)
-        return 0;
+    for (i = 0; i < def->niommus; i++) {
+        virDomainIOMMUDef *iommu = def->iommus[i];
+        g_autoptr(virJSONValue) props = NULL;
+        g_autoptr(virJSONValue) wrapperProps = NULL;
 
-    switch (iommu->model) {
-    case VIR_DOMAIN_IOMMU_MODEL_INTEL:
-        if (virJSONValueObjectAdd(&props,
-                                  "s:driver", "intel-iommu",
-                                  "s:id", iommu->info.alias,
-                                  "S:intremap", qemuOnOffAuto(iommu->intremap),
-                                  "T:caching-mode", iommu->caching_mode,
-                                  "S:eim", qemuOnOffAuto(iommu->eim),
-                                  "T:device-iotlb", iommu->iotlb,
-                                  "z:aw-bits", iommu->aw_bits,
-                                  "T:dma-translation", iommu->dma_translation,
-                                  NULL) < 0)
-            return -1;
+        switch (iommu->model) {
+        case VIR_DOMAIN_IOMMU_MODEL_INTEL:
+            if (virJSONValueObjectAdd(&props,
+                                      "s:driver", "intel-iommu",
+                                      "s:id", iommu->info.alias,
+                                      "S:intremap", qemuOnOffAuto(iommu->intremap),
+                                      "T:caching-mode", iommu->caching_mode,
+                                      "S:eim", qemuOnOffAuto(iommu->eim),
+                                      "T:device-iotlb", iommu->iotlb,
+                                      "z:aw-bits", iommu->aw_bits,
+                                      "T:dma-translation", iommu->dma_translation,
+                                      NULL) < 0)
+                return -1;
 
-        if (qemuBuildDeviceCommandlineFromJSON(cmd, props, def, qemuCaps) < 0)
-            return -1;
+            if (qemuBuildDeviceCommandlineFromJSON(cmd, props, def, qemuCaps) < 0)
+                return -1;
 
-        return 0;
+            break;
+        case VIR_DOMAIN_IOMMU_MODEL_VIRTIO:
+            if (virJSONValueObjectAdd(&props,
+                                      "s:driver", "virtio-iommu",
+                                      "s:id", iommu->info.alias,
+                                      NULL) < 0) {
+                return -1;
+            }
 
-    case VIR_DOMAIN_IOMMU_MODEL_VIRTIO:
-        if (virJSONValueObjectAdd(&props,
-                                  "s:driver", "virtio-iommu",
-                                  "s:id", iommu->info.alias,
-                                  NULL) < 0) {
+            if (qemuBuildDeviceAddressProps(props, def, &iommu->info) < 0)
+                return -1;
+
+            if (qemuBuildDeviceCommandlineFromJSON(cmd, props, def, qemuCaps) < 0)
+                return -1;
+
+            break;
+        case VIR_DOMAIN_IOMMU_MODEL_SMMUV3:
+            /* There is no -device for SMMUv3, so nothing to be done here */
+            break;
+
+        case VIR_DOMAIN_IOMMU_MODEL_AMD:
+            if (virJSONValueObjectAdd(&wrapperProps,
+                                      "s:driver", "AMDVI-PCI",
+                                      "s:id", iommu->info.alias,
+                                      NULL) < 0)
+                return -1;
+
+            if (qemuBuildDeviceAddressProps(wrapperProps, def, &iommu->info) < 0)
+                return -1;
+
+            if (qemuBuildDeviceCommandlineFromJSON(cmd, wrapperProps, def, qemuCaps) < 0)
+                return -1;
+
+            if (virJSONValueObjectAdd(&props,
+                                      "s:driver", "amd-iommu",
+                                      "s:pci-id", iommu->info.alias,
+                                      "S:intremap", qemuOnOffAuto(iommu->intremap),
+                                      "T:pt", iommu->pt,
+                                      "T:xtsup", iommu->xtsup,
+                                      "T:device-iotlb", iommu->iotlb,
+                                      NULL) < 0)
+                return -1;
+
+            if (qemuBuildDeviceCommandlineFromJSON(cmd, props, def, qemuCaps) < 0)
+                return -1;
+
+            break;
+
+        case VIR_DOMAIN_IOMMU_MODEL_LAST:
+        default:
+            virReportEnumRangeError(virDomainIOMMUModel, iommu->model);
             return -1;
         }
-
-        if (qemuBuildDeviceAddressProps(props, def, &iommu->info) < 0)
-            return -1;
-
-        if (qemuBuildDeviceCommandlineFromJSON(cmd, props, def, qemuCaps) < 0)
-            return -1;
-
-        return 0;
-
-    case VIR_DOMAIN_IOMMU_MODEL_SMMUV3:
-        /* There is no -device for SMMUv3, so nothing to be done here */
-        return 0;
-
-    case VIR_DOMAIN_IOMMU_MODEL_AMD:
-        if (virJSONValueObjectAdd(&wrapperProps,
-                                  "s:driver", "AMDVI-PCI",
-                                  "s:id", iommu->info.alias,
-                                  NULL) < 0)
-            return -1;
-
-        if (qemuBuildDeviceAddressProps(wrapperProps, def, &iommu->info) < 0)
-            return -1;
-
-        if (qemuBuildDeviceCommandlineFromJSON(cmd, wrapperProps, def, qemuCaps) < 0)
-            return -1;
-
-        if (virJSONValueObjectAdd(&props,
-                                  "s:driver", "amd-iommu",
-                                  "s:pci-id", iommu->info.alias,
-                                  "S:intremap", qemuOnOffAuto(iommu->intremap),
-                                  "T:pt", iommu->pt,
-                                  "T:xtsup", iommu->xtsup,
-                                  "T:device-iotlb", iommu->iotlb,
-                                  NULL) < 0)
-            return -1;
-
-        if (qemuBuildDeviceCommandlineFromJSON(cmd, props, def, qemuCaps) < 0)
-            return -1;
-
-        return 0;
-
-    case VIR_DOMAIN_IOMMU_MODEL_LAST:
-    default:
-        virReportEnumRangeError(virDomainIOMMUModel, iommu->model);
-        return -1;
     }
 
     return 0;
@@ -7158,8 +7157,8 @@ qemuBuildMachineCommandLine(virCommand *cmd,
     if (qemuAppendDomainFeaturesMachineParam(&buf, def, qemuCaps) < 0)
         return -1;
 
-    if (def->iommu) {
-        switch (def->iommu->model) {
+    if (def->niommus == 1) {
+        switch (def->iommus[0]->model) {
         case VIR_DOMAIN_IOMMU_MODEL_SMMUV3:
             virBufferAddLit(&buf, ",iommu=smmuv3");
             break;
@@ -7172,7 +7171,7 @@ qemuBuildMachineCommandLine(virCommand *cmd,
 
         case VIR_DOMAIN_IOMMU_MODEL_LAST:
         default:
-            virReportEnumRangeError(virDomainIOMMUModel, def->iommu->model);
+            virReportEnumRangeError(virDomainIOMMUModel, def->iommus[0]->model);
             return -1;
         }
     }

src/qemu/qemu_domain.c

@@ -8371,7 +8371,7 @@ qemuDomainGetMemLockLimitBytes(virDomainDef *def)
         int factor = nvdpa + nnvme;
 
         if (nvfio) {
-            if (def->iommu)
+            if (def->niommus > 0)
                 factor += nvfio;
             else
                 factor += 1;

src/qemu/qemu_domain_address.c

@@ -2396,9 +2396,8 @@ qemuDomainAssignDevicePCISlots(virDomainDef *def,
         /* Nada - none are PCI based (yet) */
     }
 
-    if (def->iommu) {
-        virDomainIOMMUDef *iommu = def->iommu;
-
+    for (i = 0; i < def->niommus; i++) {
+        virDomainIOMMUDef *iommu = def->iommus[i];
         switch (iommu->model) {
         case VIR_DOMAIN_IOMMU_MODEL_VIRTIO:
         case VIR_DOMAIN_IOMMU_MODEL_AMD:

src/qemu/qemu_driver.c

@@ -6902,12 +6902,12 @@ qemuDomainAttachDeviceConfig(virDomainDef *vmdef,
         break;
 
     case VIR_DOMAIN_DEVICE_IOMMU:
-        if (vmdef->iommu) {
+        if (vmdef->niommus > 0) {
             virReportError(VIR_ERR_OPERATION_INVALID, "%s",
                            _("domain already has an iommu device"));
             return -1;
         }
-        vmdef->iommu = g_steal_pointer(&dev->data.iommu);
+        VIR_APPEND_ELEMENT(vmdef->iommus, vmdef->niommus, dev->data.iommu);
         break;
 
     case VIR_DOMAIN_DEVICE_VIDEO:
@@ -7121,12 +7121,12 @@ qemuDomainDetachDeviceConfig(virDomainDef *vmdef,
         break;
 
     case VIR_DOMAIN_DEVICE_IOMMU:
-        if (!vmdef->iommu) {
+        if ((idx = virDomainIOMMUDefFind(vmdef, dev->data.iommu)) < 0) {
             virReportError(VIR_ERR_OPERATION_FAILED, "%s",
                            _("matching iommu device not found"));
             return -1;
         }
-        g_clear_pointer(&vmdef->iommu, virDomainIOMMUDefFree);
+        VIR_DELETE_ELEMENT(vmdef->iommus, idx, vmdef->niommus);
         break;
 
     case VIR_DOMAIN_DEVICE_VIDEO:

src/qemu/qemu_postparse.c

@@ -1470,7 +1470,7 @@ qemuDomainDefAddDefaultDevices(virQEMUDriver *driver,
         }
     }
 
-    if (addIOMMU && !def->iommu &&
+    if (addIOMMU && !def->iommus && def->niommus == 0 &&
         virQEMUCapsGet(qemuCaps, QEMU_CAPS_DEVICE_INTEL_IOMMU) &&
         virQEMUCapsGet(qemuCaps, QEMU_CAPS_INTEL_IOMMU_INTREMAP) &&
         virQEMUCapsGet(qemuCaps, QEMU_CAPS_INTEL_IOMMU_EIM)) {
@@ -1482,7 +1482,8 @@ qemuDomainDefAddDefaultDevices(virQEMUDriver *driver,
         iommu->intremap = VIR_TRISTATE_SWITCH_ON;
         iommu->eim = VIR_TRISTATE_SWITCH_ON;
 
-        def->iommu = g_steal_pointer(&iommu);
+        def->iommus = g_new0(virDomainIOMMUDef *, 1);
+        def->iommus[def->niommus++] = g_steal_pointer(&iommu);
     }
 
     if (qemuDomainDefAddDefaultAudioBackend(driver, def) < 0)
@@ -1558,8 +1559,8 @@ qemuDomainDefEnableDefaultFeatures(virDomainDef *def,
      * domain already has IOMMU without inremap. This will be fixed in
      * qemuDomainIOMMUDefPostParse() but there domain definition can't be
      * modified so change it now. */
-    if (def->iommu &&
-        (def->iommu->intremap == VIR_TRISTATE_SWITCH_ON ||
+    if (def->iommus && def->niommus == 1 &&
+        (def->iommus[0]->intremap == VIR_TRISTATE_SWITCH_ON ||
          qemuDomainNeedsIOMMUWithEIM(def)) &&
         def->features[VIR_DOMAIN_FEATURE_IOAPIC] == VIR_DOMAIN_IOAPIC_NONE) {
         def->features[VIR_DOMAIN_FEATURE_IOAPIC] = VIR_DOMAIN_IOAPIC_QEMU;

src/qemu/qemu_validate.c

@@ -907,7 +907,7 @@ qemuValidateDomainVCpuTopology(const virDomainDef *def, virQEMUCaps *qemuCaps)
                            QEMU_MAX_VCPUS_WITHOUT_EIM);
             return -1;
         }
-        if (!def->iommu || def->iommu->eim != VIR_TRISTATE_SWITCH_ON) {
+        if (!def->iommus || def->iommus[0]->eim != VIR_TRISTATE_SWITCH_ON) {
             virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
                            _("more than %1$d vCPUs require extended interrupt mode enabled on the iommu device"),
                            QEMU_MAX_VCPUS_WITHOUT_EIM);