qemu: add vdpa support

Enable <interface type='vdpa'> for qemu domains. This provides basic
support and does not support hotplug or migration.

Signed-off-by: Jonathon Jongsma <jjongsma@redhat.com>
Reviewed-by: Laine Stump <laine@redhat.com>

src/qemu/qemu_command.c

@@ -3431,7 +3431,8 @@ qemuBuildHostNetStr(virDomainNetDefPtr net,
                     size_t tapfdSize,
                     char **vhostfd,
                     size_t vhostfdSize,
-                    const char *slirpfd)
+                    const char *slirpfd,
+                    const char *vdpadev)
 {
     bool is_tap = false;
     virDomainNetType netType = virDomainNetGetActualType(net);
@@ -3570,6 +3571,12 @@ qemuBuildHostNetStr(virDomainNetDefPtr net,
         break;
 
     case VIR_DOMAIN_NET_TYPE_VDPA:
+        /* Caller will pass the fd to qemu with add-fd */
+        if (virJSONValueObjectCreate(&netprops, "s:type", "vhost-vdpa", NULL) < 0 ||
+            virJSONValueObjectAppendString(netprops, "vhostdev", vdpadev) < 0)
+            return NULL;
+        break;
+
     case VIR_DOMAIN_NET_TYPE_HOSTDEV:
         /* Should have been handled earlier via PCI/USB hotplug code. */
     case VIR_DOMAIN_NET_TYPE_LAST:
@@ -7902,6 +7909,8 @@ qemuBuildInterfaceCommandLine(virQEMUDriverPtr driver,
     char **tapfdName = NULL;
     char **vhostfdName = NULL;
     g_autofree char *slirpfdName = NULL;
+    g_autofree char *vdpafdName = NULL;
+    int vdpafd = -1;
     virDomainNetType actualType = virDomainNetGetActualType(net);
     const virNetDevBandwidth *actualBandwidth;
     bool requireNicdev = false;
@@ -7984,13 +7993,17 @@ qemuBuildInterfaceCommandLine(virQEMUDriverPtr driver,
 
         break;
 
+    case VIR_DOMAIN_NET_TYPE_VDPA:
+        if ((vdpafd = qemuInterfaceVDPAConnect(net)) < 0)
+            goto cleanup;
+        break;
+
     case VIR_DOMAIN_NET_TYPE_USER:
     case VIR_DOMAIN_NET_TYPE_SERVER:
     case VIR_DOMAIN_NET_TYPE_CLIENT:
     case VIR_DOMAIN_NET_TYPE_MCAST:
     case VIR_DOMAIN_NET_TYPE_INTERNAL:
     case VIR_DOMAIN_NET_TYPE_UDP:
-    case VIR_DOMAIN_NET_TYPE_VDPA:
     case VIR_DOMAIN_NET_TYPE_LAST:
         /* nada */
         break;
@@ -8108,13 +8121,29 @@ qemuBuildInterfaceCommandLine(virQEMUDriverPtr driver,
         vhostfd[i] = -1;
     }
 
+    if (vdpafd > 0) {
+        g_autofree char *fdset = NULL;
+        g_autofree char *addfdarg = NULL;
+
+        virCommandPassFD(cmd, vdpafd, VIR_COMMAND_PASS_FD_CLOSE_PARENT);
+        fdset = qemuVirCommandGetFDSet(cmd, vdpafd);
+        if (!fdset)
+            goto cleanup;
+        vdpafdName = qemuVirCommandGetDevSet(cmd, vdpafd);
+        /* set opaque to the devicepath so that we can look up the fdset later
+         * if necessary */
+        addfdarg = g_strdup_printf("%s,opaque=%s", fdset,
+                                   net->data.vdpa.devicepath);
+        virCommandAddArgList(cmd, "-add-fd", addfdarg, NULL);
+    }
+
     if (chardev)
         virCommandAddArgList(cmd, "-chardev", chardev, NULL);
 
     if (!(hostnetprops = qemuBuildHostNetStr(net,
                                              tapfdName, tapfdSize,
                                              vhostfdName, vhostfdSize,
-                                             slirpfdName)))
+                                             slirpfdName, vdpafdName)))
         goto cleanup;
 
     if (!(host = virQEMUBuildNetdevCommandlineFromJSON(hostnetprops,

src/qemu/qemu_command.h

@@ -99,7 +99,8 @@ virJSONValuePtr qemuBuildHostNetStr(virDomainNetDefPtr net,
                                     size_t tapfdSize,
                                     char **vhostfd,
                                     size_t vhostfdSize,
-                                    const char *slirpfd);
+                                    const char *slirpfd,
+                                    const char *vdpadev);
 
 /* Current, best practice */
 char *qemuBuildNicDevStr(virDomainDefPtr def,

src/qemu/qemu_domain.c

@@ -5120,8 +5120,10 @@ qemuDomainDeviceNetDefPostParse(virDomainNetDefPtr net,
                                 const virDomainDef *def,
                                 virQEMUCapsPtr qemuCaps)
 {
-    if (net->type != VIR_DOMAIN_NET_TYPE_VDPA &&
+    if (net->type == VIR_DOMAIN_NET_TYPE_VDPA &&
-        net->type != VIR_DOMAIN_NET_TYPE_HOSTDEV &&
+        !virDomainNetGetModelString(net))
+        net->model = VIR_DOMAIN_NET_MODEL_VIRTIO;
+    else if (net->type != VIR_DOMAIN_NET_TYPE_HOSTDEV &&
         !virDomainNetGetModelString(net) &&
         virDomainNetResolveActualType(net) != VIR_DOMAIN_NET_TYPE_HOSTDEV)
         net->model = qemuDomainDefaultNetModel(def, qemuCaps);

src/qemu/qemu_hotplug.c

@@ -1389,7 +1389,7 @@ qemuDomainAttachNetDevice(virQEMUDriverPtr driver,
     if (!(netprops = qemuBuildHostNetStr(net,
                                          tapfdName, tapfdSize,
                                          vhostfdName, vhostfdSize,
-                                         slirpfdName)))
+                                         slirpfdName, NULL)))
         goto cleanup;
 
     qemuDomainObjEnterMonitor(driver, vm);
@@ -3488,10 +3488,11 @@ qemuDomainChangeNet(virQEMUDriverPtr driver,
     olddev = *devslot;
 
     oldType = virDomainNetGetActualType(olddev);
-    if (oldType == VIR_DOMAIN_NET_TYPE_HOSTDEV) {
+    if (oldType == VIR_DOMAIN_NET_TYPE_HOSTDEV ||
-        /* no changes are possible to a type='hostdev' interface */
+        oldType == VIR_DOMAIN_NET_TYPE_VDPA) {
+        /* no changes are possible to a type='hostdev' or type='vdpa' interface */
         virReportError(VIR_ERR_OPERATION_UNSUPPORTED,
-                       _("cannot change config of '%s' network type"),
+                       _("cannot change config of '%s' network interface type"),
                        virDomainNetTypeToString(oldType));
         goto cleanup;
     }
@@ -3676,8 +3677,9 @@ qemuDomainChangeNet(virQEMUDriverPtr driver,
 
     newType = virDomainNetGetActualType(newdev);
 
-    if (newType == VIR_DOMAIN_NET_TYPE_HOSTDEV) {
+    if (newType == VIR_DOMAIN_NET_TYPE_HOSTDEV ||
-        /* can't turn it into a type='hostdev' interface */
+        newType == VIR_DOMAIN_NET_TYPE_VDPA) {
+        /* can't turn it into a type='hostdev' or type='vdpa' interface */
         virReportError(VIR_ERR_OPERATION_UNSUPPORTED,
                        _("cannot change network interface type to '%s'"),
                        virDomainNetTypeToString(newType));

src/qemu/qemu_interface.c

@@ -638,6 +638,29 @@ qemuInterfaceBridgeConnect(virDomainDefPtr def,
 }
 
 
+/* qemuInterfaceVDPAConnect:
+ * @net: pointer to the VM's interface description
+ *
+ * returns: file descriptor of the vdpa device
+ *
+ * Called *only* called if actualType is VIR_DOMAIN_NET_TYPE_VDPA
+ */
+int
+qemuInterfaceVDPAConnect(virDomainNetDefPtr net)
+{
+    int fd;
+
+    if ((fd = open(net->data.vdpa.devicepath, O_RDWR)) < 0) {
+        virReportSystemError(errno,
+                             _("Unable to open '%s' for vdpa device"),
+                             net->data.vdpa.devicepath);
+        return -1;
+    }
+
+    return fd;
+}
+
+
 qemuSlirpPtr
 qemuInterfacePrepareSlirp(virQEMUDriverPtr driver,
                           virDomainNetDefPtr net)

src/qemu/qemu_interface.h

@@ -58,3 +58,5 @@ int qemuInterfaceOpenVhostNet(virDomainDefPtr def,
 
 qemuSlirpPtr qemuInterfacePrepareSlirp(virQEMUDriverPtr driver,
                                        virDomainNetDefPtr net);
+
+int qemuInterfaceVDPAConnect(virDomainNetDefPtr net) G_GNUC_NO_INLINE;

src/qemu/qemu_migration.c

@@ -1367,7 +1367,15 @@ qemuMigrationSrcIsAllowed(virQEMUDriverPtr driver,
 
         for (i = 0; i < vm->def->nnets; i++) {
             virDomainNetDefPtr net = vm->def->nets[i];
-            qemuSlirpPtr slirp = QEMU_DOMAIN_NETWORK_PRIVATE(net)->slirp;
+            qemuSlirpPtr slirp;
+
+            if (net->type == VIR_DOMAIN_NET_TYPE_VDPA) {
+                virReportError(VIR_ERR_OPERATION_INVALID, "%s",
+                               _("vDPA devices cannot be migrated"));
+                return false;
+            }
+
+            slirp = QEMU_DOMAIN_NETWORK_PRIVATE(net)->slirp;
 
             if (slirp && !qemuSlirpHasFeature(slirp, QEMU_SLIRP_FEATURE_MIGRATE)) {
                 virReportError(VIR_ERR_OPERATION_INVALID, "%s",

src/qemu/qemu_validate.c

@@ -1429,6 +1429,20 @@ qemuValidateDomainDeviceDefNetwork(const virDomainNetDef *net,
                 }
             }
         }
+    } else if (net->type == VIR_DOMAIN_NET_TYPE_VDPA) {
+        if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_NETDEV_VHOST_VDPA)) {
+            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                           _("vDPA devices are not supported with this QEMU binary"));
+            return -1;
+        }
+
+        if (net->model != VIR_DOMAIN_NET_MODEL_VIRTIO) {
+            virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                           _("invalid model for interface of type '%s': '%s'"),
+                           virDomainNetTypeToString(net->type),
+                           virDomainNetModelTypeToString(net->model));
+            return -1;
+        }
     } else if (net->guestIP.nroutes || net->guestIP.nips) {
         virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                        _("Invalid attempt to set network interface "

tests/qemuxml2argvdata/net-vdpa.x86_64-latest.args

@@ -0,0 +1,38 @@
+LC_ALL=C \
+PATH=/bin \
+HOME=/tmp/lib/domain--1-QEMUGuest1 \
+USER=test \
+LOGNAME=test \
+XDG_DATA_HOME=/tmp/lib/domain--1-QEMUGuest1/.local/share \
+XDG_CACHE_HOME=/tmp/lib/domain--1-QEMUGuest1/.cache \
+XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
+QEMU_AUDIO_DRV=none \
+/usr/bin/qemu-system-i386 \
+-name guest=QEMUGuest1,debug-threads=on \
+-S \
+-object secret,id=masterKey0,format=raw,\
+file=/tmp/lib/domain--1-QEMUGuest1/master-key.aes \
+-machine pc,accel=tcg,usb=off,dump-guest-core=off,memory-backend=pc.ram \
+-cpu qemu64 \
+-m 214 \
+-object memory-backend-ram,id=pc.ram,size=224395264 \
+-overcommit mem-lock=off \
+-smp 1,sockets=1,cores=1,threads=1 \
+-uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
+-display none \
+-no-user-config \
+-nodefaults \
+-chardev socket,id=charmonitor,fd=1729,server,nowait \
+-mon chardev=charmonitor,id=monitor,mode=control \
+-rtc base=utc \
+-no-shutdown \
+-no-acpi \
+-boot strict=on \
+-device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 \
+-add-fd set=0,fd=1732,opaque=/dev/vhost-vdpa-0 \
+-netdev vhost-vdpa,vhostdev=/dev/fdset/0,id=hostnet0 \
+-device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:95:db:c0,bus=pci.0,\
+addr=0x2 \
+-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,\
+resourcecontrol=deny \
+-msg timestamp=on

tests/qemuxml2argvdata/net-vdpa.xml

@@ -0,0 +1,28 @@
+<domain type='qemu'>
+  <name>QEMUGuest1</name>
+  <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid>
+  <memory unit='KiB'>219136</memory>
+  <currentMemory unit='KiB'>219136</currentMemory>
+  <vcpu placement='static'>1</vcpu>
+  <os>
+    <type arch='i686' machine='pc'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu-system-i386</emulator>
+    <controller type='usb' index='0'/>
+    <controller type='ide' index='0'/>
+    <controller type='pci' index='0' model='pci-root'/>
+    <interface type='vdpa'>
+      <mac address='52:54:00:95:db:c0'/>
+      <source dev='/dev/vhost-vdpa-0'/>
+    </interface>
+    <input type='mouse' bus='ps2'/>
+    <input type='keyboard' bus='ps2'/>
+    <memballoon model='none'/>
+  </devices>
+</domain>

tests/qemuxml2argvmock.c

@@ -195,7 +195,7 @@ virHostGetDRMRenderNode(void)
 
 static void (*real_virCommandPassFD)(virCommandPtr cmd, int fd, unsigned int flags);
 
-static const int testCommandPassSafeFDs[] = { 1730, 1731 };
+static const int testCommandPassSafeFDs[] = { 1730, 1731, 1732 };
 
 void
 virCommandPassFD(virCommandPtr cmd,
@@ -281,3 +281,12 @@ virNetDevSetRootQDisc(const char *ifname G_GNUC_UNUSED,
 {
     return 0;
 }
+
+
+int
+qemuInterfaceVDPAConnect(virDomainNetDefPtr net G_GNUC_UNUSED)
+{
+    if (fcntl(1732, F_GETFD) != -1)
+        abort();
+    return 1732;
+}

tests/qemuxml2argvtest.c

@@ -1491,6 +1491,7 @@ mymain(void)
             QEMU_CAPS_DEVICE_VFIO_PCI);
     DO_TEST_FAILURE("net-hostdev-fail",
                     QEMU_CAPS_DEVICE_VFIO_PCI);
+    DO_TEST_CAPS_LATEST("net-vdpa");
 
     DO_TEST("hostdev-pci-multifunction",
             QEMU_CAPS_KVM,

tests/qemuxml2xmloutdata/net-vdpa.xml

@@ -0,0 +1,34 @@
+<domain type='qemu'>
+  <name>QEMUGuest1</name>
+  <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid>
+  <memory unit='KiB'>219136</memory>
+  <currentMemory unit='KiB'>219136</currentMemory>
+  <vcpu placement='static'>1</vcpu>
+  <os>
+    <type arch='i686' machine='pc'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu-system-i386</emulator>
+    <controller type='usb' index='0'>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
+    </controller>
+    <controller type='ide' index='0'>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
+    </controller>
+    <controller type='pci' index='0' model='pci-root'/>
+    <interface type='vdpa'>
+      <mac address='52:54:00:95:db:c0'/>
+      <source dev='/dev/vhost-vdpa-0'/>
+      <model type='virtio'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+    </interface>
+    <input type='mouse' bus='ps2'/>
+    <input type='keyboard' bus='ps2'/>
+    <memballoon model='none'/>
+  </devices>
+</domain>

tests/qemuxml2xmltest.c

@@ -502,6 +502,7 @@ mymain(void)
     DO_TEST("net-mtu", QEMU_CAPS_VIRTIO_NET_HOST_MTU);
     DO_TEST("net-coalesce", NONE);
     DO_TEST("net-many-models", NONE);
+    DO_TEST("net-vdpa", QEMU_CAPS_NETDEV_VHOST_VDPA);
 
     DO_TEST("serial-tcp-tlsx509-chardev", NONE);
     DO_TEST("serial-tcp-tlsx509-chardev-notls", NONE);