bhyve: add agent support

Implement QEMU Guest Agent support for bhyve. In bhyve it can configured
using the virtio-console device.

This change covers only two APIs using the agent:

 - DomainQemuAgentCommand -- the most generic one.
 - DomainGetHostname -- extended to support not only DHCP lease source,
   but an agent as well.

It shares the qemu agent implementation with the qemu driver.

Signed-off-by: Roman Bogorodskiy <bogorodskiy@gmail.com>
Reviewed-by: Michal Privoznik <mprivozn@redhat.com>

src/bhyve/bhyve_domain.c

@@ -41,6 +41,7 @@ bhyveDomainObjPrivateAlloc(void *opaque)
 {
     bhyveDomainObjPrivate *priv = g_new0(bhyveDomainObjPrivate, 1);
 
+    priv->agentTimeout = 30;
     priv->driver = opaque;
 
     return priv;
@@ -663,3 +664,34 @@ virXMLNamespace virBhyveDriverDomainXMLNamespace = {
     .uri = "http://libvirt.org/schemas/domain/bhyve/1.0",
 
 };
+
+
+int
+virBhyveDomainObjStartWorker(virDomainObj *dom)
+{
+    bhyveDomainObjPrivate *priv = dom->privateData;
+
+    if (!priv->eventThread) {
+        g_autofree char *threadName = g_strdup_printf("vm-%s", dom->def->name);
+        if (!(priv->eventThread = virEventThreadNew(threadName)))
+            return -1;
+    }
+
+    return 0;
+}
+
+
+void
+virBhyveDomainObjStopWorker(virDomainObj *dom)
+{
+    bhyveDomainObjPrivate *priv = dom->privateData;
+    virEventThread *eventThread;
+
+    if (!priv->eventThread)
+        return;
+
+    eventThread = g_steal_pointer(&priv->eventThread);
+    virObjectUnlock(dom);
+    g_object_unref(eventThread);
+    virObjectLock(dom);
+}

src/bhyve/bhyve_domain.h

@@ -22,6 +22,8 @@
 
 #include "domain_addr.h"
 #include "domain_conf.h"
+#include "vireventthread.h"
+#include "hypervisor/qemu_agent.h"
 
 #include "bhyve_monitor.h"
 
@@ -33,10 +35,22 @@ struct _bhyveDomainObjPrivate {
     bool persistentAddrs;
 
     bhyveMonitor *mon;
+
+    qemuAgent *agent;
+    bool agentError;
+    int agentTimeout;
+
+    virEventThread *eventThread;
 };
 
+#define BHYVE_DOMAIN_PRIVATE(vm) \
+    ((bhyveDomainObjPrivate *) (vm)->privateData)
+
 virDomainXMLOption *virBhyveDriverCreateXMLConf(struct _bhyveConn *);
 
 extern virDomainXMLPrivateDataCallbacks virBhyveDriverPrivateDataCallbacks;
 extern virDomainDefParserConfig virBhyveDriverDomainDefParserConfig;
 extern virXMLNamespace virBhyveDriverDomainXMLNamespace;
+
+int virBhyveDomainObjStartWorker(virDomainObj *dom);
+void virBhyveDomainObjStopWorker(virDomainObj *dom);

src/bhyve/bhyve_driver.c

@@ -53,6 +53,7 @@
 #include "virstring.h"
 #include "cpu/cpu.h"
 #include "viraccessapicheck.h"
+#include "viraccessapicheckqemu.h"
 #include "virhostcpu.h"
 #include "virhostmem.h"
 #include "virportallocator.h"
@@ -1905,6 +1906,165 @@ bhyveDomainInterfaceAddresses(virDomainPtr domain,
 }
 
 
+static qemuAgent *
+bhyveDomainObjEnterAgent(virDomainObj *obj)
+{
+    bhyveDomainObjPrivate *priv = obj->privateData;
+    qemuAgent *agent = priv->agent;
+
+    VIR_DEBUG("Entering agent (agent=%p vm=%p name=%s)",
+              priv->agent, obj, obj->def->name);
+
+    virObjectLock(agent);
+    virObjectRef(agent);
+    virObjectUnlock(obj);
+
+    return agent;
+}
+
+
+static void
+bhyveDomainObjExitAgent(virDomainObj *obj, qemuAgent *agent)
+{
+    virObjectUnlock(agent);
+    virObjectUnref(agent);
+    virObjectLock(obj);
+
+    VIR_DEBUG("Exited agent (agent=%p vm=%p name=%s)",
+              agent, obj, obj->def->name);
+}
+
+
+static bool
+bhyveDomainAgentAvailable(virDomainObj *vm,
+                          bool reportError)
+{
+    bhyveDomainObjPrivate *priv = vm->privateData;
+
+    if (virDomainObjGetState(vm, NULL) != VIR_DOMAIN_RUNNING) {
+        if (reportError) {
+            virReportError(VIR_ERR_OPERATION_INVALID, "%s",
+                           _("domain is not running"));
+        }
+        return false;
+    }
+
+    if (!priv->agent) {
+        if (bhyveFindAgentConfig(vm->def)) {
+            if (reportError) {
+                virReportError(VIR_ERR_AGENT_UNRESPONSIVE, "%s",
+                               _("QEMU guest agent is not connected"));
+            }
+            return false;
+        } else {
+            if (reportError) {
+                virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
+                               _("QEMU guest agent is not configured"));
+            }
+            return false;
+        }
+    }
+    return true;
+}
+
+
+static int
+bhyveDomainEnsureAgent(virDomainObj *vm,
+                       bool reportError)
+{
+    bhyveDomainObjPrivate *priv = vm->privateData;
+
+    if (virDomainObjGetState(vm, NULL) != VIR_DOMAIN_RUNNING) {
+        if (reportError) {
+            virReportError(VIR_ERR_OPERATION_INVALID, "%s",
+                           _("domain is not running"));
+        }
+        return -1;
+    }
+
+    if (priv->agent)
+        return 0;
+
+    if (!priv->eventThread &&
+        virBhyveDomainObjStartWorker(vm) < 0)
+        return -1;
+
+    if (bhyveConnectAgent(NULL, vm) < 0)
+        return -1;
+
+    return 0;
+}
+
+
+static int
+bhyveDomainGetHostnameAgent(virDomainObj *vm,
+                            char **hostname)
+{
+    qemuAgent *agent;
+    int ret = -1;
+
+    if (virDomainObjBeginAgentJob(vm, VIR_AGENT_JOB_QUERY) < 0)
+        return -1;
+
+    if (virDomainObjCheckActive(vm) < 0)
+        goto endjob;
+
+    if (bhyveDomainEnsureAgent(vm, true) < 0)
+        goto endjob;
+
+    agent = bhyveDomainObjEnterAgent(vm);
+    ret = qemuAgentGetHostname(agent, hostname, true);
+    bhyveDomainObjExitAgent(vm, agent);
+
+ endjob:
+    virDomainObjEndAgentJob(vm);
+    return ret;
+}
+
+
+static char *
+bhyveDomainQemuAgentCommand(virDomainPtr domain,
+                            const char *cmd,
+                            int timeout,
+                            unsigned int flags)
+{
+    virDomainObj *vm;
+    int ret = -1;
+    char *result = NULL;
+    qemuAgent *agent;
+
+    virCheckFlags(0, NULL);
+
+    if (!(vm = bhyveDomObjFromDomain(domain)))
+        goto cleanup;
+
+    if (virDomainQemuAgentCommandEnsureACL(domain->conn, vm->def) < 0)
+        goto cleanup;
+
+    if (virDomainObjBeginAgentJob(vm, VIR_AGENT_JOB_MODIFY) < 0)
+        goto cleanup;
+
+    if (virDomainObjCheckActive(vm) < 0)
+        goto endjob;
+
+    if (!bhyveDomainAgentAvailable(vm, true))
+        goto endjob;
+
+    agent = bhyveDomainObjEnterAgent(vm);
+    ret = qemuAgentArbitraryCommand(agent, cmd, &result, timeout);
+    bhyveDomainObjExitAgent(vm, agent);
+    if (ret < 0)
+        VIR_FREE(result);
+
+ endjob:
+    virDomainObjEndAgentJob(vm);
+
+ cleanup:
+    virDomainObjEndAPI(&vm);
+    return result;
+}
+
+
 static int
 bhyveDomainGetHostnameLease(virDomainObj *vm,
                             char **hostname)
@@ -1971,7 +2131,15 @@ bhyveDomainGetHostname(virDomainPtr domain,
     virDomainObj *vm = NULL;
     char *hostname = NULL;
 
-    virCheckFlags(VIR_DOMAIN_GET_HOSTNAME_LEASE, NULL);
+    virCheckFlags(VIR_DOMAIN_GET_HOSTNAME_LEASE |
+                  VIR_DOMAIN_GET_HOSTNAME_AGENT, NULL);
+
+    VIR_EXCLUSIVE_FLAGS_RET(VIR_DOMAIN_GET_HOSTNAME_LEASE,
+                            VIR_DOMAIN_GET_HOSTNAME_AGENT,
+                            NULL);
+
+    if (!(flags & VIR_DOMAIN_GET_HOSTNAME_AGENT))
+        flags |= VIR_DOMAIN_GET_HOSTNAME_LEASE;
 
     if (!(vm = bhyveDomObjFromDomain(domain)))
         return NULL;
@@ -1979,8 +2147,13 @@ bhyveDomainGetHostname(virDomainPtr domain,
     if (virDomainGetHostnameEnsureACL(domain->conn, vm->def) < 0)
         goto cleanup;
 
-    if (bhyveDomainGetHostnameLease(vm, &hostname) < 0)
-        goto cleanup;
+    if (flags & VIR_DOMAIN_GET_HOSTNAME_LEASE) {
+        if (bhyveDomainGetHostnameLease(vm, &hostname) < 0)
+            goto cleanup;
+    } else if (flags & VIR_DOMAIN_GET_HOSTNAME_AGENT) {
+        if (bhyveDomainGetHostnameAgent(vm, &hostname) < 0)
+            goto cleanup;
+    }
 
     if (!hostname) {
         virReportError(VIR_ERR_NO_HOSTNAME,
@@ -2062,6 +2235,7 @@ static virHypervisorDriver bhyveHypervisorDriver = {
     .domainGetVcpuPinInfo = bhyveDomainGetVcpuPinInfo, /* 12.1.0 */
     .domainInterfaceAddresses = bhyveDomainInterfaceAddresses, /* 12.3.0 */
     .domainGetHostname = bhyveDomainGetHostname, /* 12.3.0 */
+    .domainQemuAgentCommand = bhyveDomainQemuAgentCommand, /* 12.4.0 */
 };
 
 

src/bhyve/bhyve_process.c

@@ -171,6 +171,118 @@ bhyveSetResourceLimits(struct _bhyveConn *driver, virDomainObj *vm)
     return 0;
 }
 
+virDomainChrDef *
+bhyveFindAgentConfig(virDomainDef *def)
+{
+    size_t i;
+
+    for (i = 0; i < def->nchannels; i++) {
+        virDomainChrDef *channel = def->channels[i];
+
+        if (channel->targetType != VIR_DOMAIN_CHR_CHANNEL_TARGET_TYPE_VIRTIO)
+            continue;
+
+
+        if (STREQ_NULLABLE(channel->target.name, "org.qemu.guest_agent.0")) {
+            return channel;
+        }
+    }
+
+    return NULL;
+}
+
+static void
+bhyveProcessHandleAgentEOF(qemuAgent *agent,
+                           virDomainObj *vm)
+{
+    bhyveDomainObjPrivate *priv;
+
+    virObjectLock(vm);
+    VIR_INFO("Received EOF from agent on %p '%s'", vm, vm->def->name);
+
+    priv = vm->privateData;
+
+    if (!priv->agent) {
+        VIR_DEBUG("Agent freed already");
+        goto unlock;
+    }
+
+    qemuAgentClose(agent);
+    priv->agent = NULL;
+    priv->agentError = false;
+
+    virObjectUnlock(vm);
+    return;
+
+ unlock:
+    virObjectUnlock(vm);
+    return;
+}
+
+/*
+ * This is invoked when there is some kind of error
+ * parsing data to/from the agent. The VM can continue
+ * to run, but no further agent commands will be
+ * allowed
+ */
+static void
+bhyveProcessHandleAgentError(qemuAgent *agent G_GNUC_UNUSED,
+                             virDomainObj *vm)
+{
+    bhyveDomainObjPrivate *priv;
+
+    virObjectLock(vm);
+    VIR_INFO("Received error from agent on %p '%s'", vm, vm->def->name);
+
+    priv = vm->privateData;
+
+    priv->agentError = true;
+
+    virObjectUnlock(vm);
+}
+
+static qemuAgentCallbacks agentCallbacks = {
+    .eofNotify = bhyveProcessHandleAgentEOF,
+    .errorNotify = bhyveProcessHandleAgentError,
+};
+
+int
+bhyveConnectAgent(struct _bhyveConn *driver G_GNUC_UNUSED, virDomainObj *vm)
+{
+    bhyveDomainObjPrivate *priv = vm->privateData;
+    qemuAgent *agent = NULL;
+    virDomainChrDef *config = bhyveFindAgentConfig(vm->def);
+
+    if (!config)
+        return 0;
+
+    if (priv->agent)
+        return 0;
+
+    agent = qemuAgentOpen(vm,
+                          config->source,
+                          virEventThreadGetContext(priv->eventThread),
+                          &agentCallbacks,
+                          BHYVE_DOMAIN_PRIVATE(vm)->agentTimeout);
+
+    if (!virDomainObjIsActive(vm)) {
+        qemuAgentClose(agent);
+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                       _("guest crashed while connecting to the guest agent"));
+        return -1;
+    }
+
+    priv->agent = agent;
+    if (!priv->agent) {
+        VIR_WARN("Cannot connect to QEMU guest agent for %s", vm->def->name);
+        priv->agentError = true;
+        virResetLastError();
+    }
+
+    return 0;
+}
+
+
 static int
 virBhyveProcessStartImpl(struct _bhyveConn *driver,
                          virDomainObj *vm,
@@ -293,6 +405,9 @@ virBhyveProcessStartImpl(struct _bhyveConn *driver,
     virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, reason);
     priv->mon = bhyveMonitorOpen(vm, driver);
 
+    if (virBhyveDomainObjStartWorker(vm) < 0)
+        goto cleanup;
+
     if (virDomainObjSave(vm, driver->xmlopt,
                          BHYVE_STATE_DIR) < 0)
         goto cleanup;
@@ -714,6 +829,9 @@ virBhyveProcessReconnect(virDomainObj *vm,
         virDomainNetNotifyActualDevice(conn, vm->def, net);
     }
 
+    if (virBhyveDomainObjStartWorker(vm) < 0)
+        goto cleanup;
+
  cleanup:
     if (ret < 0) {
         /* If VM is reported to be in active state, but we cannot find

src/bhyve/bhyve_process.h

@@ -56,6 +56,10 @@ int virBhyveGetDomainTotalCpuStats(virDomainObj *vm,
 
 void virBhyveProcessReconnectAll(struct _bhyveConn *driver);
 
+int bhyveConnectAgent(struct _bhyveConn *driver, virDomainObj *vm);
+
+virDomainChrDef *bhyveFindAgentConfig(virDomainDef *def);
+
 typedef enum {
     VIR_BHYVE_PROCESS_START_AUTODESTROY = 1 << 0,
 } bhyveProcessStartFlags;