fix(xo-server-sdn-controller): better monitor of host life cycle (#4314)

CHANGELOG.unreleased.md

@@ -11,6 +11,8 @@
 
 > Users must be able to say: “I had this issue, happy to know it's fixed”
 
+- [SDN Controller] Better detect host shutting down to adapt network topology (PR [#4314](https://github.com/vatesfr/xen-orchestra/pull/4314))
+
 ### Released packages
 
 > Packages will be released in the order they are here, therefore, they should
@@ -18,5 +20,6 @@
 >
 > Rule of thumb: add packages on top.
 
+- xo-server-sdn-controller v0.1.2
 - xo-server v5.47.0
 - xo-web v5.47.0

packages/xo-server-sdn-controller/src/index.js

@@ -315,6 +315,8 @@ class SDNController extends EventEmitter {
           await this._pifUpdated(object)
         } else if ($type === 'host') {
           await this._hostUpdated(object)
+        } else if ($type === 'host_metrics') {
+          await this._hostMetricsUpdated(object)
         }
       })
     )
@@ -412,7 +414,6 @@ class SDNController extends EventEmitter {
         return
       }
 
-      const tunnels = filter(xapi.objects.all, { $type: 'tunnel' })
       const newHost = find(this._newHosts, { $ref: host.$ref })
       if (newHost != null) {
         this._newHosts.splice(this._newHosts.indexOf(newHost), 1)
@@ -424,52 +425,19 @@ class SDNController extends EventEmitter {
           )
         }
       }
-      for (const tunnel of tunnels) {
+    }
-        const accessPIF = xapi.getObjectByRef(tunnel.access_PIF)
+  }
-        if (accessPIF.host !== host.$ref) {
-          continue
-        }
 
-        const poolNetwork = find(this._poolNetworks, {
+  async _hostMetricsUpdated(hostMetrics) {
-          network: accessPIF.network,
+    const ovsdbClient = find(this._ovsdbClients, {
-        })
+      hostMetricsRef: hostMetrics.$ref,
-        if (poolNetwork == null) {
+    })
-          continue
+    const host = ovsdbClient._host
-        }
 
-        if (accessPIF.currently_attached) {
+    if (hostMetrics.live) {
-          continue
+      await this._addHostToPoolNetworks(host)
-        }
-
-        log.debug(
-          `Pluging PIF: '${accessPIF.device}' for host: '${host.name_label}' on network: '${accessPIF.$network.name_label}'`
-        )
-        try {
-          await xapi.call('PIF.plug', accessPIF.$ref)
-        } catch (error) {
-          log.error(
-            `XAPI error while pluging PIF: '${accessPIF.device}' on host: '${host.name_label}' for network: '${accessPIF.$network.name_label}'`
-          )
-        }
-
-        const starCenter = host.$xapi.getObjectByRef(poolNetwork.starCenter)
-        await this._addHostToNetwork(host, accessPIF.$network, starCenter)
-      }
     } else {
-      const poolNetworks = filter(this._poolNetworks, { starCenter: host.$ref })
+      await this._hostUnreachable(host)
-      for (const poolNetwork of poolNetworks) {
-        const network = host.$xapi.getObjectByRef(poolNetwork.network)
-        log.debug(
-          `Star center host: '${host.name_label}' of network: '${network.name_label}' in pool: '${host.$pool.name_label}' is no longer reachable, electing a new host`
-        )
-
-        const newCenter = await this._electNewCenter(network, true)
-        poolNetwork.starCenter = newCenter?.$ref
-        this._starCenters.delete(host.$id)
-        if (newCenter != null) {
-          this._starCenters.set(newCenter.$id, newCenter.$ref)
-        }
-      }
     }
   }
 
@@ -550,30 +518,30 @@ class SDNController extends EventEmitter {
 
     let newCenter = null
     const hosts = filter(pool.$xapi.objects.all, { $type: 'host' })
+
+    for (const host of hosts) {
+      const pif = find(host.$PIFs, { network: network.$ref })
+      if (pif !== undefined && pif.currently_attached && host.$metrics.live) {
+        newCenter = host
+      }
+    }
+
     await Promise.all(
       map(hosts, async host => {
-        if (resetNeeded) {
+        if (!resetNeeded) {
-          // Clean old ports and interfaces
-          const hostClient = find(this._ovsdbClients, { host: host.$ref })
-          if (hostClient != null) {
-            try {
-              await hostClient.resetForNetwork(network.uuid, network.name_label)
-            } catch (error) {
-              log.error(
-                `Couldn't reset network: '${network.name_label}' for host: '${host.name_label}' in pool: '${network.$pool.name_label}' because: ${error}`
-              )
-              return
-            }
-          }
-        }
-
-        if (newCenter != null) {
           return
         }
 
-        const pif = find(host.$PIFs, { network: network.$ref })
+        // Clean old ports and interfaces
-        if (pif != null && pif.currently_attached && host.enabled) {
+        const hostClient = find(this._ovsdbClients, { host: host.$ref })
-          newCenter = host
+        if (hostClient != null) {
+          try {
+            await hostClient.resetForNetwork(network.uuid, network.name_label)
+          } catch (error) {
+            log.error(
+              `Couldn't reset network: '${network.name_label}' for host: '${host.name_label}' in pool: '${network.$pool.name_label}' because: ${error}`
+            )
+          }
         }
       })
     )
@@ -665,6 +633,56 @@ class SDNController extends EventEmitter {
     }
   }
 
+  async _addHostToPoolNetworks(host) {
+    const xapi = host.$xapi
+
+    const tunnels = filter(xapi.objects.all, { $type: 'tunnel' })
+    for (const tunnel of tunnels) {
+      const accessPif = xapi.getObjectByRef(tunnel.access_PIF)
+      if (accessPif.host !== host.$ref) {
+        continue
+      }
+
+      const poolNetwork = find(this._poolNetworks, {
+        network: accessPif.network,
+      })
+      if (poolNetwork == null || accessPif.currently_attached) {
+        continue
+      }
+
+      log.debug(
+        `Plugging PIF: '${accessPif.device}' for host: '${host.name_label}' on network: '${accessPif.$network.name_label}'`
+      )
+      try {
+        await xapi.call('PIF.plug', accessPif.$ref)
+      } catch (error) {
+        log.error(
+          `XAPI error while plugging PIF: '${accessPif.device}' on host: '${host.name_label}' for network: '${accessPif.$network.name_label}': ${error}`
+        )
+      }
+
+      const starCenter = xapi.getObjectByRef(poolNetwork.starCenter)
+      await this._addHostToNetwork(host, accessPif.$network, starCenter)
+    }
+  }
+
+  async _hostUnreachable(host) {
+    const poolNetworks = filter(this._poolNetworks, { starCenter: host.$ref })
+    for (const poolNetwork of poolNetworks) {
+      const network = host.$xapi.getObjectByRef(poolNetwork.network)
+      log.debug(
+        `Star center host: '${host.name_label}' of network: '${network.name_label}' in pool: '${host.$pool.name_label}' is no longer reachable, electing a new host`
+      )
+
+      const newCenter = await this._electNewCenter(network, true)
+      poolNetwork.starCenter = newCenter?.$ref
+      this._starCenters.delete(host.$id)
+      if (newCenter !== null) {
+        this._starCenters.set(newCenter.$id, newCenter.$ref)
+      }
+    }
+  }
+
   // ---------------------------------------------------------------------------
 
   _createOvsdbClient(host) {

packages/xo-server-sdn-controller/src/ovsdb-client.js

@@ -35,6 +35,10 @@ export class OvsdbClient {
     return this._host.$id
   }
 
+  get hostMetricsRef() {
+    return this._host.metrics
+  }
+
   updateCertificates(clientKey, clientCert, caCert) {
     this._clientKey = clientKey
     this._clientCert = clientCert