feat(xapi/VM_{checkpoint,snapshot}): HTTP sync hook (#6423)

@xen-orchestra/xapi/docs/vm-sync-hook.md

@@ -0,0 +1,131 @@
+# VM Sync Hook
+
+> This feature is currently _unstable_ and might change or be removed in the future.
+>
+> Feedbacks are very welcome on the [project bugtracker](https://github.com/vatesfr/xen-orchestra/issues).
+
+> This feature is not currently supported for backups done with XO Proxy.
+
+Before snapshotting (with or without memory, ie checkpoint), XO can notify the VM via an HTTP request.
+
+A typical use case is to make sure the VM is in a consistent state during the snapshot process, for instance by making sure database writes are flushed to the disk.
+
+> This request will only be sent if the VM is in a running state.
+
+## Configuration
+
+The feature is opt-in via a tag on the VM: `xo:notify-on-snapshot`.
+
+By default, it will be an HTTPS request on the port `1727`, on the first IP address reported by the VM.
+
+If the _VM Tools_ (i.e. management agent) are not installed on the VM or if you which to use another URL, you can specify this in the tag: `xo:notify-on-snapshot=<URL>`.
+
+To guarantee the request comes from XO, a secret must be provided in the `xo-server`'s (and `xo-proxy` if relevant) configuration:
+
+```toml
+[xapiOptions]
+syncHookSecret = 'unique long string to ensure the request comes from XO'
+```
+
+## Specification
+
+XO will waits for the request to be answered before starting the snapshot, but will not wait longer than _1 minute_.
+
+If the request fails for any reason, XO will go ahead with snapshot immediately.
+
+```http
+GET /sync HTTP/1.1
+Authorization: Bearer dW5pcXVlIGxvbmcgc3RyaW5nIHRvIGVuc3VyZSB0aGUgcmVxdWVzdCBjb21lcyBmcm9tIFhP
+```
+
+When the snapshot is finished, another request will be sent:
+
+```http
+GET /post-sync HTTP/1.1
+Authorization: Bearer dW5pcXVlIGxvbmcgc3RyaW5nIHRvIGVuc3VyZSB0aGUgcmVxdWVzdCBjb21lcyBmcm9tIFhP
+```
+
+The created snapshot will have the special `xo:synced` tag set to make it identifiable.
+
+## Example server in Node
+
+`index.cjs`:
+
+```js
+const exec = require('node:util').promisify(require('node:child_process').execFile)
+
+const SECRET = 'unique long string to ensure the request comes from XO'
+
+const HANDLERS = {
+  __proto__: null,
+
+  async '/sync'() {
+    // actions to do before the VM is snapshotted
+
+    // in this example, the Linux command `sync` is called:
+    await exec('sync')
+  },
+
+  async '/post-sync'() {
+    // actions to do after the VM is snapshotted
+  },
+}
+
+function checkAuthorization(req) {
+  try {
+    const { authorization } = req.headers
+    if (authorization !== undefined) {
+      const parts = authorization.split('  ')
+      if (parts.length >= 1 && parts[0].toLowerCase() === 'bearer') {
+        return Buffer.from(parts[1], 'base64').toString() === SECRET
+      }
+    }
+  } catch (error) {
+    console.warn('checkAuthorization', error)
+  }
+  return false
+}
+
+async function main() {
+  // generate a self-signed certificate
+  const [, key, cert] =
+    /^(-----BEGIN PRIVATE KEY-----.+-----END PRIVATE KEY-----\n)(-----BEGIN CERTIFICATE-----.+-----END CERTIFICATE-----\n)$/s.exec(
+      (await exec('openssl', ['req', '-batch', '-new', '-x509', '-nodes', '-newkey', 'rsa:2048', '-keyout', '-']))
+        .stdout
+    )
+
+  const server = require('node:https').createServer({ cert, key }, async function onRequest(req, res) {
+    if (!checkAuthorization(req)) {
+      res.statusCode = 403
+      return res.end('Forbidden')
+    }
+
+    const handler = HANDLERS[req.url.split('?')[0]]
+    if (handler === undefined || req.method !== 'GET') {
+      res.statusCode = 404
+      return res.end('Not Found')
+    }
+
+    try {
+      await handler()
+
+      res.statusCode = 200
+      res.end('Ok')
+    } catch (error) {
+      console.warn(error)
+
+      if (!res.headersSent) {
+        res.statusCode = 500
+        res.write('Internal Error')
+      }
+      res.end()
+    }
+  })
+
+  await new Promise((resolve, reject) => {
+    server.on('close', resolve).on('error', reject).listen(1727)
+  })
+}
+
+main().catch(console.warn)
+```

@xen-orchestra/xapi/index.js

@@ -102,6 +102,8 @@ class Xapi extends Base {
   constructor({
     callRetryWhenTooManyPendingTasks = { delay: 5e3, tries: 10 },
     maxUncoalescedVdis,
+    syncHookSecret,
+    syncHookTimeout,
     vdiDestroyRetryWhenInUse = { delay: 5e3, tries: 10 },
     ...opts
   }) {
@@ -112,6 +114,8 @@ class Xapi extends Base {
       when: { code: 'TOO_MANY_PENDING_TASKS' },
     }
     this._maxUncoalescedVdis = maxUncoalescedVdis
+    this._syncHookSecret = syncHookSecret
+    this._syncHookTimeout = syncHookTimeout
     this._vdiDestroyRetryWhenInUse = {
       ...vdiDestroyRetryWhenInUse,
       onRetry,

@xen-orchestra/xapi/package.json

@@ -26,6 +26,7 @@
     "@xen-orchestra/log": "^0.3.0",
     "d3-time-format": "^3.0.0",
     "golike-defer": "^0.5.1",
+    "http-request-plus": "^0.14.0",
     "json-rpc-protocol": "^0.13.2",
     "lodash": "^4.17.15",
     "promise-toolbox": "^0.21.0",

@xen-orchestra/xapi/vm.js

@@ -2,6 +2,7 @@
 
 const CancelToken = require('promise-toolbox/CancelToken')
 const groupBy = require('lodash/groupBy.js')
+const hrp = require('http-request-plus')
 const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const pickBy = require('lodash/pickBy.js')
 const omit = require('lodash/omit.js')
@@ -46,6 +47,31 @@ const cleanBiosStrings = biosStrings => {
   }
 }
 
+// See: https://github.com/xapi-project/xen-api/blob/324bc6ee6664dd915c0bbe57185f1d6243d9ed7e/ocaml/xapi/xapi_guest_agent.ml#L59-L81
+//
+// Returns <min(n)>/ip || <min(n)>/ipv4/<min(m)> || <min(n)>/ipv6/<min(m)> || undefined
+// where n corresponds to the network interface and m to its IP
+const IPV4_KEY_RE = /^\d+\/ip(?:v4\/\d+)?$/
+const IPV6_KEY_RE = /^\d+\/ipv6\/\d+$/
+function getVmAddress(networks) {
+  if (networks !== undefined) {
+    let ipv6
+    for (const key of Object.keys(networks).sort()) {
+      if (IPV4_KEY_RE.test(key)) {
+        return networks[key]
+      }
+
+      if (ipv6 === undefined && IPV6_KEY_RE.test(key)) {
+        ipv6 = networks[key]
+      }
+    }
+    if (ipv6 !== undefined) {
+      return ipv6
+    }
+  }
+  throw new Error('no VM address found')
+}
+
 async function listNobakVbds(xapi, vbdRefs) {
   const vbds = []
   await asyncMap(vbdRefs, async vbdRef => {
@@ -132,6 +158,51 @@ class Vm {
     }
   }
 
+  async _httpHook({ guest_metrics, power_state, tags, uuid }, pathname) {
+    if (power_state !== 'Running') {
+      return
+    }
+
+    let url
+    let i = tags.length
+    do {
+      if (i === 0) {
+        return
+      }
+      const tag = tags[--i]
+      if (tag === 'xo:notify-on-snapshot') {
+        const { networks } = await this.getRecord('VM_guest_metrics', guest_metrics)
+        url = Object.assign(new URL('https://locahost'), {
+          hostname: getVmAddress(networks),
+          port: 1727,
+        })
+      } else {
+        const prefix = 'xo:notify-on-snapshot='
+        if (tag.startsWith(prefix)) {
+          url = new URL(tag.slice(prefix.length))
+        }
+      }
+    } while (url === undefined)
+
+    url.pathname = pathname
+
+    const headers = {}
+    const secret = this._asyncHookSecret
+    if (secret !== undefined) {
+      headers.authorization = 'Bearer ' + Buffer.from(secret).toString('base64')
+    }
+
+    try {
+      await hrp(url, {
+        headers,
+        rejectUnauthorized: false,
+        timeout: this._syncHookTimeout ?? 60e3,
+      })
+    } catch (error) {
+      warn('HTTP hook failed', { error, url, vm: uuid })
+    }
+  }
+
   async assertHealthyVdiChains(vmRef, tolerance = this._maxUncoalescedVdis) {
     const vdiRefs = {}
     ;(await this.getRecords('VBD', await this.getField('VM', vmRef, 'VBDs'))).forEach(({ VDI: ref }) => {
@@ -148,6 +219,8 @@ class Vm {
   async checkpoint($defer, vmRef, { cancelToken = CancelToken.none, ignoreNobakVdis = false, name_label } = {}) {
     const vm = await this.getRecord('VM', vmRef)
 
+    await this._httpHook(vm, '/sync')
+
     let destroyNobakVdis = false
 
     if (ignoreNobakVdis) {
@@ -168,6 +241,9 @@ class Vm {
     try {
       const ref = await this.callAsync(cancelToken, 'VM.checkpoint', vmRef, name_label).then(extractOpaqueRef)
 
+      // detached async
+      this._httpHook(vm, '/post-sync').catch(noop)
+
       // VM checkpoints are marked as templates, unfortunately it does not play well with XVA export/import
       // which will import them as templates and not VM checkpoints or plain VMs
       await pCatch.call(
@@ -544,6 +620,8 @@ class Vm {
   ) {
     const vm = await this.getRecord('VM', vmRef)
 
+    await this._httpHook(vm, '/sync')
+
     const isHalted = vm.power_state === 'Halted'
 
     // requires the VM to be halted because it's not possible to re-plug VUSB on a live VM
@@ -646,6 +724,9 @@ class Vm {
       ref = await this.callAsync(cancelToken, 'VM.snapshot', vmRef, name_label).then(extractOpaqueRef)
     } while (false)
 
+    // detached async
+    this._httpHook(vm, '/post-sync').catch(noop)
+
     // VM snapshots are marked as templates, unfortunately it does not play well with XVA export/import
     // which will import them as templates and not VM snapshots or plain VMs
     await pCatch.call(

CHANGELOG.unreleased.md

@@ -10,6 +10,7 @@
 - [Backup/Restore file] Implement File level restore for s3 and encrypted backups (PR [#6409](https://github.com/vatesfr/xen-orchestra/pull/6409))
 - [Backup] Improve listing speed by updating caches instead of regenerating them on backup creation/deletion (PR [#6411](https://github.com/vatesfr/xen-orchestra/pull/6411))
 - [Backup] Add `mergeBlockConcurrency` and `writeBlockConcurrency` to allow tuning of backup resources consumptions (PR [#6416](https://github.com/vatesfr/xen-orchestra/pull/6416))
+- [Sync hook] VM can now be notified before being snapshot, please [see the documentation](https://github.com/vatesfr/xen-orchestra/blob/master/@xen-orchestra/xapi/docs/vm-sync-hook.md) (PR [#6423](https://github.com/vatesfr/xen-orchestra/pull/6423))
 
 ### Bug fixes
 
@@ -40,6 +41,7 @@
 
 - @vates/fuse-vhd major
 - @xen-orchestra/backups minor
+- @xen-orchestra/xapi minor
 - vhd-lib minor
 - xo-server-auth-saml patch
 - xo-server minor