diff --git a/@xen-orchestra/xapi/docs/vm-sync-hook.md b/@xen-orchestra/xapi/docs/vm-sync-hook.md new file mode 100644 index 000000000..f8cb8c3da --- /dev/null +++ b/@xen-orchestra/xapi/docs/vm-sync-hook.md @@ -0,0 +1,131 @@ +# VM Sync Hook + +> This feature is currently _unstable_ and might change or be removed in the future. +> +> Feedbacks are very welcome on the [project bugtracker](https://github.com/vatesfr/xen-orchestra/issues). + +> This feature is not currently supported for backups done with XO Proxy. + +Before snapshotting (with or without memory, ie checkpoint), XO can notify the VM via an HTTP request. + +A typical use case is to make sure the VM is in a consistent state during the snapshot process, for instance by making sure database writes are flushed to the disk. + +> This request will only be sent if the VM is in a running state. + +## Configuration + +The feature is opt-in via a tag on the VM: `xo:notify-on-snapshot`. + +By default, it will be an HTTPS request on the port `1727`, on the first IP address reported by the VM. + +If the _VM Tools_ (i.e. management agent) are not installed on the VM or if you which to use another URL, you can specify this in the tag: `xo:notify-on-snapshot=`. + +To guarantee the request comes from XO, a secret must be provided in the `xo-server`'s (and `xo-proxy` if relevant) configuration: + +```toml +[xapiOptions] +syncHookSecret = 'unique long string to ensure the request comes from XO' +``` + +## Specification + +XO will waits for the request to be answered before starting the snapshot, but will not wait longer than _1 minute_. + +If the request fails for any reason, XO will go ahead with snapshot immediately. + +```http +GET /sync HTTP/1.1 +Authorization: Bearer dW5pcXVlIGxvbmcgc3RyaW5nIHRvIGVuc3VyZSB0aGUgcmVxdWVzdCBjb21lcyBmcm9tIFhP +``` + +When the snapshot is finished, another request will be sent: + +```http +GET /post-sync HTTP/1.1 +Authorization: Bearer dW5pcXVlIGxvbmcgc3RyaW5nIHRvIGVuc3VyZSB0aGUgcmVxdWVzdCBjb21lcyBmcm9tIFhP +``` + +The created snapshot will have the special `xo:synced` tag set to make it identifiable. + +## Example server in Node + +`index.cjs`: + +```js +const exec = require('node:util').promisify(require('node:child_process').execFile) + +const SECRET = 'unique long string to ensure the request comes from XO' + +const HANDLERS = { + __proto__: null, + + async '/sync'() { + // actions to do before the VM is snapshotted + + // in this example, the Linux command `sync` is called: + await exec('sync') + }, + + async '/post-sync'() { + // actions to do after the VM is snapshotted + }, +} + +function checkAuthorization(req) { + try { + const { authorization } = req.headers + if (authorization !== undefined) { + const parts = authorization.split(' ') + if (parts.length >= 1 && parts[0].toLowerCase() === 'bearer') { + return Buffer.from(parts[1], 'base64').toString() === SECRET + } + } + } catch (error) { + console.warn('checkAuthorization', error) + } + return false +} + +async function main() { + // generate a self-signed certificate + const [, key, cert] = + /^(-----BEGIN PRIVATE KEY-----.+-----END PRIVATE KEY-----\n)(-----BEGIN CERTIFICATE-----.+-----END CERTIFICATE-----\n)$/s.exec( + (await exec('openssl', ['req', '-batch', '-new', '-x509', '-nodes', '-newkey', 'rsa:2048', '-keyout', '-'])) + .stdout + ) + + const server = require('node:https').createServer({ cert, key }, async function onRequest(req, res) { + if (!checkAuthorization(req)) { + res.statusCode = 403 + return res.end('Forbidden') + } + + const handler = HANDLERS[req.url.split('?')[0]] + if (handler === undefined || req.method !== 'GET') { + res.statusCode = 404 + return res.end('Not Found') + } + + try { + await handler() + + res.statusCode = 200 + res.end('Ok') + } catch (error) { + console.warn(error) + + if (!res.headersSent) { + res.statusCode = 500 + res.write('Internal Error') + } + res.end() + } + }) + + await new Promise((resolve, reject) => { + server.on('close', resolve).on('error', reject).listen(1727) + }) +} + +main().catch(console.warn) +``` diff --git a/@xen-orchestra/xapi/index.js b/@xen-orchestra/xapi/index.js index 7f1a7d676..8126066d2 100644 --- a/@xen-orchestra/xapi/index.js +++ b/@xen-orchestra/xapi/index.js @@ -102,6 +102,8 @@ class Xapi extends Base { constructor({ callRetryWhenTooManyPendingTasks = { delay: 5e3, tries: 10 }, maxUncoalescedVdis, + syncHookSecret, + syncHookTimeout, vdiDestroyRetryWhenInUse = { delay: 5e3, tries: 10 }, ...opts }) { @@ -112,6 +114,8 @@ class Xapi extends Base { when: { code: 'TOO_MANY_PENDING_TASKS' }, } this._maxUncoalescedVdis = maxUncoalescedVdis + this._syncHookSecret = syncHookSecret + this._syncHookTimeout = syncHookTimeout this._vdiDestroyRetryWhenInUse = { ...vdiDestroyRetryWhenInUse, onRetry, diff --git a/@xen-orchestra/xapi/package.json b/@xen-orchestra/xapi/package.json index 95265f499..63eaaf2b8 100644 --- a/@xen-orchestra/xapi/package.json +++ b/@xen-orchestra/xapi/package.json @@ -26,6 +26,7 @@ "@xen-orchestra/log": "^0.3.0", "d3-time-format": "^3.0.0", "golike-defer": "^0.5.1", + "http-request-plus": "^0.14.0", "json-rpc-protocol": "^0.13.2", "lodash": "^4.17.15", "promise-toolbox": "^0.21.0", diff --git a/@xen-orchestra/xapi/vm.js b/@xen-orchestra/xapi/vm.js index 083f5a4a4..1e54afa80 100644 --- a/@xen-orchestra/xapi/vm.js +++ b/@xen-orchestra/xapi/vm.js @@ -2,6 +2,7 @@ const CancelToken = require('promise-toolbox/CancelToken') const groupBy = require('lodash/groupBy.js') +const hrp = require('http-request-plus') const ignoreErrors = require('promise-toolbox/ignoreErrors') const pickBy = require('lodash/pickBy.js') const omit = require('lodash/omit.js') @@ -46,6 +47,31 @@ const cleanBiosStrings = biosStrings => { } } +// See: https://github.com/xapi-project/xen-api/blob/324bc6ee6664dd915c0bbe57185f1d6243d9ed7e/ocaml/xapi/xapi_guest_agent.ml#L59-L81 +// +// Returns /ip || /ipv4/ || /ipv6/ || undefined +// where n corresponds to the network interface and m to its IP +const IPV4_KEY_RE = /^\d+\/ip(?:v4\/\d+)?$/ +const IPV6_KEY_RE = /^\d+\/ipv6\/\d+$/ +function getVmAddress(networks) { + if (networks !== undefined) { + let ipv6 + for (const key of Object.keys(networks).sort()) { + if (IPV4_KEY_RE.test(key)) { + return networks[key] + } + + if (ipv6 === undefined && IPV6_KEY_RE.test(key)) { + ipv6 = networks[key] + } + } + if (ipv6 !== undefined) { + return ipv6 + } + } + throw new Error('no VM address found') +} + async function listNobakVbds(xapi, vbdRefs) { const vbds = [] await asyncMap(vbdRefs, async vbdRef => { @@ -132,6 +158,51 @@ class Vm { } } + async _httpHook({ guest_metrics, power_state, tags, uuid }, pathname) { + if (power_state !== 'Running') { + return + } + + let url + let i = tags.length + do { + if (i === 0) { + return + } + const tag = tags[--i] + if (tag === 'xo:notify-on-snapshot') { + const { networks } = await this.getRecord('VM_guest_metrics', guest_metrics) + url = Object.assign(new URL('https://locahost'), { + hostname: getVmAddress(networks), + port: 1727, + }) + } else { + const prefix = 'xo:notify-on-snapshot=' + if (tag.startsWith(prefix)) { + url = new URL(tag.slice(prefix.length)) + } + } + } while (url === undefined) + + url.pathname = pathname + + const headers = {} + const secret = this._asyncHookSecret + if (secret !== undefined) { + headers.authorization = 'Bearer ' + Buffer.from(secret).toString('base64') + } + + try { + await hrp(url, { + headers, + rejectUnauthorized: false, + timeout: this._syncHookTimeout ?? 60e3, + }) + } catch (error) { + warn('HTTP hook failed', { error, url, vm: uuid }) + } + } + async assertHealthyVdiChains(vmRef, tolerance = this._maxUncoalescedVdis) { const vdiRefs = {} ;(await this.getRecords('VBD', await this.getField('VM', vmRef, 'VBDs'))).forEach(({ VDI: ref }) => { @@ -148,6 +219,8 @@ class Vm { async checkpoint($defer, vmRef, { cancelToken = CancelToken.none, ignoreNobakVdis = false, name_label } = {}) { const vm = await this.getRecord('VM', vmRef) + await this._httpHook(vm, '/sync') + let destroyNobakVdis = false if (ignoreNobakVdis) { @@ -168,6 +241,9 @@ class Vm { try { const ref = await this.callAsync(cancelToken, 'VM.checkpoint', vmRef, name_label).then(extractOpaqueRef) + // detached async + this._httpHook(vm, '/post-sync').catch(noop) + // VM checkpoints are marked as templates, unfortunately it does not play well with XVA export/import // which will import them as templates and not VM checkpoints or plain VMs await pCatch.call( @@ -544,6 +620,8 @@ class Vm { ) { const vm = await this.getRecord('VM', vmRef) + await this._httpHook(vm, '/sync') + const isHalted = vm.power_state === 'Halted' // requires the VM to be halted because it's not possible to re-plug VUSB on a live VM @@ -646,6 +724,9 @@ class Vm { ref = await this.callAsync(cancelToken, 'VM.snapshot', vmRef, name_label).then(extractOpaqueRef) } while (false) + // detached async + this._httpHook(vm, '/post-sync').catch(noop) + // VM snapshots are marked as templates, unfortunately it does not play well with XVA export/import // which will import them as templates and not VM snapshots or plain VMs await pCatch.call( diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 8f7628502..a22267ec4 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -10,6 +10,7 @@ - [Backup/Restore file] Implement File level restore for s3 and encrypted backups (PR [#6409](https://github.com/vatesfr/xen-orchestra/pull/6409)) - [Backup] Improve listing speed by updating caches instead of regenerating them on backup creation/deletion (PR [#6411](https://github.com/vatesfr/xen-orchestra/pull/6411)) - [Backup] Add `mergeBlockConcurrency` and `writeBlockConcurrency` to allow tuning of backup resources consumptions (PR [#6416](https://github.com/vatesfr/xen-orchestra/pull/6416)) +- [Sync hook] VM can now be notified before being snapshot, please [see the documentation](https://github.com/vatesfr/xen-orchestra/blob/master/@xen-orchestra/xapi/docs/vm-sync-hook.md) (PR [#6423](https://github.com/vatesfr/xen-orchestra/pull/6423)) ### Bug fixes @@ -40,6 +41,7 @@ - @vates/fuse-vhd major - @xen-orchestra/backups minor +- @xen-orchestra/xapi minor - vhd-lib minor - xo-server-auth-saml patch - xo-server minor