NuKVM/xen-orchestra
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(xapi/VM_{checkpoint,snapshot}): HTTP sync hook (#6423)

Browse Source
This commit is contained in:
Julien Fontanet 2022-09-26 12:23:51 +02:00 committed by GitHub
parent f1ab62524c
commit f82eb8aeb4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 219 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
@xen-orchestra/xapi/docs/vm-sync-hook.md Normal file
View File

@ -0,0 +1,131 @@
# VM Sync Hook
> This feature is currently _unstable_ and might change or be removed in the future.
>
> Feedbacks are very welcome on the [project bugtracker](https://github.com/vatesfr/xen-orchestra/issues).
> This feature is not currently supported for backups done with XO Proxy.
Before snapshotting (with or without memory, ie checkpoint), XO can notify the VM via an HTTP request.
A typical use case is to make sure the VM is in a consistent state during the snapshot process, for instance by making sure database writes are flushed to the disk.
> This request will only be sent if the VM is in a running state.
## Configuration
The feature is opt-in via a tag on the VM: `xo:notify-on-snapshot`.
By default, it will be an HTTPS request on the port `1727`, on the first IP address reported by the VM.
If the _VM Tools_ (i.e. management agent) are not installed on the VM or if you which to use another URL, you can specify this in the tag: `xo:notify-on-snapshot=<URL>`.
To guarantee the request comes from XO, a secret must be provided in the `xo-server`'s (and `xo-proxy` if relevant) configuration:
```toml
[xapiOptions]
syncHookSecret = 'unique long string to ensure the request comes from XO'
```
## Specification
XO will waits for the request to be answered before starting the snapshot, but will not wait longer than _1 minute_.
If the request fails for any reason, XO will go ahead with snapshot immediately.
```http
GET /sync HTTP/1.1
Authorization: Bearer dW5pcXVlIGxvbmcgc3RyaW5nIHRvIGVuc3VyZSB0aGUgcmVxdWVzdCBjb21lcyBmcm9tIFhP
```
When the snapshot is finished, another request will be sent:
```http
GET /post-sync HTTP/1.1
Authorization: Bearer dW5pcXVlIGxvbmcgc3RyaW5nIHRvIGVuc3VyZSB0aGUgcmVxdWVzdCBjb21lcyBmcm9tIFhP
```
The created snapshot will have the special `xo:synced` tag set to make it identifiable.
## Example server in Node
`index.cjs`:
```js
const exec = require('node:util').promisify(require('node:child_process').execFile)
const SECRET = 'unique long string to ensure the request comes from XO'
const HANDLERS = {
__proto__: null,
async '/sync'() {
// actions to do before the VM is snapshotted
// in this example, the Linux command `sync` is called:
await exec('sync')
},
async '/post-sync'() {
// actions to do after the VM is snapshotted
},
}
function checkAuthorization(req) {
try {
const { authorization } = req.headers
if (authorization !== undefined) {
const parts = authorization.split(' ')
if (parts.length >= 1 && parts[0].toLowerCase() === 'bearer') {
return Buffer.from(parts[1], 'base64').toString() === SECRET
}
}
} catch (error) {
console.warn('checkAuthorization', error)
}
return false
}
async function main() {
// generate a self-signed certificate
const [, key, cert] =
/^(-----BEGIN PRIVATE KEY-----.+-----END PRIVATE KEY-----\n)(-----BEGIN CERTIFICATE-----.+-----END CERTIFICATE-----\n)$/s.exec(
(await exec('openssl', ['req', '-batch', '-new', '-x509', '-nodes', '-newkey', 'rsa:2048', '-keyout', '-']))
.stdout
)
const server = require('node:https').createServer({ cert, key }, async function onRequest(req, res) {
if (!checkAuthorization(req)) {
res.statusCode = 403
return res.end('Forbidden')
}
const handler = HANDLERS[req.url.split('?')[0]]
if (handler === undefined || req.method !== 'GET') {
res.statusCode = 404
return res.end('Not Found')
}
try {
await handler()
res.statusCode = 200
res.end('Ok')
} catch (error) {
console.warn(error)
if (!res.headersSent) {
res.statusCode = 500
res.write('Internal Error')
}
res.end()
}
})
await new Promise((resolve, reject) => {
server.on('close', resolve).on('error', reject).listen(1727)
})
}
main().catch(console.warn)
```

4
@xen-orchestra/xapi/index.js
View File

@ -102,6 +102,8 @@ class Xapi extends Base {
constructor({
callRetryWhenTooManyPendingTasks = { delay: 5e3, tries: 10 },
maxUncoalescedVdis,
syncHookSecret,
syncHookTimeout,
vdiDestroyRetryWhenInUse = { delay: 5e3, tries: 10 },
...opts
}) {
@ -112,6 +114,8 @@ class Xapi extends Base {
when: { code: 'TOO_MANY_PENDING_TASKS' },
}
this._maxUncoalescedVdis = maxUncoalescedVdis
this._syncHookSecret = syncHookSecret
this._syncHookTimeout = syncHookTimeout
this._vdiDestroyRetryWhenInUse = {
...vdiDestroyRetryWhenInUse,
onRetry,

1
@xen-orchestra/xapi/package.json
View File

@ -26,6 +26,7 @@
"@xen-orchestra/log": "^0.3.0",
"d3-time-format": "^3.0.0",
"golike-defer": "^0.5.1",
"http-request-plus": "^0.14.0",
"json-rpc-protocol": "^0.13.2",
"lodash": "^4.17.15",
"promise-toolbox": "^0.21.0",

81
@xen-orchestra/xapi/vm.js
View File

@ -2,6 +2,7 @@
const CancelToken = require('promise-toolbox/CancelToken')
const groupBy = require('lodash/groupBy.js')
const hrp = require('http-request-plus')
const ignoreErrors = require('promise-toolbox/ignoreErrors')
const pickBy = require('lodash/pickBy.js')
const omit = require('lodash/omit.js')
@ -46,6 +47,31 @@ const cleanBiosStrings = biosStrings => {
}
}
// See: https://github.com/xapi-project/xen-api/blob/324bc6ee6664dd915c0bbe57185f1d6243d9ed7e/ocaml/xapi/xapi_guest_agent.ml#L59-L81
//
// Returns <min(n)>/ip || <min(n)>/ipv4/<min(m)> || <min(n)>/ipv6/<min(m)> || undefined
// where n corresponds to the network interface and m to its IP
const IPV4_KEY_RE = /^\d+\/ip(?:v4\/\d+)?$/
const IPV6_KEY_RE = /^\d+\/ipv6\/\d+$/
function getVmAddress(networks) {
if (networks !== undefined) {
let ipv6
for (const key of Object.keys(networks).sort()) {
if (IPV4_KEY_RE.test(key)) {
return networks[key]
}
if (ipv6 === undefined && IPV6_KEY_RE.test(key)) {
ipv6 = networks[key]
}
}
if (ipv6 !== undefined) {
return ipv6
}
}
throw new Error('no VM address found')
}
async function listNobakVbds(xapi, vbdRefs) {
const vbds = []
await asyncMap(vbdRefs, async vbdRef => {
@ -132,6 +158,51 @@ class Vm {
}
}
async _httpHook({ guest_metrics, power_state, tags, uuid }, pathname) {
if (power_state !== 'Running') {
return
}
let url
let i = tags.length
do {
if (i === 0) {
return
}
const tag = tags[--i]
if (tag === 'xo:notify-on-snapshot') {
const { networks } = await this.getRecord('VM_guest_metrics', guest_metrics)
url = Object.assign(new URL('https://locahost'), {
hostname: getVmAddress(networks),
port: 1727,
})
} else {
const prefix = 'xo:notify-on-snapshot='
if (tag.startsWith(prefix)) {
url = new URL(tag.slice(prefix.length))
}
}
} while (url === undefined)
url.pathname = pathname
const headers = {}
const secret = this._asyncHookSecret
if (secret !== undefined) {
headers.authorization = 'Bearer ' + Buffer.from(secret).toString('base64')
}
try {
await hrp(url, {
headers,
rejectUnauthorized: false,
timeout: this._syncHookTimeout ?? 60e3,
})
} catch (error) {
warn('HTTP hook failed', { error, url, vm: uuid })
}
}
async assertHealthyVdiChains(vmRef, tolerance = this._maxUncoalescedVdis) {
const vdiRefs = {}
;(await this.getRecords('VBD', await this.getField('VM', vmRef, 'VBDs'))).forEach(({ VDI: ref }) => {
@ -148,6 +219,8 @@ class Vm {
async checkpoint($defer, vmRef, { cancelToken = CancelToken.none, ignoreNobakVdis = false, name_label } = {}) {
const vm = await this.getRecord('VM', vmRef)
await this._httpHook(vm, '/sync')
let destroyNobakVdis = false
if (ignoreNobakVdis) {
@ -168,6 +241,9 @@ class Vm {
try {
const ref = await this.callAsync(cancelToken, 'VM.checkpoint', vmRef, name_label).then(extractOpaqueRef)
// detached async
this._httpHook(vm, '/post-sync').catch(noop)
// VM checkpoints are marked as templates, unfortunately it does not play well with XVA export/import
// which will import them as templates and not VM checkpoints or plain VMs
await pCatch.call(
@ -544,6 +620,8 @@ class Vm {
) {
const vm = await this.getRecord('VM', vmRef)
await this._httpHook(vm, '/sync')
const isHalted = vm.power_state === 'Halted'
// requires the VM to be halted because it's not possible to re-plug VUSB on a live VM
@ -646,6 +724,9 @@ class Vm {
ref = await this.callAsync(cancelToken, 'VM.snapshot', vmRef, name_label).then(extractOpaqueRef)
} while (false)
// detached async
this._httpHook(vm, '/post-sync').catch(noop)
// VM snapshots are marked as templates, unfortunately it does not play well with XVA export/import
// which will import them as templates and not VM snapshots or plain VMs
await pCatch.call(

2
CHANGELOG.unreleased.md
View File

@ -10,6 +10,7 @@
- [Backup/Restore file] Implement File level restore for s3 and encrypted backups (PR [#6409](https://github.com/vatesfr/xen-orchestra/pull/6409))
- [Backup] Improve listing speed by updating caches instead of regenerating them on backup creation/deletion (PR [#6411](https://github.com/vatesfr/xen-orchestra/pull/6411))
- [Backup] Add `mergeBlockConcurrency` and `writeBlockConcurrency` to allow tuning of backup resources consumptions (PR [#6416](https://github.com/vatesfr/xen-orchestra/pull/6416))
- [Sync hook] VM can now be notified before being snapshot, please [see the documentation](https://github.com/vatesfr/xen-orchestra/blob/master/@xen-orchestra/xapi/docs/vm-sync-hook.md) (PR [#6423](https://github.com/vatesfr/xen-orchestra/pull/6423))
### Bug fixes
@ -40,6 +41,7 @@
- @vates/fuse-vhd major
- @xen-orchestra/backups minor
- @xen-orchestra/xapi minor
- vhd-lib minor
- xo-server-auth-saml patch
- xo-server minor