feat(backups/health check): support custom checks via XenStore (#6784)

@xen-orchestra/backups/HealthCheckVmBackup.js

@@ -3,12 +3,14 @@
 const { Task } = require('./Task')
 
 exports.HealthCheckVmBackup = class HealthCheckVmBackup {
-  #xapi
   #restoredVm
+  #timeout
+  #xapi
 
-  constructor({ restoredVm, xapi }) {
+  constructor({ restoredVm, timeout = 10 * 60 * 1000, xapi }) {
     this.#restoredVm = restoredVm
     this.#xapi = xapi
+    this.#timeout = timeout
   }
 
   async run() {
@@ -23,7 +25,12 @@ exports.HealthCheckVmBackup = class HealthCheckVmBackup {
 
         // remove vifs
         await Promise.all(restoredVm.$VIFs.map(vif => xapi.callAsync('VIF.destroy', vif.$ref)))
-
+        const waitForScript = restoredVm.tags.includes('xo-backup-health-check-xenstore')
+        if (waitForScript) {
+          await restoredVm.set_xenstore_data({
+            'vm-data/xo-backup-health-check': 'planned',
+          })
+        }
         const start = new Date()
         // start Vm
 
@@ -34,7 +41,7 @@ exports.HealthCheckVmBackup = class HealthCheckVmBackup {
           false // Skip pre-boot checks?
         )
         const started = new Date()
-        const timeout = 10 * 60 * 1000
+        const timeout = this.#timeout
         const startDuration = started - start
 
         let remainingTimeout = timeout - startDuration
@@ -52,12 +59,52 @@ exports.HealthCheckVmBackup = class HealthCheckVmBackup {
         remainingTimeout -= running - started
 
         if (remainingTimeout < 0) {
-          throw new Error(`local xapi  did not get Runnig state for VM ${restoredId} after ${timeout / 1000} second`)
+          throw new Error(`local xapi  did not get Running state for VM ${restoredId} after ${timeout / 1000} second`)
         }
         // wait for the guest tool version to be defined
         await xapi.waitObjectState(restoredVm.guest_metrics, gm => gm?.PV_drivers_version?.major !== undefined, {
           timeout: remainingTimeout,
         })
+
+        const guestToolsReady = new Date()
+        remainingTimeout -= guestToolsReady - running
+        if (remainingTimeout < 0) {
+          throw new Error(`local xapi  did not get he guest tools check ${restoredId} after ${timeout / 1000} second`)
+        }
+
+        if (waitForScript) {
+          const startedRestoredVm = await xapi.waitObjectState(
+            restoredVm.$ref,
+            vm =>
+              vm?.xenstore_data !== undefined &&
+              (vm.xenstore_data['vm-data/xo-backup-health-check'] === 'success' ||
+                vm.xenstore_data['vm-data/xo-backup-health-check'] === 'failure'),
+            {
+              timeout: remainingTimeout,
+            }
+          )
+          const scriptOk = new Date()
+          remainingTimeout -= scriptOk - guestToolsReady
+          if (remainingTimeout < 0) {
+            throw new Error(
+              `Backup health check script did not update vm-data/xo-backup-health-check of ${restoredId} after ${
+                timeout / 1000
+              } second, got ${
+                startedRestoredVm.xenstore_data['vm-data/xo-backup-health-check']
+              } instead of 'success' or 'failure'`
+            )
+          }
+
+          if (startedRestoredVm.xenstore_data['vm-data/xo-backup-health-check'] !== 'success') {
+            const message = startedRestoredVm.xenstore_data['vm-data/xo-backup-health-check-error']
+            if (message) {
+              throw new Error(`Backup health check script failed with message ${message} for VM ${restoredId} `)
+            } else {
+              throw new Error(`Backup health check script failed for VM ${restoredId} `)
+            }
+          }
+          Task.info('Backup health check script successfully executed')
+        }
       }
     )
   }

@xen-orchestra/backups/docs/healthcheck/example.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+# This script must be executed at the start of the machine.
+#
+# It must run as root to be able to use xenstore-read and xenstore-write
+
+# fail in case of error or undefined variable
+set -eu
+
+# stop there if a health check is not in progress
+if [ "$(xenstore-read vm-data/xo-backup-health-check 2>&1)" != planned ]
+then
+  exit
+fi
+
+# not necessary, but informs XO that this script has started which helps diagnose issues
+xenstore-write vm-data/xo-backup-health-check running
+
+# put your test here
+#
+# in this example, the command `sqlite3` is used to validate the health of a database
+# and its output is captured and passed to XO via the XenStore in case of error
+if output=$(sqlite3 ~/my-database.sqlite3 .table 2>&1)
+then
+  # inform XO everything is ok
+  xenstore-write vm-data/xo-backup-health-check success
+else
+  # inform XO there is an issue
+  xenstore-write vm-data/xo-backup-health-check failure
+
+  # more info about the issue can be written to `vm-data/health-check-error`
+  #
+  # it will be shown in XO
+  xenstore-write vm-data/xo-backup-health-check-error "$output"
+fi

@xen-orchestra/backups/writers/DeltaReplicationWriter.js

@@ -50,6 +50,7 @@ exports.DeltaReplicationWriter = class DeltaReplicationWriter extends MixinRepli
       },
     })
     this.transfer = task.wrapFn(this.transfer)
+    this.healthCheck = task.wrapFn(this.healthCheck)
     this.cleanup = task.wrapFn(this.cleanup, true)
 
     return task.run(() => this._prepare())

@xen-orchestra/backups/writers/_MixinBackupWriter.js

@@ -80,7 +80,7 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
       assert.notStrictEqual(
         this._metadataFileName,
         undefined,
-        'Metadata file name should be defined before making a healthcheck'
+        'Metadata file name should be defined before making a health check'
       )
       return Task.run(
         {

CHANGELOG.unreleased.md

@@ -7,6 +7,8 @@
 
 > Users must be able to say: “Nice enhancement, I'm eager to test it”
 
+- [Backup/Health check] [Opt-in XenStore API](https://xen-orchestra.com/docs/backups.html#backup-health-check) to execute custom checks inside the VM (PR [#6784](https://github.com/vatesfr/xen-orchestra/pull/6784))
+
 ### Bug fixes
 
 > Users must be able to say: “I had this issue, happy to know it's fixed”
@@ -28,5 +30,8 @@
 <!--packages-start-->
 
 - @vates/task patch
+- @xen-orchestra/backups minor
+- xo-server minor
+- xo-web minor
 
 <!--packages-end-->

docs/backups.md

@@ -296,8 +296,9 @@ When it's done exporting, we'll remove the snapshot. Note: this operation will t
 Concurrency is a parameter that let you define how many VMs your backup job will manage simultaneously.
 
 :::tip
+
 - Default concurrency value is 2 if left empty.
-:::
+  :::
 
 Let's say you want to backup 50 VMs (each with 1x disk) at 3:00 AM. There are **2 different strategies**:
 
@@ -335,6 +336,7 @@ When a backup job is configured using Normal snapshot mode, it's possible to use
 
 - **xo-offline-backup** to apply offline snapshotting mode (VM with be shut down prior to snapshot)
 - **xo-memory-backup** to apply RAM-enabled snapshotting
+- **xo-backup-healthcheck-xenstore** to use a script during [backup healthcheck](#backup-health-check)
 
 For example, you could have a regular backup job with 10 VMs configured with Normal snapshotting, including two which are database servers. Since database servers are generally more sensitive to being restored from snapshots, you could apply the **xo-memory-backup** tag to those two VMs and only those will be backed up in RAM-enabled mode. This will avoid the need to manage a separate backup job and schedule.
 
@@ -342,10 +344,10 @@ For example, you could have a regular backup job with 10 VMs configured with Nor
 
 Just a refresher/summary: You can select multiple backup methods for the same job:
 
-- Full: *Backup* and *Disaster Recovery* (DR)
-- Deltas: *Delta Backup* and *Continuous Replication* (CR)
+- Full: _Backup_ and _Disaster Recovery_ (DR)
+- Deltas: _Delta Backup_ and _Continuous Replication_ (CR)
 
-The Full and Delta options are mutually exclusive; Rolling Snapshots are compatible with both. The Backup and Delta Backup go to a remote Target (e.g, NFS); DR and CR back up to another XCP-ng storage repository (i.e., not the one on which the VM's being backed up reside). In the Schedule configuration, you will have the option to select the number of "Backup Retention" if your backup includes a *Backup* (or *Delta Backup*); you will have the option to select the number "Replication Retention" if you have selected *DR* or *CR* in the backup configuration.
+The Full and Delta options are mutually exclusive; Rolling Snapshots are compatible with both. The Backup and Delta Backup go to a remote Target (e.g, NFS); DR and CR back up to another XCP-ng storage repository (i.e., not the one on which the VM's being backed up reside). In the Schedule configuration, you will have the option to select the number of "Backup Retention" if your backup includes a _Backup_ (or _Delta Backup_); you will have the option to select the number "Replication Retention" if you have selected _DR_ or _CR_ in the backup configuration.
 
 ### Rolling Snapshots
 
@@ -366,3 +368,56 @@ It is often a good idea to configure retention of older backups with decreasing
 - a monthly backup (retaining 12)
 
 Again, all of these can be assigned to the same backup job. Note that if you do a weekly and a monthly backup, at some point, these will fall on the same day. Xen Orchestra is designed to fail gracefully (with an error message) if a backup job for a VM is already running. For this reason, you will want to set the time on the monthly job to run before the weekly job so that if one fails, it will be the weekly rather than the monthly one; if the weekly one fails, the monthly will be there for that spot in the retention plan; if the monthly one fails, the weekly one will only be retained for 4 weeks, and then there will be a gap in the monthly retention.
+
+## Backup Health Check
+
+Backup health check ensures the backups are ready to be restored.
+
+### Different level of checking
+
+#### Check for boot
+
+XO will restore the VM, either by downloading it for a delta/full backup or by cloning it for a disaster recovery of continous replication and then wait for the guest tools to be loaded before the end of a timeout of 10 minutes (boot + guest tools).
+
+A VM without guest tools will fail its health check.
+
+The restored VM is then deleted.
+
+#### Execute a script
+
+If a VM has the tag **xo-backup-healthcheck-xenstore** during a backup health check, then XO will wait for a script to change the value of the xenstore `vm-data/xo-backup-health-check` key to be either `success` or `failure`.
+
+In case of `failure`, it will mark the health check as failed, and will show the (optional) message contained in `vm-data/xo-backup-health-check-error`
+
+The script needs to be planned on boot. It can check if the record `vm-data/xo-backup-health-check` of the local xenstore contains `planned`
+to differenciate a normal boot and a boot during health check.
+On success it must write `success`in `vm-data/xo-backup-health-check`.
+On failure it must write `failure` in `vm-data/xo-backup-health-check`, and may optionally add details in `vm-data/xo-backup-health-check-error` .
+
+The total timeout of a backup health check (boot + guest tools + scripts) is 10min.
+
+The restored VM is then deleted.
+
+An example in bash is shown in `@xen-orchestra/backups/docs/healtcheck example/wait30seconds.sh`
+
+### Running Health checks
+
+#### Checking a backup
+
+Go to backup > restore and click on the tick to launch a health check.
+
+![](./assets/restorehealthcheck.png)
+
+Then, you will select the backup to be checked and a destination SR, which must have enough space for the full restore.
+
+#### Scheduling health check after backups
+
+Go to Backup > overview > edit.
+
+Then edit the schedule and check the healthcheck box.
+
+![](./assets/scheduled_healthcheck.png)
+
+You will then need to select the SR used, which must have enough space to restore the VMs. Healthcheck will be done after each VM backup, before starting the next one.
+
+You can filter the VMs list by providing tags, only the VMs with these tags will be checked.

packages/xo-server/src/xo-mixins/backups-ng/index.mjs

@@ -191,7 +191,7 @@ export default class BackupNg {
           vmIds.forEach(handleRecord)
           unboxIdsFromPattern(job.srs).forEach(handleRecord)
 
-          // add xapi specific to the healthcheck SR if needed
+          // add xapi specific to the health check SR if needed
           if (job.settings[schedule.id].healthCheckSr !== undefined) {
             handleRecord(job.settings[schedule.id].healthCheckSr)
           }

packages/xo-web/src/xo-app/backup/new/new-schedule.js

@@ -136,7 +136,17 @@ const New = decorate([
         )}
         <FormGroup>
           <label>
-            <strong>{_('healthCheck')}</strong>{' '}
+            <strong>
+              <a
+                className='text-info'
+                rel='noreferrer'
+                href='https://xen-orchestra.com/docs/backups.html#backup-health-check'
+                target='_blank'
+              >
+                <Icon icon='info' />
+              </a>{' '}
+              {_('healthCheck')}
+            </strong>{' '}
             {conditionalTooltip(
               <input
                 checked={schedule.healthCheckVmsWithTags !== undefined}
@@ -157,7 +167,7 @@ const New = decorate([
               <p className='h2'>
                 <Tags labels={schedule.healthCheckVmsWithTags} onChange={effects.setHealthCheckTags} />
               </p>
-              <strong>{_('sr')}</strong>
+              <strong>{_('healthCheckChooseSr')}</strong>
               <SelectSr
                 onChange={effects.setHealthCheckSr}
                 placeholder={_('healthCheckChooseSr')}

packages/xo-web/src/xo-app/backup/restore/index.js

@@ -201,7 +201,14 @@ export default class Restore extends Component {
   _restoreHealthCheck = data =>
     confirm({
       title: _('checkVmBackupsTitle', { vm: data.last.vm.name_label }),
-      body: <RestoreBackupsModalBody data={data} showGenerateNewMacAddress={false} showStartAfterBackup={false} />,
+      body: (
+        <RestoreBackupsModalBody
+          data={data}
+          showGenerateNewMacAddress={false}
+          showStartAfterBackup={false}
+          backupHealthCheck
+        />
+      ),
       icon: 'restore',
     })
       .then(({ backup, targetSrs: { mainSr, mapVdisSrs } }) => {

packages/xo-web/src/xo-app/backup/restore/restore-backups-modal-body.js

@@ -1,4 +1,5 @@
 import _ from 'intl'
+import Icon from 'icon'
 import React from 'react'
 import ChooseSrForEachVdisModal from 'xo/choose-sr-for-each-vdis-modal'
 import Component from 'base-component'
@@ -33,6 +34,16 @@ export default class RestoreBackupsModalBody extends Component {
   render() {
     return (
       <div>
+        {this.props.backupHealthCheck && (
+          <a
+            className='text-info'
+            rel='noreferrer'
+            href='https://xen-orchestra.com/docs/backups.html#backup-health-check'
+            target='_blank'
+          >
+            <Icon icon='info' />
+          </a>
+        )}
         <div className='mb-1'>
           <Select
             optionRenderer={BACKUP_RENDERER}
@@ -77,6 +88,7 @@ export default class RestoreBackupsModalBody extends Component {
 RestoreBackupsModalBody.defaultProps = {
   showGenerateNewMacAddress: true,
   showStartAfterBackup: true,
+  backupHealthCheck: false,
 }
 
 export class RestoreBackupsBulkModalBody extends Component {

packages/xo-web/src/xo-app/logs/backup-ng/log-alert-body.js

@@ -319,6 +319,7 @@ const HealthCheckTask = ({ children, className, task }) => (
 const HealthCheckVmStartTask = ({ children, className, task }) => (
   <li className={className}>
     <Icon icon='run' /> {task.message} <TaskStateInfos status={task.status} />
+    <TaskInfos infos={task.infos} />
     <TaskStart task={task} />
     <TaskEnd task={task} />
     <TaskError task={task} />