feat(xo-server,xo-web): checkpoint backups (#4252)

Fixes #645

CHANGELOG.unreleased.md

@@ -7,6 +7,8 @@
 
 > Users must be able to say: “Nice enhancement, I'm eager to test it”
 
+- [Backup] **BETA** Ability to backup running VMs with their memory [#645](https://github.com/vatesfr/xen-orchestra/issues/645) (PR [#4252](https://github.com/vatesfr/xen-orchestra/pull/4252))
+
 ### Bug fixes
 
 > Users must be able to say: “I had this issue, happy to know it's fixed”

packages/xo-server/src/xapi/index.js

@@ -28,6 +28,7 @@ import {
   flatMap,
   flatten,
   groupBy,
+  identity,
   includes,
   isEmpty,
   noop,
@@ -502,51 +503,63 @@ export default class Xapi extends XapiBase {
   }
 
   // Low level create VM.
-  _createVmRecord({
+  _createVmRecord(
-    actions_after_crash,
+    {
-    actions_after_reboot,
+      actions_after_crash,
-    actions_after_shutdown,
+      actions_after_reboot,
-    affinity,
+      actions_after_shutdown,
-    // appliance,
+      affinity,
-    blocked_operations,
+      // appliance,
-    generation_id,
+      blocked_operations,
-    ha_always_run,
+      domain_type, // Used when the VM is created Suspended
-    ha_restart_priority,
+      generation_id,
-    has_vendor_device = false, // Avoid issue with some Dundee builds.
+      ha_always_run,
-    hardware_platform_version,
+      ha_restart_priority,
-    HVM_boot_params,
+      has_vendor_device = false, // Avoid issue with some Dundee builds.
-    HVM_boot_policy,
+      hardware_platform_version,
-    HVM_shadow_multiplier,
+      HVM_boot_params,
-    is_a_template,
+      HVM_boot_policy,
-    memory_dynamic_max,
+      HVM_shadow_multiplier,
-    memory_dynamic_min,
+      is_a_template,
-    memory_static_max,
+      last_boot_CPU_flags, // Used when the VM is created Suspended
-    memory_static_min,
+      last_booted_record, // Used when the VM is created Suspended
-    name_description,
+      memory_dynamic_max,
-    name_label,
+      memory_dynamic_min,
-    order,
+      memory_static_max,
-    other_config,
+      memory_static_min,
-    PCI_bus,
+      name_description,
-    platform,
+      name_label,
-    protection_policy,
+      order,
-    PV_args,
+      other_config,
-    PV_bootloader,
+      PCI_bus,
-    PV_bootloader_args,
+      platform,
-    PV_kernel,
+      protection_policy,
-    PV_legacy_args,
+      PV_args,
-    PV_ramdisk,
+      PV_bootloader,
-    recommendations,
+      PV_bootloader_args,
-    shutdown_delay,
+      PV_kernel,
-    start_delay,
+      PV_legacy_args,
-    // suspend_SR,
+      PV_ramdisk,
-    tags,
+      recommendations,
-    user_version,
+      shutdown_delay,
-    VCPUs_at_startup,
+      start_delay,
-    VCPUs_max,
+      // suspend_SR,
-    VCPUs_params,
+      tags,
-    version,
+      user_version,
-    xenstore_data,
+      VCPUs_at_startup,
-  }) {
+      VCPUs_max,
+      VCPUs_params,
+      version,
+      xenstore_data,
+    },
+    {
+      // if set, will create the VM in Suspended power_state with this VDI
+      //
+      // it's a separate param because it's not supported for all versions of
+      // XCP-ng/XenServer and should be passed explicitly
+      suspend_VDI,
+    } = {}
+  ) {
     log.debug(`Creating VM ${name_label}`)
 
     return this.call(
@@ -598,6 +611,13 @@ export default class Xapi extends XapiBase {
         tags,
         version: asInteger(version),
         xenstore_data,
+
+        // VM created Suspended
+        power_state: suspend_VDI !== undefined ? 'Suspended' : undefined,
+        suspend_VDI,
+        domain_type,
+        last_boot_CPU_flags,
+        last_booted_record,
       })
     )
   }
@@ -894,6 +914,17 @@ export default class Xapi extends XapiBase {
         this._exportVdi($cancelToken, vdi, baseVdi, VDI_FORMAT_VHD)
     })
 
+    const suspendVdi = vm.$suspend_VDI
+    if (suspendVdi !== undefined) {
+      const vdiRef = suspendVdi.$ref
+      vdis[vdiRef] = {
+        ...suspendVdi,
+        $SR$uuid: suspendVdi.$SR.uuid,
+      }
+      streams[`${vdiRef}.vhd`] = () =>
+        this._exportVdi($cancelToken, suspendVdi, undefined, VDI_FORMAT_VHD)
+    }
+
     const vifs = {}
     forEach(vm.$VIFs, vif => {
       const network = vif.$network
@@ -980,23 +1011,42 @@ export default class Xapi extends XapiBase {
         }
       })
 
+    // 0. Create suspend_VDI
+    let suspendVdi
+    if (delta.vm.power_state === 'Suspended') {
+      const vdi = delta.vdis[delta.vm.suspend_VDI]
+      suspendVdi = await this.createVdi({
+        ...vdi,
+        other_config: {
+          ...vdi.other_config,
+          [TAG_BASE_DELTA]: undefined,
+          [TAG_COPY_SRC]: vdi.uuid,
+        },
+        sr: mapVdisSrs[vdi.uuid] || srId,
+      })
+      $defer.onFailure.call(this, '_deleteVdi', suspendVdi.$ref)
+    }
+
     // 1. Create the VMs.
     const vm = await this._getOrWaitObject(
-      await this._createVmRecord({
+      await this._createVmRecord(
-        ...delta.vm,
+        {
-        affinity: null,
+          ...delta.vm,
-        blocked_operations: {
+          affinity: null,
-          ...delta.vm.blocked_operations,
+          blocked_operations: {
-          start: 'Importing…',
+            ...delta.vm.blocked_operations,
+            start: 'Importing…',
+          },
+          ha_always_run: false,
+          is_a_template: false,
+          name_label: `[Importing…] ${name_label}`,
+          other_config: {
+            ...delta.vm.other_config,
+            [TAG_COPY_SRC]: delta.vm.uuid,
+          },
         },
-        ha_always_run: false,
+        { suspend_VDI: suspendVdi?.$ref }
-        is_a_template: false,
+      )
-        name_label: `[Importing…] ${name_label}`,
-        other_config: {
-          ...delta.vm.other_config,
-          [TAG_COPY_SRC]: delta.vm.uuid,
-        },
-      })
     )
     $defer.onFailure(() => this._deleteVm(vm))
 
@@ -1004,8 +1054,10 @@ export default class Xapi extends XapiBase {
     await asyncMap(vm.$VBDs, vbd => this._deleteVbd(vbd))::ignoreErrors()
 
     // 3. Create VDIs & VBDs.
+    //
+    // TODO: move all VDIs creation before the VM and simplify the code
     const vbds = groupBy(delta.vbds, 'VDI')
-    const newVdis = await map(delta.vdis, async (vdi, vdiId) => {
+    const newVdis = await map(delta.vdis, async (vdi, vdiRef) => {
       let newVdi
 
       const remoteBaseVdiUuid = detectBase && vdi.other_config[TAG_BASE_DELTA]
@@ -1022,6 +1074,9 @@ export default class Xapi extends XapiBase {
         $defer.onFailure(() => this._deleteVdi(newVdi.$ref))
 
         await newVdi.update_other_config(TAG_COPY_SRC, vdi.uuid)
+      } else if (vdiRef === delta.vm.suspend_VDI) {
+        // suspend VDI has been already created
+        newVdi = suspendVdi
       } else {
         newVdi = await this.createVdi({
           ...vdi,
@@ -1035,7 +1090,7 @@ export default class Xapi extends XapiBase {
         $defer.onFailure(() => this._deleteVdi(newVdi.$ref))
       }
 
-      await asyncMap(vbds[vdiId], vbd =>
+      await asyncMap(vbds[vdiRef], vbd =>
         this.createVbd({
           ...vbd,
           vdi: newVdi,
@@ -1653,6 +1708,8 @@ export default class Xapi extends XapiBase {
 
   async createVbd({
     bootable = false,
+    currently_attached = false,
+    device = '',
     other_config = {},
     qos_algorithm_params = {},
     qos_algorithm_type = '',
@@ -1693,9 +1750,13 @@ export default class Xapi extends XapiBase {
       }
     }
 
+    const ifVmSuspended = vm.power_state === 'Suspended' ? identity : noop
+
     // By default a VBD is unpluggable.
     const vbdRef = await this.call('VBD.create', {
       bootable: Boolean(bootable),
+      currently_attached: ifVmSuspended(currently_attached),
+      device: ifVmSuspended(device),
       empty: Boolean(empty),
       mode,
       other_config,
@@ -2045,6 +2106,8 @@ export default class Xapi extends XapiBase {
     const vifRef = await this.call(
       'VIF.create',
       filterUndefineds({
+        currently_attached:
+          vm.power_state === 'Suspended' ? currently_attached : undefined,
         device,
         ipv4_allowed,
         ipv6_allowed,

packages/xo-server/src/xapi/mixins/vm.js

@@ -1,6 +1,6 @@
 import deferrable from 'golike-defer'
 import { find, gte, includes, isEmpty, lte, noop } from 'lodash'
-import { ignoreErrors, pCatch } from 'promise-toolbox'
+import { cancelable, ignoreErrors, pCatch } from 'promise-toolbox'
 import { NULL_REF } from 'xen-api'
 
 import { forEach, mapToArray, parseSize } from '../../utils'
@@ -18,10 +18,12 @@ const XEN_VIDEORAM_VALUES = [1, 2, 4, 8, 16]
 
 export default {
   // https://xapi-project.github.io/xen-api/classes/vm.html#checkpoint
-  async checkpointVm(vmId, nameLabel) {
+  @cancelable
+  async checkpointVm($cancelToken, vmId, nameLabel) {
     const vm = this.getObject(vmId)
     try {
       const ref = await this.callAsync(
+        $cancelToken,
         'VM.checkpoint',
         vm.$ref,
         nameLabel != null ? nameLabel : vm.name_label
@@ -29,7 +31,7 @@ export default {
       return this.barrier(ref)
     } catch (error) {
       if (error.code === 'VM_BAD_POWER_STATE') {
-        return this._snapshotVm(vm, nameLabel)
+        return this._snapshotVm($cancelToken, vm, nameLabel)
       }
       throw error
     }

packages/xo-server/src/xo-mixins/backups-ng/index.js

@@ -74,6 +74,7 @@ export type ReportWhen = 'always' | 'failure' | 'never'
 
 type Settings = {|
   bypassVdiChainsCheck?: boolean,
+  checkpointSnapshot?: boolean,
   concurrency?: number,
   deleteFirst?: boolean,
   copyRetention?: number,
@@ -149,6 +150,7 @@ const getOldEntries = <T>(retention: number, entries?: T[]): T[] =>
 
 const defaultSettings: Settings = {
   bypassVdiChainsCheck: false,
+  checkpointSnapshot: false,
   concurrency: 0,
   deleteFirst: false,
   exportRetention: 0,
@@ -1208,6 +1210,9 @@ export default class BackupNg {
         )
       }
 
+      const checkpointSnapshot =
+        !offlineSnapshot &&
+        getSetting(settings, 'checkpointSnapshot', [vmUuid, ''])
       exported = (await wrapTask(
         {
           logger,
@@ -1215,11 +1220,17 @@ export default class BackupNg {
           parentId: taskId,
           result: _ => _.uuid,
         },
-        xapi._snapshotVm(
+        checkpointSnapshot
-          $cancelToken,
+          ? xapi.checkpointVm(
-          vm,
+              $cancelToken,
-          `[XO Backup ${job.name}] ${vm.name_label}`
+              vm.$id,
-        )
+              `[XO Backup ${job.name}] ${vm.name_label}`
+            )
+          : xapi._snapshotVm(
+              $cancelToken,
+              vm,
+              `[XO Backup ${job.name}] ${vm.name_label}`
+            )
       ): any)
 
       if (startAfterSnapshot) {
@@ -1641,7 +1652,12 @@ export default class BackupNg {
           deltaExport.vdis,
           vdi =>
             `vdis/${jobId}/${
-              (xapi.getObject(vdi.snapshot_of): Object).uuid
+              (vdi.type === 'suspend'
+                ? // doesn't make sense to group by parent for memory because we
+                  // don't do delta for it
+                  vdi
+                : (xapi.getObject(vdi.snapshot_of): Object)
+              ).uuid
             }/${basename}.vhd`
         ),
         vm,

packages/xo-web/src/common/intl/messages.js

@@ -72,6 +72,10 @@ const messages = {
   altered: 'Altered',
   missing: 'Missing',
   verified: 'Verified',
+  snapshotMode: 'Snapshot mode',
+  normal: 'Normal',
+  withMemory: 'With memory',
+  offline: 'Offline',
 
   // ----- Modals -----
   alertOk: 'OK',
@@ -479,8 +483,8 @@ const messages = {
   smartBackup: 'Smart backup',
   snapshotRetention: 'Snapshot retention',
   backupName: 'Name',
+  checkpointSnapshot: 'Checkpoint snapshot',
   offlineSnapshot: 'Offline snapshot',
-  offlineSnapshotInfo: 'Shutdown VMs before snapshotting them',
   offlineBackup: 'Offline backup',
   offlineBackupInfo:
     'Export VMs without snapshotting them. The VMs will be shutdown during the export.',

packages/xo-web/src/xo-app/backup/_getSettingsWithNonDefaultValue.js

@@ -3,6 +3,7 @@ import { pickBy } from 'lodash'
 const DEFAULTS = {
   __proto__: null,
 
+  checkpointSnapshot: false,
   compression: '',
   concurrency: 0,
   fullInterval: 0,

packages/xo-web/src/xo-app/backup/new/_selectSnapshotMode.js

@@ -0,0 +1,73 @@
+import _ from 'intl'
+import decorate from 'apply-decorators'
+import PropTypes from 'prop-types'
+import React from 'react'
+import { CURRENT, PREMIUM } from 'xoa-plans'
+import { generateId } from 'reaclette-utils'
+import { injectState, provideState } from 'reaclette'
+import { Select } from 'form'
+
+import { FormGroup } from '../utils'
+
+const OPTIONS = [
+  {
+    label: _('normal'),
+    value: '',
+  },
+  {
+    disabled: CURRENT.value < PREMIUM.value,
+    label: _('withMemory'),
+    value: 'checkpointSnapshot',
+  },
+  {
+    label: _('offline'),
+    value: 'offlineSnapshot',
+  },
+]
+
+const SelectSnapshotMode = decorate([
+  provideState({
+    effects: {
+      setMode(_, value) {
+        this.props.setGlobalSettings({
+          offlineSnapshot: value === 'offlineSnapshot',
+          checkpointSnapshot: value === 'checkpointSnapshot',
+        })
+      },
+    },
+    computed: {
+      idSelect: generateId,
+      value: (_, { checkpointSnapshot, offlineSnapshot }) =>
+        checkpointSnapshot
+          ? 'checkpointSnapshot'
+          : offlineSnapshot
+          ? 'offlineSnapshot'
+          : '',
+    },
+  }),
+  injectState,
+  ({ state, effects, ...props }) => (
+    <FormGroup>
+      <label htmlFor={state.idSelect}>
+        <strong>{_('snapshotMode')}</strong>
+      </label>{' '}
+      <Select
+        {...props}
+        id={state.idSelect}
+        onChange={effects.setMode}
+        options={OPTIONS}
+        required
+        simpleValue
+        value={state.value}
+      />
+    </FormGroup>
+  ),
+])
+
+SelectSnapshotMode.propTypes = {
+  checkpointSnapshot: PropTypes.bool,
+  offlineSnapshot: PropTypes.bool,
+  setGlobalSettings: PropTypes.func.isRequired,
+}
+
+export { SelectSnapshotMode as default }

packages/xo-web/src/xo-app/backup/new/index.js

@@ -54,6 +54,8 @@ import NewSchedule from './new-schedule'
 import ReportWhen from './_reportWhen'
 import Schedules from './schedules'
 import SmartBackup from './smart-backup'
+import SelectSnapshotMode from './_selectSnapshotMode'
+
 import getSettingsWithNonDefaultValue from '../_getSettingsWithNonDefaultValue'
 import {
   canDeltaBackup,
@@ -623,13 +625,13 @@ export default decorate([
       toggleDisplayAdvancedSettings: () => ({ displayAdvancedSettings }) => ({
         _displayAdvancedSettings: !displayAdvancedSettings,
       }),
-      setGlobalSettings: (_, { name, value }) => ({
+      setGlobalSettings: (_, globalSettings) => ({
         propSettings,
         settings = propSettings,
       }) => ({
         settings: settings.update('', setting => ({
           ...setting,
-          [name]: value,
+          ...globalSettings,
         })),
       }),
       addReportRecipient({ setGlobalSettings }, value) {
@@ -640,8 +642,7 @@ export default decorate([
         )
         if (!reportRecipients.includes(value)) {
           setGlobalSettings({
-            name: 'reportRecipients',
+            reportRecipients: (reportRecipients.push(value), reportRecipients),
-            value: (reportRecipients.push(value), reportRecipients),
           })
         }
       },
@@ -649,50 +650,35 @@ export default decorate([
         const { propSettings, settings = propSettings } = this.state
         const reportRecipients = settings.getIn(['', 'reportRecipients'])
         setGlobalSettings({
-          name: 'reportRecipients',
+          reportRecipients: (reportRecipients.splice(key, 1), reportRecipients),
-          value: (reportRecipients.splice(key, 1), reportRecipients),
         })
       },
       setReportWhen: ({ setGlobalSettings }, { value }) => () => {
         setGlobalSettings({
-          name: 'reportWhen',
+          reportWhen: value,
-          value,
         })
       },
-      setConcurrency: ({ setGlobalSettings }, value) => () => {
+      setConcurrency: ({ setGlobalSettings }, concurrency) => () => {
         setGlobalSettings({
-          name: 'concurrency',
+          concurrency,
-          value,
         })
       },
       setTimeout: ({ setGlobalSettings }, value) => () => {
         setGlobalSettings({
-          name: 'timeout',
+          timeout: value && value * 3600e3,
-          value: value && value * 3600e3,
         })
       },
-      setFullInterval({ setGlobalSettings }, value) {
+      setFullInterval({ setGlobalSettings }, fullInterval) {
         setGlobalSettings({
-          name: 'fullInterval',
+          fullInterval,
-          value,
-        })
-      },
-      setOfflineSnapshot: (
-        { setGlobalSettings },
-        { target: { checked: value } }
-      ) => () => {
-        setGlobalSettings({
-          name: 'offlineSnapshot',
-          value,
         })
       },
       setOfflineBackup: (
         { setGlobalSettings },
-        { target: { checked: value } }
+        { target: { checked: offlineBackup } }
       ) => () => {
         setGlobalSettings({
-          name: 'offlineBackup',
+          offlineBackup,
-          value,
         })
       },
     },
@@ -819,6 +805,7 @@ export default decorate([
     const { propSettings, settings = propSettings } = state
     const compression = defined(state.compression, job.compression, '')
     const {
+      checkpointSnapshot,
       concurrency,
       fullInterval,
       offlineBackup,
@@ -1150,6 +1137,7 @@ export default decorate([
                               </Tooltip>{' '}
                               <input
                                 checked={offlineBackup}
+                                disabled={offlineSnapshot || checkpointSnapshot}
                                 onChange={effects.setOfflineBackup}
                                 type='checkbox'
                               />
@@ -1157,21 +1145,12 @@ export default decorate([
                           </FormGroup>
                         </div>
                       )}
-                      {!state.offlineBackupActive && (
+                      <SelectSnapshotMode
-                        <FormGroup>
+                        checkpointSnapshot={checkpointSnapshot}
-                          <label>
+                        disabled={state.offlineBackupActive}
-                            <strong>{_('offlineSnapshot')}</strong>{' '}
+                        offlineSnapshot={offlineSnapshot}
-                            <Tooltip content={_('offlineSnapshotInfo')}>
+                        setGlobalSettings={effects.setGlobalSettings}
-                              <Icon icon='info' />
+                      />
-                            </Tooltip>{' '}
-                            <input
-                              checked={offlineSnapshot}
-                              onChange={effects.setOfflineSnapshot}
-                              type='checkbox'
-                            />
-                          </label>
-                        </FormGroup>
-                      )}
                     </div>
                   )}
                 </CardBlock>

packages/xo-web/src/xo-app/backup/overview/tab-jobs.js

@@ -247,6 +247,7 @@ class JobsTable extends React.Component {
       {
         itemRenderer: job => {
           const {
+            checkpointSnapshot,
             compression,
             concurrency,
             fullInterval,
@@ -294,6 +295,14 @@ class JobsTable extends React.Component {
                   )}
                 </Li>
               )}
+              {checkpointSnapshot !== undefined && (
+                <Li>
+                  {_.keyValue(
+                    _('checkpointSnapshot'),
+                    _(checkpointSnapshot ? 'stateEnabled' : 'stateDisabled')
+                  )}
+                </Li>
+              )}
               {compression !== undefined && (
                 <Li>
                   {_.keyValue(