feat(xo-web/backup): add 'nRetriesVmBackupFailures' input

@xen-orchestra/backups/_runners/VmsRemote.mjs

@@ -6,11 +6,12 @@ import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
 import { Task } from '../Task.mjs'
 import createStreamThrottle from './_createStreamThrottle.mjs'
 import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
-import { runTask } from './_runTask.mjs'
 import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
 import { FullRemote } from './_vmRunners/FullRemote.mjs'
 import { IncrementalRemote } from './_vmRunners/IncrementalRemote.mjs'
 
+const noop = Function.prototype
+
 const DEFAULT_REMOTE_VM_SETTINGS = {
   concurrency: 2,
   copyRetention: 0,
@@ -20,6 +21,7 @@ const DEFAULT_REMOTE_VM_SETTINGS = {
   healthCheckVmsWithTags: [],
   maxExportRate: 0,
   maxMergedDeltasPerRun: Infinity,
+  nRetriesVmBackupFailures: 0,
   timeout: 0,
   validateVhdStreams: false,
   vmTimeout: 0,
@@ -41,6 +43,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
     const throttleStream = createStreamThrottle(settings.maxExportRate)
 
     const config = this._config
+
     await Disposable.use(
       () => this._getAdapter(job.sourceRemote),
       () => (settings.healthCheckSr !== undefined ? this._getRecord('SR', settings.healthCheckSr) : undefined),
@@ -62,8 +65,19 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
         const allSettings = this._job.settings
         const baseSettings = this._baseSettings
 
+        const queue = new Set(vmsUuids)
+        const taskByVmId = {}
+        const nTriesByVmId = {}
+
         const handleVm = vmUuid => {
+          if (nTriesByVmId[vmUuid] === undefined) {
+            nTriesByVmId[vmUuid] = 0
+          }
+          nTriesByVmId[vmUuid]++
+
           const taskStart = { name: 'backup VM', data: { type: 'VM', id: vmUuid } }
+          const vmSettings = { ...settings, ...allSettings[vmUuid] }
+          const isLastRun = nTriesByVmId[vmUuid] === vmSettings.nRetriesVmBackupFailures + 1
 
           const opts = {
             baseSettings,
@@ -72,7 +86,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
             healthCheckSr,
             remoteAdapters,
             schedule,
-            settings: { ...settings, ...allSettings[vmUuid] },
+            settings: vmSettings,
             sourceRemoteAdapter,
             throttleStream,
             vmUuid,
@@ -86,10 +100,39 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
             throw new Error(`Job mode ${job.mode} not implemented for mirror backup`)
           }
 
-          return runTask(taskStart, () => vmBackup.run())
+          if (taskByVmId[vmUuid] === undefined) {
+            taskByVmId[vmUuid] = new Task(taskStart)
+          }
+          const task = taskByVmId[vmUuid]
+          return task
+            .run(async () => {
+              try {
+                const result = await vmBackup.run()
+                task.success(result)
+                return result
+              } catch (error) {
+                if (isLastRun) {
+                  throw error
+                } else {
+                  Task.warning(`Retry the VM mirror backup due to an error`, {
+                    attempt: nTriesByVmId[vmUuid],
+                    error: error.message,
+                  })
+                  queue.add(vmUuid)
+                }
+              }
+            })
+            .catch(noop)
         }
         const { concurrency } = settings
-        await asyncMapSettled(vmsUuids, !concurrency ? handleVm : limitConcurrency(concurrency)(handleVm))
+        const _handleVm = !concurrency ? handleVm : limitConcurrency(concurrency)(handleVm)
+
+        while (queue.size > 0) {
+          const vmIds = Array.from(queue)
+          queue.clear()
+
+          await asyncMapSettled(vmIds, _handleVm)
+        }
       }
     )
   }

@xen-orchestra/backups/_runners/VmsXapi.mjs

@@ -11,6 +11,8 @@ import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
 import { IncrementalXapi } from './_vmRunners/IncrementalXapi.mjs'
 import { FullXapi } from './_vmRunners/FullXapi.mjs'
 
+const noop = Function.prototype
+
 const DEFAULT_XAPI_VM_SETTINGS = {
   bypassVdiChainsCheck: false,
   checkpointSnapshot: false,
@@ -24,6 +26,7 @@ const DEFAULT_XAPI_VM_SETTINGS = {
   healthCheckVmsWithTags: [],
   maxExportRate: 0,
   maxMergedDeltasPerRun: Infinity,
+  nRetriesVmBackupFailures: 0,
   offlineBackup: false,
   offlineSnapshot: false,
   snapshotRetention: 0,
@@ -53,6 +56,7 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
     const throttleStream = createStreamThrottle(settings.maxExportRate)
 
     const config = this._config
+
     await Disposable.use(
       Disposable.all(
         extractIdsFromSimplePattern(job.srs).map(id =>
@@ -89,48 +93,98 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
         const allSettings = this._job.settings
         const baseSettings = this._baseSettings
 
+        const queue = new Set(vmIds)
+        const taskByVmId = {}
+        const nTriesByVmId = {}
+
         const handleVm = vmUuid => {
+          const getVmTask = () => {
+            if (taskByVmId[vmUuid] === undefined) {
+              taskByVmId[vmUuid] = new Task(taskStart)
+            }
+            return taskByVmId[vmUuid]
+          }
+          const vmBackupFailed = error => {
+            if (isLastRun) {
+              throw error
+            } else {
+              Task.warning(`Retry the VM backup due to an error`, {
+                attempt: nTriesByVmId[vmUuid],
+                error: error.message,
+              })
+              queue.add(vmUuid)
+            }
+          }
+
+          if (nTriesByVmId[vmUuid] === undefined) {
+            nTriesByVmId[vmUuid] = 0
+          }
+          nTriesByVmId[vmUuid]++
+
+          const vmSettings = { ...settings, ...allSettings[vmUuid] }
           const taskStart = { name: 'backup VM', data: { type: 'VM', id: vmUuid } }
+          const isLastRun = nTriesByVmId[vmUuid] === vmSettings.nRetriesVmBackupFailures + 1
 
           return this._getRecord('VM', vmUuid).then(
             disposableVm =>
-              Disposable.use(disposableVm, vm => {
-                taskStart.data.name_label = vm.name_label
-                return runTask(taskStart, () => {
-                  const opts = {
-                    baseSettings,
-                    config,
-                    getSnapshotNameLabel,
-                    healthCheckSr,
-                    job,
-                    remoteAdapters,
-                    schedule,
-                    settings: { ...settings, ...allSettings[vm.uuid] },
-                    srs,
-                    throttleStream,
-                    vm,
-                  }
-                  let vmBackup
-                  if (job.mode === 'delta') {
-                    vmBackup = new IncrementalXapi(opts)
-                  } else {
-                    if (job.mode === 'full') {
-                      vmBackup = new FullXapi(opts)
-                    } else {
-                      throw new Error(`Job mode ${job.mode} not implemented`)
+              Disposable.use(disposableVm, async vm => {
+                if (taskStart.data.name_label === undefined) {
+                  taskStart.data.name_label = vm.name_label
+                }
+
+                const task = getVmTask()
+                return task
+                  .run(async () => {
+                    const opts = {
+                      baseSettings,
+                      config,
+                      getSnapshotNameLabel,
+                      healthCheckSr,
+                      job,
+                      remoteAdapters,
+                      schedule,
+                      settings: vmSettings,
+                      srs,
+                      throttleStream,
+                      vm,
                     }
-                  }
-                  return vmBackup.run()
-                })
+
+                    let vmBackup
+                    if (job.mode === 'delta') {
+                      vmBackup = new IncrementalXapi(opts)
+                    } else {
+                      if (job.mode === 'full') {
+                        vmBackup = new FullXapi(opts)
+                      } else {
+                        throw new Error(`Job mode ${job.mode} not implemented`)
+                      }
+                    }
+
+                    try {
+                      const result = await vmBackup.run()
+                      task.success(result)
+                      return result
+                    } catch (error) {
+                      vmBackupFailed(error)
+                    }
+                  })
+                  .catch(noop) // errors are handled by logs
               }),
             error =>
-              runTask(taskStart, () => {
-                throw error
+              getVmTask().run(() => {
+                vmBackupFailed(error)
               })
           )
         }
         const { concurrency } = settings
-        await asyncMapSettled(vmIds, concurrency === 0 ? handleVm : limitConcurrency(concurrency)(handleVm))
+        const _handleVm = concurrency === 0 ? handleVm : limitConcurrency(concurrency)(handleVm)
+
+        while (queue.size > 0) {
+          const vmIds = Array.from(queue)
+          queue.clear()
+
+          await asyncMapSettled(vmIds, _handleVm)
+        }
       }
     )
   }

CHANGELOG.unreleased.md

@@ -12,7 +12,7 @@
 - [REST API] Export host [SMT](https://en.wikipedia.org/wiki/Simultaneous_multithreading) status at `/hosts/:id/smt` [Forum#71374](https://xcp-ng.org/forum/post/71374)
 - [Home & REST API] `$container` field of an halted VM now points to a host if a VDI is on a local storage [Forum#71769](https://xcp-ng.org/forum/post/71769)
 - [Size Input] Ability to select two new units in the dropdown (`TiB`, `PiB`) (PR [#7382](https://github.com/vatesfr/xen-orchestra/pull/7382))
-
+- [Backup] Ability to set a number of retries for VM backup failures [#2139](https://github.com/vatesfr/xen-orchestra/issues/2139) (PR [#7308](https://github.com/vatesfr/xen-orchestra/pull/7308))
 
 ### Bug fixes
 

packages/xo-server/src/api/backup-ng.mjs

@@ -27,6 +27,11 @@ const SCHEMA_SETTINGS = {
           minimum: 1,
           optional: true,
         },
+        nRetriesVmBackupFailures: {
+          minimum: 0,
+          optional: true,
+          type: 'number',
+        },
         preferNbd: {
           type: 'boolean',
           optional: true,

packages/xo-web/src/common/intl/messages.js

@@ -141,6 +141,7 @@ const messages = {
   removeColor: 'Remove color',
   xcpNg: 'XCP-ng',
   noFileSelected: 'No file selected',
+  nRetriesVmBackupFailures: 'Number of retries if VM backup fails',
 
   // ----- Modals -----
   alertOk: 'OK',

packages/xo-web/src/xo-app/backup/new/index.js

@@ -189,6 +189,7 @@ const getInitialState = ({ preSelectedVmIds, setHomeVmIdsSelection, suggestedExc
     drMode: false,
     name: '',
     nbdConcurrency: 1,
+    nRetriesVmBackupFailures: 0,
     preferNbd: false,
     remotes: [],
     schedules: {},
@@ -635,6 +636,11 @@ const New = decorate([
           nbdConcurrency,
         })
       },
+      setNRetriesVmBackupFailures({ setGlobalSettings }, nRetries) {
+        setGlobalSettings({
+          nRetriesVmBackupFailures: nRetries,
+        })
+      },
     },
     computed: {
       compressionId: generateId,
@@ -644,6 +650,7 @@ const New = decorate([
       inputMaxExportRate: generateId,
       inputPreferNbd: generateId,
       inputNbdConcurrency: generateId,
+      inputNRetriesVmBackupFailures: generateId,
       inputTimeoutId: generateId,
 
       // In order to keep the user preference, the offline backup is kept in the DB
@@ -756,6 +763,7 @@ const New = decorate([
       fullInterval,
       maxExportRate,
       nbdConcurrency = 1,
+      nRetriesVmBackupFailures = 0,
       offlineBackup,
       offlineSnapshot,
       preferNbd,
@@ -990,6 +998,17 @@ const New = decorate([
                           value={concurrency}
                         />
                       </FormGroup>
+                      <FormGroup>
+                        <label htmlFor={state.inputNRetriesVmBackupFailures}>
+                          <strong>{_('nRetriesVmBackupFailures')}</strong>
+                        </label>
+                        <Number
+                          id={state.inputNRetriesVmBackupFailures}
+                          min={0}
+                          onChange={effects.setNRetriesVmBackupFailures}
+                          value={nRetriesVmBackupFailures}
+                        />
+                      </FormGroup>
                       <FormGroup>
                         <label htmlFor={state.inputTimeoutId}>
                           <strong>{_('timeout')}</strong>

packages/xo-web/src/xo-app/backup/new/mirror/index.js

@@ -124,6 +124,8 @@ const NewMirrorBackup = decorate([
         setAdvancedSettings({ timeout: timeout !== undefined ? timeout * 3600e3 : undefined }),
       setMaxExportRate: ({ setAdvancedSettings }, rate) =>
         setAdvancedSettings({ maxExportRate: rate !== undefined ? rate * (1024 * 1024) : undefined }),
+      setNRetriesVmBackupFailures: ({ setAdvancedSettings }, nRetriesVmBackupFailures) =>
+        setAdvancedSettings({ nRetriesVmBackupFailures }),
       setSourceRemote: (_, obj) => () => ({
         sourceRemote: obj === null ? {} : obj.value,
       }),
@@ -204,6 +206,7 @@ const NewMirrorBackup = decorate([
       inputConcurrencyId: generateId,
       inputTimeoutId: generateId,
       inputMaxExportRateId: generateId,
+      inputNRetriesVmBackupFailures: generateId,
       isBackupInvalid: state =>
         state.isMissingName || state.isMissingBackupMode || state.isMissingSchedules || state.isMissingRetention,
       isFull: state => state.mode === 'full',
@@ -231,7 +234,7 @@ const NewMirrorBackup = decorate([
   }),
   injectState,
   ({ state, effects, intl: { formatMessage } }) => {
-    const { concurrency, timeout, maxExportRate } = state.advancedSettings
+    const { concurrency, timeout, maxExportRate, nRetriesVmBackupFailures = 0 } = state.advancedSettings
     return (
       <form id={state.formId}>
         <Container>
@@ -314,6 +317,17 @@ const NewMirrorBackup = decorate([
                           value={concurrency}
                         />
                       </FormGroup>
+                      <FormGroup>
+                        <label htmlFor={state.inputNRetriesVmBackupFailures}>
+                          <strong>{_('nRetriesVmBackupFailures')}</strong>
+                        </label>
+                        <Number
+                          id={state.inputNRetriesVmBackupFailures}
+                          min={0}
+                          onChange={effects.setNRetriesVmBackupFailures}
+                          value={nRetriesVmBackupFailures}
+                        />
+                      </FormGroup>
                       <FormGroup>
                         <label htmlFor={state.inputTimeoutId}>
                           <strong>{_('timeout')}</strong>

packages/xo-web/src/xo-app/backup/overview/tab-jobs.js

@@ -319,6 +319,7 @@ class JobsTable extends React.Component {
             compression,
             concurrency,
             fullInterval,
+            nRetriesVmBackupFailures,
             offlineBackup,
             offlineSnapshot,
             proxyId,
@@ -349,6 +350,9 @@ class JobsTable extends React.Component {
               {compression !== undefined && (
                 <Li>{_.keyValue(_('compression'), compression === 'native' ? 'GZIP' : compression)}</Li>
               )}
+              {nRetriesVmBackupFailures > 0 && (
+                <Li>{_.keyValue(_('nRetriesVmBackupFailures'), nRetriesVmBackupFailures)}</Li>
+              )}
             </Ul>
           )
         },