feat(xo-web/backup): add 'nRetriesVmBackupFailures' input

Similar to 757a8915d9

Starting from XAPI 23.31, stats are in valid JSON but numbers are encoded as strings.

.eslintrc.js

@@ -65,10 +65,11 @@ module.exports = {
           typescript: true,
           'eslint-import-resolver-custom-alias': {
             alias: {
+              '@core': '../web-core/lib',
               '@': './src',
             },
             extensions: ['.ts'],
-            packages: ['@xen-orchestra/lite'],
+            packages: ['@xen-orchestra/lite', '@xen-orchestra/web'],
           },
         },
       },

@xen-orchestra/backups/ImportVmBackup.mjs

@@ -160,10 +160,10 @@ export class ImportVmBackup {
           // update the stream with the negative vhd stream
           stream = await negativeVhd.stream()
           vdis[vdiRef].baseVdi = snapshotCandidate
-        } catch (err) {
+        } catch (error) {
           // can be a broken VHD chain, a vhd chain with a key backup, ....
           // not an irrecuperable error, don't dispose parentVhd, and fallback to full restore
-          warn(`can't use differential restore`, err)
+          warn(`can't use differential restore`, { error })
           disposableDescendants?.dispose()
         }
       }

@xen-orchestra/backups/_runners/VmsRemote.mjs

@@ -6,11 +6,12 @@ import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
 import { Task } from '../Task.mjs'
 import createStreamThrottle from './_createStreamThrottle.mjs'
 import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
-import { runTask } from './_runTask.mjs'
 import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
 import { FullRemote } from './_vmRunners/FullRemote.mjs'
 import { IncrementalRemote } from './_vmRunners/IncrementalRemote.mjs'
 
+const noop = Function.prototype
+
 const DEFAULT_REMOTE_VM_SETTINGS = {
   concurrency: 2,
   copyRetention: 0,
@@ -20,6 +21,7 @@ const DEFAULT_REMOTE_VM_SETTINGS = {
   healthCheckVmsWithTags: [],
   maxExportRate: 0,
   maxMergedDeltasPerRun: Infinity,
+  nRetriesVmBackupFailures: 0,
   timeout: 0,
   validateVhdStreams: false,
   vmTimeout: 0,
@@ -41,6 +43,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
     const throttleStream = createStreamThrottle(settings.maxExportRate)
 
     const config = this._config
+
     await Disposable.use(
       () => this._getAdapter(job.sourceRemote),
       () => (settings.healthCheckSr !== undefined ? this._getRecord('SR', settings.healthCheckSr) : undefined),
@@ -62,8 +65,19 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
         const allSettings = this._job.settings
         const baseSettings = this._baseSettings
 
+        const queue = new Set(vmsUuids)
+        const taskByVmId = {}
+        const nTriesByVmId = {}
+
         const handleVm = vmUuid => {
+          if (nTriesByVmId[vmUuid] === undefined) {
+            nTriesByVmId[vmUuid] = 0
+          }
+          nTriesByVmId[vmUuid]++
+
           const taskStart = { name: 'backup VM', data: { type: 'VM', id: vmUuid } }
+          const vmSettings = { ...settings, ...allSettings[vmUuid] }
+          const isLastRun = nTriesByVmId[vmUuid] === vmSettings.nRetriesVmBackupFailures + 1
 
           const opts = {
             baseSettings,
@@ -72,7 +86,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
             healthCheckSr,
             remoteAdapters,
             schedule,
-            settings: { ...settings, ...allSettings[vmUuid] },
+            settings: vmSettings,
             sourceRemoteAdapter,
             throttleStream,
             vmUuid,
@@ -86,10 +100,39 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
             throw new Error(`Job mode ${job.mode} not implemented for mirror backup`)
           }
 
-          return runTask(taskStart, () => vmBackup.run())
+          if (taskByVmId[vmUuid] === undefined) {
+            taskByVmId[vmUuid] = new Task(taskStart)
+          }
+          const task = taskByVmId[vmUuid]
+          return task
+            .run(async () => {
+              try {
+                const result = await vmBackup.run()
+                task.success(result)
+                return result
+              } catch (error) {
+                if (isLastRun) {
+                  throw error
+                } else {
+                  Task.warning(`Retry the VM mirror backup due to an error`, {
+                    attempt: nTriesByVmId[vmUuid],
+                    error: error.message,
+                  })
+                  queue.add(vmUuid)
+                }
+              }
+            })
+            .catch(noop)
         }
         const { concurrency } = settings
-        await asyncMapSettled(vmsUuids, !concurrency ? handleVm : limitConcurrency(concurrency)(handleVm))
+        const _handleVm = !concurrency ? handleVm : limitConcurrency(concurrency)(handleVm)
+
+        while (queue.size > 0) {
+          const vmIds = Array.from(queue)
+          queue.clear()
+
+          await asyncMapSettled(vmIds, _handleVm)
+        }
       }
     )
   }

@xen-orchestra/backups/_runners/VmsXapi.mjs

@@ -11,6 +11,8 @@ import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
 import { IncrementalXapi } from './_vmRunners/IncrementalXapi.mjs'
 import { FullXapi } from './_vmRunners/FullXapi.mjs'
 
+const noop = Function.prototype
+
 const DEFAULT_XAPI_VM_SETTINGS = {
   bypassVdiChainsCheck: false,
   checkpointSnapshot: false,
@@ -24,6 +26,7 @@ const DEFAULT_XAPI_VM_SETTINGS = {
   healthCheckVmsWithTags: [],
   maxExportRate: 0,
   maxMergedDeltasPerRun: Infinity,
+  nRetriesVmBackupFailures: 0,
   offlineBackup: false,
   offlineSnapshot: false,
   snapshotRetention: 0,
@@ -53,6 +56,7 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
     const throttleStream = createStreamThrottle(settings.maxExportRate)
 
     const config = this._config
+
     await Disposable.use(
       Disposable.all(
         extractIdsFromSimplePattern(job.srs).map(id =>
@@ -89,48 +93,98 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
         const allSettings = this._job.settings
         const baseSettings = this._baseSettings
 
+        const queue = new Set(vmIds)
+        const taskByVmId = {}
+        const nTriesByVmId = {}
+
         const handleVm = vmUuid => {
+          const getVmTask = () => {
+            if (taskByVmId[vmUuid] === undefined) {
+              taskByVmId[vmUuid] = new Task(taskStart)
+            }
+            return taskByVmId[vmUuid]
+          }
+          const vmBackupFailed = error => {
+            if (isLastRun) {
+              throw error
+            } else {
+              Task.warning(`Retry the VM backup due to an error`, {
+                attempt: nTriesByVmId[vmUuid],
+                error: error.message,
+              })
+              queue.add(vmUuid)
+            }
+          }
+
+          if (nTriesByVmId[vmUuid] === undefined) {
+            nTriesByVmId[vmUuid] = 0
+          }
+          nTriesByVmId[vmUuid]++
+
+          const vmSettings = { ...settings, ...allSettings[vmUuid] }
           const taskStart = { name: 'backup VM', data: { type: 'VM', id: vmUuid } }
+          const isLastRun = nTriesByVmId[vmUuid] === vmSettings.nRetriesVmBackupFailures + 1
 
           return this._getRecord('VM', vmUuid).then(
             disposableVm =>
-              Disposable.use(disposableVm, vm => {
-                taskStart.data.name_label = vm.name_label
-                return runTask(taskStart, () => {
-                  const opts = {
-                    baseSettings,
-                    config,
-                    getSnapshotNameLabel,
-                    healthCheckSr,
-                    job,
-                    remoteAdapters,
-                    schedule,
-                    settings: { ...settings, ...allSettings[vm.uuid] },
-                    srs,
-                    throttleStream,
-                    vm,
-                  }
-                  let vmBackup
-                  if (job.mode === 'delta') {
-                    vmBackup = new IncrementalXapi(opts)
-                  } else {
-                    if (job.mode === 'full') {
-                      vmBackup = new FullXapi(opts)
-                    } else {
-                      throw new Error(`Job mode ${job.mode} not implemented`)
+              Disposable.use(disposableVm, async vm => {
+                if (taskStart.data.name_label === undefined) {
+                  taskStart.data.name_label = vm.name_label
+                }
+
+                const task = getVmTask()
+                return task
+                  .run(async () => {
+                    const opts = {
+                      baseSettings,
+                      config,
+                      getSnapshotNameLabel,
+                      healthCheckSr,
+                      job,
+                      remoteAdapters,
+                      schedule,
+                      settings: vmSettings,
+                      srs,
+                      throttleStream,
+                      vm,
                     }
-                  }
-                  return vmBackup.run()
-                })
+
+                    let vmBackup
+                    if (job.mode === 'delta') {
+                      vmBackup = new IncrementalXapi(opts)
+                    } else {
+                      if (job.mode === 'full') {
+                        vmBackup = new FullXapi(opts)
+                      } else {
+                        throw new Error(`Job mode ${job.mode} not implemented`)
+                      }
+                    }
+
+                    try {
+                      const result = await vmBackup.run()
+                      task.success(result)
+                      return result
+                    } catch (error) {
+                      vmBackupFailed(error)
+                    }
+                  })
+                  .catch(noop) // errors are handled by logs
               }),
             error =>
-              runTask(taskStart, () => {
-                throw error
+              getVmTask().run(() => {
+                vmBackupFailed(error)
               })
           )
         }
         const { concurrency } = settings
-        await asyncMapSettled(vmIds, concurrency === 0 ? handleVm : limitConcurrency(concurrency)(handleVm))
+        const _handleVm = concurrency === 0 ? handleVm : limitConcurrency(concurrency)(handleVm)
+
+        while (queue.size > 0) {
+          const vmIds = Array.from(queue)
+          queue.clear()
+
+          await asyncMapSettled(vmIds, _handleVm)
+        }
       }
     )
   }

@xen-orchestra/backups/_runners/_writers/IncrementalRemoteWriter.mjs

@@ -7,7 +7,7 @@ import { chainVhd, checkVhdChain, openVhd, VhdAbstract } from 'vhd-lib'
 import { createLogger } from '@xen-orchestra/log'
 import { decorateClass } from '@vates/decorate-with'
 import { defer } from 'golike-defer'
-import { dirname, basename as pathBasename } from 'node:path'
+import { dirname } from 'node:path'
 
 import { formatFilenameDate } from '../../_filenameDate.mjs'
 import { getOldEntries } from '../../_getOldEntries.mjs'
@@ -143,8 +143,10 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
 
     let metadataContent = await this._isAlreadyTransferred(timestamp)
     if (metadataContent !== undefined) {
-      // @todo : should skip backup while being vigilant to not stuck the forked stream
+      // skip backup while being vigilant to not stuck the forked stream
       Task.info('This backup has already been transfered')
+      Object.values(deltaExport.streams).forEach(stream => stream.destroy())
+      return { size: 0 }
     }
 
     const basename = formatFilenameDate(timestamp)
@@ -199,10 +201,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
               undefined,
               `missing parent of ${id} in ${dirname(path)}, looking for ${vdi.other_config[TAG_BASE_DELTA]}`
             )
-            assert.ok(
-              pathBasename(parentPath) < pathBasename(path),
-              `vhd must be sorted to be chained`
-            )
+
             parentPath = parentPath.slice(1) // remove leading slash
 
             // TODO remove when this has been done before the export

@xen-orchestra/backups/_runners/_writers/_MixinRemoteWriter.mjs

@@ -113,13 +113,13 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
       )
     }
 
-    _isAlreadyTransferred(timestamp) {
+    async _isAlreadyTransferred(timestamp) {
       const vmUuid = this._vmUuid
       const adapter = this._adapter
       const backupDir = getVmBackupDir(vmUuid)
       try {
         const actualMetadata = JSON.parse(
-          adapter._handler.readFile(`${backupDir}/${formatFilenameDate(timestamp)}.json`)
+          await adapter._handler.readFile(`${backupDir}/${formatFilenameDate(timestamp)}.json`)
         )
         return actualMetadata
       } catch (error) {}

@xen-orchestra/fs/src/path.js

@@ -20,5 +20,7 @@ export function split(path) {
   return parts
 }
 
-export const relativeFromFile = (file, path) => relative(dirname(file), path)
+// paths are made absolute otherwise fs.relative() would resolve them against working directory
+export const relativeFromFile = (file, path) => relative(dirname(normalize(file)), normalize(path))
+
 export const resolveFromFile = (file, path) => resolve('/', dirname(file), path).slice(1)

@xen-orchestra/fs/src/path.test.js

@@ -0,0 +1,17 @@
+import { describe, it } from 'test'
+import { strict as assert } from 'assert'
+
+import { relativeFromFile } from './path.js'
+
+describe('relativeFromFile()', function () {
+  for (const [title, args] of Object.entries({
+    'file absolute and path absolute': ['/foo/bar/file.vhd', '/foo/baz/path.vhd'],
+    'file relative and path absolute': ['foo/bar/file.vhd', '/foo/baz/path.vhd'],
+    'file absolute and path relative': ['/foo/bar/file.vhd', 'foo/baz/path.vhd'],
+    'file relative and path relative': ['foo/bar/file.vhd', 'foo/baz/path.vhd'],
+  })) {
+    it('works with ' + title, function () {
+      assert.equal(relativeFromFile(...args), '../baz/path.vhd')
+    })
+  }
+})

@xen-orchestra/immutable-backups/protectRemotes.mjs

@@ -54,10 +54,10 @@ async function handleExistingFile(root, indexPath, path) {
         await indexFile(fullPath, indexPath)
       }
     }
-  } catch (err) {
-    if (err.code !== 'EEXIST') {
+  } catch (error) {
+    if (error.code !== 'EEXIST') {
       // there can be a symbolic link in the tree
-      warn('handleExistingFile', err)
+      warn('handleExistingFile', { error })
     }
   }
 }
@@ -106,7 +106,7 @@ export async function watchRemote(remoteId, { root, immutabilityDuration, rebuil
     await File.liftImmutability(settingPath)
   } catch (error) {
     // file may not exists, and it's not really a problem
-    info('lifting immutability on current settings', error)
+    info('lifting immutability on current settings', { error })
   }
   await fs.writeFile(
     settingPath,

@xen-orchestra/lite/src/libs/xapi-stats.ts

@@ -50,7 +50,17 @@ const RRD_POINTS_PER_STEP: { [key in RRD_STEP]: number } = {
 // Utils
 // -------------------------------------------------------------------
 
-function convertNanToNull(value: number) {
+function parseNumber(value: number | string) {
+  // Starting from XAPI 23.31, numbers in the JSON payload are encoded as
+  // strings to support NaN, Infinity and -Infinity
+  if (typeof value === 'string') {
+    const asNumber = +value
+    if (isNaN(asNumber) && value !== 'NaN') {
+      throw new Error('cannot parse number: ' + value)
+    }
+    value = asNumber
+  }
+
   return isNaN(value) ? null : value
 }
 
@@ -59,7 +69,7 @@ function convertNanToNull(value: number) {
 // -------------------------------------------------------------------
 
 const computeValues = (dataRow: any, legendIndex: number, transformValue = identity) =>
-  map(dataRow, ({ values }) => transformValue(convertNanToNull(values[legendIndex])))
+  map(dataRow, ({ values }) => transformValue(parseNumber(values[legendIndex])))
 
 const createGetProperty = (obj: object, property: string, defaultValue: unknown) =>
   defaults(obj, { [property]: defaultValue })[property] as any
@@ -319,8 +329,14 @@ export default class XapiStats {
       },
       abortSignal,
     })
-    // eslint-disable-next-line import/no-named-as-default-member -- https://github.com/json5/json5/issues/287
-    return JSON5.parse(await resp.text())
+    const text = await resp.text()
+    try {
+      // starting from XAPI 23.31, the response is valid JSON
+      return JSON.parse(text)
+    } catch (error) {
+      // eslint-disable-next-line import/no-named-as-default-member -- https://github.com/json5/json5/issues/287
+      return JSON5.parse(text)
+    }
   }
 
   // To avoid multiple requests, we keep a cache for the stats and
@@ -383,7 +399,10 @@ export default class XapiStats {
         abortSignal,
       })
 
-      const actualStep = json.meta.step as number
+      const actualStep = parseNumber(json.meta.step)
+      if (actualStep !== step) {
+        throw new FaultyGranularity(`Unable to get the true granularity: ${actualStep}`)
+      }
 
       if (json.data.length > 0) {
         // fetched data is organized from the newest to the oldest
@@ -407,14 +426,15 @@ export default class XapiStats {
 
           let stepStats = xoObjectStats[actualStep]
           let cacheStepStats = cacheXoObjectStats[actualStep]
-          if (stepStats === undefined || stepStats.endTimestamp !== json.meta.end) {
+          const endTimestamp = parseNumber(json.meta.end)
+          if (stepStats === undefined || stepStats.endTimestamp !== endTimestamp) {
             stepStats = xoObjectStats[actualStep] = {
-              endTimestamp: json.meta.end,
+              endTimestamp,
               interval: actualStep,
               canBeExpired: false,
             }
             cacheStepStats = cacheXoObjectStats[actualStep] = {
-              endTimestamp: json.meta.end,
+              endTimestamp,
               interval: actualStep,
               canBeExpired: true,
             }
@@ -438,10 +458,6 @@ export default class XapiStats {
           })
         })
       }
-
-      if (actualStep !== step) {
-        throw new FaultyGranularity(`Unable to get the true granularity: ${actualStep}`)
-      }
     } catch (error) {
       if (error instanceof Error && error.name === 'AbortError') {
         return

@xen-orchestra/lite/tsconfig.app.json

@@ -1,13 +1,15 @@
 {
   "extends": "@vue/tsconfig/tsconfig.dom.json",
-  "include": ["env.d.ts", "src/**/*", "src/**/*.vue"],
+  "include": ["env.d.ts", "src/**/*", "src/**/*.vue", "../web-core/lib/**/*", "../web-core/lib/**/*.vue"],
   "exclude": ["src/**/__tests__/*"],
   "compilerOptions": {
     "composite": true,
     "noEmit": true,
     "baseUrl": ".",
+    "rootDir": "..",
     "paths": {
-      "@/*": ["./src/*"]
+      "@/*": ["./src/*"],
+      "@core/*": ["../web-core/lib/*"]
     }
   }
 }

@xen-orchestra/lite/vite.config.ts

@@ -23,6 +23,7 @@ export default defineConfig({
   resolve: {
     alias: {
       '@': fileURLToPath(new URL('./src', import.meta.url)),
+      '@core': fileURLToPath(new URL('../web-core/lib', import.meta.url)),
     },
   },
 

@xen-orchestra/log/.USAGE.md

@@ -27,6 +27,16 @@ log.error('could not join server', {
 })
 ```
 
+A logging method has the following signature:
+
+```ts
+interface LoggingMethod {
+  (error): void
+
+  (message: string, data?: { error?: Error; [property: string]: any }): void
+}
+```
+
 ### Consumer
 
 Then, at application level, configure the logs are handled:

@xen-orchestra/log/README.md

@@ -45,6 +45,16 @@ log.error('could not join server', {
 })
 ```
 
+A logging method has the following signature:
+
+```ts
+interface LoggingMethod {
+  (error): void
+
+  (message: string, data?: { error?: Error; [property: string]: any }): void
+}
+```
+
 ### Consumer
 
 Then, at application level, configure the logs are handled:

@xen-orchestra/web-core/package.json

@@ -10,7 +10,8 @@
     }
   },
   "devDependencies": {
-    "vue": "^3.4.13"
+    "vue": "^3.4.13",
+    "@vue/tsconfig": "^0.5.1"
   },
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/web-core",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
@@ -25,6 +26,6 @@
   },
   "license": "AGPL-3.0-or-later",
   "engines": {
-    "node": ">=8.10"
+    "node": ">=18"
   }
 }

@xen-orchestra/web-core/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "@vue/tsconfig/tsconfig.dom.json",
+  "include": ["env.d.ts", "lib/**/*", "lib/**/*.vue"],
+  "exclude": ["lib/**/__tests__/*"],
+  "compilerOptions": {
+    "noEmit": true,
+    "baseUrl": ".",
+    "paths": {
+      "@core/*": ["./lib/*"]
+    }
+  }
+}

@xen-orchestra/web/tsconfig.app.json

@@ -1,13 +1,22 @@
 {
   "extends": "@vue/tsconfig/tsconfig.dom.json",
-  "include": ["env.d.ts", "typed-router.d.ts", "src/**/*", "src/**/*.vue"],
+  "include": [
+    "env.d.ts",
+    "typed-router.d.ts",
+    "src/**/*",
+    "src/**/*.vue",
+    "../web-core/lib/**/*",
+    "../web-core/lib/**/*.vue"
+  ],
   "exclude": ["src/**/__tests__/*"],
   "compilerOptions": {
     "composite": true,
     "noEmit": true,
     "baseUrl": ".",
+    "rootDir": "..",
     "paths": {
-      "@/*": ["./src/*"]
+      "@/*": ["./src/*"],
+      "@core/*": ["../web-core/lib/*"]
     }
   }
 }

@xen-orchestra/web/vite.config.ts

@@ -11,6 +11,7 @@ export default defineConfig({
   resolve: {
     alias: {
       '@': fileURLToPath(new URL('./src', import.meta.url)),
+      '@core': fileURLToPath(new URL('../web-core/lib', import.meta.url)),
     },
   },
 })

@xen-orchestra/xapi/vif.mjs

@@ -21,12 +21,23 @@ export default class Vif {
       MAC = '',
     } = {}
   ) {
+    if (device === undefined) {
+      const allowedDevices = await this.call('VM.get_allowed_VIF_devices', VM)
+      if (allowedDevices.length === 0) {
+        const error = new Error('could not find an allowed VIF device')
+        error.poolUuid = this.pool.uuid
+        error.vmRef = VM
+        throw error
+      }
+
+      device = allowedDevices[0]
+    }
+
     const [powerState, ...rest] = await Promise.all([
       this.getField('VM', VM, 'power_state'),
-      device ?? (await this.call('VM.get_allowed_VIF_devices', VM))[0],
-      MTU ?? (await this.getField('network', network, 'MTU')),
+      MTU ?? this.getField('network', network, 'MTU'),
     ])
-    ;[device, MTU] = rest
+    ;[MTU] = rest
 
     const vifRef = await this.call('VIF.create', {
       currently_attached: powerState === 'Suspended' ? currently_attached : undefined,

CHANGELOG.unreleased.md

@@ -8,6 +8,11 @@
 > Users must be able to say: “Nice enhancement, I'm eager to test it”
 
 - Disable search engine indexing via a `robots.txt`
+- [Stats] Support format used by XAPI 23.31
+- [REST API] Export host [SMT](https://en.wikipedia.org/wiki/Simultaneous_multithreading) status at `/hosts/:id/smt` [Forum#71374](https://xcp-ng.org/forum/post/71374)
+- [Home & REST API] `$container` field of an halted VM now points to a host if a VDI is on a local storage [Forum#71769](https://xcp-ng.org/forum/post/71769)
+- [Size Input] Ability to select two new units in the dropdown (`TiB`, `PiB`) (PR [#7382](https://github.com/vatesfr/xen-orchestra/pull/7382))
+- [Backup] Ability to set a number of retries for VM backup failures [#2139](https://github.com/vatesfr/xen-orchestra/issues/2139) (PR [#7308](https://github.com/vatesfr/xen-orchestra/pull/7308))
 
 ### Bug fixes
 
@@ -16,6 +21,7 @@
 - [Settings/XO Config] Sort backups from newest to oldest
 - [Plugins/audit] Don't log `tag.getAllConfigured` calls
 - [Remotes] Correctly clear error when the remote is tested with success
+- [Import/VMWare] Fix importing last snapshot (PR [#7370](https://github.com/vatesfr/xen-orchestra/pull/7370))
 
 ### Packages to release
 
@@ -34,8 +40,11 @@
 <!--packages-start-->
 
 - @xen-orchestra/backups patch
-- xo-server patch
+- @xen-orchestra/fs patch
+- @xen-orchestra/xapi patch
+- vhd-lib patch
+- xo-server minor
 - xo-server-audit patch
-- xo-web patch
+- xo-web minor
 
 <!--packages-end-->

docs/xoa.md

@@ -93,6 +93,21 @@ Follow the instructions:
 
 You can also download XOA from xen-orchestra.com in an XVA file. Once you've got the XVA file, you can import it with `xe vm-import filename=xoa_unified.xva` or via XenCenter.
 
+If you want to use static IP address for your appliance:
+
+```sh
+xe vm-param-set uuid="$uuid" \
+  xenstore-data:vm-data/ip="$ip" \
+  xenstore-data:vm-data/netmask="$netmask" \
+  xenstore-data:vm-data/gateway="$gateway"
+```
+
+If you want to replace the default DNS server:
+
+```sh
+xe vm-param-set uuid="$uuid" xenstore-data:vm-data/dns="$dns"
+```
+
 After the VM is imported, you just need to start it with `xe vm-start vm="XOA"` or with XenCenter.
 
 ## First console connection

packages/vhd-lib/chain.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { dirname, relative } = require('path')
+const { relativeFromFile } = require('@xen-orchestra/fs/path')
 
 const { openVhd } = require('./openVhd')
 const { DISK_TYPES } = require('./_constants')
@@ -21,7 +21,7 @@ module.exports = async function chain(parentHandler, parentPath, childHandler, c
       }
       await childVhd.readBlockAllocationTable()
 
-      const parentName = relative(dirname(childPath), parentPath)
+      const parentName = relativeFromFile(childPath, parentPath)
       header.parentUuid = parentVhd.footer.uuid
       header.parentUnicodeName = parentName
       await childVhd.setUniqueParentLocator(parentName)

packages/xo-server/src/_handleBackupLog.mjs

@@ -27,7 +27,7 @@ async function sendToNagios(app, jobName, vmBackupInfo) {
       jobName
     )
   } catch (error) {
-    warn('sendToNagios:', error)
+    warn('sendToNagios:', { error })
   }
 }
 

packages/xo-server/src/api/backup-ng.mjs

@@ -27,6 +27,11 @@ const SCHEMA_SETTINGS = {
           minimum: 1,
           optional: true,
         },
+        nRetriesVmBackupFailures: {
+          minimum: 0,
+          optional: true,
+          type: 'number',
+        },
         preferNbd: {
           type: 'boolean',
           optional: true,

packages/xo-server/src/xapi-object-to-xo.mjs

@@ -328,6 +328,34 @@ const TRANSFORMS = {
 
     const { creation } = xoData.extract(obj) ?? {}
 
+    let $container
+    if (obj.resident_on !== 'OpaqueRef:NULL') {
+      // resident_on is set when the VM is running (or paused or suspended on a host)
+      $container = link(obj, 'resident_on')
+    } else {
+      // if the VM is halted, the $container is the pool
+      $container = link(obj, 'pool')
+
+      // unless one of its VDI is on a non shared SR
+      //
+      // linked objects may not be there when this code run, and it will only be
+      // refreshed when the VM XAPI record change, this value is not guaranteed
+      // to be up-to-date, but it practice it appears to work fine thanks to
+      // `VBDs` and `current_operations` changing when a VDI is
+      // added/removed/migrated
+      for (const vbd of obj.$VBDs) {
+        const sr = vbd?.$VDI?.$SR
+        if (sr !== undefined && !sr.shared) {
+          const pbd = sr.$PBDs[0]
+          const hostId = pbd && link(pbd, 'host')
+          if (hostId !== undefined) {
+            $container = hostId
+            break
+          }
+        }
+      }
+    }
+
     const vm = {
       // type is redefined after for controllers/, templates &
       // snapshots.
@@ -422,8 +450,7 @@ const TRANSFORMS = {
       xenTools,
       ...getVmGuestToolsProps(obj),
 
-      // TODO: handle local VMs (`VM.get_possible_hosts()`).
-      $container: isRunning ? link(obj, 'resident_on') : link(obj, 'pool'),
+      $container,
       $VBDs: link(obj, 'VBDs'),
 
       // TODO: dedupe

packages/xo-server/src/xapi-stats.mjs

@@ -45,7 +45,17 @@ const RRD_POINTS_PER_STEP = {
 // Utils
 // -------------------------------------------------------------------
 
-function convertNanToNull(value) {
+function parseNumber(value) {
+  // Starting from XAPI 23.31, numbers in the JSON payload are encoded as
+  // strings to support NaN, Infinity and -Infinity
+  if (typeof value === 'string') {
+    const asNumber = +value
+    if (isNaN(asNumber) && value !== 'NaN') {
+      throw new Error('cannot parse number: ' + value)
+    }
+    value = asNumber
+  }
+
   return isNaN(value) ? null : value
 }
 
@@ -58,7 +68,7 @@ async function getServerTimestamp(xapi, hostRef) {
 // -------------------------------------------------------------------
 
 const computeValues = (dataRow, legendIndex, transformValue = identity) =>
-  map(dataRow, ({ values }) => transformValue(convertNanToNull(values[legendIndex])))
+  map(dataRow, ({ values }) => transformValue(parseNumber(values[legendIndex])))
 
 const combineStats = (stats, path, combineValues) => zipWith(...map(stats, path), (...values) => combineValues(values))
 
@@ -245,7 +255,15 @@ export default class XapiStats {
           start: timestamp,
         },
       })
-      .then(response => response.text().then(JSON5.parse))
+      .then(response => response.text())
+      .then(data => {
+        try {
+          // starting from XAPI 23.31, the response is valid JSON
+          return JSON.parse(data)
+        } catch (_) {
+          return JSON5.parse(data)
+        }
+      })
       .catch(err => {
         delete this.#hostCache[hostUuid][step]
         throw err
@@ -299,7 +317,7 @@ export default class XapiStats {
     // To avoid crossing over the boundary, we ask for one less step
     const optimumTimestamp = currentTimeStamp - maxDuration + step
     const json = await this._getJson(xapi, host, optimumTimestamp, step)
-    const actualStep = json.meta.step
+    const actualStep = parseNumber(json.meta.step)
 
     if (actualStep !== step) {
       throw new FaultyGranularity(`Unable to get the true granularity: ${actualStep}`)
@@ -326,9 +344,10 @@ export default class XapiStats {
           return
         }
 
-        if (stepStats === undefined || stepStats.endTimestamp !== json.meta.end) {
+        const endTimestamp = parseNumber(json.meta.end)
+        if (stepStats === undefined || stepStats.endTimestamp !== endTimestamp) {
           stepStats = {
-            endTimestamp: json.meta.end,
+            endTimestamp,
             interval: actualStep,
             stats: {},
           }

packages/xo-server/src/xo-mixins/migrate-vm.mjs

@@ -280,7 +280,7 @@ export default class MigrateVm {
             const stream = vhd.stream()
             await vdi.$importContent(stream, { format: VDI_FORMAT_VHD })
           }
-          return vhd
+          return { vdi, vhd }
         })
       )
     )

packages/xo-server/src/xo-mixins/remotes.mjs

@@ -1,5 +1,6 @@
 import asyncMapSettled from '@xen-orchestra/async-map/legacy.js'
 import { basename } from 'path'
+import { createLogger } from '@xen-orchestra/log'
 import { format, parse } from 'xo-remote-parser'
 import {
   DEFAULT_ENCRYPTION_ALGORITHM,
@@ -17,17 +18,35 @@ import { Remotes } from '../models/remote.mjs'
 
 // ===================================================================
 
+const { warn } = createLogger('xo:mixins:remotes')
+
 const obfuscateRemote = ({ url, ...remote }) => {
   const parsedUrl = parse(url)
   remote.url = format(sensitiveValues.obfuscate(parsedUrl))
   return remote
 }
 
-function validatePath(url) {
-  const { path } = parse(url)
+// these properties should be defined on the remote object itself and not as
+// part of the remote URL
+//
+// there is a bug somewhere that keep putting them into the URL, this list
+// is here to help track it
+const INVALID_URL_PARAMS = ['benchmarks', 'id', 'info', 'name', 'proxy', 'enabled', 'error', 'url']
+
+function validateUrl(url) {
+  const parsedUrl = parse(url)
+
+  const { path } = parsedUrl
   if (path !== undefined && basename(path) === 'xo-vm-backups') {
     throw invalidParameters('remote url should not end with xo-vm-backups')
   }
+
+  for (const param of INVALID_URL_PARAMS) {
+    if (Object.hasOwn(parsedUrl, param)) {
+      // log with stack trace
+      warn(new Error('invalid remote URL param ' + param))
+    }
+  }
 }
 
 export default class {
@@ -182,6 +201,22 @@ export default class {
     if (remote === undefined) {
       throw noSuchObject(id, 'remote')
     }
+
+    const parsedUrl = parse(remote.url)
+    let fixed = false
+    for (const param of INVALID_URL_PARAMS) {
+      if (Object.hasOwn(parsedUrl, param)) {
+        // delete the value to trace its real origin when it's added back
+        // with `updateRemote()`
+        delete parsedUrl[param]
+        fixed = true
+      }
+    }
+    if (fixed) {
+      remote.url = format(parsedUrl)
+      this._remotes.update(remote).catch(warn)
+    }
+
     return remote
   }
 
@@ -202,7 +237,7 @@ export default class {
   }
 
   async createRemote({ name, options, proxy, url }) {
-    validatePath(url)
+    validateUrl(url)
 
     const params = {
       enabled: false,
@@ -219,6 +254,10 @@ export default class {
   }
 
   updateRemote(id, { enabled, name, options, proxy, url }) {
+    if (url !== undefined) {
+      validateUrl(url)
+    }
+
     const handlers = this._handlers
     const handler = handlers[id]
     if (handler !== undefined) {
@@ -238,7 +277,7 @@ export default class {
   @synchronized()
   async _updateRemote(id, { url, ...props }) {
     if (url !== undefined) {
-      validatePath(url)
+      validateUrl(url)
     }
 
     const remote = await this._getRemote(id)

packages/xo-server/src/xo-mixins/rest-api.mjs

@@ -253,6 +253,10 @@ export default class RestApi {
           const host = req.xapiObject
           res.json(await host.$xapi.listMissingPatches(host))
         },
+
+        async smt({ xapiObject }, res) {
+          res.json({ enabled: await xapiObject.$xapi.isHyperThreadingEnabled(xapiObject.$id) })
+        },
       }
 
       collections.pools.routes = {

packages/xo-web/src/common/form/index.js

@@ -138,7 +138,7 @@ export class Range extends Component {
 
 export Toggle from './toggle'
 
-const UNITS = ['kiB', 'MiB', 'GiB']
+const UNITS = ['kiB', 'MiB', 'GiB', 'TiB', 'PiB']
 const DEFAULT_UNIT = 'GiB'
 
 export class SizeInput extends BaseComponent {

packages/xo-web/src/common/intl/messages.js

@@ -141,6 +141,7 @@ const messages = {
   removeColor: 'Remove color',
   xcpNg: 'XCP-ng',
   noFileSelected: 'No file selected',
+  nRetriesVmBackupFailures: 'Number of retries if VM backup fails',
 
   // ----- Modals -----
   alertOk: 'OK',

packages/xo-web/src/common/report-bug-button.js

@@ -75,7 +75,7 @@ export const reportOnSupportPanel = async ({ files = [], formatMessage = identit
     ADDITIONAL_FILES.map(({ fetch, name }) =>
       timeout.call(fetch(), ADDITIONAL_FILES_FETCH_TIMEOUT).then(
         file => formData.append('attachments', createBlobFromString(file), name),
-        error => logger.warn(`cannot get ${name}`, error)
+        error => logger.warn(`cannot get ${name}`, { error })
       )
     )
   )

packages/xo-web/src/xo-app/backup/new/index.js

@@ -189,6 +189,7 @@ const getInitialState = ({ preSelectedVmIds, setHomeVmIdsSelection, suggestedExc
     drMode: false,
     name: '',
     nbdConcurrency: 1,
+    nRetriesVmBackupFailures: 0,
     preferNbd: false,
     remotes: [],
     schedules: {},
@@ -635,6 +636,11 @@ const New = decorate([
           nbdConcurrency,
         })
       },
+      setNRetriesVmBackupFailures({ setGlobalSettings }, nRetries) {
+        setGlobalSettings({
+          nRetriesVmBackupFailures: nRetries,
+        })
+      },
     },
     computed: {
       compressionId: generateId,
@@ -644,6 +650,7 @@ const New = decorate([
       inputMaxExportRate: generateId,
       inputPreferNbd: generateId,
       inputNbdConcurrency: generateId,
+      inputNRetriesVmBackupFailures: generateId,
       inputTimeoutId: generateId,
 
       // In order to keep the user preference, the offline backup is kept in the DB
@@ -756,6 +763,7 @@ const New = decorate([
       fullInterval,
       maxExportRate,
       nbdConcurrency = 1,
+      nRetriesVmBackupFailures = 0,
       offlineBackup,
       offlineSnapshot,
       preferNbd,
@@ -990,6 +998,17 @@ const New = decorate([
                           value={concurrency}
                         />
                       </FormGroup>
+                      <FormGroup>
+                        <label htmlFor={state.inputNRetriesVmBackupFailures}>
+                          <strong>{_('nRetriesVmBackupFailures')}</strong>
+                        </label>
+                        <Number
+                          id={state.inputNRetriesVmBackupFailures}
+                          min={0}
+                          onChange={effects.setNRetriesVmBackupFailures}
+                          value={nRetriesVmBackupFailures}
+                        />
+                      </FormGroup>
                       <FormGroup>
                         <label htmlFor={state.inputTimeoutId}>
                           <strong>{_('timeout')}</strong>

packages/xo-web/src/xo-app/backup/new/mirror/index.js

@@ -124,6 +124,8 @@ const NewMirrorBackup = decorate([
         setAdvancedSettings({ timeout: timeout !== undefined ? timeout * 3600e3 : undefined }),
       setMaxExportRate: ({ setAdvancedSettings }, rate) =>
         setAdvancedSettings({ maxExportRate: rate !== undefined ? rate * (1024 * 1024) : undefined }),
+      setNRetriesVmBackupFailures: ({ setAdvancedSettings }, nRetriesVmBackupFailures) =>
+        setAdvancedSettings({ nRetriesVmBackupFailures }),
       setSourceRemote: (_, obj) => () => ({
         sourceRemote: obj === null ? {} : obj.value,
       }),
@@ -204,6 +206,7 @@ const NewMirrorBackup = decorate([
       inputConcurrencyId: generateId,
       inputTimeoutId: generateId,
       inputMaxExportRateId: generateId,
+      inputNRetriesVmBackupFailures: generateId,
       isBackupInvalid: state =>
         state.isMissingName || state.isMissingBackupMode || state.isMissingSchedules || state.isMissingRetention,
       isFull: state => state.mode === 'full',
@@ -231,7 +234,7 @@ const NewMirrorBackup = decorate([
   }),
   injectState,
   ({ state, effects, intl: { formatMessage } }) => {
-    const { concurrency, timeout, maxExportRate } = state.advancedSettings
+    const { concurrency, timeout, maxExportRate, nRetriesVmBackupFailures = 0 } = state.advancedSettings
     return (
       <form id={state.formId}>
         <Container>
@@ -314,6 +317,17 @@ const NewMirrorBackup = decorate([
                           value={concurrency}
                         />
                       </FormGroup>
+                      <FormGroup>
+                        <label htmlFor={state.inputNRetriesVmBackupFailures}>
+                          <strong>{_('nRetriesVmBackupFailures')}</strong>
+                        </label>
+                        <Number
+                          id={state.inputNRetriesVmBackupFailures}
+                          min={0}
+                          onChange={effects.setNRetriesVmBackupFailures}
+                          value={nRetriesVmBackupFailures}
+                        />
+                      </FormGroup>
                       <FormGroup>
                         <label htmlFor={state.inputTimeoutId}>
                           <strong>{_('timeout')}</strong>

packages/xo-web/src/xo-app/backup/overview/tab-jobs.js

@@ -319,6 +319,7 @@ class JobsTable extends React.Component {
             compression,
             concurrency,
             fullInterval,
+            nRetriesVmBackupFailures,
             offlineBackup,
             offlineSnapshot,
             proxyId,
@@ -349,6 +350,9 @@ class JobsTable extends React.Component {
               {compression !== undefined && (
                 <Li>{_.keyValue(_('compression'), compression === 'native' ? 'GZIP' : compression)}</Li>
               )}
+              {nRetriesVmBackupFailures > 0 && (
+                <Li>{_.keyValue(_('nRetriesVmBackupFailures'), nRetriesVmBackupFailures)}</Li>
+              )}
             </Ul>
           )
         },