usecollection wip

Introduced by 1718649e0

.eslintrc.js

@@ -65,10 +65,11 @@ module.exports = {
           typescript: true,
           'eslint-import-resolver-custom-alias': {
             alias: {
+              '@core': '../web-core/lib',
               '@': './src',
             },
             extensions: ['.ts'],
-            packages: ['@xen-orchestra/lite'],
+            packages: ['@xen-orchestra/lite', '@xen-orchestra/web'],
           },
         },
       },

@xen-orchestra/backups/ImportVmBackup.mjs

@@ -160,10 +160,10 @@ export class ImportVmBackup {
           // update the stream with the negative vhd stream
           stream = await negativeVhd.stream()
           vdis[vdiRef].baseVdi = snapshotCandidate
-        } catch (err) {
+        } catch (error) {
           // can be a broken VHD chain, a vhd chain with a key backup, ....
           // not an irrecuperable error, don't dispose parentVhd, and fallback to full restore
-          warn(`can't use differential restore`, err)
+          warn(`can't use differential restore`, { error })
           disposableDescendants?.dispose()
         }
       }

@xen-orchestra/backups/RemoteAdapter.mjs

@@ -191,13 +191,14 @@ export class RemoteAdapter {
   // check if we will be allowed to merge a a vhd created in this adapter
   // with the vhd at path `path`
   async isMergeableParent(packedParentUid, path) {
-    return await Disposable.use(openVhd(this.handler, path), vhd => {
+    return await Disposable.use(VhdSynthetic.fromVhdChain(this.handler, path), vhd => {
       // this baseUuid is not linked with this vhd
       if (!vhd.footer.uuid.equals(packedParentUid)) {
         return false
       }
 
-      const isVhdDirectory = vhd instanceof VhdDirectory
+      // check if all the chain is composed of vhd directory
+      const isVhdDirectory = vhd.checkVhdsClass(VhdDirectory)
       return isVhdDirectory
         ? this.useVhdDirectory() && this.#getCompressionType() === vhd.compressionType
         : !this.useVhdDirectory()

@xen-orchestra/backups/_runners/_vmRunners/IncrementalRemote.mjs

@@ -2,6 +2,7 @@ import { asyncEach } from '@vates/async-each'
 import { decorateMethodsWith } from '@vates/decorate-with'
 import { defer } from 'golike-defer'
 import assert from 'node:assert'
+import * as UUID from 'uuid'
 import isVhdDifferencingDisk from 'vhd-lib/isVhdDifferencingDisk.js'
 import mapValues from 'lodash/mapValues.js'
 
@@ -9,11 +10,48 @@ import { AbstractRemote } from './_AbstractRemote.mjs'
 import { forkDeltaExport } from './_forkDeltaExport.mjs'
 import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs'
 import { Task } from '../../Task.mjs'
+import { Disposable } from 'promise-toolbox'
+import { openVhd } from 'vhd-lib'
+import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
 
 class IncrementalRemoteVmBackupRunner extends AbstractRemote {
   _getRemoteWriter() {
     return IncrementalRemoteWriter
   }
+  async _selectBaseVm(metadata) {
+    // for each disk , get the parent
+    const baseUuidToSrcVdi = new Map()
+
+    // no previous backup for a base( =key) backup
+    if (metadata.isBase) {
+      return
+    }
+    await asyncEach(Object.entries(metadata.vdis), async ([id, vdi]) => {
+      const isDifferencing = metadata.isVhdDifferencing[`${id}.vhd`]
+      if (isDifferencing) {
+        const vmDir = getVmBackupDir(metadata.vm.uuid)
+        const path = `${vmDir}/${metadata.vhds[id]}`
+        // don't catch error : we can't recover if the source vhd are missing
+        await Disposable.use(openVhd(this._sourceRemoteAdapter._handler, path), vhd => {
+          baseUuidToSrcVdi.set(UUID.stringify(vhd.header.parentUuid), vdi.$snapshot_of$uuid)
+        })
+      }
+    })
+
+    const presentBaseVdis = new Map(baseUuidToSrcVdi)
+    await this._callWriters(
+      writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis),
+      'writer.checkBaseVdis()',
+      false
+    )
+    // check if the parent vdi are present in all the remotes
+    baseUuidToSrcVdi.forEach((srcVdiUuid, baseUuid) => {
+      if (!presentBaseVdis.has(baseUuid)) {
+        throw new Error(`Missing vdi ${baseUuid} which is a base for a delta`)
+      }
+    })
+    // yeah , let's go
+  }
   async _run($defer) {
     const transferList = await this._computeTransferList(({ mode }) => mode === 'delta')
     await this._callWriters(async writer => {
@@ -26,7 +64,7 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
     if (transferList.length > 0) {
       for (const metadata of transferList) {
         assert.strictEqual(metadata.mode, 'delta')
-
+        await this._selectBaseVm(metadata)
         await this._callWriters(writer => writer.prepare({ isBase: metadata.isBase }), 'writer.prepare()')
         const incrementalExport = await this._sourceRemoteAdapter.readIncrementalVmBackup(metadata, undefined, {
           useChain: false,
@@ -50,6 +88,17 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
             }),
           'writer.transfer()'
         )
+        // this will update parent name with the needed alias
+        await this._callWriters(
+          writer =>
+            writer.updateUuidAndChain({
+              isVhdDifferencing,
+              timestamp: metadata.timestamp,
+              vdis: incrementalExport.vdis,
+            }),
+          'writer.updateUuidAndChain()'
+        )
+
         await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
         // for healthcheck
         this._tags = metadata.vm.tags

@xen-orchestra/backups/_runners/_vmRunners/IncrementalXapi.mjs

@@ -78,6 +78,18 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
       'writer.transfer()'
     )
 
+    // we want to control the uuid of the vhd in the chain
+    // and ensure they are correctly chained
+    await this._callWriters(
+      writer =>
+        writer.updateUuidAndChain({
+          isVhdDifferencing,
+          timestamp,
+          vdis: deltaExport.vdis,
+        }),
+      'writer.updateUuidAndChain()'
+    )
+
     this._baseVm = exportedVm
 
     if (baseVm !== undefined) {
@@ -133,7 +145,7 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
       ])
       const srcVdi = srcVdis[snapshotOf]
       if (srcVdi !== undefined) {
-        baseUuidToSrcVdi.set(baseUuid, srcVdi)
+        baseUuidToSrcVdi.set(baseUuid, srcVdi.uuid)
       } else {
         debug('ignore snapshot VDI because no longer present on VM', {
           vdi: baseUuid,
@@ -154,18 +166,18 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
     }
 
     const fullVdisRequired = new Set()
-    baseUuidToSrcVdi.forEach((srcVdi, baseUuid) => {
+    baseUuidToSrcVdi.forEach((srcVdiUuid, baseUuid) => {
       if (presentBaseVdis.has(baseUuid)) {
         debug('found base VDI', {
           base: baseUuid,
-          vdi: srcVdi.uuid,
+          vdi: srcVdiUuid,
         })
       } else {
         debug('missing base VDI', {
           base: baseUuid,
-          vdi: srcVdi.uuid,
+          vdi: srcVdiUuid,
         })
-        fullVdisRequired.add(srcVdi.uuid)
+        fullVdisRequired.add(srcVdiUuid)
       }
     })
 

@xen-orchestra/backups/_runners/_writers/IncrementalRemoteWriter.mjs

@@ -1,17 +1,15 @@
 import assert from 'node:assert'
 import mapValues from 'lodash/mapValues.js'
-import ignoreErrors from 'promise-toolbox/ignoreErrors'
 import { asyncEach } from '@vates/async-each'
 import { asyncMap } from '@xen-orchestra/async-map'
-import { chainVhd, checkVhdChain, openVhd, VhdAbstract } from 'vhd-lib'
+import { chainVhd, openVhd } from 'vhd-lib'
 import { createLogger } from '@xen-orchestra/log'
 import { decorateClass } from '@vates/decorate-with'
 import { defer } from 'golike-defer'
-import { dirname } from 'node:path'
+import { dirname, basename } from 'node:path'
 
 import { formatFilenameDate } from '../../_filenameDate.mjs'
 import { getOldEntries } from '../../_getOldEntries.mjs'
-import { TAG_BASE_DELTA } from '../../_incrementalVm.mjs'
 import { Task } from '../../Task.mjs'
 
 import { MixinRemoteWriter } from './_MixinRemoteWriter.mjs'
@@ -23,42 +21,45 @@ import { Disposable } from 'promise-toolbox'
 const { warn } = createLogger('xo:backups:DeltaBackupWriter')
 
 export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrementalWriter) {
+  #parentVdiPaths
+  #vhds
   async checkBaseVdis(baseUuidToSrcVdi) {
+    this.#parentVdiPaths = {}
     const { handler } = this._adapter
     const adapter = this._adapter
 
     const vdisDir = `${this._vmBackupDir}/vdis/${this._job.id}`
 
-    await asyncMap(baseUuidToSrcVdi, async ([baseUuid, srcVdi]) => {
-      let found = false
+    await asyncMap(baseUuidToSrcVdi, async ([baseUuid, srcVdiUuid]) => {
+      let parentDestPath
+      const vhdDir = `${vdisDir}/${srcVdiUuid}`
       try {
-        const vhds = await handler.list(`${vdisDir}/${srcVdi.uuid}`, {
+        const vhds = await handler.list(vhdDir, {
           filter: _ => _[0] !== '.' && _.endsWith('.vhd'),
           ignoreMissing: true,
           prependDir: true,
         })
         const packedBaseUuid = packUuid(baseUuid)
-        await asyncMap(vhds, async path => {
-          try {
-            await checkVhdChain(handler, path)
-            // Warning, this should not be written as found = found || await adapter.isMergeableParent(packedBaseUuid, path)
-            //
-            // since all the checks of a path are done in parallel, found would be containing
-            // only the last answer of isMergeableParent which is probably not the right one
-            // this led to the support tickets  https://help.vates.fr/#ticket/zoom/4751 , 4729, 4665 and 4300
+        // the last one is probably the right one
 
-            const isMergeable = await adapter.isMergeableParent(packedBaseUuid, path)
-            found = found || isMergeable
+        for (let i = vhds.length - 1; i >= 0 && parentDestPath === undefined; i--) {
+          const path = vhds[i]
+          try {
+            if (await adapter.isMergeableParent(packedBaseUuid, path)) {
+              parentDestPath = path
+            }
           } catch (error) {
             warn('checkBaseVdis', { error })
-            await ignoreErrors.call(VhdAbstract.unlink(handler, path))
           }
-        })
+        }
       } catch (error) {
         warn('checkBaseVdis', { error })
       }
-      if (!found) {
+      // no usable parent => the runner will have to decide to fall back to a full or stop backup
+      if (parentDestPath === undefined) {
         baseUuidToSrcVdi.delete(baseUuid)
+      } else {
+        this.#parentVdiPaths[vhdDir] = parentDestPath
       }
     })
   }
@@ -123,6 +124,44 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
     }
   }
 
+  async updateUuidAndChain({ isVhdDifferencing, vdis }) {
+    assert.notStrictEqual(
+      this.#vhds,
+      undefined,
+      '_transfer must be called before updateUuidAndChain for incremental backups'
+    )
+
+    const parentVdiPaths = this.#parentVdiPaths
+    const { handler } = this._adapter
+    const vhds = this.#vhds
+    await asyncEach(Object.entries(vdis), async ([id, vdi]) => {
+      const isDifferencing = isVhdDifferencing[`${id}.vhd`]
+      const path = `${this._vmBackupDir}/${vhds[id]}`
+      if (isDifferencing) {
+        assert.notStrictEqual(
+          parentVdiPaths,
+          'checkbasevdi must be called before updateUuidAndChain for incremental backups'
+        )
+        const parentPath = parentVdiPaths[dirname(path)]
+        // we are in a incremental backup
+        // we already computed the chain in checkBaseVdis
+        assert.notStrictEqual(parentPath, undefined, 'A differential VHD must have a parent')
+        // forbid any kind of loop
+        assert.ok(basename(parentPath) < basename(path), `vhd must be sorted to be chained`)
+        await chainVhd(handler, parentPath, handler, path)
+      }
+
+      // set the correct UUID in the VHD if needed
+      await Disposable.use(openVhd(handler, path), async vhd => {
+        if (!vhd.footer.uuid.equals(packUuid(vdi.uuid))) {
+          vhd.footer.uuid = packUuid(vdi.uuid)
+          await vhd.readBlockAllocationTable() // required by writeFooter()
+          await vhd.writeFooter()
+        }
+      })
+    })
+  }
+
   async _deleteOldEntries() {
     const adapter = this._adapter
     const oldEntries = this._oldEntries
@@ -141,14 +180,10 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
     const jobId = job.id
     const handler = adapter.handler
 
-    let metadataContent = await this._isAlreadyTransferred(timestamp)
-    if (metadataContent !== undefined) {
-      // @todo : should skip backup while being vigilant to not stuck the forked stream
-      Task.info('This backup has already been transfered')
-    }
-
     const basename = formatFilenameDate(timestamp)
-    const vhds = mapValues(
+    // update this.#vhds before eventually skipping transfer, so that
+    // updateUuidAndChain has all the mandatory data
+    const vhds = (this.#vhds = mapValues(
       deltaExport.vdis,
       vdi =>
         `vdis/${jobId}/${
@@ -158,7 +193,15 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
               vdi.uuid
             : vdi.$snapshot_of$uuid
         }/${adapter.getVhdFileName(basename)}`
-    )
+    ))
+
+    let metadataContent = await this._isAlreadyTransferred(timestamp)
+    if (metadataContent !== undefined) {
+      // skip backup while being vigilant to not stuck the forked stream
+      Task.info('This backup has already been transfered')
+      Object.values(deltaExport.streams).forEach(stream => stream.destroy())
+      return { size: 0 }
+    }
 
     metadataContent = {
       isVhdDifferencing,
@@ -174,38 +217,13 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
       vm,
       vmSnapshot,
     }
+
     const { size } = await Task.run({ name: 'transfer' }, async () => {
       let transferSize = 0
       await asyncEach(
-        Object.entries(deltaExport.vdis),
-        async ([id, vdi]) => {
+        Object.keys(deltaExport.vdis),
+        async id => {
           const path = `${this._vmBackupDir}/${vhds[id]}`
-
-          const isDifferencing = isVhdDifferencing[`${id}.vhd`]
-          let parentPath
-          if (isDifferencing) {
-            const vdiDir = dirname(path)
-            parentPath = (
-              await handler.list(vdiDir, {
-                filter: filename => filename[0] !== '.' && filename.endsWith('.vhd'),
-                prependDir: true,
-              })
-            )
-              .sort()
-              .pop()
-
-            assert.notStrictEqual(
-              parentPath,
-              undefined,
-              `missing parent of ${id} in ${dirname(path)}, looking for ${vdi.other_config[TAG_BASE_DELTA]}`
-            )
-
-            parentPath = parentPath.slice(1) // remove leading slash
-
-            // TODO remove when this has been done before the export
-            await checkVhd(handler, parentPath)
-          }
-
           // don't write it as transferSize += await async function
           // since i += await asyncFun lead to race condition
           // as explained : https://eslint.org/docs/latest/rules/require-atomic-updates
@@ -217,17 +235,6 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
             writeBlockConcurrency: this._config.writeBlockConcurrency,
           })
           transferSize += transferSizeOneDisk
-
-          if (isDifferencing) {
-            await chainVhd(handler, parentPath, handler, path)
-          }
-
-          // set the correct UUID in the VHD
-          await Disposable.use(openVhd(handler, path), async vhd => {
-            vhd.footer.uuid = packUuid(vdi.uuid)
-            await vhd.readBlockAllocationTable() // required by writeFooter()
-            await vhd.writeFooter()
-          })
         },
         {
           concurrency: settings.diskPerVmConcurrency,

@xen-orchestra/backups/_runners/_writers/IncrementalXapiWriter.mjs

@@ -1,3 +1,4 @@
+import assert from 'node:assert'
 import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
 import ignoreErrors from 'promise-toolbox/ignoreErrors'
 import { formatDateTime } from '@xen-orchestra/xapi'
@@ -14,6 +15,7 @@ import find from 'lodash/find.js'
 
 export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWriter) {
   async checkBaseVdis(baseUuidToSrcVdi, baseVm) {
+    assert.notStrictEqual(baseVm, undefined)
     const sr = this._sr
     const replicatedVm = listReplicatedVms(sr.$xapi, this._job.id, sr.uuid, this._vmUuid).find(
       vm => vm.other_config[TAG_COPY_SRC] === baseVm.uuid
@@ -36,7 +38,9 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
       }
     }
   }
-
+  updateUuidAndChain() {
+    // nothing to do, the chaining is not modified in this case
+  }
   prepare({ isFull }) {
     // create the task related to this export and ensure all methods are called in this context
     const task = new Task({

@xen-orchestra/backups/_runners/_writers/_AbstractIncrementalWriter.mjs

@@ -5,6 +5,10 @@ export class AbstractIncrementalWriter extends AbstractWriter {
     throw new Error('Not implemented')
   }
 
+  updateUuidAndChain() {
+    throw new Error('Not implemented')
+  }
+
   cleanup() {
     throw new Error('Not implemented')
   }

@xen-orchestra/backups/_runners/_writers/_MixinRemoteWriter.mjs

@@ -113,13 +113,13 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
       )
     }
 
-    _isAlreadyTransferred(timestamp) {
+    async _isAlreadyTransferred(timestamp) {
       const vmUuid = this._vmUuid
       const adapter = this._adapter
       const backupDir = getVmBackupDir(vmUuid)
       try {
         const actualMetadata = JSON.parse(
-          adapter._handler.readFile(`${backupDir}/${formatFilenameDate(timestamp)}.json`)
+          await adapter._handler.readFile(`${backupDir}/${formatFilenameDate(timestamp)}.json`)
         )
         return actualMetadata
       } catch (error) {}

@xen-orchestra/backups/docs/VM backups/README.md

@@ -230,6 +230,7 @@ Settings are described in [`@xen-orchestra/backups/\_runners/VmsXapi.mjs``](http
     - `checkBaseVdis(baseUuidToSrcVdi, baseVm)`
     - `prepare({ isFull })`
     - `transfer({ timestamp, deltaExport, sizeContainers })`
+    - `updateUuidAndChain({ isVhdDifferencing, vdis })`
     - `cleanup()`
     - `healthCheck()` // is not executed if no health check sr or tag doesn't match
   - **Full**

@xen-orchestra/fs/src/path.js

@@ -20,5 +20,7 @@ export function split(path) {
   return parts
 }
 
-export const relativeFromFile = (file, path) => relative(dirname(file), path)
+// paths are made absolute otherwise fs.relative() would resolve them against working directory
+export const relativeFromFile = (file, path) => relative(dirname(normalize(file)), normalize(path))
+
 export const resolveFromFile = (file, path) => resolve('/', dirname(file), path).slice(1)

@xen-orchestra/fs/src/path.test.js

@@ -0,0 +1,17 @@
+import { describe, it } from 'test'
+import { strict as assert } from 'assert'
+
+import { relativeFromFile } from './path.js'
+
+describe('relativeFromFile()', function () {
+  for (const [title, args] of Object.entries({
+    'file absolute and path absolute': ['/foo/bar/file.vhd', '/foo/baz/path.vhd'],
+    'file relative and path absolute': ['foo/bar/file.vhd', '/foo/baz/path.vhd'],
+    'file absolute and path relative': ['/foo/bar/file.vhd', 'foo/baz/path.vhd'],
+    'file relative and path relative': ['foo/bar/file.vhd', 'foo/baz/path.vhd'],
+  })) {
+    it('works with ' + title, function () {
+      assert.equal(relativeFromFile(...args), '../baz/path.vhd')
+    })
+  }
+})

@xen-orchestra/immutable-backups/protectRemotes.mjs

@@ -54,10 +54,10 @@ async function handleExistingFile(root, indexPath, path) {
         await indexFile(fullPath, indexPath)
       }
     }
-  } catch (err) {
-    if (err.code !== 'EEXIST') {
+  } catch (error) {
+    if (error.code !== 'EEXIST') {
       // there can be a symbolic link in the tree
-      warn('handleExistingFile', err)
+      warn('handleExistingFile', { error })
     }
   }
 }
@@ -106,7 +106,7 @@ export async function watchRemote(remoteId, { root, immutabilityDuration, rebuil
     await File.liftImmutability(settingPath)
   } catch (error) {
     // file may not exists, and it's not really a problem
-    info('lifting immutability on current settings', error)
+    info('lifting immutability on current settings', { error })
   }
   await fs.writeFile(
     settingPath,

@xen-orchestra/lite/src/libs/xapi-stats.ts

@@ -50,7 +50,17 @@ const RRD_POINTS_PER_STEP: { [key in RRD_STEP]: number } = {
 // Utils
 // -------------------------------------------------------------------
 
-function convertNanToNull(value: number) {
+function parseNumber(value: number | string) {
+  // Starting from XAPI 23.31, numbers in the JSON payload are encoded as
+  // strings to support NaN, Infinity and -Infinity
+  if (typeof value === 'string') {
+    const asNumber = +value
+    if (isNaN(asNumber) && value !== 'NaN') {
+      throw new Error('cannot parse number: ' + value)
+    }
+    value = asNumber
+  }
+
   return isNaN(value) ? null : value
 }
 
@@ -59,7 +69,7 @@ function convertNanToNull(value: number) {
 // -------------------------------------------------------------------
 
 const computeValues = (dataRow: any, legendIndex: number, transformValue = identity) =>
-  map(dataRow, ({ values }) => transformValue(convertNanToNull(values[legendIndex])))
+  map(dataRow, ({ values }) => transformValue(parseNumber(values[legendIndex])))
 
 const createGetProperty = (obj: object, property: string, defaultValue: unknown) =>
   defaults(obj, { [property]: defaultValue })[property] as any
@@ -319,8 +329,14 @@ export default class XapiStats {
       },
       abortSignal,
     })
-    // eslint-disable-next-line import/no-named-as-default-member -- https://github.com/json5/json5/issues/287
-    return JSON5.parse(await resp.text())
+    const text = await resp.text()
+    try {
+      // starting from XAPI 23.31, the response is valid JSON
+      return JSON.parse(text)
+    } catch (error) {
+      // eslint-disable-next-line import/no-named-as-default-member -- https://github.com/json5/json5/issues/287
+      return JSON5.parse(text)
+    }
   }
 
   // To avoid multiple requests, we keep a cache for the stats and
@@ -383,7 +399,10 @@ export default class XapiStats {
         abortSignal,
       })
 
-      const actualStep = json.meta.step as number
+      const actualStep = parseNumber(json.meta.step)
+      if (actualStep !== step) {
+        throw new FaultyGranularity(`Unable to get the true granularity: ${actualStep}`)
+      }
 
       if (json.data.length > 0) {
         // fetched data is organized from the newest to the oldest
@@ -407,14 +426,15 @@ export default class XapiStats {
 
           let stepStats = xoObjectStats[actualStep]
           let cacheStepStats = cacheXoObjectStats[actualStep]
-          if (stepStats === undefined || stepStats.endTimestamp !== json.meta.end) {
+          const endTimestamp = parseNumber(json.meta.end)
+          if (stepStats === undefined || stepStats.endTimestamp !== endTimestamp) {
             stepStats = xoObjectStats[actualStep] = {
-              endTimestamp: json.meta.end,
+              endTimestamp,
               interval: actualStep,
               canBeExpired: false,
             }
             cacheStepStats = cacheXoObjectStats[actualStep] = {
-              endTimestamp: json.meta.end,
+              endTimestamp,
               interval: actualStep,
               canBeExpired: true,
             }
@@ -438,10 +458,6 @@ export default class XapiStats {
           })
         })
       }
-
-      if (actualStep !== step) {
-        throw new FaultyGranularity(`Unable to get the true granularity: ${actualStep}`)
-      }
     } catch (error) {
       if (error instanceof Error && error.name === 'AbortError') {
         return

@xen-orchestra/lite/tsconfig.app.json

@@ -1,13 +1,15 @@
 {
   "extends": "@vue/tsconfig/tsconfig.dom.json",
-  "include": ["env.d.ts", "src/**/*", "src/**/*.vue"],
+  "include": ["env.d.ts", "src/**/*", "src/**/*.vue", "../web-core/lib/**/*", "../web-core/lib/**/*.vue"],
   "exclude": ["src/**/__tests__/*"],
   "compilerOptions": {
     "composite": true,
     "noEmit": true,
     "baseUrl": ".",
+    "rootDir": "..",
     "paths": {
-      "@/*": ["./src/*"]
+      "@/*": ["./src/*"],
+      "@core/*": ["../web-core/lib/*"]
     }
   }
 }

@xen-orchestra/lite/vite.config.ts

@@ -23,6 +23,7 @@ export default defineConfig({
   resolve: {
     alias: {
       '@': fileURLToPath(new URL('./src', import.meta.url)),
+      '@core': fileURLToPath(new URL('../web-core/lib', import.meta.url)),
     },
   },
 

@xen-orchestra/log/.USAGE.md

@@ -27,6 +27,16 @@ log.error('could not join server', {
 })
 ```
 
+A logging method has the following signature:
+
+```ts
+interface LoggingMethod {
+  (error): void
+
+  (message: string, data?: { error?: Error; [property: string]: any }): void
+}
+```
+
 ### Consumer
 
 Then, at application level, configure the logs are handled:

@xen-orchestra/log/README.md

@@ -45,6 +45,16 @@ log.error('could not join server', {
 })
 ```
 
+A logging method has the following signature:
+
+```ts
+interface LoggingMethod {
+  (error): void
+
+  (message: string, data?: { error?: Error; [property: string]: any }): void
+}
+```
+
 ### Consumer
 
 Then, at application level, configure the logs are handled:

@xen-orchestra/web-core/docs/composables/collection-navigation.composable.md

@@ -0,0 +1,61 @@
+# `useCollectionNavigation`
+
+The `useCollectionNavigation` composable handles the navigation across a collection (i.e., changing the active item).
+
+It is mainly used to navigate between items in a collection with a keyboard.
+
+## Usage
+
+```ts
+const definition = defineCollection(/* ... */)
+
+const { items, activeItem } = useCollection(definitions)
+
+const { moveUp, moveDown, moveLeft, moveRight, handleKeydown } = useCollectionNavigation(items, activeItem)
+```
+
+## `moveUp`
+
+The `moveUp` function set the `activeItem` to the previous one, if any, or `undefined` otherwise.
+
+If the previous item is a `Group` and it is expanded, the `moveUp` function will set the `activeItem` to the last item
+of that group.
+
+## `moveDown`
+
+The `moveDown` function set the `activeItem` to the next one, if any, or `undefined` otherwise.
+
+If the current `activeItem` is a `Group` and it is expanded, the `moveDown` function will set the `activeItem` to the
+first item of that group.
+
+## `moveLeft`
+
+If the current `activeItem` is a `Group` and it is expanded, the `moveLeft` function will collapse the group.
+
+In any other case, the `moveLeft` function will set the `activeItem` to the parent `Group`, if any, or will do nothing
+otherwise.
+
+## `moveRight`
+
+If the current `activeItem` is a `Group` and it is collapsed, the `moveRight` function will expand the group.
+
+In any other case, the `moveRight` function will act the same as `moveDown`.
+
+## `handleKeydown`
+
+The `handleKeydown` function is a helper function that can be used with the `@keydown` event binding.
+
+```html
+<div @keydown="handleKeydown" tabindex="0">...</div>
+
+<!-- Is equivalent to -->
+<div
+  @keydown.left.prevent="moveLeft"
+  @keydown.right.prevent="moveRight"
+  @keydown.up.prevent="moveUp"
+  @keydown.down.prevent="moveDown"
+  tabindex="0"
+>
+  ...
+</div>
+```

@xen-orchestra/web-core/docs/composables/collection.composable.md

@@ -0,0 +1,492 @@
+# `useCollection` composable
+
+The `useCollection` composable handles a collection of items (called `Leaf` and `Group`) in a tree structure.
+
+`Leaf` and `Group` can be _selected_, _activated_, and/or _filtered_.
+
+Additionally, `Group` can be _expanded_ and contains `Leaf` and/or `Group` children.
+
+Multiple items can be selected at the same time (if `allowMultiSelect` is `true`). But only one item can be activated at
+a time.
+
+## Usage
+
+The `useCollection` composable takes an array of definitions (called `LeafDefinition` and `GroupDefinition`) as first
+argument, and an optional object of options as second argument.
+
+```ts
+useCollection(definitions)
+useCollection(definitions, options)
+```
+
+|                            | Required | Type                                    | Default |                                                       |
+| -------------------------- | :------: | --------------------------------------- | ------- | ----------------------------------------------------- |
+| `definitions`              |    ✓     | `(LeafDefinition \| GroupDefinition)[]` |         | The definitions of the items in the collection        |
+| `options.allowMultiSelect` |          | `boolean`                               | `false` | Whether more than one item can be selected at a time. |
+| `options.expanded`         |          | `boolean`                               | `true`  | Whether all groups are initially expanded.            |
+
+## `useCollection` return values
+
+|                 | Type                                      |                                                                           |
+| --------------- | ----------------------------------------- | ------------------------------------------------------------------------- |
+| `items`         | `(Leaf \| Group)[]`                       | Array of visible `Leaf` and `Group` instances (See Item Visibility below) |
+| `activeItem`    | `ComputedRef<Leaf \| Group \| undefined>` | The active item instance                                                  |
+| `selectedItems` | `ComputedRef<(Leaf \| Group)[]>`          | Array of selected item instances                                          |
+| `expandedItems` | `ComputedRef<Group[]>`                    | Array of expanded group instances                                         |
+
+## `LeafDefinition`
+
+```ts
+new LeafDefinition(id, data)
+new LeafDefinition(id, data, options)
+```
+
+|                         | Required | Type                                | Default     |                                                                                        |
+| ----------------------- | :------: | ----------------------------------- | ----------- | -------------------------------------------------------------------------------------- |
+| `id`                    |    ✓     | `string`                            |             | unique identifier across the whole collection of leafs and groups                      |
+| `data`                  |    ✓     | `T`                                 |             | data to be stored in the item                                                          |
+| `options.discriminator` |          | `string`                            | `undefined` | discriminator for the item when you mix different data types (see Discriminator below) |
+| `options.passesFilter`  |          | `(data: T) => boolean \| undefined` | `undefined` | filter function (see Filtering below)                                                  |
+
+### Example
+
+```ts
+const definition = new LeafDefinition('jd-1', { name: 'John Doe', age: 30 })
+```
+
+## `GroupDefinition`
+
+A `GroupDefinition` is very similar to a `LeafDefinition`, but it contains a collection of children definitions.
+
+```ts
+new GroupDefinition(id, data, children)
+new GroupDefinition(id, data, options, children)
+```
+
+|                         |     | Type                                    | Default     |                                                                                        |
+| ----------------------- | --- | --------------------------------------- | ----------- | -------------------------------------------------------------------------------------- |
+| `id`                    | ✓   | `string`                                |             | unique identifier across the whole collection of leafs and groups                      |
+| `data`                  | ✓   | `any`                                   |             | data to be stored in the item                                                          |
+| `options.discriminator` |     | `string`                                | `undefined` | discriminator for the item when you mix different data types (see Discriminator below) |
+| `options.passesFilter`  |     | `(data) => boolean \| undefined`        | `undefined` | filter function (see Filtering below)                                                  |
+| `children`              | ✓   | `(LeafDefinition \| GroupDefinition)[]` |             | array of items that are contained in this group                                        |
+
+### Example
+
+```ts
+const definition = new GroupDefinition('smithes', { name: 'The Smithes' }, [
+  new ItemDefinition('jd-1', { name: 'John Smith', age: 30 }),
+  new ItemDefinition('jd-2', { name: 'Jane Smith', age: 28 }),
+])
+```
+
+## Discriminator
+
+The `discriminator` is a string used to differentiate between different types of items. This is useful when you want to
+mix different types of items at the same collection depth.
+
+### Mixed data without discriminator
+
+```ts
+const definitions = [
+  new LeafDefinition('jd-1', { name: 'John Doe', age: 30 }),
+  new LeafDefinition('rx-1', { name: 'Rex', breed: 'Golden Retriever' }),
+]
+
+const { items } = useCollection(definitions)
+
+items.value.forEach(item => {
+  // item.data.<cursor> neither 'age' nor 'breed' are available here because we can't know the type of the item
+})
+```
+
+### Using the discriminator
+
+```ts
+const definitions = [
+  new LeafDefinition('jd-1', { name: 'John Doe', age: 30 }, { discriminator: 'person' }),
+  new LeafDefinition('rx-1', { name: 'Rex', breed: 'Golden Retriever' }, { discriminator: 'animal' }),
+]
+
+const { items } = useCollection(definitions)
+
+items.value.forEach(item => {
+  if (item.discriminator === 'person') {
+    // item.data.<cursor> `name` and `age` are available here
+  } else {
+    // item.data.<cursor> `name` and `breed` are available here
+  }
+})
+```
+
+### Mixing `GroupDefinition` and `LeafDefinition` (of same types each)
+
+If you mix `LeafDefinition` and `GroupDefinition` (of same types each), you don't need to use the discriminator because
+the `isGroup` property will serve the same purpose.
+
+```ts
+const definitions = [
+  new LeafDefinition('jd-1', { name: 'John Doe', age: 30 }),
+  new GroupDefinition('dogs', { name: 'Dogs', legs: 4 }, [
+    /* ... */
+  ]),
+]
+
+const { items } = useCollection(definitions)
+
+items.value.forEach(item => {
+  if (item.isGroup) {
+    // item.data.<cursor> `name` and `legs` are available here
+  } else {
+    // item.data.<cursor> `name` and `age` are available here
+  }
+})
+```
+
+## Filtering
+
+The optional `passesFilter` function is used to filter the item across the collection and can affect its visibility (see
+Item Visibility below).
+
+It takes the `data` as first argument and will return:
+
+- `true` if the item explicitly passes the filter
+- `false` if the item explicitly doesn't pass the filter
+- `undefined` if the filter is ignored
+
+## `defineCollection` helper
+
+The `defineCollection` helper creates a collection of definitions in a more convenient way.
+
+```ts
+defineCollection(entries)
+defineCollection(entries, options)
+defineCollection(entries, getChildren)
+defineCollection(entries, options, getChildren)
+```
+
+|                         | Required | Type                             | Default     |                                                                                |
+| ----------------------- | :------: | -------------------------------- | ----------- | ------------------------------------------------------------------------------ |
+| `entries`               |    ✓     | `T[]`                            |             | array of items to be stored in the collection                                  |
+| `options.idField`       |          | `keyof T`                        | `id`        | field to be used as the unique identifier for the items.                       |
+| `options.discriminator` |          | `string`                         | `undefined` | discriminator for the item when you mix different data types                   |
+| `options.passesFilter`  |          | `(data) => boolean \| undefined` | `undefined` | filter function that takes the data as first argument                          |
+| `getChildren`           |    ✓     | `(data: T) => Definition[]`      |             | function that returns an array of definitions that are contained in this group |
+
+Let's take this `families` example:
+
+```ts
+const families = [
+  {
+    id: 'does',
+    name: 'The Does',
+    members: [
+      {
+        id: 'jd-1',
+        name: 'John Doe',
+        age: 30,
+        animals: [
+          {
+            id: 'jd-1-dog',
+            name: 'Rex',
+          },
+        ],
+      },
+      {
+        id: 'jd-2',
+        name: 'Jane Doe',
+        age: 28,
+        animals: [],
+      },
+    ],
+  },
+  {
+    id: 'smiths',
+    name: 'The Smiths',
+    members: [
+      {
+        id: 'js-1',
+        name: 'John Smith',
+        age: 35,
+        animals: [
+          {
+            id: 'js-1-cat',
+            name: 'Whiskers',
+          },
+          {
+            id: 'js-1-dog',
+            name: 'Fido',
+          },
+        ],
+      },
+      {
+        id: 'js-2',
+        name: 'Jane Smith',
+        age: 33,
+        animals: [
+          {
+            id: 'js-2-cat',
+            name: 'Mittens',
+          },
+        ],
+      },
+    ],
+  },
+]
+```
+
+You can use the `defineCollection` helper:
+
+```ts
+const definitions = defineCollection(families, family =>
+  defineCollection(family.members, person => defineCollection(person.animals))
+)
+```
+
+This is the equivalent of the following code:
+
+```ts
+const definitions = families.map(
+  family =>
+    new GroupDefinition(
+      family.id,
+      family,
+      family.members.map(
+        person =>
+          new GroupDefinition(
+            person.id,
+            person,
+            person.animals.map(animal => new ItemDefinition(animal.id, animal))
+          )
+      )
+    )
+)
+```
+
+## `Leaf` and `Group` instances
+
+`Leaf` and `Group` instances have the following properties:
+
+|                 |                             |                                                                   |
+| --------------- | --------------------------- | ----------------------------------------------------------------- |
+| `id`            | `string`                    | unique identifier across the whole collection of leafs and groups |
+| `isGroup`       | `boolean`                   | `true`for `Group` instances, `false` for `Leaf` instances         |
+| `discriminator` | `string` \| `undefined`     | discriminator for the item when you mix different data types      |
+| `data`          | `T`                         | data stored in the item                                           |
+| `depth`         | `number`                    | depth of the item in the collection                               |
+| `isSelected`    | `boolean`                   | whether the item is selected                                      |
+| `isActive`      | `boolean`                   | whether the item is active                                        |
+| `isVisible`     | `boolean`                   | whether the item is visible (see Item Visibility below)           |
+| `activate`      | `() => void`                | function to activate the item                                     |
+| `toggleSelect`  | `(force?: boolean) => void` | function to toggle the selection of the item                      |
+| `labelClasses`  | `{ [name]: boolean }`       | object of classes to be used in the template (see below)          |
+
+### `labelClasses`
+
+The `labelClasses` properties are classes to be used in the template `:class`.
+
+For a `Leaf` instance, it contains the following properties:
+
+- `selected`: whether the leaf is selected
+- `active`: whether the leaf is active
+- `matches`: whether the leaf matches the filter
+
+## `Group` instances
+
+Additionally, `Group` instances have the following properties:
+
+|                                |           |                                                 |
+| ------------------------------ | --------- | ----------------------------------------------- |
+| `isExpanded`                   | `boolean` | whether the item is expanded                    |
+| `areChildrenFullySelected`     | `boolean` | whether all children are selected               |
+| `areChildrenPartiallySelected` | `boolean` | whether some children are selected              |
+| `rawChildren`                  | `Item[]`  | array of all children instances                 |
+| `children`                     | `Item[]`  | array of visible children instances (see below) |
+
+### `labelClasses`
+
+For a `Group` instance, it contains the following properties:
+
+- `selected`: whether the group is selected
+- `selected-partial`: whether the group is partially selected (i.e., some children are selected)
+- `selected-full`: whether the group is fully selected (i.e., all children are selected)
+- `expanded`: whether the group is expanded
+- `active`: whether the group is active
+- `matches`: whether the group matches the filter
+
+## Item Visibility
+
+Here are the rules to determine whether an item is visible or not.
+
+**Note**: Only the first matching rule determines an item's visibility. Subsequent rules are not evaluated.
+
+1. If `passesFilter` returns `true` => _visible_
+2. If any of its ancestors `passesFilter` returns `true` => _visible_
+3. _(`Group` only)_ If any of its descendants `passesFilter` returns `true` => _visible_
+4. If `passesFilter` returns `false` => _**not** visible_
+5. If it doesn't have a parent => _visible_
+6. If the parent's `isExpanded` is `true` => _visible_
+7. If the parent's `isExpanded` is `false` => _**not** visible_
+
+## Example 1: Tree View
+
+```html
+<template>
+  <ul>
+    <li v-for="family in items" :key="family.id">
+      <div class="label" @click="family.toggleExpand()">{{ family.isExpanded ? '↓' : '→' }} {{ family.data.name }}</div>
+      <ul v-if="family.isExpanded" class="persons">
+        <li v-for="person in family.children" :key="person.id">
+          <div class="label" @click="person.toggleExpand()">
+            {{ person.isExpanded ? '↓' : '→' }} {{ person.data.name }} ({{ person.data.age }})
+          </div>
+          <ul v-if="person.isExpanded" class="animals">
+            <li v-for="animal in person.children" :key="animal.id">{{ animal.data.name }}</li>
+          </ul>
+        </li>
+      </ul>
+    </li>
+  </ul>
+</template>
+
+<script lang="ts" setup>
+  const definitions = defineCollection(families, ({ members }) =>
+    defineCollection(members, ({ animals }) => defineCollection(animals))
+  )
+
+  const { items } = useCollection(definitions)
+</script>
+
+<style lang="postcss" scoped>
+  .persons,
+  .animals {
+    padding-left: 20px;
+  }
+
+  .animals li {
+    padding-left: 10px;
+  }
+
+  .label {
+    cursor: pointer;
+  }
+</style>
+```
+
+## Example 2: Multi-select
+
+```html
+<template>
+  <ul>
+    <li v-for="family in items" :key="family.id">
+      <div
+        class="label family"
+        :class="family.labelClasses"
+        @mouseenter="family.activate()"
+        @click="family.toggleChildrenSelect()"
+      >
+        {{ family.data.name }}
+      </div>
+      <ul class="persons">
+        <li v-for="person in family.children" :key="person.id">
+          <div
+            class="label person"
+            :class="person.labelClasses"
+            @mouseenter="person.activate()"
+            @click="person.toggleSelect()"
+          >
+            {{ person.data.name }} ({{ person.data.age }})
+          </div>
+        </li>
+      </ul>
+    </li>
+  </ul>
+</template>
+
+<script lang="ts" setup>
+  const definitions = defineCollection(families, ({ members }) => defineCollection(members))
+
+  const { items } = useCollection(definitions, { allowMultiSelect: true })
+</script>
+
+<style lang="postcss" scoped>
+  .persons {
+    padding-left: 20px;
+  }
+
+  .family {
+    background-color: #eaeaea;
+
+    &.selected-full {
+      background-color: #add8e6;
+    }
+
+    &.active {
+      filter: brightness(1.1);
+    }
+  }
+
+  .person {
+    background-color: #f5f5f5;
+
+    &.selected {
+      background-color: #b5e2f1;
+    }
+
+    &.active {
+      filter: brightness(1.07);
+    }
+  }
+</style>
+```
+
+### Example 3: Filtering
+
+```html
+<template>
+  <div>
+    <input v-model="filter" placeholder="Filter" />
+  </div>
+  <ul>
+    <li v-for="family in items" :key="family.id">
+      <div :class="family.labelClasses">{{ family.data.name }}</div>
+      <ul class="sub">
+        <li v-for="person in family.children" :key="person.id">
+          <div :class="person.labelClasses">{{ person.data.name }} ({{ person.data.age }})</div>
+          <ul class="sub">
+            <li v-for="animal in person.children" :key="animal.id">
+              <div :class="animal.labelClasses">{{ animal.data.name }}</div>
+            </li>
+          </ul>
+        </li>
+      </ul>
+    </li>
+  </ul>
+</template>
+
+<script lang="ts" setup>
+  const filter = ref<string>()
+
+  const predicate = ({ name }: { name: string }) => {
+    const filterValue = filter.value?.trim().toLocaleLowerCase() ?? false
+
+    return !filterValue ? undefined : name.toLocaleLowerCase().includes(filterValue)
+  }
+
+  const definitions = defineCollection(families, { predicate }, ({ members }) =>
+    defineCollection(members, { predicate }, ({ animals }) => defineCollection(animals, { predicate }))
+  )
+
+  const { items } = useCollection(definitions, { expand: false })
+</script>
+
+<style lang="postcss" scoped>
+  .sub {
+    padding-left: 20px;
+  }
+
+  .matches {
+    font-weight: bold;
+  }
+</style>
+```

@xen-orchestra/web-core/lib/composables/collection-navigation.composable.ts

@@ -0,0 +1,73 @@
+import { Group } from '@core/composables/collection/group'
+import type { Item } from '@core/composables/collection/types'
+import { computed, type ComputedRef } from 'vue'
+
+export function useCollectionNavigation<TItem extends Item>(
+  items: ComputedRef<TItem[]>,
+  activeItem: ComputedRef<TItem | undefined>
+) {
+  const flatItems = computed(() => {
+    const result = [] as any[]
+
+    function add(item: Item) {
+      result.push(item)
+
+      if (item instanceof Group) {
+        item.children.forEach(child => add(child))
+      }
+    }
+
+    items.value.forEach(item => add(item))
+
+    return result
+  }) as ComputedRef<TItem[]>
+
+  const activeIndex = computed(() => {
+    const id = activeItem.value?.id
+
+    return id === undefined ? -1 : flatItems.value.findIndex(item => item.id === id)
+  })
+
+  const moveDown = () => {
+    flatItems.value[activeIndex.value === -1 ? 0 : activeIndex.value + 1]?.activate()
+  }
+
+  const moveUp = () => {
+    flatItems.value[activeIndex.value - 1]?.activate()
+  }
+
+  const moveLeft = () => {
+    if (activeItem.value instanceof Group && activeItem.value.isExpanded) {
+      return activeItem.value.toggleExpand(false, true)
+    }
+
+    activeItem.value?.parent?.activate()
+  }
+
+  const moveRight = () => {
+    if (activeItem.value instanceof Group && !activeItem.value.isExpanded) {
+      return activeItem.value.toggleExpand(true)
+    }
+
+    moveDown()
+  }
+
+  const handleKeydown = (event: KeyboardEvent) => {
+    if (['ArrowDown', 'ArrowUp', 'ArrowLeft', 'ArrowRight'].includes(event.key)) {
+      event.preventDefault()
+    }
+
+    switch (event.key) {
+      case 'ArrowDown':
+        return moveDown()
+      case 'ArrowUp':
+        return moveUp()
+      case 'ArrowLeft':
+        return moveLeft()
+      case 'ArrowRight':
+        return moveRight()
+    }
+  }
+
+  return { moveUp, moveDown, moveLeft, moveRight, handleKeydown }
+}

@xen-orchestra/web-core/lib/composables/collection.composable.ts

@@ -0,0 +1,33 @@
+import { buildCollection } from '@core/composables/collection/build-collection'
+import type { CollectionContext, Definition, Item } from '@core/composables/collection/types'
+import { computed, type MaybeRefOrGetter, reactive, type Ref, ref, toValue } from 'vue'
+
+export function useCollection<TDefinition extends Definition>(
+  definitions: MaybeRefOrGetter<TDefinition[]>,
+  options?: { allowMultiSelect?: boolean; expand?: boolean }
+) {
+  const selected = ref(new Map()) as Ref<Map<string, Item>>
+  const expanded = ref(new Map()) as Ref<Map<string, Item>>
+  const active = ref() as Ref<Item | undefined>
+
+  const context = reactive({
+    allowMultiSelect: options?.allowMultiSelect ?? false,
+    selected,
+    expanded,
+    active,
+  }) as CollectionContext
+
+  const rawItems = computed(() => buildCollection(toValue(definitions), context))
+  const items = computed(() => rawItems.value.filter(item => item.isVisible))
+
+  if (options?.expand !== false) {
+    items.value.forEach(item => item.isGroup && item.toggleExpand(true, true))
+  }
+
+  return {
+    items,
+    activeItem: computed(() => context.active),
+    selectedItems: computed(() => Array.from(context.selected.values())),
+    expandedItems: computed(() => Array.from(context.expanded.values())),
+  }
+}

@xen-orchestra/web-core/lib/composables/collection/base.ts

@@ -0,0 +1,69 @@
+import type { Group } from '@core/composables/collection/group'
+import type { CollectionContext, ItemOptions } from '@core/composables/collection/types'
+
+export abstract class Base<T = any, TDiscriminator = any> {
+  abstract readonly isGroup: boolean
+  abstract passesFilterDownwards: boolean
+  abstract isVisible: boolean
+  abstract labelClasses: Record<string, boolean>
+
+  readonly id: string
+  readonly data: T
+  readonly depth: number
+  readonly discriminator: TDiscriminator | undefined
+
+  readonly parent: Group | undefined
+  readonly context: CollectionContext
+  readonly predicate: undefined | ((data: T) => boolean | undefined)
+
+  constructor(
+    id: string,
+    data: T,
+    parent: Group | undefined,
+    context: CollectionContext,
+    depth: number,
+    options?: ItemOptions<T, TDiscriminator>
+  ) {
+    this.id = id
+    this.data = data
+    this.parent = parent
+    this.context = context
+    this.depth = depth
+    this.discriminator = options?.discriminator
+    this.predicate = options?.predicate
+  }
+
+  get passesFilter() {
+    return this.predicate?.(this.data)
+  }
+
+  get isSelected() {
+    return this.context.selected.has(this.id)
+  }
+
+  get isActive() {
+    return this.context.active?.id === this.id
+  }
+
+  get passesFilterUpwards(): boolean {
+    return this.passesFilter || (this.parent?.passesFilterUpwards ?? false)
+  }
+
+  activate() {
+    this.context.active = this
+  }
+
+  toggleSelect(force?: boolean) {
+    const shouldSelect = force ?? !this.isSelected
+
+    if (shouldSelect) {
+      if (!this.context.allowMultiSelect) {
+        this.context.selected.clear()
+      }
+
+      this.context.selected.set(this.id, this)
+    } else {
+      this.context.selected.delete(this.id)
+    }
+  }
+}

@xen-orchestra/web-core/lib/composables/collection/build-collection.ts

@@ -0,0 +1,21 @@
+import { Group } from '@core/composables/collection/group'
+import { GroupDefinition } from '@core/composables/collection/group-definition'
+import { Leaf } from '@core/composables/collection/leaf'
+import type { CollectionContext, Definition, DefinitionToItem, Item } from '@core/composables/collection/types'
+
+export function buildCollection<TDefinition extends Definition>(
+  definitions: TDefinition[],
+  context: CollectionContext
+): DefinitionToItem<TDefinition>[] {
+  function create(definitions: Definition[], parent: Group | undefined, depth: number): Item[] {
+    return definitions.map(definition =>
+      definition instanceof GroupDefinition
+        ? new Group(definition.id, definition.data, parent, context, depth, definition.options, thisGroup =>
+            create(definition.children, thisGroup, depth + 1)
+          )
+        : new Leaf(definition.id, definition.data, parent, context, depth, definition.options)
+    )
+  }
+
+  return create(definitions, undefined, 0) as DefinitionToItem<TDefinition>[]
+}

@xen-orchestra/web-core/lib/composables/collection/define-collection.ts

@@ -0,0 +1,43 @@
+import { GroupDefinition } from '@core/composables/collection/group-definition'
+import { LeafDefinition } from '@core/composables/collection/leaf-definition'
+import type { DefineCollectionOptions, Definition } from '@core/composables/collection/types'
+
+// Overload 1: Leaf with no options
+export function defineCollection<T, const TDiscriminator>(entries: T[]): LeafDefinition<T, TDiscriminator>[]
+
+// Overload 2: Leaf with options
+export function defineCollection<T, const TDiscriminator>(
+  entries: T[],
+  options: DefineCollectionOptions<T, TDiscriminator>
+): LeafDefinition<T, TDiscriminator>[]
+
+// Overload 3: Group with no options
+export function defineCollection<T, TChildDefinition extends Definition, const TDiscriminator>(
+  entries: T[],
+  getChildren: (data: T) => TChildDefinition[]
+): GroupDefinition<T, TChildDefinition, TDiscriminator>[]
+
+// Overload 4: Group with options
+export function defineCollection<T, TChildDefinition extends Definition, const TDiscriminator>(
+  entries: T[],
+  options: DefineCollectionOptions<T, TDiscriminator>,
+  getChildren: (data: T) => TChildDefinition[]
+): GroupDefinition<T, TChildDefinition, TDiscriminator>[]
+
+// Implementation
+export function defineCollection<T, TChildDefinition extends Definition, const TDiscriminator>(
+  entries: T[],
+  optionsOrGetChildren?: DefineCollectionOptions<T, TDiscriminator> | ((data: T) => TChildDefinition[]),
+  getChildren?: (data: T) => TChildDefinition[]
+) {
+  const options = typeof optionsOrGetChildren === 'function' ? {} : optionsOrGetChildren ?? {}
+  const getChildrenFn = typeof optionsOrGetChildren === 'function' ? optionsOrGetChildren : getChildren
+
+  const { idField = 'id' as keyof T, ...otherOptions } = options
+
+  if (getChildrenFn !== undefined) {
+    return entries.map(data => new GroupDefinition(data[idField] as string, data, otherOptions, getChildrenFn(data)))
+  }
+
+  return entries.map(data => new LeafDefinition(data[idField] as string, data, options))
+}

@xen-orchestra/web-core/lib/composables/collection/definition-base.ts

@@ -0,0 +1,13 @@
+import type { ItemOptions } from '@core/composables/collection/types'
+
+export abstract class DefinitionBase<T, TDiscriminator> {
+  id: string
+  data: T
+  options: ItemOptions<T, TDiscriminator>
+
+  constructor(id: string, data: T, options: ItemOptions<T, TDiscriminator> = {}) {
+    this.data = data
+    this.options = options
+    this.id = id
+  }
+}

@xen-orchestra/web-core/lib/composables/collection/group-definition.ts

@@ -0,0 +1,23 @@
+import { DefinitionBase } from '@core/composables/collection/definition-base'
+import type { Definition, ItemOptions } from '@core/composables/collection/types'
+
+export class GroupDefinition<
+  T = any,
+  TChildDefinition extends Definition = Definition,
+  const TDiscriminator = any,
+> extends DefinitionBase<T, TDiscriminator> {
+  children: TChildDefinition[]
+
+  constructor(id: string, data: T, children: TChildDefinition[])
+  constructor(id: string, data: T, options: ItemOptions<T, TDiscriminator>, children: TChildDefinition[])
+  constructor(
+    id: string,
+    data: T,
+    optionsOrChildren: ItemOptions<T, TDiscriminator> | TChildDefinition[],
+    children?: TChildDefinition[]
+  ) {
+    super(id, data, Array.isArray(optionsOrChildren) ? {} : optionsOrChildren)
+
+    this.children = Array.isArray(optionsOrChildren) ? optionsOrChildren : children!
+  }
+}

@xen-orchestra/web-core/lib/composables/collection/group.ts

@@ -0,0 +1,106 @@
+import { Base } from '@core/composables/collection/base'
+import type { CollectionContext, Item, ItemOptions } from '@core/composables/collection/types'
+
+export class Group<T = any, TChild extends Item = Item, const TDiscriminator = any> extends Base<T, TDiscriminator> {
+  readonly isGroup = true
+  readonly rawChildren: TChild[]
+
+  constructor(
+    id: string,
+    data: T,
+    parent: Group | undefined,
+    context: CollectionContext,
+    depth: number,
+    options: ItemOptions<T, TDiscriminator> | undefined,
+    getChildren: (thisGroup: Group<T, TChild, TDiscriminator>) => TChild[]
+  ) {
+    super(id, data, parent, context, depth, options)
+    this.rawChildren = getChildren(this)
+  }
+
+  get children() {
+    return this.rawChildren.filter(child => child.isVisible)
+  }
+
+  get passesFilterDownwards(): boolean {
+    return this.passesFilter || this.rawChildren.some(child => child.passesFilterDownwards)
+  }
+
+  get isVisible() {
+    if (this.passesFilterUpwards || this.passesFilterDownwards) {
+      return true
+    }
+
+    if (this.passesFilter === false) {
+      return false
+    }
+
+    return this.parent?.isExpanded ?? true
+  }
+
+  get isExpanded() {
+    return this.context.expanded.has(this.id) || this.passesFilterDownwards || this.passesFilterUpwards
+  }
+
+  get areChildrenFullySelected(): boolean {
+    if (!this.context.allowMultiSelect) {
+      throw new Error('allowMultiSelect must be enabled to use areChildrenFullySelected')
+    }
+
+    return this.rawChildren.every(child => (child.isGroup ? child.areChildrenFullySelected : child.isSelected))
+  }
+
+  get areChildrenPartiallySelected(): boolean {
+    if (!this.context.allowMultiSelect) {
+      throw new Error('allowMultiSelect must be enabled to use areChildrenPartiallySelected')
+    }
+
+    if (this.areChildrenFullySelected) {
+      return false
+    }
+
+    return this.rawChildren.some(child => (child.isGroup ? child.areChildrenPartiallySelected : child.isSelected))
+  }
+
+  get labelClasses() {
+    return {
+      active: this.isActive,
+      selected: this.isSelected,
+      matches: this.passesFilter === true,
+      'selected-partial': this.context.allowMultiSelect && this.areChildrenPartiallySelected,
+      'selected-full': this.context.allowMultiSelect && this.areChildrenFullySelected,
+      expanded: this.isExpanded,
+    }
+  }
+
+  toggleExpand(force?: boolean, recursive?: boolean) {
+    const shouldExpand = force ?? !this.isExpanded
+
+    if (shouldExpand) {
+      this.context.expanded.set(this.id, this)
+    } else {
+      this.context.expanded.delete(this.id)
+    }
+
+    const shouldPropagate = recursive ?? !shouldExpand
+
+    if (shouldPropagate) {
+      this.rawChildren.forEach(child => {
+        if (child.isGroup) {
+          child.toggleExpand(shouldExpand, recursive)
+        }
+      })
+    }
+  }
+
+  toggleChildrenSelect(force?: boolean) {
+    if (!this.context.allowMultiSelect) {
+      throw new Error('allowMultiSelect must be enabled to use toggleChildrenSelect')
+    }
+
+    const shouldSelect = force ?? !this.areChildrenFullySelected
+    this.rawChildren.forEach(child => {
+      child instanceof Group ? child.toggleChildrenSelect(shouldSelect) : child.toggleSelect(shouldSelect)
+    })
+  }
+}

@xen-orchestra/web-core/lib/composables/collection/leaf-definition.ts

@@ -0,0 +1,3 @@
+import { DefinitionBase } from '@core/composables/collection/definition-base'
+
+export class LeafDefinition<T = any, const TDiscriminator = any> extends DefinitionBase<T, TDiscriminator> {}

@xen-orchestra/web-core/lib/composables/collection/leaf.ts

@@ -0,0 +1,29 @@
+import { Base } from '@core/composables/collection/base'
+
+export class Leaf<T = any, const TDiscriminator = any> extends Base<T, TDiscriminator> {
+  readonly isGroup = false
+
+  get passesFilterDownwards(): boolean {
+    return this.passesFilter ?? false
+  }
+
+  get isVisible() {
+    if (this.passesFilterUpwards) {
+      return true
+    }
+
+    if (this.passesFilter === false) {
+      return false
+    }
+
+    return this.parent?.isExpanded ?? true
+  }
+
+  get labelClasses() {
+    return {
+      active: this.isActive,
+      selected: this.isSelected,
+      matches: this.passesFilter === true,
+    }
+  }
+}

@xen-orchestra/web-core/lib/composables/collection/types.ts

@@ -0,0 +1,32 @@
+import type { Base } from '@core/composables/collection/base'
+import type { Group } from '@core/composables/collection/group'
+import type { GroupDefinition } from '@core/composables/collection/group-definition'
+import type { Leaf } from '@core/composables/collection/leaf'
+import type { LeafDefinition } from '@core/composables/collection/leaf-definition'
+
+export type ItemOptions<T, TDiscriminator> = {
+  discriminator?: TDiscriminator
+  predicate?: (data: T) => boolean | undefined
+}
+
+export type DefineCollectionOptions<T, TDiscriminator> = ItemOptions<T, TDiscriminator> & {
+  idField?: keyof T
+}
+
+export type Definition = LeafDefinition | GroupDefinition
+
+export type CollectionContext = {
+  allowMultiSelect: boolean
+  selected: Map<string, Base>
+  expanded: Map<string, Base>
+  active: Base | undefined
+}
+
+export type DefinitionToItem<TDefinition> =
+  TDefinition extends GroupDefinition<infer T, infer TChildDefinition, infer TDiscriminator>
+    ? Group<T, DefinitionToItem<TChildDefinition>, TDiscriminator>
+    : TDefinition extends LeafDefinition<infer T, infer TDiscriminator>
+      ? Leaf<T, TDiscriminator>
+      : never
+
+export type Item = Leaf | Group

@xen-orchestra/web-core/package.json

@@ -10,7 +10,8 @@
     }
   },
   "devDependencies": {
-    "vue": "^3.4.13"
+    "vue": "^3.4.13",
+    "@vue/tsconfig": "^0.5.1"
   },
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/web-core",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
@@ -25,6 +26,6 @@
   },
   "license": "AGPL-3.0-or-later",
   "engines": {
-    "node": ">=8.10"
+    "node": ">=18"
   }
 }

@xen-orchestra/web-core/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "@vue/tsconfig/tsconfig.dom.json",
+  "include": ["env.d.ts", "lib/**/*", "lib/**/*.vue"],
+  "exclude": ["lib/**/__tests__/*"],
+  "compilerOptions": {
+    "noEmit": true,
+    "baseUrl": ".",
+    "paths": {
+      "@core/*": ["./lib/*"]
+    }
+  }
+}

@xen-orchestra/web/tsconfig.app.json

@@ -1,13 +1,22 @@
 {
   "extends": "@vue/tsconfig/tsconfig.dom.json",
-  "include": ["env.d.ts", "typed-router.d.ts", "src/**/*", "src/**/*.vue"],
+  "include": [
+    "env.d.ts",
+    "typed-router.d.ts",
+    "src/**/*",
+    "src/**/*.vue",
+    "../web-core/lib/**/*",
+    "../web-core/lib/**/*.vue"
+  ],
   "exclude": ["src/**/__tests__/*"],
   "compilerOptions": {
     "composite": true,
     "noEmit": true,
     "baseUrl": ".",
+    "rootDir": "..",
     "paths": {
-      "@/*": ["./src/*"]
+      "@/*": ["./src/*"],
+      "@core/*": ["../web-core/lib/*"]
     }
   }
 }

@xen-orchestra/web/vite.config.ts

@@ -11,6 +11,7 @@ export default defineConfig({
   resolve: {
     alias: {
       '@': fileURLToPath(new URL('./src', import.meta.url)),
+      '@core': fileURLToPath(new URL('../web-core/lib', import.meta.url)),
     },
   },
 })

@xen-orchestra/xapi/vif.mjs

@@ -21,12 +21,23 @@ export default class Vif {
       MAC = '',
     } = {}
   ) {
+    if (device === undefined) {
+      const allowedDevices = await this.call('VM.get_allowed_VIF_devices', VM)
+      if (allowedDevices.length === 0) {
+        const error = new Error('could not find an allowed VIF device')
+        error.poolUuid = this.pool.uuid
+        error.vmRef = VM
+        throw error
+      }
+
+      device = allowedDevices[0]
+    }
+
     const [powerState, ...rest] = await Promise.all([
       this.getField('VM', VM, 'power_state'),
-      device ?? (await this.call('VM.get_allowed_VIF_devices', VM))[0],
-      MTU ?? (await this.getField('network', network, 'MTU')),
+      MTU ?? this.getField('network', network, 'MTU'),
     ])
-    ;[device, MTU] = rest
+    ;[MTU] = rest
 
     const vifRef = await this.call('VIF.create', {
       currently_attached: powerState === 'Suspended' ? currently_attached : undefined,

CHANGELOG.unreleased.md

@@ -8,11 +8,22 @@
 > Users must be able to say: “Nice enhancement, I'm eager to test it”
 
 - Disable search engine indexing via a `robots.txt`
+- [Stats] Support format used by XAPI 23.31
+- [REST API] Export host [SMT](https://en.wikipedia.org/wiki/Simultaneous_multithreading) status at `/hosts/:id/smt` [Forum#71374](https://xcp-ng.org/forum/post/71374)
+- [Home & REST API] `$container` field of an halted VM now points to a host if a VDI is on a local storage [Forum#71769](https://xcp-ng.org/forum/post/71769)
+- [Size Input] Ability to select two new units in the dropdown (`TiB`, `PiB`) (PR [#7382](https://github.com/vatesfr/xen-orchestra/pull/7382))
+
 
 ### Bug fixes
 
 > Users must be able to say: “I had this issue, happy to know it's fixed”
 
+- [Settings/XO Config] Sort backups from newest to oldest
+- [Plugins/audit] Don't log `tag.getAllConfigured` calls
+- [Remotes] Correctly clear error when the remote is tested with success
+- [Import/VMWare] Fix importing last snapshot (PR [#7370](https://github.com/vatesfr/xen-orchestra/pull/7370))
+- [Host/Reboot] Fix false positive warning when restarting an host after updates (PR [#7366](https://github.com/vatesfr/xen-orchestra/pull/7366))
+
 ### Packages to release
 
 > When modifying a package, add it here with its release type.
@@ -29,6 +40,12 @@
 
 <!--packages-start-->
 
-- xo-server patch
+- @xen-orchestra/backups patch
+- @xen-orchestra/fs patch
+- @xen-orchestra/xapi patch
+- vhd-lib patch
+- xo-server minor
+- xo-server-audit patch
+- xo-web minor
 
 <!--packages-end-->

docs/xoa.md

@@ -93,6 +93,21 @@ Follow the instructions:
 
 You can also download XOA from xen-orchestra.com in an XVA file. Once you've got the XVA file, you can import it with `xe vm-import filename=xoa_unified.xva` or via XenCenter.
 
+If you want to use static IP address for your appliance:
+
+```sh
+xe vm-param-set uuid="$uuid" \
+  xenstore-data:vm-data/ip="$ip" \
+  xenstore-data:vm-data/netmask="$netmask" \
+  xenstore-data:vm-data/gateway="$gateway"
+```
+
+If you want to replace the default DNS server:
+
+```sh
+xe vm-param-set uuid="$uuid" xenstore-data:vm-data/dns="$dns"
+```
+
 After the VM is imported, you just need to start it with `xe vm-start vm="XOA"` or with XenCenter.
 
 ## First console connection

package.json

@@ -1,4 +1,6 @@
 {
+  "name": "xen-orchestra",
+  "version": "0.0.0",
   "devDependencies": {
     "@babel/core": "^7.0.0",
     "@babel/eslint-parser": "^7.13.8",
@@ -94,7 +96,7 @@
   },
   "private": true,
   "scripts": {
-    "build": "turbo run build --scope xo-server --scope xo-server-'*' --scope xo-web",
+    "build": "TURBO_TELEMETRY_DISABLED=1 turbo run build --scope xo-server --scope xo-server-'*' --scope xo-web",
     "build:xo-lite": "turbo run build --scope @xen-orchestra/lite",
     "clean": "scripts/run-script.js --parallel clean",
     "dev": "scripts/run-script.js --parallel --concurrency 0 --verbose dev",

packages/vhd-lib/Vhd/VhdSynthetic.js

@@ -46,9 +46,9 @@ const VhdSynthetic = class VhdSynthetic extends VhdAbstract {
   }
 
   get compressionType() {
-    const compressionType = this.vhds[0].compressionType
-    for (let i = 0; i < this.vhds.length; i++) {
-      if (compressionType !== this.vhds[i].compressionType) {
+    const compressionType = this.#vhds[0].compressionType
+    for (let i = 0; i < this.#vhds.length; i++) {
+      if (compressionType !== this.#vhds[i].compressionType) {
         return 'MIXED'
       }
     }

packages/vhd-lib/chain.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { dirname, relative } = require('path')
+const { relativeFromFile } = require('@xen-orchestra/fs/path')
 
 const { openVhd } = require('./openVhd')
 const { DISK_TYPES } = require('./_constants')
@@ -21,7 +21,7 @@ module.exports = async function chain(parentHandler, parentPath, childHandler, c
       }
       await childVhd.readBlockAllocationTable()
 
-      const parentName = relative(dirname(childPath), parentPath)
+      const parentName = relativeFromFile(childPath, parentPath)
       header.parentUuid = parentVhd.footer.uuid
       header.parentUnicodeName = parentName
       await childVhd.setUniqueParentLocator(parentName)

packages/xo-server-audit/src/index.js

@@ -72,6 +72,7 @@ const DEFAULT_BLOCKED_LIST = {
   'system.getServerTimezone': true,
   'system.getServerVersion': true,
   'system.getVersion': true,
+  'tag.getAllConfigured': true,
   'test.getPermissionsForUser': true,
   'user.getAll': true,
   'user.getAuthenticationTokens': true,

packages/xo-server/src/_handleBackupLog.mjs

@@ -27,7 +27,7 @@ async function sendToNagios(app, jobName, vmBackupInfo) {
       jobName
     )
   } catch (error) {
-    warn('sendToNagios:', error)
+    warn('sendToNagios:', { error })
   }
 }
 

packages/xo-server/src/api/host.mjs

@@ -1,3 +1,4 @@
+import semver from 'semver'
 import { createLogger } from '@xen-orchestra/log'
 import assert from 'assert'
 import { format } from 'json-rpc-peer'
@@ -136,13 +137,38 @@ export async function restart({
     const pool = this.getObject(host.$poolId, 'pool')
     const master = this.getObject(pool.master, 'host')
     const hostRebootRequired = host.rebootRequired
-    if (hostRebootRequired && host.id !== master.id && host.version === master.version) {
-      throw incorrectState({
-        actual: hostRebootRequired,
-        expected: false,
-        object: master.id,
-        property: 'rebootRequired',
-      })
+
+    // we are currently in an host upgrade process
+    if (hostRebootRequired && host.id !== master.id) {
+      // this error is not ideal but it means that the pool master must be fully upgraded/rebooted before the current host can be rebooted.
+      //
+      // there is a single error for the 3 cases because the client must handle them the same way
+      const throwError = () =>
+        incorrectState({
+          actual: hostRebootRequired,
+          expected: false,
+          object: master.id,
+          property: 'rebootRequired',
+        })
+
+      if (semver.lt(master.version, host.version)) {
+        log.error(`master version (${master.version}) is older than the host version (${host.version})`, {
+          masterId: master.id,
+          hostId: host.id,
+        })
+        throwError()
+      }
+
+      if (semver.eq(master.version, host.version)) {
+        if ((await this.getXapi(host).listMissingPatches(master._xapiId)).length > 0) {
+          log.error('master has missing patches', { masterId: master.id })
+          throwError()
+        }
+        if (master.rebootRequired) {
+          log.error('master needs to reboot')
+          throwError()
+        }
+      }
     }
   }
 

packages/xo-server/src/xapi-object-to-xo.mjs

@@ -328,6 +328,34 @@ const TRANSFORMS = {
 
     const { creation } = xoData.extract(obj) ?? {}
 
+    let $container
+    if (obj.resident_on !== 'OpaqueRef:NULL') {
+      // resident_on is set when the VM is running (or paused or suspended on a host)
+      $container = link(obj, 'resident_on')
+    } else {
+      // if the VM is halted, the $container is the pool
+      $container = link(obj, 'pool')
+
+      // unless one of its VDI is on a non shared SR
+      //
+      // linked objects may not be there when this code run, and it will only be
+      // refreshed when the VM XAPI record change, this value is not guaranteed
+      // to be up-to-date, but it practice it appears to work fine thanks to
+      // `VBDs` and `current_operations` changing when a VDI is
+      // added/removed/migrated
+      for (const vbd of obj.$VBDs) {
+        const sr = vbd?.$VDI?.$SR
+        if (sr !== undefined && !sr.shared) {
+          const pbd = sr.$PBDs[0]
+          const hostId = pbd && link(pbd, 'host')
+          if (hostId !== undefined) {
+            $container = hostId
+            break
+          }
+        }
+      }
+    }
+
     const vm = {
       // type is redefined after for controllers/, templates &
       // snapshots.
@@ -422,8 +450,7 @@ const TRANSFORMS = {
       xenTools,
       ...getVmGuestToolsProps(obj),
 
-      // TODO: handle local VMs (`VM.get_possible_hosts()`).
-      $container: isRunning ? link(obj, 'resident_on') : link(obj, 'pool'),
+      $container,
       $VBDs: link(obj, 'VBDs'),
 
       // TODO: dedupe

packages/xo-server/src/xapi-stats.mjs

@@ -45,7 +45,17 @@ const RRD_POINTS_PER_STEP = {
 // Utils
 // -------------------------------------------------------------------
 
-function convertNanToNull(value) {
+function parseNumber(value) {
+  // Starting from XAPI 23.31, numbers in the JSON payload are encoded as
+  // strings to support NaN, Infinity and -Infinity
+  if (typeof value === 'string') {
+    const asNumber = +value
+    if (isNaN(asNumber) && value !== 'NaN') {
+      throw new Error('cannot parse number: ' + value)
+    }
+    value = asNumber
+  }
+
   return isNaN(value) ? null : value
 }
 
@@ -58,7 +68,7 @@ async function getServerTimestamp(xapi, hostRef) {
 // -------------------------------------------------------------------
 
 const computeValues = (dataRow, legendIndex, transformValue = identity) =>
-  map(dataRow, ({ values }) => transformValue(convertNanToNull(values[legendIndex])))
+  map(dataRow, ({ values }) => transformValue(parseNumber(values[legendIndex])))
 
 const combineStats = (stats, path, combineValues) => zipWith(...map(stats, path), (...values) => combineValues(values))
 
@@ -245,7 +255,15 @@ export default class XapiStats {
           start: timestamp,
         },
       })
-      .then(response => response.text().then(JSON5.parse))
+      .then(response => response.text())
+      .then(data => {
+        try {
+          // starting from XAPI 23.31, the response is valid JSON
+          return JSON.parse(data)
+        } catch (_) {
+          return JSON5.parse(data)
+        }
+      })
       .catch(err => {
         delete this.#hostCache[hostUuid][step]
         throw err
@@ -299,7 +317,7 @@ export default class XapiStats {
     // To avoid crossing over the boundary, we ask for one less step
     const optimumTimestamp = currentTimeStamp - maxDuration + step
     const json = await this._getJson(xapi, host, optimumTimestamp, step)
-    const actualStep = json.meta.step
+    const actualStep = parseNumber(json.meta.step)
 
     if (actualStep !== step) {
       throw new FaultyGranularity(`Unable to get the true granularity: ${actualStep}`)
@@ -326,9 +344,10 @@ export default class XapiStats {
           return
         }
 
-        if (stepStats === undefined || stepStats.endTimestamp !== json.meta.end) {
+        const endTimestamp = parseNumber(json.meta.end)
+        if (stepStats === undefined || stepStats.endTimestamp !== endTimestamp) {
           stepStats = {
-            endTimestamp: json.meta.end,
+            endTimestamp,
             interval: actualStep,
             stats: {},
           }

packages/xo-server/src/xo-mixins/migrate-vm.mjs

@@ -280,7 +280,7 @@ export default class MigrateVm {
             const stream = vhd.stream()
             await vdi.$importContent(stream, { format: VDI_FORMAT_VHD })
           }
-          return vhd
+          return { vdi, vhd }
         })
       )
     )

packages/xo-server/src/xo-mixins/remotes.mjs

@@ -1,5 +1,6 @@
 import asyncMapSettled from '@xen-orchestra/async-map/legacy.js'
 import { basename } from 'path'
+import { createLogger } from '@xen-orchestra/log'
 import { format, parse } from 'xo-remote-parser'
 import {
   DEFAULT_ENCRYPTION_ALGORITHM,
@@ -17,17 +18,35 @@ import { Remotes } from '../models/remote.mjs'
 
 // ===================================================================
 
+const { warn } = createLogger('xo:mixins:remotes')
+
 const obfuscateRemote = ({ url, ...remote }) => {
   const parsedUrl = parse(url)
   remote.url = format(sensitiveValues.obfuscate(parsedUrl))
   return remote
 }
 
-function validatePath(url) {
-  const { path } = parse(url)
+// these properties should be defined on the remote object itself and not as
+// part of the remote URL
+//
+// there is a bug somewhere that keep putting them into the URL, this list
+// is here to help track it
+const INVALID_URL_PARAMS = ['benchmarks', 'id', 'info', 'name', 'proxy', 'enabled', 'error', 'url']
+
+function validateUrl(url) {
+  const parsedUrl = parse(url)
+
+  const { path } = parsedUrl
   if (path !== undefined && basename(path) === 'xo-vm-backups') {
     throw invalidParameters('remote url should not end with xo-vm-backups')
   }
+
+  for (const param of INVALID_URL_PARAMS) {
+    if (Object.hasOwn(parsedUrl, param)) {
+      // log with stack trace
+      warn(new Error('invalid remote URL param ' + param))
+    }
+  }
 }
 
 export default class {
@@ -182,6 +201,22 @@ export default class {
     if (remote === undefined) {
       throw noSuchObject(id, 'remote')
     }
+
+    const parsedUrl = parse(remote.url)
+    let fixed = false
+    for (const param of INVALID_URL_PARAMS) {
+      if (Object.hasOwn(parsedUrl, param)) {
+        // delete the value to trace its real origin when it's added back
+        // with `updateRemote()`
+        delete parsedUrl[param]
+        fixed = true
+      }
+    }
+    if (fixed) {
+      remote.url = format(parsedUrl)
+      this._remotes.update(remote).catch(warn)
+    }
+
     return remote
   }
 
@@ -202,7 +237,7 @@ export default class {
   }
 
   async createRemote({ name, options, proxy, url }) {
-    validatePath(url)
+    validateUrl(url)
 
     const params = {
       enabled: false,
@@ -219,6 +254,10 @@ export default class {
   }
 
   updateRemote(id, { enabled, name, options, proxy, url }) {
+    if (url !== undefined) {
+      validateUrl(url)
+    }
+
     const handlers = this._handlers
     const handler = handlers[id]
     if (handler !== undefined) {
@@ -238,7 +277,7 @@ export default class {
   @synchronized()
   async _updateRemote(id, { url, ...props }) {
     if (url !== undefined) {
-      validatePath(url)
+      validateUrl(url)
     }
 
     const remote = await this._getRemote(id)

packages/xo-server/src/xo-mixins/rest-api.mjs

@@ -253,6 +253,10 @@ export default class RestApi {
           const host = req.xapiObject
           res.json(await host.$xapi.listMissingPatches(host))
         },
+
+        async smt({ xapiObject }, res) {
+          res.json({ enabled: await xapiObject.$xapi.isHyperThreadingEnabled(xapiObject.$id) })
+        },
       }
 
       collections.pools.routes = {

packages/xo-web/src/common/form/index.js

@@ -138,7 +138,7 @@ export class Range extends Component {
 
 export Toggle from './toggle'
 
-const UNITS = ['kiB', 'MiB', 'GiB']
+const UNITS = ['kiB', 'MiB', 'GiB', 'TiB', 'PiB']
 const DEFAULT_UNIT = 'GiB'
 
 export class SizeInput extends BaseComponent {

packages/xo-web/src/common/iso-device.js

@@ -54,13 +54,9 @@ export default class IsoDevice extends Component {
     () => this.props.vm.$pool,
     () => this.props.vm.$container,
     (vmPool, vmContainer) => sr => {
-      const vmRunning = vmContainer !== vmPool
-      const sameHost = vmContainer === sr.$container
-      const samePool = vmPool === sr.$pool
-
       return (
-        samePool &&
-        (vmRunning ? sr.shared || sameHost : true) &&
+        vmPool === sr.$pool &&
+        (sr.shared || vmContainer === sr.$container) &&
         (sr.SR_type === 'iso' || (sr.SR_type === 'udev' && sr.size))
       )
     }

packages/xo-web/src/common/report-bug-button.js

@@ -75,7 +75,7 @@ export const reportOnSupportPanel = async ({ files = [], formatMessage = identit
     ADDITIONAL_FILES.map(({ fetch, name }) =>
       timeout.call(fetch(), ADDITIONAL_FILES_FETCH_TIMEOUT).then(
         file => formData.append('attachments', createBlobFromString(file), name),
-        error => logger.warn(`cannot get ${name}`, error)
+        error => logger.warn(`cannot get ${name}`, { error })
       )
     )
   )

packages/xo-web/src/common/select-objects.js

@@ -1099,7 +1099,9 @@ export const SelectXoCloudConfig = makeSubscriptionSelect(
   subscriber =>
     subscribeCloudXoConfigBackups(configs => {
       const xoObjects = groupBy(
-        map(configs, config => ({ ...config, type: 'xoConfig' })),
+        map(configs, config => ({ ...config, type: 'xoConfig' }))
+          // from newest to oldest
+          .sort((a, b) => b.createdAt - a.createdAt),
         'xoaId'
       )
       subscriber({

packages/xo-web/src/xo-app/host/tab-general.js

@@ -3,7 +3,6 @@ import _ from 'intl'
 import Copiable from 'copiable'
 import decorate from 'apply-decorators'
 import Icon from 'icon'
-import map from 'lodash/map'
 import React from 'react'
 import store from 'store'
 import HomeTags from 'home-tags'
@@ -24,10 +23,21 @@ export default decorate([
   provideState({
     computed: {
       areHostsVersionsEqual: ({ areHostsVersionsEqualByPool }, { host }) => areHostsVersionsEqualByPool[host.$pool],
+      inMemoryVms: (_, { vms }) => {
+        const result = []
+        for (const key of Object.keys(vms)) {
+          const vm = vms[key]
+          const { power_state } = vm
+          if (power_state === 'Running' || power_state === 'Paused') {
+            result.push(vm)
+          }
+        }
+        return result
+      },
     },
   }),
   injectState,
-  ({ statsOverview, host, nVms, vmController, vms, state: { areHostsVersionsEqual } }) => {
+  ({ statsOverview, host, nVms, vmController, state: { areHostsVersionsEqual, inMemoryVms } }) => {
     const pool = getObject(store.getState(), host.$pool)
     const vmsFilter = encodeURIComponent(new CM.Property('$container', new CM.String(host.id)).toString())
     return (
@@ -120,7 +130,7 @@ export default decorate([
                 tooltip={`${host.productBrand} (${formatSize(vmController.memory.size)})`}
                 value={vmController.memory.size}
               />
-              {map(vms, vm => (
+              {inMemoryVms.map(vm => (
                 <UsageElement
                   tooltip={`${vm.name_label} (${formatSize(vm.memory.size)})`}
                   key={vm.id}

packages/xo-web/src/xo-app/settings/config/xo-cloud-config/index.js

@@ -5,10 +5,9 @@ import decorate from 'apply-decorators'
 import Icon from 'icon'
 import React from 'react'
 import { confirm } from 'modal'
-import { getApiApplianceInfo, subscribeCloudXoConfig, subscribeCloudXoConfigBackups } from 'xo'
-import { groupBy, sortBy } from 'lodash'
 import { injectState, provideState } from 'reaclette'
 import { SelectXoCloudConfig } from 'select-objects'
+import { subscribeCloudXoConfig, subscribeCloudXoConfigBackups } from 'xo'
 
 import BackupXoConfigModal from './backup-xo-config-modal'
 import RestoreXoConfigModal from './restore-xo-config-modal'
@@ -88,15 +87,7 @@ const CloudConfig = decorate([
         },
     },
     computed: {
-      applianceId: async () => {
-        const { id } = await getApiApplianceInfo()
-        return id
-      },
-      groupedConfigs: ({ applianceId, sortedConfigs }) =>
-        sortBy(groupBy(sortedConfigs, 'xoaId'), config => (config[0].xoaId === applianceId ? -1 : 1)),
       isConfigDefined: ({ config }) => config != null,
-      sortedConfigs: (_, { cloudXoConfigBackups }) =>
-        cloudXoConfigBackups?.sort((config, nextConfig) => config.createdAt - nextConfig.createdAt),
     },
   }),
   injectState,

packages/xo-web/src/xo-app/settings/remotes/index.js

@@ -33,7 +33,7 @@ const formatError = error => (typeof error === 'string' ? error : JSON.stringify
 
 const _changeUrlElement = (value, { remote, element }) =>
   editRemote(remote, {
-    url: format({ ...remote, [element]: value === null ? undefined : value }),
+    url: format({ ...parse(remote.url), [element]: value === null ? undefined : value }),
   })
 const _showError = remote => alert(_('remoteConnectionFailed'), <pre>{formatError(remote.error)}</pre>)
 const _editRemoteName = (name, { remote }) => editRemote(remote, { name })