review

See https://xcp-ng.org/forum/topic/6070/netbox-plugin-enhancements

.gitignore

@@ -10,8 +10,6 @@
 /packages/*/dist/
 /packages/*/node_modules/
 
-/packages/vhd-cli/src/commands/index.js
-
 /packages/xen-api/examples/node_modules/
 /packages/xen-api/plot.dat
 

@vates/async-each/.USAGE.md

@@ -14,7 +14,7 @@ Returns a promise wich rejects as soon as a call to `iteratee` throws or a promi
 
 `opts` is an object that can contains the following options:
 
-- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
+- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `10`. The value `0` means no concurrency limit.
 - `signal`: an abort signal to stop the iteration
 - `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
 

@vates/async-each/README.md

@@ -32,7 +32,7 @@ Returns a promise wich rejects as soon as a call to `iteratee` throws or a promi
 
 `opts` is an object that can contains the following options:
 
-- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
+- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `10`. The value `0` means no concurrency limit.
 - `signal`: an abort signal to stop the iteration
 - `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
 

@vates/async-each/index.js

@@ -9,7 +9,16 @@ class AggregateError extends Error {
   }
 }
 
-exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 1, signal, stopOnError = true } = {}) {
+/**
+ * @template Item
+ * @param {Iterable<Item>} iterable
+ * @param {(item: Item, index: number, iterable: Iterable<Item>) => Promise<void>} iteratee
+ * @returns {Promise<void>}
+ */
+exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 10, signal, stopOnError = true } = {}) {
+  if (concurrency === 0) {
+    concurrency = Infinity
+  }
   return new Promise((resolve, reject) => {
     const it = (iterable[Symbol.iterator] || iterable[Symbol.asyncIterator]).call(iterable)
     const errors = []

@vates/async-each/index.spec.js

@@ -36,7 +36,7 @@ describe('asyncEach', () => {
       it('works', async () => {
         const iteratee = jest.fn(async () => {})
 
-        await asyncEach.call(thisArg, iterable, iteratee)
+        await asyncEach.call(thisArg, iterable, iteratee, { concurrency: 1 })
 
         expect(iteratee.mock.instances).toEqual(Array.from(values, () => thisArg))
         expect(iteratee.mock.calls).toEqual(Array.from(values, (value, index) => [value, index, iterable]))
@@ -66,7 +66,7 @@ describe('asyncEach', () => {
           }
         })
 
-        expect(await rejectionOf(asyncEach(iterable, iteratee, { stopOnError: true }))).toBe(error)
+        expect(await rejectionOf(asyncEach(iterable, iteratee, { concurrency: 1, stopOnError: true }))).toBe(error)
         expect(iteratee).toHaveBeenCalledTimes(2)
       })
 
@@ -91,7 +91,9 @@ describe('asyncEach', () => {
           }
         })
 
-        await expect(asyncEach(iterable, iteratee, { signal: ac.signal })).rejects.toThrow('asyncEach aborted')
+        await expect(asyncEach(iterable, iteratee, { concurrency: 1, signal: ac.signal })).rejects.toThrow(
+          'asyncEach aborted'
+        )
         expect(iteratee).toHaveBeenCalledTimes(2)
       })
     })

@vates/event-listeners-manager/index.js

@@ -9,7 +9,7 @@ exports.EventListenersManager = class EventListenersManager {
   }
 
   add(type, listener) {
-    let listeners = this._listeners[type]
+    let listeners = this._listeners.get(type)
     if (listeners === undefined) {
       listeners = new Set()
       this._listeners.set(type, listeners)

@vates/event-listeners-manager/index.spec.js

@@ -0,0 +1,67 @@
+'use strict'
+
+const t = require('tap')
+const { EventEmitter } = require('events')
+
+const { EventListenersManager } = require('./')
+
+const noop = Function.prototype
+
+// function spy (impl = Function.prototype) {
+//   function spy() {
+//     spy.calls.push([Array.from(arguments), this])
+//   }
+//   spy.calls = []
+//   return spy
+// }
+
+function assertListeners(t, event, listeners) {
+  t.strictSame(t.context.ee.listeners(event), listeners)
+}
+
+t.beforeEach(function (t) {
+  t.context.ee = new EventEmitter()
+  t.context.em = new EventListenersManager(t.context.ee)
+})
+
+t.test('.add adds a listener', function (t) {
+  t.context.em.add('foo', noop)
+
+  assertListeners(t, 'foo', [noop])
+
+  t.end()
+})
+
+t.test('.add does not add a duplicate listener', function (t) {
+  t.context.em.add('foo', noop).add('foo', noop)
+
+  assertListeners(t, 'foo', [noop])
+
+  t.end()
+})
+
+t.test('.remove removes a listener', function (t) {
+  t.context.em.add('foo', noop).remove('foo', noop)
+
+  assertListeners(t, 'foo', [])
+
+  t.end()
+})
+
+t.test('.removeAll removes all listeners of a given type', function (t) {
+  t.context.em.add('foo', noop).add('bar', noop).removeAll('foo')
+
+  assertListeners(t, 'foo', [])
+  assertListeners(t, 'bar', [noop])
+
+  t.end()
+})
+
+t.test('.removeAll removes all listeners', function (t) {
+  t.context.em.add('foo', noop).add('bar', noop).removeAll()
+
+  assertListeners(t, 'foo', [])
+  assertListeners(t, 'bar', [])
+
+  t.end()
+})

@vates/event-listeners-manager/package.json

@@ -35,8 +35,12 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "scripts": {
-    "postversion": "npm publish --access public"
+    "postversion": "npm publish --access public",
+    "test": "tap --branches=72"
+  },
+  "devDependencies": {
+    "tap": "^16.2.0"
   }
 }

@vates/read-chunk/.USAGE.md

@@ -1,6 +1,9 @@
+### `readChunk(stream, [size])`
+
 - returns the next available chunk of data
 - like `stream.read()`, a number of bytes can be specified
-- returns `null` if the stream has ended
+- returns with less data than expected if stream has ended
+- returns `null` if the stream has ended and no data has been read
 
 ```js
 import { readChunk } from '@vates/read-chunk'
@@ -11,3 +14,13 @@ import { readChunk } from '@vates/read-chunk'
   }
 })()
 ```
+
+### `readChunkStrict(stream, [size])`
+
+Similar behavior to `readChunk` but throws if the stream ended before the requested data could be read.
+
+```js
+import { readChunkStrict } from '@vates/read-chunk'
+
+const chunk = await readChunkStrict(stream, 1024)
+```

@vates/read-chunk/README.md

@@ -16,9 +16,12 @@ Installation of the [npm package](https://npmjs.org/package/@vates/read-chunk):
 
 ## Usage
 
+### `readChunk(stream, [size])`
+
 - returns the next available chunk of data
 - like `stream.read()`, a number of bytes can be specified
-- returns `null` if the stream has ended
+- returns with less data than expected if stream has ended
+- returns `null` if the stream has ended and no data has been read
 
 ```js
 import { readChunk } from '@vates/read-chunk'
@@ -30,6 +33,16 @@ import { readChunk } from '@vates/read-chunk'
 })()
 ```
 
+### `readChunkStrict(stream, [size])`
+
+Similar behavior to `readChunk` but throws if the stream ended before the requested data could be read.
+
+```js
+import { readChunkStrict } from '@vates/read-chunk'
+
+const chunk = await readChunkStrict(stream, 1024)
+```
+
 ## Contributions
 
 Contributions are _very_ welcomed, either on the documentation or on

@vates/read-chunk/index.js

@@ -30,3 +30,22 @@ const readChunk = (stream, size) =>
         onReadable()
       })
 exports.readChunk = readChunk
+
+exports.readChunkStrict = async function readChunkStrict(stream, size) {
+  const chunk = await readChunk(stream, size)
+  if (chunk === null) {
+    throw new Error('stream has ended without data')
+  }
+
+  if (size !== undefined && chunk.length !== size) {
+    const error = new Error('stream has ended with not enough data')
+    Object.defineProperties(error, {
+      chunk: {
+        value: chunk,
+      },
+    })
+    throw error
+  }
+
+  return chunk
+}

@vates/read-chunk/index.spec.js

@@ -4,7 +4,7 @@
 
 const { Readable } = require('stream')
 
-const { readChunk } = require('./')
+const { readChunk, readChunkStrict } = require('./')
 
 const makeStream = it => Readable.from(it, { objectMode: false })
 makeStream.obj = Readable.from
@@ -43,3 +43,27 @@ describe('readChunk', () => {
     })
   })
 })
+
+const rejectionOf = promise =>
+  promise.then(
+    value => {
+      throw value
+    },
+    error => error
+  )
+
+describe('readChunkStrict', function () {
+  it('throws if stream is empty', async () => {
+    const error = await rejectionOf(readChunkStrict(makeStream([])))
+    expect(error).toBeInstanceOf(Error)
+    expect(error.message).toBe('stream has ended without data')
+    expect(error.chunk).toEqual(undefined)
+  })
+
+  it('throws if stream ends with not enough data', async () => {
+    const error = await rejectionOf(readChunkStrict(makeStream(['foo', 'bar']), 10))
+    expect(error).toBeInstanceOf(Error)
+    expect(error.message).toBe('stream has ended with not enough data')
+    expect(error.chunk).toEqual(Buffer.from('foobar'))
+  })
+})

@vates/read-chunk/package.json

@@ -19,7 +19,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.1.2",
+  "version": "1.0.0",
   "engines": {
     "node": ">=8.10"
   },

@xen-orchestra/backups-cli/package.json

@@ -7,8 +7,8 @@
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.23.0",
-    "@xen-orchestra/fs": "^1.0.1",
+    "@xen-orchestra/backups": "^0.27.0",
+    "@xen-orchestra/fs": "^1.1.0",
     "filenamify": "^4.1.0",
     "getopts": "^2.2.5",
     "lodash": "^4.17.15",
@@ -27,7 +27,7 @@
   "scripts": {
     "postversion": "npm publish --access public"
   },
-  "version": "0.7.1",
+  "version": "0.7.5",
   "license": "AGPL-3.0-or-later",
   "author": {
     "name": "Vates SAS",

@xen-orchestra/backups/Backup.js

@@ -6,7 +6,7 @@ const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { compileTemplate } = require('@xen-orchestra/template')
 const { limitConcurrency } = require('limit-concurrency-decorator')
 
-const { extractIdsFromSimplePattern } = require('./_extractIdsFromSimplePattern.js')
+const { extractIdsFromSimplePattern } = require('./extractIdsFromSimplePattern.js')
 const { PoolMetadataBackup } = require('./_PoolMetadataBackup.js')
 const { Task } = require('./Task.js')
 const { VmBackup } = require('./_VmBackup.js')
@@ -36,6 +36,8 @@ const DEFAULT_VM_SETTINGS = {
   deleteFirst: false,
   exportRetention: 0,
   fullInterval: 0,
+  healthCheckSr: undefined,
+  healthCheckVmsWithTags: [],
   maxMergedDeltasPerRun: 2,
   offlineBackup: false,
   offlineSnapshot: false,
@@ -215,6 +217,7 @@ exports.Backup = class Backup {
     const schedule = this._schedule
 
     const config = this._config
+    const settings = this._settings
     await Disposable.use(
       Disposable.all(
         extractIdsFromSimplePattern(job.srs).map(id =>
@@ -242,15 +245,14 @@ exports.Backup = class Backup {
           })
         )
       ),
-      async (srs, remoteAdapters) => {
+      () => settings.healthCheckSr !== undefined ? this._getRecord('SR', settings.healthCheckSr) : undefined,
+      async (srs, remoteAdapters, healthCheckSr) => {
         // remove adapters that failed (already handled)
         remoteAdapters = remoteAdapters.filter(_ => _ !== undefined)
 
         // remove srs that failed (already handled)
         srs = srs.filter(_ => _ !== undefined)
 
-        const settings = this._settings
-
         if (remoteAdapters.length === 0 && srs.length === 0 && settings.snapshotRetention === 0) {
           return
         }
@@ -271,6 +273,7 @@ exports.Backup = class Backup {
                 baseSettings,
                 config,
                 getSnapshotNameLabel,
+                healthCheckSr,
                 job,
                 remoteAdapters,
                 schedule,

@xen-orchestra/backups/HealthCheckVmBackup.js

@@ -0,0 +1,64 @@
+'use strict'
+
+const { Task } = require('./Task')
+
+exports.HealthCheckVmBackup = class HealthCheckVmBackup {
+  #xapi
+  #restoredVm
+
+  constructor({ restoredVm, xapi }) {
+    this.#restoredVm = restoredVm
+    this.#xapi = xapi
+  }
+
+  async run() {
+    return Task.run(
+      {
+        name: 'vmstart',
+      },
+      async () => {
+        let restoredVm = this.#restoredVm
+        const xapi = this.#xapi
+        const restoredId = restoredVm.uuid
+
+        // remove vifs
+        await Promise.all(restoredVm.$VIFs.map(vif => xapi.callAsync('VIF.destroy', vif.$ref)))
+
+        const start = new Date()
+        // start Vm
+
+        await xapi.callAsync(
+          'VM.start',
+          restoredVm.$ref,
+          false, // Start paused?
+          false // Skip pre-boot checks?
+        )
+        const started = new Date()
+        const timeout = 10 * 60 * 1000
+        const startDuration = started - start
+
+        let remainingTimeout = timeout - startDuration
+
+        if (remainingTimeout < 0) {
+          throw new Error(`VM ${restoredId} not started after ${timeout / 1000} second`)
+        }
+
+        // wait for the 'Running' event to be really stored in local xapi object cache
+        restoredVm = await xapi.waitObjectState(restoredVm.$ref, vm => vm.power_state === 'Running', {
+          timeout: remainingTimeout,
+        })
+
+        const running = new Date()
+        remainingTimeout -= running - started
+
+        if (remainingTimeout < 0) {
+          throw new Error(`local xapi  did not get Runnig state for VM ${restoredId} after ${timeout / 1000} second`)
+        }
+        // wait for the guest tool version to be defined
+        await xapi.waitObjectState(restoredVm.guest_metrics, gm => gm?.PV_drivers_version?.major !== undefined, {
+          timeout: remainingTimeout,
+        })
+      }
+    )
+  }
+}

@xen-orchestra/backups/RemoteAdapter.js

@@ -15,7 +15,7 @@ const { deduped } = require('@vates/disposable/deduped.js')
 const { decorateMethodsWith } = require('@vates/decorate-with')
 const { compose } = require('@vates/compose')
 const { execFile } = require('child_process')
-const { readdir, stat } = require('fs-extra')
+const { readdir, lstat } = require('fs-extra')
 const { v4: uuidv4 } = require('uuid')
 const { ZipFile } = require('yazl')
 const zlib = require('zlib')
@@ -47,13 +47,12 @@ const resolveSubpath = (root, path) => resolve(root, `.${resolve('/', path)}`)
 const RE_VHDI = /^vhdi(\d+)$/
 
 async function addDirectory(files, realPath, metadataPath) {
-  try {
-    const subFiles = await readdir(realPath)
-    await asyncMap(subFiles, file => addDirectory(files, realPath + '/' + file, metadataPath + '/' + file))
-  } catch (error) {
-    if (error == null || error.code !== 'ENOTDIR') {
-      throw error
-    }
+  const stats = await lstat(realPath)
+  if (stats.isDirectory()) {
+    await asyncMap(await readdir(realPath), file =>
+      addDirectory(files, realPath + '/' + file, metadataPath + '/' + file)
+    )
+  } else if (stats.isFile()) {
     files.push({
       realPath,
       metadataPath,
@@ -292,7 +291,7 @@ class RemoteAdapter {
   }
 
   #useVhdDirectory() {
-    return this.handler.type === 's3'
+    return this.handler.useVhdDirectory()
   }
 
   #useAlias() {
@@ -383,8 +382,12 @@ class RemoteAdapter {
       const entriesMap = {}
       await asyncMap(await readdir(path), async name => {
         try {
-          const stats = await stat(`${path}/${name}`)
-          entriesMap[stats.isDirectory() ? `${name}/` : name] = {}
+          const stats = await lstat(`${path}/${name}`)
+          if (stats.isDirectory()) {
+            entriesMap[name + '/'] = {}
+          } else if (stats.isFile()) {
+            entriesMap[name] = {}
+          }
         } catch (error) {
           if (error == null || error.code !== 'ENOENT') {
             throw error

@xen-orchestra/backups/_VmBackup.js

@@ -45,7 +45,18 @@ const forkDeltaExport = deltaExport =>
   })
 
 class VmBackup {
-  constructor({ baseSettings, config, getSnapshotNameLabel, job, remoteAdapters, schedule, settings, srs, vm }) {
+  constructor({
+    config,
+    getSnapshotNameLabel,
+    healthCheckSr,
+    job,
+    remoteAdapters,
+    remotes,
+    schedule,
+    settings,
+    srs,
+    vm,
+  }) {
     if (vm.other_config['xo:backup:job'] === job.id && 'start' in vm.blocked_operations) {
       // don't match replicated VMs created by this very job otherwise they
       // will be replicated again and again
@@ -68,6 +79,7 @@ class VmBackup {
     this._fullVdisRequired = undefined
     this._getSnapshotNameLabel = getSnapshotNameLabel
     this._isDelta = job.mode === 'delta'
+    this._healthCheckSr = healthCheckSr
     this._jobId = job.id
     this._jobSnapshots = undefined
     this._xapi = vm.$xapi
@@ -94,7 +106,6 @@ class VmBackup {
         : [FullBackupWriter, FullReplicationWriter]
 
       const allSettings = job.settings
-
       Object.keys(remoteAdapters).forEach(remoteId => {
         const targetSettings = {
           ...settings,
@@ -142,6 +153,13 @@ class VmBackup {
         errors.push(error)
         this.delete(writer)
         warn(warnMessage, { error, writer: writer.constructor.name })
+
+        // these two steps are the only one that are not already in their own sub tasks
+        if (warnMessage === 'writer.checkBaseVdis()' || warnMessage === 'writer.beforeBackup()') {
+          Task.warning(
+            `the writer ${writer.constructor.name} has failed the step ${warnMessage} with error ${error.message}. It won't be used anymore in this job execution.`
+          )
+        }
       }
     })
     if (writers.size === 0) {
@@ -185,6 +203,7 @@ class VmBackup {
         const snapshotRef = await vm[settings.checkpointSnapshot ? '$checkpoint' : '$snapshot']({
           ignoreNobakVdis: true,
           name_label: this._getSnapshotNameLabel(vm),
+          unplugVusbs: true,
         })
         this.timestamp = Date.now()
 
@@ -395,6 +414,24 @@ class VmBackup {
     this._fullVdisRequired = fullVdisRequired
   }
 
+  async _healthCheck() {
+    const settings = this._settings
+
+    if (this._healthCheckSr === undefined) {
+      return
+    }
+
+    // check if current VM has tags
+    const { tags } = this.vm
+    const intersect = settings.healthCheckVmsWithTags.some(t => tags.includes(t))
+
+    if (settings.healthCheckVmsWithTags.length !== 0 && !intersect) {
+      return
+    }
+
+    await this._callWriters(writer => writer.healthCheck(this._healthCheckSr), 'writer.healthCheck()')
+  }
+
   async run($defer) {
     const settings = this._settings
     assert(
@@ -404,7 +441,9 @@ class VmBackup {
 
     await this._callWriters(async writer => {
       await writer.beforeBackup()
-      $defer(() => writer.afterBackup())
+      $defer(async () => {
+        await writer.afterBackup()
+      })
     }, 'writer.beforeBackup()')
 
     await this._fetchJobSnapshots()
@@ -440,6 +479,7 @@ class VmBackup {
       await this._fetchJobSnapshots()
       await this._removeUnusedSnapshots()
     }
+    await this._healthCheck()
   }
 }
 exports.VmBackup = VmBackup

@xen-orchestra/backups/_cleanVm.js

@@ -2,6 +2,7 @@
 
 const assert = require('assert')
 const sum = require('lodash/sum')
+const UUID = require('uuid')
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { Constants, mergeVhd, openVhd, VhdAbstract, VhdFile } = require('vhd-lib')
 const { isVhdAlias, resolveVhdAlias } = require('vhd-lib/aliases')
@@ -47,42 +48,32 @@ const computeVhdsSize = (handler, vhdPaths) =>
 //         |                              |
 //         \___________rename_____________/
 
-async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
+async function mergeVhdChain(chain, { handler, logInfo, remove, merge }) {
   assert(chain.length >= 2)
   const chainCopy = [...chain]
-  const parent = chainCopy.pop()
+  const parent = chainCopy.shift()
   const children = chainCopy
 
   if (merge) {
-    onLog(`merging ${children.length} children into ${parent}`)
+    logInfo(`merging children into parent`, { childrenCount: children.length, parent })
 
     let done, total
     const handle = setInterval(() => {
       if (done !== undefined) {
-        onLog(`merging ${children.join(',')} into ${parent}: ${done}/${total}`)
+        logInfo(`merging children in progress`, { children, parent, doneCount: done, totalCount: total })
       }
     }, 10e3)
 
     const mergedSize = await mergeVhd(handler, parent, handler, children, {
+      logInfo,
       onProgress({ done: d, total: t }) {
         done = d
         total = t
       },
+      remove,
     })
 
     clearInterval(handle)
-    const mergeTargetChild = children.shift()
-    await Promise.all([
-      VhdAbstract.rename(handler, parent, mergeTargetChild),
-      asyncMap(children, child => {
-        onLog(`the VHD ${child} is already merged`)
-        if (remove) {
-          onLog(`deleting merged VHD ${child}`)
-          return VhdAbstract.unlink(handler, child)
-        }
-      }),
-    ])
-
     return mergedSize
   }
 }
@@ -125,14 +116,19 @@ const listVhds = async (handler, vmDir) => {
   return { vhds, interruptedVhds, aliases }
 }
 
-async function checkAliases(aliasPaths, targetDataRepository, { handler, onLog = noop, remove = false }) {
+async function checkAliases(
+  aliasPaths,
+  targetDataRepository,
+  { handler, logInfo = noop, logWarn = console.warn, remove = false }
+) {
   const aliasFound = []
   for (const path of aliasPaths) {
     const target = await resolveVhdAlias(handler, path)
 
     if (!isVhdFile(target)) {
-      onLog(`Alias ${path} references a non vhd target:  ${target}`)
+      logWarn('alias references non VHD target', { path, target })
       if (remove) {
+        logInfo('removing alias and non VHD target', { path, target })
         await handler.unlink(target)
         await handler.unlink(path)
       }
@@ -147,13 +143,13 @@ async function checkAliases(aliasPaths, targetDataRepository, { handler, onLog =
         // error during dispose should not trigger a deletion
       }
     } catch (error) {
-      onLog(`target ${target} of alias ${path} is missing or broken`, { error })
+      logWarn('missing or broken alias target', { target, path, error })
       if (remove) {
         try {
           await VhdAbstract.unlink(handler, path)
-        } catch (e) {
-          if (e.code !== 'ENOENT') {
-            onLog(`Error while deleting target ${target} of alias ${path}`, { error: e })
+        } catch (error) {
+          if (error.code !== 'ENOENT') {
+            logWarn('error deleting alias target', { target, path, error })
           }
         }
       }
@@ -170,26 +166,29 @@ async function checkAliases(aliasPaths, targetDataRepository, { handler, onLog =
 
   entries.forEach(async entry => {
     if (!aliasFound.includes(entry)) {
-      onLog(`the Vhd  ${entry} is not referenced by a an alias`)
+      logWarn('no alias references VHD', { entry })
       if (remove) {
+        logInfo('deleting unaliased VHD')
         await VhdAbstract.unlink(handler, entry)
       }
     }
   })
 }
+
 exports.checkAliases = checkAliases
 
 const defaultMergeLimiter = limitConcurrency(1)
 
 exports.cleanVm = async function cleanVm(
   vmDir,
-  { fixMetadata, remove, merge, mergeLimiter = defaultMergeLimiter, onLog = noop }
+  { fixMetadata, remove, merge, mergeLimiter = defaultMergeLimiter, logInfo = noop, logWarn = console.warn }
 ) {
   const limitedMergeVhdChain = mergeLimiter(mergeVhdChain)
 
   const handler = this._handler
 
   const vhdsToJSons = new Set()
+  const vhdById = new Map()
   const vhdParents = { __proto__: null }
   const vhdChildren = { __proto__: null }
 
@@ -211,12 +210,33 @@ exports.cleanVm = async function cleanVm(
           }
           vhdChildren[parent] = path
         }
+        // Detect VHDs with the same UUIDs
+        //
+        // Due to a bug introduced in a1bcd35e2
+        const duplicate = vhdById.get(UUID.stringify(vhd.footer.uuid))
+        let vhdKept = vhd
+        if (duplicate !== undefined) {
+          logWarn('uuid is duplicated', { uuid: UUID.stringify(vhd.footer.uuid) })
+          if (duplicate.containsAllDataOf(vhd)) {
+            logWarn(`should delete ${path}`)
+            vhdKept = duplicate
+            vhds.delete(path)
+          } else if (vhd.containsAllDataOf(duplicate)) {
+            logWarn(`should delete ${duplicate._path}`)
+            vhds.delete(duplicate._path)
+          } else {
+            logWarn(`same ids but different content`)
+          }
+        } else {
+          logInfo('not duplicate', UUID.stringify(vhd.footer.uuid), path)
+        }
+        vhdById.set(UUID.stringify(vhdKept.footer.uuid), vhdKept)
       })
     } catch (error) {
       vhds.delete(path)
-      onLog(`error while checking the VHD with path ${path}`, { error })
+      logWarn('VHD check error', { path, error })
       if (error?.code === 'ERR_ASSERTION' && remove) {
-        onLog(`deleting broken ${path}`)
+        logInfo('deleting broken path', { path })
         return VhdAbstract.unlink(handler, path)
       }
     }
@@ -228,12 +248,12 @@ exports.cleanVm = async function cleanVm(
       const statePath = interruptedVhds.get(interruptedVhd)
       interruptedVhds.delete(interruptedVhd)
 
-      onLog('orphan merge state', {
+      logWarn('orphan merge state', {
         mergeStatePath: statePath,
         missingVhdPath: interruptedVhd,
       })
       if (remove) {
-        onLog(`deleting orphan merge state ${statePath}`)
+        logInfo('deleting orphan merge state', { statePath })
         await handler.unlink(statePath)
       }
     }
@@ -242,7 +262,7 @@ exports.cleanVm = async function cleanVm(
   // check if alias are correct
   // check if all vhd in data subfolder have a corresponding alias
   await asyncMap(Object.keys(aliases), async dir => {
-    await checkAliases(aliases[dir], `${dir}/data`, { handler, onLog, remove })
+    await checkAliases(aliases[dir], `${dir}/data`, { handler, logInfo, logWarn, remove })
   })
 
   // remove VHDs with missing ancestors
@@ -264,9 +284,9 @@ exports.cleanVm = async function cleanVm(
       if (!vhds.has(parent)) {
         vhds.delete(vhdPath)
 
-        onLog(`the parent ${parent} of the VHD ${vhdPath} is missing`)
+        logWarn('parent VHD is missing', { parent, vhdPath })
         if (remove) {
-          onLog(`deleting orphan VHD ${vhdPath}`)
+          logInfo('deleting orphan VHD', { vhdPath })
           deletions.push(VhdAbstract.unlink(handler, vhdPath))
         }
       }
@@ -303,7 +323,7 @@ exports.cleanVm = async function cleanVm(
     // check is not good enough to delete the file, the best we can do is report
     // it
     if (!(await this.isValidXva(path))) {
-      onLog(`the XVA with path ${path} is potentially broken`)
+      logWarn('XVA might be broken', { path })
     }
   })
 
@@ -317,7 +337,7 @@ exports.cleanVm = async function cleanVm(
     try {
       metadata = JSON.parse(await handler.readFile(json))
     } catch (error) {
-      onLog(`failed to read metadata file ${json}`, { error })
+      logWarn('failed to read metadata file', { json, error })
       jsons.delete(json)
       return
     }
@@ -328,9 +348,9 @@ exports.cleanVm = async function cleanVm(
       if (xvas.has(linkedXva)) {
         unusedXvas.delete(linkedXva)
       } else {
-        onLog(`the XVA linked to the metadata ${json} is missing`)
+        logWarn('metadata XVA is missing', { json })
         if (remove) {
-          onLog(`deleting incomplete backup ${json}`)
+          logInfo('deleting incomplete backup', { json })
           jsons.delete(json)
           await handler.unlink(json)
         }
@@ -351,9 +371,9 @@ exports.cleanVm = async function cleanVm(
           vhdsToJSons[path] = json
         })
       } else {
-        onLog(`Some VHDs linked to the metadata ${json} are missing`, { missingVhds })
+        logWarn('some metadata VHDs are missing', { json, missingVhds })
         if (remove) {
-          onLog(`deleting incomplete backup ${json}`)
+          logInfo('deleting incomplete backup', { json })
           jsons.delete(json)
           await handler.unlink(json)
         }
@@ -365,7 +385,7 @@ exports.cleanVm = async function cleanVm(
   const unusedVhdsDeletion = []
   const toMerge = []
   {
-    // VHD chains (as list from child to ancestor) to merge indexed by last
+    // VHD chains (as list from oldest to most recent) to merge indexed by most recent
     // ancestor
     const vhdChainsToMerge = { __proto__: null }
 
@@ -389,14 +409,14 @@ exports.cleanVm = async function cleanVm(
       if (child !== undefined) {
         const chain = getUsedChildChainOrDelete(child)
         if (chain !== undefined) {
-          chain.push(vhd)
+          chain.unshift(vhd)
           return chain
         }
       }
 
-      onLog(`the VHD ${vhd} is unused`)
+      logWarn('unused VHD', { vhd })
       if (remove) {
-        onLog(`deleting unused VHD ${vhd}`)
+        logInfo('deleting unused VHD', { vhd })
         unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))
       }
     }
@@ -420,7 +440,7 @@ exports.cleanVm = async function cleanVm(
   const metadataWithMergedVhd = {}
   const doMerge = async () => {
     await asyncMap(toMerge, async chain => {
-      const merged = await limitedMergeVhdChain(chain, { handler, onLog, remove, merge })
+      const merged = await limitedMergeVhdChain(chain, { handler, logInfo, logWarn, remove, merge })
       if (merged !== undefined) {
         const metadataPath = vhdsToJSons[chain[0]] // all the chain should have the same metada file
         metadataWithMergedVhd[metadataPath] = true
@@ -432,18 +452,18 @@ exports.cleanVm = async function cleanVm(
     ...unusedVhdsDeletion,
     toMerge.length !== 0 && (merge ? Task.run({ name: 'merge' }, doMerge) : doMerge()),
     asyncMap(unusedXvas, path => {
-      onLog(`the XVA ${path} is unused`)
+      logWarn('unused XVA', { path })
       if (remove) {
-        onLog(`deleting unused XVA ${path}`)
+        logInfo('deleting unused XVA', { path })
         return handler.unlink(path)
       }
     }),
     asyncMap(xvaSums, path => {
       // no need to handle checksums for XVAs deleted by the script, they will be handled by `unlink()`
       if (!xvas.has(path.slice(0, -'.checksum'.length))) {
-        onLog(`the XVA checksum ${path} is unused`)
+        logInfo('unused XVA checksum', { path })
         if (remove) {
-          onLog(`deleting unused XVA checksum ${path}`)
+          logInfo('deleting unused XVA checksum', { path })
           return handler.unlink(path)
         }
       }
@@ -477,11 +497,11 @@ exports.cleanVm = async function cleanVm(
 
         // don't warn if the size has changed after a merge
         if (!merged && fileSystemSize !== size) {
-          onLog(`incorrect size in metadata: ${size ?? 'none'} instead of ${fileSystemSize}`)
+          logWarn('incorrect size in metadata', { size: size ?? 'none', fileSystemSize })
         }
       }
     } catch (error) {
-      onLog(`failed to get size of ${metadataPath}`, { error })
+      logWarn('failed to get metadata size', { metadataPath, error })
       return
     }
 
@@ -491,7 +511,7 @@ exports.cleanVm = async function cleanVm(
       try {
         await handler.writeFile(metadataPath, JSON.stringify(metadata), { flags: 'w' })
       } catch (error) {
-        onLog(`failed to update size in backup metadata ${metadataPath} after merge`, { error })
+        logWarn('metadata size update failed', { metadataPath, error })
       }
     }
   })

@xen-orchestra/backups/docs/VM backups/README.md

@@ -69,6 +69,8 @@ job.start(data: { mode: Mode, reportWhen: ReportWhen })
 ├─ task.warning(message: string)
 ├─ task.start(data: { type: 'VM', id: string })
 │  ├─ task.warning(message: string)
+|  ├─ task.start(message: 'clean-vm')
+│  │  └─ task.end
 │  ├─ task.start(message: 'snapshot')
 │  │  └─ task.end
 │  ├─ task.start(message: 'export', data: { type: 'SR' | 'remote', id: string, isFull: boolean })
@@ -77,16 +79,20 @@ job.start(data: { mode: Mode, reportWhen: ReportWhen })
 │  │  │  ├─ task.warning(message: string)
 │  │  │  └─ task.end(result: { size: number })
 │  │  │
+│  │  │  // in case there is a healthcheck scheduled for this vm in this job
+│  │  ├─ task.start(message: 'health check')
+│  │  │  ├─ task.start(message: 'transfer')
+│  │  │  │  └─ task.end(result: { size: number })
+│  │  │  ├─ task.start(message: 'vmstart')
+│  │  │  │  └─ task.end
+│  │  │  └─ task.end
+│  │  │
 │  │  │  // in case of full backup, DR and CR
 │  │  ├─ task.start(message: 'clean')
 │  │  │  ├─ task.warning(message: string)
 │  │  │  └─ task.end
-│  │  │
-│  │  │ // in case of delta backup
-│  │  ├─ task.start(message: 'merge')
-│  │  │  ├─ task.warning(message: string)
-│  │  │  └─ task.end(result: { size: number })
-│  │  │
+│  │  └─ task.end
+|  ├─ task.start(message: 'clean-vm')
 │  │  └─ task.end
 │  └─ task.end
 └─ job.end
@@ -195,6 +201,7 @@ Settings are described in [`@xen-orchestra/backups/Backup.js](https://github.com
     - `prepare({ isFull })`
     - `transfer({ timestamp, deltaExport, sizeContainers })`
     - `cleanup()`
+    - `healthCheck(sr)`
   - **Full**
     - `run({ timestamp, sizeContainer, stream })`
 - `afterBackup()`

@xen-orchestra/backups/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.23.0",
+  "version": "0.27.0",
   "engines": {
     "node": ">=14.6"
   },
@@ -22,7 +22,7 @@
     "@vates/disposable": "^0.1.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^1.0.1",
+    "@xen-orchestra/fs": "^1.1.0",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/template": "^0.1.0",
     "compare-versions": "^4.0.1",
@@ -38,7 +38,7 @@
     "promise-toolbox": "^0.21.0",
     "proper-lockfile": "^4.1.2",
     "uuid": "^8.3.2",
-    "vhd-lib": "^3.1.0",
+    "vhd-lib": "^3.3.2",
     "yazl": "^2.5.1"
   },
   "devDependencies": {
@@ -46,7 +46,7 @@
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^1.0.0"
+    "@xen-orchestra/xapi": "^1.4.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

@xen-orchestra/backups/writers/DeltaBackupWriter.js

@@ -19,6 +19,8 @@ const { AbstractDeltaWriter } = require('./_AbstractDeltaWriter.js')
 const { checkVhd } = require('./_checkVhd.js')
 const { packUuid } = require('./_packUuid.js')
 const { Disposable } = require('promise-toolbox')
+const { HealthCheckVmBackup } = require('../HealthCheckVmBackup.js')
+const { ImportVmBackup } = require('../ImportVmBackup.js')
 
 const { warn } = createLogger('xo:backups:DeltaBackupWriter')
 
@@ -69,6 +71,35 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
     return this._cleanVm({ merge: true })
   }
 
+  healthCheck(sr) {
+    return Task.run(
+      {
+        name: 'health check',
+      },
+      async () => {
+        const xapi = sr.$xapi
+        const srUuid = sr.uuid
+        const adapter = this._adapter
+        const metadata = await adapter.readVmBackupMetadata(this._metadataFileName)
+        const { id: restoredId } = await new ImportVmBackup({
+          adapter,
+          metadata,
+          srUuid,
+          xapi,
+        }).run()
+        const restoredVm = xapi.getObject(restoredId)
+        try {
+          await new HealthCheckVmBackup({
+            restoredVm,
+            xapi,
+          }).run()
+        } finally {
+          await xapi.VM_destroy(restoredVm.$ref)
+        }
+      }
+    )
+  }
+
   prepare({ isFull }) {
     // create the task related to this export and ensure all methods are called in this context
     const task = new Task({
@@ -80,7 +111,9 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
       },
     })
     this.transfer = task.wrapFn(this.transfer)
-    this.cleanup = task.wrapFn(this.cleanup, true)
+    this.healthCheck = task.wrapFn(this.healthCheck)
+    this.cleanup = task.wrapFn(this.cleanup)
+    this.afterBackup = task.wrapFn(this.afterBackup, true)
 
     return task.run(() => this._prepare())
   }
@@ -156,7 +189,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
         }/${adapter.getVhdFileName(basename)}`
     )
 
-    const metadataFilename = `${backupDir}/${basename}.json`
+    const metadataFilename = (this._metadataFileName = `${backupDir}/${basename}.json`)
     const metadataContent = {
       jobId,
       mode: job.mode,

@xen-orchestra/backups/writers/_AbstractWriter.js

@@ -9,4 +9,6 @@ exports.AbstractWriter = class AbstractWriter {
   beforeBackup() {}
 
   afterBackup() {}
+
+  healthCheck(sr) {}
 }

@xen-orchestra/backups/writers/_MixinBackupWriter.js

@@ -6,8 +6,9 @@ const { join } = require('path')
 const { getVmBackupDir } = require('../_getVmBackupDir.js')
 const MergeWorker = require('../merge-worker/index.js')
 const { formatFilenameDate } = require('../_filenameDate.js')
+const { Task } = require('../Task.js')
 
-const { warn } = createLogger('xo:backups:MixinBackupWriter')
+const { info, warn } = createLogger('xo:backups:MixinBackupWriter')
 
 exports.MixinBackupWriter = (BaseClass = Object) =>
   class MixinBackupWriter extends BaseClass {
@@ -25,11 +26,17 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
 
     async _cleanVm(options) {
       try {
-        return await this._adapter.cleanVm(this.#vmBackupDir, {
-          ...options,
-          fixMetadata: true,
-          onLog: warn,
-          lock: false,
+        return await Task.run({ name: 'clean-vm' }, () => {
+          return this._adapter.cleanVm(this.#vmBackupDir, {
+            ...options,
+            fixMetadata: true,
+            logInfo: info,
+            logWarn: (message, data) => {
+              warn(message, data)
+              Task.warning(message, data)
+            },
+            lock: false,
+          })
         })
       } catch (error) {
         warn(error)

@xen-orchestra/cr-seed-cli/package.json

@@ -18,7 +18,7 @@
   "preferGlobal": true,
   "dependencies": {
     "golike-defer": "^0.5.1",
-    "xen-api": "^1.2.0"
+    "xen-api": "^1.2.1"
   },
   "scripts": {
     "postversion": "npm publish"

@xen-orchestra/emit-async/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "@xen-orchestra/emit-async",
-  "version": "0.1.0",
+  "version": "1.0.0",
   "license": "ISC",
   "description": "Emit an event for async listeners to settle",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/emit-async",

@xen-orchestra/fs/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "@xen-orchestra/fs",
-  "version": "1.0.1",
+  "version": "1.1.0",
   "license": "AGPL-3.0-or-later",
   "description": "The File System for Xen Orchestra backups.",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
@@ -42,7 +42,7 @@
     "proper-lockfile": "^4.1.2",
     "readable-stream": "^3.0.6",
     "through2": "^4.0.2",
-    "xo-remote-parser": "^0.8.0"
+    "xo-remote-parser": "^0.9.1"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

@xen-orchestra/fs/src/abstract.js

@@ -1,6 +1,7 @@
 import asyncMapSettled from '@xen-orchestra/async-map/legacy'
 import getStream from 'get-stream'
 import { coalesceCalls } from '@vates/coalesce-calls'
+import { createLogger } from '@xen-orchestra/log'
 import { fromCallback, fromEvent, ignoreErrors, timeout } from 'promise-toolbox'
 import { limitConcurrency } from 'limit-concurrency-decorator'
 import { parse } from 'xo-remote-parser'
@@ -11,6 +12,8 @@ import { synchronized } from 'decorator-synchronized'
 import { basename, dirname, normalize as normalizePath } from './_path'
 import { createChecksumStream, validChecksumOfReadStream } from './checksum'
 
+const { warn } = createLogger('@xen-orchestra:fs')
+
 const checksumFile = file => file + '.checksum'
 const computeRate = (hrtime, size) => {
   const seconds = hrtime[0] + hrtime[1] / 1e9
@@ -357,11 +360,12 @@ export default class RemoteHandlerAbstract {
         readRate: computeRate(readDuration, SIZE),
       }
     } catch (error) {
+      warn(`error while testing the remote at step ${step}`, { error })
       return {
         success: false,
         step,
         file: testFileName,
-        error: error.message || String(error),
+        error,
       }
     } finally {
       ignoreErrors.call(this._unlink(testFileName))
@@ -420,6 +424,10 @@ export default class RemoteHandlerAbstract {
 
   // Methods that can be implemented by inheriting classes
 
+  useVhdDirectory() {
+    return this._remote.useVhdDirectory ?? false
+  }
+
   async _closeFile(fd) {
     throw new Error('Not implemented')
   }

@xen-orchestra/fs/src/s3.js

@@ -525,4 +525,8 @@ export default class S3Handler extends RemoteHandlerAbstract {
   }
 
   async _closeFile(fd) {}
+
+  useVhdDirectory() {
+    return true
+  }
 }

@xen-orchestra/mixins/Config.mjs

@@ -7,6 +7,9 @@ import { watch } from 'app-conf'
 
 const { warn } = createLogger('xo:mixins:config')
 
+// if path is undefined, an empty string or an empty array, returns the root value
+const niceGet = (value, path) => (path === undefined || path.length === 0 ? value : get(value, path))
+
 export default class Config {
   constructor(app, { appDir, appName, config }) {
     this._config = config
@@ -30,7 +33,7 @@ export default class Config {
   }
 
   get(path) {
-    const value = get(this._config, path)
+    const value = niceGet(this._config, path)
     if (value === undefined) {
       throw new TypeError('missing config entry: ' + path)
     }
@@ -42,17 +45,23 @@ export default class Config {
   }
 
   getOptional(path) {
-    return get(this._config, path)
+    return niceGet(this._config, path)
   }
 
   watch(path, cb) {
+    // short syntax for the whole config: watch(cb)
+    if (typeof path === 'function') {
+      cb = path
+      path = undefined
+    }
+
     // internal arg
     const processor = arguments.length > 2 ? arguments[2] : identity
 
     let prev
     const watcher = config => {
       try {
-        const value = processor(get(config, path))
+        const value = processor(niceGet(config, path))
         if (!isEqual(value, prev)) {
           const previous = prev
           prev = value

@xen-orchestra/mixins/SslCertificate.mjs

@@ -0,0 +1,219 @@
+import { createLogger } from '@xen-orchestra/log'
+import { genSelfSignedCert } from '@xen-orchestra/self-signed'
+import pRetry from 'promise-toolbox/retry'
+
+import { X509Certificate } from 'crypto'
+import fs from 'node:fs/promises'
+import { dirname } from 'path'
+import pw from 'pw'
+import tls from 'node:tls'
+
+const { debug, info, warn } = createLogger('xo:mixins:sslCertificate')
+
+async function outputFile(path, content) {
+  await fs.mkdir(dirname(path), { recursive: true })
+  await fs.writeFile(path, content, { flag: 'w', mode: 0o400 })
+}
+
+class SslCertificate {
+  #app
+  #configKey
+  #updateSslCertificatePromise
+  #secureContext
+  #validTo
+
+  constructor(app, configKey) {
+    this.#app = app
+    this.#configKey = configKey
+  }
+
+  #createSecureContext(cert, key, passphrase) {
+    return tls.createSecureContext({
+      cert,
+      key,
+      passphrase,
+    })
+  }
+
+  // load on register
+  async #loadSslCertificate(config) {
+    const certPath = config.cert
+    const keyPath = config.key
+    let key, cert, passphrase
+    try {
+      ;[cert, key] = await Promise.all([fs.readFile(certPath), fs.readFile(keyPath)])
+      if (keyPath.includes('ENCRYPTED')) {
+        if (config.autoCert) {
+          throw new Error(`encrytped certificates aren't compatible with autoCert option`)
+        }
+        passphrase = await new Promise(resolve => {
+          // eslint-disable-next-line no-console
+          process.stdout.write(`Enter pass phrase: `)
+          pw(resolve)
+        })
+      }
+    } catch (error) {
+      if (!(config.autoCert && error.code === 'ENOENT')) {
+        throw error
+      }
+      // self signed certificate or let's encrypt will be generated on demand
+    }
+
+    // create secure context also make a validation of the certificate
+    const secureContext = this.#createSecureContext(cert, key, passphrase)
+    this.#secureContext = secureContext
+
+    // will be tested and eventually renewed on first query
+    const { validTo } = new X509Certificate(cert)
+    this.#validTo = new Date(validTo)
+  }
+
+  #getConfig() {
+    const config = this.#app.config.get(this.#configKey)
+    if (config === undefined) {
+      throw new Error(`config for key ${this.#configKey} is unavailable`)
+    }
+    return config
+  }
+
+  async #getSelfSignedContext(config) {
+    return pRetry(
+      async () => {
+        const { cert, key } = await genSelfSignedCert()
+        info('new certificates generated', { cert, key })
+        try {
+          await Promise.all([outputFile(config.cert, cert), outputFile(config.key, key)])
+        } catch (error) {
+          warn(`can't save self signed certificates `, { error, config })
+        }
+
+        // create secure context also make a validation of the certificate
+        const { validTo } = new X509Certificate(cert)
+        return { secureContext: this.#createSecureContext(cert, key), validTo: new Date(validTo) }
+      },
+      {
+        tries: 2,
+        when: e => e.code === 'ERR_SSL_EE_KEY_TOO_SMALL',
+        onRetry: () => {
+          warn('got ERR_SSL_EE_KEY_TOO_SMALL while generating self signed certificate ')
+        },
+      }
+    )
+  }
+
+  // get the current certificate for this hostname
+  async getSecureContext(hostName) {
+    const config = this.#getConfig()
+    if (config === undefined) {
+      throw new Error(`config for key ${this.#configKey} is unavailable`)
+    }
+
+    if (this.#updateSslCertificatePromise) {
+      debug('certificate is already refreshing')
+      return this.#updateSslCertificatePromise
+    }
+
+    let certificateIsValid = this.#validTo !== undefined
+    let shouldRenew = !certificateIsValid
+
+    if (certificateIsValid) {
+      certificateIsValid = this.#validTo >= new Date()
+      shouldRenew = !certificateIsValid || this.#validTo - new Date() < 30 * 24 * 60 * 60 * 1000
+    }
+
+    let promise = Promise.resolve()
+    if (shouldRenew) {
+      try {
+        // @todo : should also handle let's encrypt
+        if (config.autoCert === true) {
+          promise = promise.then(() => this.#getSelfSignedContext(config))
+        }
+
+        this.#updateSslCertificatePromise = promise
+
+        // cleanup and store
+        promise = promise.then(
+          ({ secureContext, validTo }) => {
+            this.#validTo = validTo
+            this.#secureContext = secureContext
+            this.#updateSslCertificatePromise = undefined
+            return secureContext
+          },
+          async error => {
+            console.warn('error while updating ssl certificate', { error })
+            this.#updateSslCertificatePromise = undefined
+            if (!certificateIsValid) {
+              // we couldn't generate a valid certificate
+              // only throw if the current certificate is invalid
+              warn('deleting invalid certificate')
+              this.#secureContext = undefined
+              this.#validTo = undefined
+              await Promise.all([fs.unlink(config.cert), fs.unlink(config.key)])
+              throw error
+            }
+          }
+        )
+      } catch (error) {
+        warn('error while refreshing ssl certificate', { error })
+        throw error
+      }
+    }
+    if (certificateIsValid) {
+      // still valid  : does not need to wait for the refresh
+      return this.#secureContext
+    }
+
+    if (this.#updateSslCertificatePromise === undefined) {
+      throw new Error(`Invalid certificate and no strategy defined to renew it. Try activating autoCert in the config`)
+    }
+
+    // invalid cert : wait for refresh
+    return this.#updateSslCertificatePromise
+  }
+
+  async register() {
+    await this.#loadSslCertificate(this.#getConfig())
+  }
+}
+
+export default class SslCertificates {
+  #app
+  #handlers = {}
+  constructor(app, { httpServer }) {
+    // don't setup the proxy if httpServer is not present
+    //
+    // that can happen when the app is instanciated in another context like xo-server-recover-account
+    if (httpServer === undefined) {
+      return
+    }
+    this.#app = app
+
+    httpServer.getSecureContext = this.getSecureContext.bind(this)
+  }
+
+  async getSecureContext(hostname, configKey) {
+    const config = this.#app.config.get(`http.listen.${configKey}`)
+    if (!config || !config.cert || !config.key) {
+      throw new Error(`HTTPS configuration does no exists for key http.listen.${configKey}`)
+    }
+
+    if (this.#handlers[configKey] === undefined) {
+      throw new Error(`the SslCertificate handler for key  http.listen.${configKey} does not exists.`)
+    }
+    return this.#handlers[configKey].getSecureContext(hostname, config)
+  }
+
+  async register() {
+    // http.listen can be an array or an object
+    const configs = this.#app.config.get('http.listen') || []
+    const configKeys = Object.keys(configs) || []
+    await Promise.all(
+      configKeys
+        .filter(configKey => configs[configKey].cert !== undefined && configs[configKey].key !== undefined)
+        .map(async configKey => {
+          this.#handlers[configKey] = new SslCertificate(this.#app, `http.listen.${configKey}`)
+          return this.#handlers[configKey].register(configs[configKey])
+        })
+    )
+  }
+}

@xen-orchestra/mixins/package.json

@@ -14,18 +14,20 @@
     "url": "https://vates.fr"
   },
   "license": "AGPL-3.0-or-later",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "engines": {
-    "node": ">=12"
+    "node": ">=14"
   },
   "dependencies": {
-    "@vates/event-listeners-manager": "^1.0.0",
+    "@vates/event-listeners-manager": "^1.0.1",
     "@vates/parse-duration": "^0.1.1",
-    "@xen-orchestra/emit-async": "^0.1.0",
+    "@xen-orchestra/emit-async": "^1.0.0",
     "@xen-orchestra/log": "^0.3.0",
+    "@xen-orchestra/self-signed": "^0.1.3",
     "app-conf": "^2.1.0",
     "lodash": "^4.17.21",
-    "promise-toolbox": "^0.21.0"
+    "promise-toolbox": "^0.21.0",
+    "pw": "^0.0.4"
   },
   "scripts": {
     "postversion": "npm publish --access public"

@xen-orchestra/openflow/package.json

@@ -9,7 +9,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.1.1",
+  "version": "0.1.2",
   "engines": {
     "node": ">=8.10"
   },
@@ -30,7 +30,7 @@
     "rimraf": "^3.0.0"
   },
   "dependencies": {
-    "@vates/read-chunk": "^0.1.2"
+    "@vates/read-chunk": "^1.0.0"
   },
   "author": {
     "name": "Vates SAS",

@xen-orchestra/proxy-cli/index.mjs

@@ -33,26 +33,19 @@ async function main(argv) {
     ignoreUnknownFormats: true,
   })
 
-  const { hostname = 'localhost', port } = config?.http?.listen?.https ?? {}
-
-  const {
-    _: args,
-    file,
-    help,
-    host,
-    raw,
-    token,
-  } = getopts(argv, {
+  const opts = getopts(argv, {
     alias: { file: 'f', help: 'h' },
     boolean: ['help', 'raw'],
     default: {
       token: config.authenticationToken,
     },
     stopEarly: true,
-    string: ['file', 'host', 'token'],
+    string: ['file', 'host', 'token', 'url'],
   })
 
-  if (help || (file === '' && args.length === 0)) {
+  const { _: args, file } = opts
+
+  if (opts.help || (file === '' && args.length === 0)) {
     return console.log(
       '%s',
       `Usage:
@@ -77,18 +70,29 @@ ${pkg.name} v${pkg.version}`
   const baseRequest = {
     headers: {
       'content-type': 'application/json',
-      cookie: `authenticationToken=${token}`,
     },
     pathname: '/api/v1',
-    protocol: 'https:',
     rejectUnauthorized: false,
   }
-  if (host !== '') {
-    baseRequest.host = host
+  let { token } = opts
+  if (opts.url !== '') {
+    const { protocol, host, username } = new URL(opts.url)
+    Object.assign(baseRequest, { protocol, host })
+    if (username !== '') {
+      token = username
+    }
   } else {
-    baseRequest.hostname = hostname
-    baseRequest.port = port
+    baseRequest.protocol = 'https:'
+    if (opts.host !== '') {
+      baseRequest.host = opts.host
+    } else {
+      const { hostname = 'localhost', port } = config?.http?.listen?.https ?? {}
+      baseRequest.hostname = hostname
+      baseRequest.port = port
+    }
   }
+  baseRequest.headers.cookie = `authenticationToken=${token}`
+
   const call = async ({ method, params }) => {
     if (callPath.length !== 0) {
       process.stderr.write(`\n${colors.bold(`--- call #${callPath.join('.')}`)} ---\n\n`)
@@ -127,7 +131,7 @@ ${pkg.name} v${pkg.version}`
           stdout.write(inspect(JSON.parse(line), { colors: true, depth: null }))
           stdout.write('\n')
         }
-      } else if (raw && typeof result === 'string') {
+      } else if (opts.raw && typeof result === 'string') {
         stdout.write(result)
       } else {
         stdout.write(inspect(result, { colors: true, depth: null }))

@xen-orchestra/proxy-cli/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "@xen-orchestra/proxy-cli",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "license": "AGPL-3.0-or-later",
   "description": "CLI for @xen-orchestra/proxy",
   "keywords": [
@@ -26,7 +26,7 @@
   },
   "dependencies": {
     "@iarna/toml": "^2.2.0",
-    "@vates/read-chunk": "^0.1.2",
+    "@vates/read-chunk": "^1.0.0",
     "ansi-colors": "^4.1.1",
     "app-conf": "^2.1.0",
     "content-type": "^1.0.4",

@xen-orchestra/proxy/index.mjs

@@ -3,13 +3,11 @@
 import forOwn from 'lodash/forOwn.js'
 import fse from 'fs-extra'
 import getopts from 'getopts'
-import pRetry from 'promise-toolbox/retry'
 import { catchGlobalErrors } from '@xen-orchestra/log/configure.js'
 import { create as createServer } from 'http-server-plus'
 import { createCachedLookup } from '@vates/cached-dns.lookup'
 import { createLogger } from '@xen-orchestra/log'
 import { createSecureServer } from 'http2'
-import { genSelfSignedCert } from '@xen-orchestra/self-signed'
 import { load as loadConfig } from 'app-conf'
 
 // -------------------------------------------------------------------
@@ -56,41 +54,21 @@ ${APP_NAME} v${APP_VERSION}
     createSecureServer: opts => createSecureServer({ ...opts, allowHTTP1: true }),
   })
 
-  forOwn(config.http.listen, async ({ autoCert, cert, key, ...opts }) => {
+  forOwn(config.http.listen, async ({ autoCert, cert, key, ...opts }, listenKey) => {
     try {
-      const niceAddress = await pRetry(
-        async () => {
-          if (cert !== undefined && key !== undefined) {
-            try {
-              opts.cert = fse.readFileSync(cert)
-              opts.key = fse.readFileSync(key)
-            } catch (error) {
-              if (!(autoCert && error.code === 'ENOENT')) {
-                throw error
-              }
-
-              const pems = await genSelfSignedCert()
-              fse.outputFileSync(cert, pems.cert, { flag: 'wx', mode: 0o400 })
-              fse.outputFileSync(key, pems.key, { flag: 'wx', mode: 0o400 })
-              info('new certificate generated', { cert, key })
-              opts.cert = pems.cert
-              opts.key = pems.key
-            }
+      if (cert !== undefined && key !== undefined) {
+        opts.SNICallback = async (serverName, callback) => {
+          // injected by @xen-orchestr/mixins/sslCertificate.mjs
+          try {
+            const secureContext = await httpServer.getSecureContext(serverName, listenKey)
+            callback(null, secureContext)
+          } catch (error) {
+            warn('An error occured during certificate context creation', { error, listenKey, serverName })
+            callback(error)
           }
-
-          return httpServer.listen(opts)
-        },
-        {
-          tries: 2,
-          when: e => autoCert && e.code === 'ERR_SSL_EE_KEY_TOO_SMALL',
-          onRetry: () => {
-            warn('deleting invalid certificate')
-            fse.unlinkSync(cert)
-            fse.unlinkSync(key)
-          },
         }
-      )
-
+      }
+      const niceAddress = httpServer.listen(opts)
       info(`Web server listening on ${niceAddress}`)
     } catch (error) {
       if (error.niceAddress !== undefined) {
@@ -138,6 +116,8 @@ ${APP_NAME} v${APP_VERSION}
   const { default: fromCallback } = await import('promise-toolbox/fromCallback')
   app.hooks.on('stop', () => fromCallback(cb => httpServer.stop(cb)))
 
+  await app.sslCertificate.register()
+
   await app.hooks.start()
 
   // Gracefully shutdown on signals.
@@ -146,6 +126,7 @@ ${APP_NAME} v${APP_VERSION}
     process.on(signal, () => {
       if (alreadyCalled) {
         warn('forced exit')
+        // eslint-disable-next-line n/no-process-exit
         process.exit(1)
       }
       alreadyCalled = true
@@ -163,7 +144,7 @@ main(process.argv.slice(2)).then(
   },
   error => {
     fatal(error)
-
+    // eslint-disable-next-line n/no-process-exit
     process.exit(1)
   }
 )

@xen-orchestra/proxy/package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "@xen-orchestra/proxy",
-  "version": "0.22.1",
+  "version": "0.23.5",
   "license": "AGPL-3.0-or-later",
   "description": "XO Proxy used to remotely execute backup jobs",
   "keywords": [
@@ -26,19 +26,18 @@
   },
   "dependencies": {
     "@iarna/toml": "^2.2.0",
-    "@koa/router": "^10.0.0",
+    "@koa/router": "^11.0.1",
     "@vates/cached-dns.lookup": "^1.0.0",
     "@vates/compose": "^2.1.0",
     "@vates/decorate-with": "^2.0.0",
     "@vates/disposable": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.23.0",
-    "@xen-orchestra/fs": "^1.0.1",
+    "@xen-orchestra/backups": "^0.27.0",
+    "@xen-orchestra/fs": "^1.1.0",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/mixin": "^0.1.0",
-    "@xen-orchestra/mixins": "^0.4.0",
-    "@xen-orchestra/self-signed": "^0.1.0",
-    "@xen-orchestra/xapi": "^1.0.0",
+    "@xen-orchestra/mixins": "^0.5.0",
+    "@xen-orchestra/xapi": "^1.4.0",
     "ajv": "^8.0.3",
     "app-conf": "^2.1.0",
     "async-iterator-to-stream": "^1.1.0",
@@ -46,7 +45,7 @@
     "get-stream": "^6.0.0",
     "getopts": "^2.2.3",
     "golike-defer": "^0.5.1",
-    "http-server-plus": "^0.11.0",
+    "http-server-plus": "^0.11.1",
     "http2-proxy": "^5.0.53",
     "json-rpc-protocol": "^0.13.1",
     "jsonrpc-websocket-client": "^0.7.2",
@@ -60,7 +59,7 @@
     "source-map-support": "^0.5.16",
     "stoppable": "^1.0.6",
     "xdg-basedir": "^5.1.0",
-    "xen-api": "^1.2.0",
+    "xen-api": "^1.2.1",
     "xo-common": "^0.8.0"
   },
   "devDependencies": {

@xen-orchestra/self-signed/index.js

@@ -2,22 +2,23 @@
 
 const { execFile } = require('child_process')
 
-const openssl = (cmd, args, { input, ...opts } = {}) =>
+const RE =
+  /^(-----BEGIN PRIVATE KEY-----.+-----END PRIVATE KEY-----\n)(-----BEGIN CERTIFICATE-----.+-----END CERTIFICATE-----\n)$/s
+exports.genSelfSignedCert = async ({ days = 360 } = {}) =>
   new Promise((resolve, reject) => {
-    const child = execFile('openssl', [cmd, ...args], opts, (error, stdout) =>
-      error != null ? reject(error) : resolve(stdout)
+    execFile(
+      'openssl',
+      ['req', '-batch', '-new', '-x509', '-days', String(days), '-nodes', '-newkey', 'rsa:2048', '-keyout', '-'],
+      (error, stdout) => {
+        if (error != null) {
+          return reject(error)
+        }
+        const matches = RE.exec(stdout)
+        if (matches === null) {
+          return reject(new Error('stdout does not match regular expression'))
+        }
+        const [, key, cert] = matches
+        resolve({ cert, key })
+      }
     )
-    if (input !== undefined) {
-      child.stdin.end(input)
-    }
   })
-
-exports.genSelfSignedCert = async ({ days = 360 } = {}) => {
-  const key = await openssl('genrsa', ['2048'])
-  return {
-    cert: await openssl('req', ['-batch', '-new', '-key', '-', '-x509', '-days', String(days), '-nodes'], {
-      input: key,
-    }),
-    key,
-  }
-}

@xen-orchestra/self-signed/package.json

@@ -9,7 +9,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.1.0",
+  "version": "0.1.3",
   "engines": {
     "node": ">=8.10"
   },

@xen-orchestra/template/.babelrc.js

@@ -1,3 +0,0 @@
-'use strict'
-
-module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))

@xen-orchestra/template/.eslintrc.js

@@ -1 +0,0 @@
-../../scripts/babel-eslintrc.js

@xen-orchestra/template/index.js

@@ -1,8 +1,10 @@
-import escapeRegExp from 'lodash/escapeRegExp'
+'use strict'
+
+const escapeRegExp = require('lodash/escapeRegExp')
 
 const compareLengthDesc = (a, b) => b.length - a.length
 
-export function compileTemplate(pattern, rules) {
+exports.compileTemplate = function compileTemplate(pattern, rules) {
   const matches = Object.keys(rules).sort(compareLengthDesc).map(escapeRegExp).join('|')
   const regExp = new RegExp(`\\\\(?:\\\\|${matches})|${matches}`, 'g')
   return (...params) =>

@xen-orchestra/template/index.spec.js

@@ -1,5 +1,8 @@
 /* eslint-env jest */
-import { compileTemplate } from '.'
+
+'use strict'
+
+const { compileTemplate } = require('.')
 
 it("correctly replaces the template's variables", () => {
   const replacer = compileTemplate('{property}_\\{property}_\\\\{property}_{constant}_%_FOO', {

@xen-orchestra/template/package.json

@@ -14,31 +14,13 @@
     "name": "Vates SAS",
     "url": "https://vates.fr"
   },
-  "preferGlobal": false,
-  "main": "dist/",
-  "browserslist": [
-    ">2%"
-  ],
   "engines": {
     "node": ">=6"
   },
-  "devDependencies": {
-    "@babel/cli": "^7.0.0",
-    "@babel/core": "^7.0.0",
-    "@babel/preset-env": "^7.0.0",
-    "cross-env": "^7.0.2",
-    "rimraf": "^3.0.0"
-  },
-  "scripts": {
-    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
-    "clean": "rimraf dist/",
-    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
-    "prebuild": "yarn run clean",
-    "predev": "yarn run prebuild",
-    "prepublishOnly": "yarn run build",
-    "postversion": "npm publish --access public"
-  },
   "dependencies": {
     "lodash": "^4.17.15"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
   }
 }

@xen-orchestra/test-e2e/.eslintrc.json

@@ -1,5 +0,0 @@
-{
-  "extends": [
-    "plugin:cypress/recommended"
-  ]
-}

@xen-orchestra/test-e2e/.gitignore

@@ -1,4 +0,0 @@
-cypress/downloads
-cypress/screenshots
-cypress/videos
-cypress.json

@xen-orchestra/test-e2e/.npmignore

@@ -1 +0,0 @@
-../../scripts/npmignore

@xen-orchestra/test-e2e/cypress.json.dist

@@ -1,28 +0,0 @@
-{
-  "baseUrl": "http://ci-test-xen-orchestra.localdomain/",
-  "env": {
-    "xoLab": {
-      "url": "",
-      "username": "",
-      "password": ""
-    },
-    "xenOrchestra": {
-      "host": "ci-test-xen-orchestra.localdomain",
-      "username": "",
-      "password": "",
-      "snapshotId": ""
-    },
-    "xcpNgLts": {
-      "host": "ci-test-xcp-ng-lts.localdomain",
-      "username": "",
-      "password": "",
-      "snapshotId": ""
-    },
-    "xenServerLts": {
-      "host": "ci-test-xenserver-lts.localdomain",
-      "username": "",
-      "password": "",
-      "snapshotId": ""
-    }
-  }
-}

@xen-orchestra/test-e2e/cypress/integration/login.spec.js

@@ -1,26 +0,0 @@
-'use strict'
-
-describe('Sign In', () => {
-  it('should not be able to sign in with bad credentials', () => {
-    cy.visit('/')
-    cy.get('[name="username"]').type('bad-user')
-    cy.get('[name="password"]').type('bad-password')
-    cy.get('.btn-info').click()
-    cy.get('.text-danger')
-    cy.url().should('not.include', '/#/home')
-  })
-
-  it('should be able to sign in', () => {
-    cy.visit('/')
-    cy.get('[name="username"]').type(Cypress.env('xenOrchestra').username)
-    cy.get('[name="password"]').type(Cypress.env('xenOrchestra').password)
-    cy.get('.btn-info').click()
-    cy.url().should('include', '/#/home')
-  })
-
-  it('should sign in without UI', () => {
-    cy.login()
-    cy.visit('/')
-    cy.url().should('include', '/#/home')
-  })
-})

@xen-orchestra/test-e2e/cypress/integration/remote.js

@@ -1,20 +0,0 @@
-'use strict'
-
-describe('Remote', function () {
-  beforeEach('login', () => {
-    cy.login()
-  })
-
-  it('should add a remote', function () {
-    cy.addServers()
-    cy.visit('/#/settings/remotes')
-    cy.contains('New file system remote').should('exist')
-    cy.get('select[name="type"]').select('Local')
-    cy.get('input[name="name"]').type('Test local file remote 2')
-    cy.get('input[name="path"]').type('var/tmp/test-remote')
-    cy.contains('Save configuration').click()
-    cy.contains('Local remote selected')
-    cy.get('button:contains("OK")').click()
-    cy.get('td:contains("Test local file remote 2")').closest('tr').find('button:contains("Enabled")')
-  })
-})

@xen-orchestra/test-e2e/cypress/integration/server.spec.js

@@ -1,38 +0,0 @@
-'use strict'
-
-describe('Server', () => {
-  beforeEach('login', () => {
-    cy.login()
-  })
-
-  it('should add a server', () => {
-    cy.visit('/#/settings/servers')
-
-    cy.get('#form-add-server .form-group:nth-child(1) input').type('XCP-ng LTS')
-    cy.get('#form-add-server .form-group:nth-child(2) input').type(Cypress.env('xcpNgLts').host)
-    cy.get('#form-add-server .form-group:nth-child(3) input').type(Cypress.env('xcpNgLts').username)
-    cy.get('#form-add-server .form-group:nth-child(4) input').type(Cypress.env('xcpNgLts').password)
-    cy.get('#form-add-server .form-group:nth-child(5) .xo-icon-toggle-off').click()
-
-    cy.get('span:contains("Connect")').click()
-    cy.get(`td:contains("XCP-ng LTS")`)
-  })
-
-  it('should remove a server', () => {
-    cy.get(`td:contains("XCP-ng LTS")`).closest('tr').find('.btn-danger').click()
-    cy.get(`td:contains("XCP-ng LTS")`).should('not.exist')
-  })
-
-  it('should disable a server', () => {
-    cy.addServers();
-    cy.visit('/#/settings/servers')
-    cy.contains('ci-test-xcp-ng-lts.localdomain').closest('tr').find('button:contains("Enabled")').click()
-    cy.contains('ci-test-xcp-ng-lts.localdomain').closest('tr').find('button:contains("Disabled")')
-  })
-
-  it('should enable a server', () => {
-    cy.visit('/#/settings/servers')
-    cy.contains('ci-test-xcp-ng-lts.localdomain').closest('tr').find('button:contains("Disabled")').click()
-    cy.contains('ci-test-xcp-ng-lts.localdomain').closest('tr').find('button:contains("Enabled")')
-  })
-})

@xen-orchestra/test-e2e/cypress/integration/vm.spec.js

@@ -1,17 +0,0 @@
-'use strict'
-
-describe('VM', function () {
-  beforeEach('login', () => {
-    cy.login()
-  })
-
-  it('should add a VM', function () {
-    cy.addServers()
-    cy.visit('/#/home?t=VM')
-    cy.get('a:contains("New VM")').click()
-    cy.contains('Create a new VM on').should('exist')
-    cy.contains('Select pool').closest('.Select-control').click()
-    cy.contains('ci-test-xcp-ng-lts').should('exist').click()
-    cy.get('h4:contains("Performance")').should('exist')
-  })
-})

@xen-orchestra/test-e2e/cypress/plugins/index.js

@@ -1,6 +0,0 @@
-/// <reference types="cypress" />
-'use strict'
-
-module.exports = (on, config) => {
-  // configure plugins here
-}

@xen-orchestra/test-e2e/cypress/support/commands.js

@@ -1,39 +0,0 @@
-'use strict'
-// https://on.cypress.io/custom-commands
-
-const { default: Xo } = require('xo-lib')
-const { username: xoUsername, password: xoPassword } = Cypress.env('xenOrchestra')
-
-Cypress.Commands.add('login', (username = xoUsername, password = xoPassword) => {
-  cy.request({
-    method: 'POST',
-    url: '/signin/local',
-    form: true,
-    body: {
-      username,
-      password,
-    },
-  })
-  cy.setCookie('previousDisclaimer', Date.now().toString())
-})
-
-Cypress.Commands.add('addServers', async () => {
-  const xo = new Xo({ url: Cypress.config('baseUrl') })
-
-  await xo.open()
-
-  await xo.signIn({
-    email: xoUsername,
-    password: xoPassword,
-  })
-
-  const { host, username, password } = Cypress.env('xcpNgLts')
-
-  await xo.call('server.add', {
-    host,
-    username,
-    password,
-    label: 'XCP-ng LTS',
-    allowUnauthorized: true,
-  })
-})

@xen-orchestra/test-e2e/cypress/support/index.d.ts

@@ -1,9 +0,0 @@
-/// <reference types="cypress" />
-
-declare namespace Cypress {
-  interface Chainable<Subject> {
-    login(username: string, password: string): Chainable<any>
-    logout(): Chainable<any>
-    addServers(): Chainable<any>
-  }
-}

@xen-orchestra/test-e2e/cypress/support/index.js

@@ -1,10 +0,0 @@
-'use strict'
-
-import './commands'
-
-before('Restore VMs from snapshot', () => {
-  cy.exec(
-    `node scripts/restore-vm.js && wait-on ${Cypress.config('baseUrl')} && wait-on tcp:${Cypress.env('xcpNgLts').host}:80`,
-    { timeout: 300e3 },
-  )
-})

@xen-orchestra/test-e2e/package.json

@@ -1,32 +0,0 @@
-{
-  "private": true,
-  "name": "@xen-orchestra/test-e2e",
-  "version": "0.0.0",
-  "license": "AGPL-3.0-or-later",
-  "description": "E2E Tests for Xen Orchestra",
-  "repository": {
-    "directory": "@xen-orchestra/test-e2e",
-    "type": "git",
-    "url": "https://github.com/vatesfr/xen-orchestra.git"
-  },
-  "engines": {
-    "node": ">=14"
-  },
-  "author": {
-    "name": "Vates SAS",
-    "url": "https://vates.fr"
-  },
-  "bin": {
-    "restore-vm": "./scripts/restore-vm.js"
-  },
-  "devDependencies": {
-    "cypress": "^9.7.0",
-    "eslint-plugin-chai-friendly": "^0.7.2",
-    "eslint-plugin-cypress": "^2.12.1",
-    "wait-on": "^6.0.1",
-    "xo-lib": "^0.11.1"
-  },
-  "scripts": {
-    "test": "cypress run"
-  }
-}

@xen-orchestra/test-e2e/scripts/restore-vm.js

@@ -1,40 +0,0 @@
-#!/usr/bin/env node
-'use strict'
-
-/* eslint-disable no-console */
-
-const Xo = require('xo-lib').default
-const path = require('path')
-const cypressJson = require(path.resolve(__dirname, '..', 'cypress.json'))
-
-async function main() {
-  const xo = new Xo({ url: cypressJson.env.xoLab.url })
-
-  await xo.open()
-
-  await xo.signIn({
-    email: cypressJson.env.xoLab.username,
-    password: cypressJson.env.xoLab.password,
-  })
-
-  console.log('Reverting Test VMs from snapshots')
-
-  try {
-    await Promise.all([
-      xo.call('vm.revert', { snapshot: cypressJson.env.xenServerLts.snapshotId }),
-      xo.call('vm.revert', { snapshot: cypressJson.env.xcpNgLts.snapshotId }),
-      xo.call('vm.revert', { snapshot: cypressJson.env.xenOrchestra.snapshotId }),
-    ])
-  } catch (error) {
-    console.error('Error happened while reverting VMs')
-    throw error
-  }
-
-  xo.close()
-
-  console.log('VMs reverted successfully.')
-}
-
-main()
-
-/* eslint-enable no-console */

@xen-orchestra/upload-ova/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xen-orchestra/upload-ova",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "license": "AGPL-3.0-or-later",
   "description": "Basic CLI to upload ova files to Xen-Orchestra",
   "keywords": [
@@ -43,7 +43,7 @@
     "pw": "^0.0.4",
     "xdg-basedir": "^4.0.0",
     "xo-lib": "^0.11.1",
-    "xo-vmdk-to-vhd": "^2.3.0"
+    "xo-vmdk-to-vhd": "^2.4.2"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

@xen-orchestra/xapi/_AggregateError.js

@@ -0,0 +1,9 @@
+'use strict'
+
+// TODO: remove when Node >=15.0
+module.exports = class AggregateError extends Error {
+  constructor(errors, message) {
+    super(message)
+    this.errors = errors
+  }
+}

@xen-orchestra/xapi/index.js

@@ -230,8 +230,9 @@ function mixin(mixins) {
   defineProperties(xapiProto, descriptors)
 }
 mixin({
-  task: require('./task.js'),
   host: require('./host.js'),
+  SR: require('./sr.js'),
+  task: require('./task.js'),
   VBD: require('./vbd.js'),
   VDI: require('./vdi.js'),
   VIF: require('./vif.js'),

@xen-orchestra/xapi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xen-orchestra/xapi",
-  "version": "1.0.0",
+  "version": "1.4.0",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/xapi",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
@@ -15,7 +15,7 @@
     "node": ">=14"
   },
   "peerDependencies": {
-    "xen-api": "^1.2.0"
+    "xen-api": "^1.2.1"
   },
   "scripts": {
     "postversion": "npm publish --access public"
@@ -26,8 +26,10 @@
     "@xen-orchestra/log": "^0.3.0",
     "d3-time-format": "^3.0.0",
     "golike-defer": "^0.5.1",
+    "json-rpc-protocol": "^0.13.2",
     "lodash": "^4.17.15",
     "promise-toolbox": "^0.21.0",
+    "vhd-lib": "^3.3.2",
     "xo-common": "^0.8.0"
   },
   "private": false,

@xen-orchestra/xapi/sr.js

@@ -0,0 +1,164 @@
+'use strict'
+
+const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
+const { decorateClass } = require('@vates/decorate-with')
+const { defer } = require('golike-defer')
+const { incorrectState } = require('xo-common/api-errors')
+const { VDI_FORMAT_RAW } = require('./index.js')
+const peekFooterFromStream = require('vhd-lib/peekFooterFromVhdStream')
+
+const AggregateError = require('./_AggregateError.js')
+
+const { warn } = require('@xen-orchestra/log').createLogger('xo:xapi:sr')
+
+const OC_MAINTENANCE = 'xo:maintenanceState'
+
+class Sr {
+  async create({
+    content_type = 'user', // recommended by Citrix
+    device_config,
+    host,
+    name_description = '',
+    name_label,
+    physical_size = 0,
+    shared,
+    sm_config = {},
+    type,
+  }) {
+    const ref = await this.call(
+      'SR.create',
+      host,
+      device_config,
+      physical_size,
+      name_label,
+      name_description,
+      type,
+      content_type,
+      shared,
+      sm_config
+    )
+
+    // https://developer-docs.citrix.com/projects/citrix-hypervisor-sdk/en/latest/xc-api-extensions/#sr
+    this.setFieldEntry('SR', ref, 'other_config', 'auto-scan', 'true').catch(warn)
+
+    return ref
+  }
+
+  // Switch the SR to maintenance mode:
+  // - shutdown all running VMs with a VDI on this SR
+  //   - their UUID is saved into SR.other_config[OC_MAINTENANCE].shutdownVms
+  //   - clean shutdown is attempted, and falls back to a hard shutdown
+  // - unplug all connected hosts from this SR
+  async enableMaintenanceMode($defer, ref, { vmsToShutdown = [] } = {}) {
+    const state = { timestamp: Date.now() }
+
+    // will throw if already in maintenance mode
+    await this.call('SR.add_to_other_config', ref, OC_MAINTENANCE, JSON.stringify(state))
+
+    await $defer.onFailure.call(this, 'call', 'SR.remove_from_other_config', ref, OC_MAINTENANCE)
+
+    const runningVms = new Map()
+    const handleVbd = async ref => {
+      const vmRef = await this.getField('VBD', ref, 'VM')
+      if (!runningVms.has(vmRef)) {
+        const power_state = await this.getField('VM', vmRef, 'power_state')
+        const isPaused = power_state === 'Paused'
+        if (isPaused || power_state === 'Running') {
+          runningVms.set(vmRef, isPaused)
+        }
+      }
+    }
+    await asyncMap(await this.getField('SR', ref, 'VDIs'), async ref => {
+      await asyncMap(await this.getField('VDI', ref, 'VBDs'), handleVbd)
+    })
+
+    {
+      const runningVmUuids = await asyncMap(runningVms.keys(), ref => this.getField('VM', ref, 'uuid'))
+
+      const set = new Set(vmsToShutdown)
+      for (const vmUuid of runningVmUuids) {
+        if (!set.has(vmUuid)) {
+          throw incorrectState({
+            actual: vmsToShutdown,
+            expected: runningVmUuids,
+            property: 'vmsToShutdown',
+          })
+        }
+      }
+    }
+
+    state.shutdownVms = {}
+
+    await asyncMapSettled(runningVms, async ([ref, isPaused]) => {
+      state.shutdownVms[await this.getField('VM', ref, 'uuid')] = isPaused
+
+      try {
+        await this.callAsync('VM.clean_shutdown', ref)
+      } catch (error) {
+        warn('SR_enableMaintenanceMode, VM clean shutdown', { error })
+        await this.callAsync('VM.hard_shutdown', ref)
+      }
+
+      $defer.onFailure.call(this, 'callAsync', 'VM.start', ref, isPaused, true)
+    })
+
+    state.unpluggedPbds = []
+    await asyncMapSettled(await this.getField('SR', ref, 'PBDs'), async ref => {
+      if (await this.getField('PBD', ref, 'currently_attached')) {
+        state.unpluggedPbds.push(await this.getField('PBD', ref, 'uuid'))
+
+        await this.callAsync('PBD.unplug', ref)
+
+        $defer.onFailure.call(this, 'callAsync', 'PBD.plug', ref)
+      }
+    })
+
+    await this.setFieldEntry('SR', ref, 'other_config', OC_MAINTENANCE, JSON.stringify(state))
+  }
+
+  // this method is best effort and will not stop on first error
+  async disableMaintenanceMode(ref) {
+    const state = JSON.parse((await this.getField('SR', ref, 'other_config'))[OC_MAINTENANCE])
+
+    // will throw if not in maintenance mode
+    await this.call('SR.remove_from_other_config', ref, OC_MAINTENANCE)
+
+    const errors = []
+
+    await asyncMap(state.unpluggedPbds, async uuid => {
+      try {
+        await this.callAsync('PBD.plug', await this.call('PBD.get_by_uuid', uuid))
+      } catch (error) {
+        errors.push(error)
+      }
+    })
+
+    await asyncMap(Object.entries(state.shutdownVms), async ([uuid, isPaused]) => {
+      try {
+        await this.callAsync('VM.start', await this.call('VM.get_by_uuid', uuid), isPaused, true)
+      } catch (error) {
+        errors.push(error)
+      }
+    })
+
+    if (errors.length !== 0) {
+      throw new AggregateError(errors)
+    }
+  }
+
+  async importVdi(
+    $defer,
+    ref,
+    stream,
+    { name_label = '[XO] Imported disk - ' + new Date().toISOString(), ...vdiCreateOpts } = {}
+  ) {
+    const footer = await peekFooterFromStream(stream)
+    const vdiRef = await this.VDI_create({ ...vdiCreateOpts, name_label, SR: ref, virtual_size: footer.currentSize })
+    $defer.onFailure.call(this, 'callAsync', 'VDI.destroy', vdiRef)
+    await this.VDI_importContent(vdiRef, stream, { format: VDI_FORMAT_RAW })
+    return vdiRef
+  }
+}
+module.exports = Sr
+
+decorateClass(Sr, { enableMaintenanceMode: defer, importVdi: defer })

@xen-orchestra/xapi/vbd.js

@@ -6,6 +6,8 @@ const { Ref } = require('xen-api')
 
 const isVmRunning = require('./_isVmRunning.js')
 
+const { warn } = require('@xen-orchestra/log').createLogger('xo:xapi:vbd')
+
 const noop = Function.prototype
 
 module.exports = class Vbd {
@@ -66,8 +68,10 @@ module.exports = class Vbd {
     })
 
     if (isVmRunning(powerState)) {
-      await this.callAsync('VBD.plug', vbdRef)
+      this.callAsync('VBD.plug', vbdRef).catch(warn)
     }
+
+    return vbdRef
   }
 
   async unplug(ref) {

@xen-orchestra/xapi/vdi.js

@@ -30,8 +30,7 @@ class Vdi {
       other_config = {},
       read_only = false,
       sharable = false,
-      sm_config,
-      SR,
+      SR = this.pool.default_SR,
       tags,
       type = 'user',
       virtual_size,
@@ -39,10 +38,10 @@ class Vdi {
     },
     {
       // blindly copying `sm_config` from another VDI can create problems,
-      // therefore it is ignored by default by this method
+      // therefore it should be passed explicitly
       //
       // see https://github.com/vatesfr/xen-orchestra/issues/4482
-      setSmConfig = false,
+      sm_config,
     } = {}
   ) {
     return this.call('VDI.create', {
@@ -51,7 +50,7 @@ class Vdi {
       other_config,
       read_only,
       sharable,
-      sm_config: setSmConfig ? sm_config : undefined,
+      sm_config,
       SR,
       tags,
       type,

@xen-orchestra/xapi/vm.js

@@ -11,7 +11,8 @@ const { asyncMap } = require('@xen-orchestra/async-map')
 const { createLogger } = require('@xen-orchestra/log')
 const { decorateClass } = require('@vates/decorate-with')
 const { defer } = require('golike-defer')
-const { incorrectState } = require('xo-common/api-errors.js')
+const { incorrectState, forbiddenOperation } = require('xo-common/api-errors.js')
+const { JsonRpcError } = require('json-rpc-protocol')
 const { Ref } = require('xen-api')
 
 const extractOpaqueRef = require('./_extractOpaqueRef.js')
@@ -343,7 +344,13 @@ class Vm {
     const vm = await this.getRecord('VM', vmRef)
 
     if (!bypassBlockedOperation && 'destroy' in vm.blocked_operations) {
-      throw new Error('destroy is blocked')
+      throw forbiddenOperation(
+        `destroy is blocked: ${
+          vm.blocked_operations.destroy === 'true'
+            ? 'protected from accidental deletion'
+            : vm.blocked_operations.destroy
+        }`
+      )
     }
 
     if (!forceDeleteDefaultTemplate && isDefaultTemplate(vm)) {
@@ -503,6 +510,22 @@ class Vm {
       }
       return ref
     } catch (error) {
+      if (
+        // xxhash is the new form consistency hashing in CH 8.1 which uses a faster,
+        // more efficient hashing algorithm to generate the consistency checks
+        // in order to support larger files without the consistency checking process taking an incredibly long time
+        error.code === 'IMPORT_ERROR' &&
+        error.params?.some(
+          param =>
+            param.includes('INTERNAL_ERROR') &&
+            param.includes('Expected to find an inline checksum') &&
+            param.includes('.xxhash')
+        )
+      ) {
+        warn('import', { error })
+        throw new JsonRpcError('Importing this VM requires XCP-ng or Citrix Hypervisor >=8.1')
+      }
+
       // augment the error with as much relevant info as possible
       const [poolMaster, sr] = await Promise.all([
         safeGetRecord(this, 'host', this.pool.master),
@@ -514,12 +537,31 @@ class Vm {
     }
   }
 
-  async snapshot($defer, vmRef, { cancelToken = CancelToken.none, ignoreNobakVdis = false, name_label } = {}) {
+  async snapshot(
+    $defer,
+    vmRef,
+    { cancelToken = CancelToken.none, ignoreNobakVdis = false, name_label, unplugVusbs = false } = {}
+  ) {
     const vm = await this.getRecord('VM', vmRef)
 
+    const isHalted = vm.power_state === 'Halted'
+
+    // requires the VM to be halted because it's not possible to re-plug VUSB on a live VM
+    if (unplugVusbs && isHalted) {
+      // vm.VUSBs can be undefined (e.g. on XS 7.0.0)
+      const vusbs = vm.VUSBs
+      if (vusbs !== undefined) {
+        await asyncMap(vusbs, async ref => {
+          const vusb = await this.getRecord('VUSB', ref)
+          await vusb.$call('destroy')
+          $defer.call(this, 'call', 'VUSB.create', vusb.VM, vusb.USB_group, vusb.other_config)
+        })
+      }
+    }
+
     let destroyNobakVdis = false
     if (ignoreNobakVdis) {
-      if (vm.power_state === 'Halted') {
+      if (isHalted) {
         await asyncMap(await listNobakVbds(this, vm.VBDs), async vbd => {
           await this.VBD_destroy(vbd.$ref)
           $defer.call(this, 'VBD_create', vbd)

CHANGELOG.md

@@ -1,5 +1,153 @@
 # ChangeLog
 
+## **5.72.1** (2022-07-11)
+
+<img id="latest" src="https://badgen.net/badge/channel/latest/yellow" alt="Channel: latest" />
+
+### Enhancements
+
+- [SR] When SR is in maintenance, add "Maintenance mode" badge next to its name (PR [#6313](https://github.com/vatesfr/xen-orchestra/pull/6313))
+
+### Bug fixes
+
+- [Tasks] Fix tasks not displayed when running CR backup job [Forum#6038](https://xcp-ng.org/forum/topic/6038/not-seeing-tasks-any-more-as-admin) (PR [#6315](https://github.com/vatesfr/xen-orchestra/pull/6315))
+- [Backup] Fix failing merge multiple VHDs at once (PR [#6317](https://github.com/vatesfr/xen-orchestra/pull/6317))
+- [VM/Console] Fix _Connect with SSH/RDP_ when address is IPv6
+- [Audit] Ignore side-effects free API methods `xoa.check`, `xoa.clearCheckCache` and `xoa.getHVSupportedVersions`
+
+### Released packages
+
+- @xen-orchestra/backups 0.27.0
+- @xen-orchestra/backups-cli 0.7.5
+- @xen-orchestra/proxy 0.23.5
+- vhd-lib 3.3.2
+- xo-server 5.98.1
+- xo-server-audit 0.10.0
+- xo-web 5.100.0
+
+## **5.72.0** (2022-06-30)
+
+### Highlights
+
+- [Backup] Merge delta backups without copying data when using VHD directories on NFS/SMB/local remote(https://github.com/vatesfr/xen-orchestra/pull/6271))
+- [Proxies] Ability to copy the proxy access URL (PR [#6287](https://github.com/vatesfr/xen-orchestra/pull/6287))
+- [SR/Advanced] Ability to enable/disable _Maintenance Mode_ [#6215](https://github.com/vatesfr/xen-orchestra/issues/6215) (PRs [#6308](https://github.com/vatesfr/xen-orchestra/pull/6308), [#6297](https://github.com/vatesfr/xen-orchestra/pull/6297))
+- [User] User tokens management through XO interface (PR [#6276](https://github.com/vatesfr/xen-orchestra/pull/6276))
+- [Tasks, VM/General] Self Service users: show tasks related to their pools, hosts, SRs, networks and VMs (PR [#6217](https://github.com/vatesfr/xen-orchestra/pull/6217))
+
+### Enhancements
+
+> Users must be able to say: “Nice enhancement, I'm eager to test it”
+
+- [Backup/Restore] Clearer error message when importing a VM backup requires XCP-n/CH >= 8.1 (PR [#6304](https://github.com/vatesfr/xen-orchestra/pull/6304))
+- [Backup] Users can use VHD directory on any remote type (PR [#6273](https://github.com/vatesfr/xen-orchestra/pull/6273))
+
+### Bug fixes
+
+> Users must be able to say: “I had this issue, happy to know it's fixed”
+
+- [VDI Import] Fix `this._getOrWaitObject is not a function`
+- [VM] Attempting to delete a protected VM should display a modal with the error and the ability to bypass it (PR [#6290](https://github.com/vatesfr/xen-orchestra/pull/6290))
+- [OVA Import] Fix import stuck after first disk
+- [File restore] Ignore symbolic links
+
+### Released packages
+
+- @vates/event-listeners-manager 1.0.1
+- @vates/read-chunk 1.0.0
+- @xen-orchestra/backups 0.26.0
+- @xen-orchestra/backups-cli 0.7.4
+- xo-remote-parser 0.9.1
+- @xen-orchestra/fs 1.1.0
+- @xen-orchestra/openflow 0.1.2
+- @xen-orchestra/xapi 1.4.0
+- @xen-orchestra/proxy 0.23.4
+- @xen-orchestra/proxy-cli 0.3.1
+- vhd-lib 3.3.1
+- vhd-cli 0.8.0
+- xo-vmdk-to-vhd 2.4.2
+- xo-server 5.98.0
+- xo-web 5.99.0
+
+## **5.71.1 (2022-06-13)**
+
+<img id="stable" src="https://badgen.net/badge/channel/stable/green" alt="Channel: stable" />
+
+### Enhancements
+
+- Show raw errors to administrators instead of _unknown error from the peer_ (PR [#6260](https://github.com/vatesfr/xen-orchestra/pull/6260))
+
+### Bug fixes
+
+- [New SR] Fix `method.startsWith is not a function` when creating an _ext_ SR
+- Import VDI content now works when there is a HTTP proxy between XO and the host (PR [#6261](https://github.com/vatesfr/xen-orchestra/pull/6261))
+- [Backup] Fix `undefined is not iterable (cannot read property Symbol(Symbol.iterator))` on XS 7.0.0
+- [Backup] Ensure a warning is shown if a target preparation step fails (PR [#6266](https://github.com/vatesfr/xen-orchestra/pull/6266))
+- [OVA Export] Avoid creating a zombie task (PR [#6267](https://github.com/vatesfr/xen-orchestra/pull/6267))
+- [OVA Export] Increase speed by lowering compression to acceptable level (PR [#6267](https://github.com/vatesfr/xen-orchestra/pull/6267))
+- [OVA Export] Fix broken OVAs due to special characters in VM name (PR [#6267](https://github.com/vatesfr/xen-orchestra/pull/6267))
+
+### Released packages
+
+- @xen-orchestra/backups 0.25.0
+- @xen-orchestra/backups-cli 0.7.3
+- xen-api 1.2.1
+- @xen-orchestra/xapi 1.2.0
+- @xen-orchestra/proxy 0.23.2
+- @xen-orchestra/proxy-cli 0.3.0
+- xo-cli 0.14.0
+- xo-vmdk-to-vhd 2.4.1
+- xo-server 5.96.0
+- xo-web 5.97.2
+
+## **5.71.0 (2022-05-31)**
+
+### Highlights
+
+- [Backup] _Restore Health Check_ can now be configured to be run automatically during a backup schedule (PRs [#6227](https://github.com/vatesfr/xen-orchestra/pull/6227), [#6228](https://github.com/vatesfr/xen-orchestra/pull/6228), [#6238](https://github.com/vatesfr/xen-orchestra/pull/6238) & [#6242](https://github.com/vatesfr/xen-orchestra/pull/6242))
+- [Backup] VMs with USB Pass-through devices are now supported! The advanced _Offline Snapshot Mode_ setting must be enabled. For Full Backup or Disaster Recovery jobs, Rolling Snapshot needs to be anabled as well. (PR [#6239](https://github.com/vatesfr/xen-orchestra/pull/6239))
+- [Backup] Implement file cache for listing the backups of a VM (PR [#6220](https://github.com/vatesfr/xen-orchestra/pull/6220))
+- [RPU/Host] If some backup jobs are running on the pool, ask for confirmation before starting an RPU, shutdown/rebooting a host or restarting a host's toolstack (PR [6232](https://github.com/vatesfr/xen-orchestra/pull/6232))
+- [XO Web] Add ability to configure a default filter for Storage [#6236](https://github.com/vatesfr/xen-orchestra/issues/6236) (PR [#6237](https://github.com/vatesfr/xen-orchestra/pull/6237))
+- [REST API] Support VDI creation via VHD import
+
+### Enhancements
+
+- [Backup] Merge multiple VHDs at once which will speed up the merging phase after reducing the retention of a backup job(PR [#6184](https://github.com/vatesfr/xen-orchestra/pull/6184))
+- [Backup] Add setting `backups.metadata.defaultSettings.unconditionalSnapshot` in `xo-server`'s configuration file to force a snapshot even when not required by the backup, this is useful to avoid locking the VM halted during the backup (PR [#6221](https://github.com/vatesfr/xen-orchestra/pull/6221))
+- [VM migration] Ensure the VM can be migrated before performing the migration to avoid issues [#5301](https://github.com/vatesfr/xen-orchestra/issues/5301) (PR [#6245](https://github.com/vatesfr/xen-orchestra/pull/6245))
+- [Backup] Show any detected errors on existing backups instead of fixing them silently (PR [#6207](https://github.com/vatesfr/xen-orchestra/pull/6225))
+- Created SRs will now have auto-scan enabled similarly to what XenCenter does (PR [#6246](https://github.com/vatesfr/xen-orchestra/pull/6246))
+- [RPU] Disable scheduled backup jobs during RPU (PR [#6244](https://github.com/vatesfr/xen-orchestra/pull/6244))
+
+### Bug fixes
+
+- [S3] Fix S3 remote with empty directory not showing anything to restore (PR [#6218](https://github.com/vatesfr/xen-orchestra/pull/6218))
+- [S3] remote fom did not save the `https` and `allow unatuhorized`during remote creation (PR [#6219](https://github.com/vatesfr/xen-orchestra/pull/6219))
+- [VM/advanced] Fix various errors when adding ACLs [#6213](https://github.com/vatesfr/xen-orchestra/issues/6213) (PR [#6230](https://github.com/vatesfr/xen-orchestra/pull/6230))
+- [Home/Self] Don't make VM's resource set name clickable for non admin users as they aren't allowed to view the Self Service page (PR [#6252](https://github.com/vatesfr/xen-orchestra/pull/6252))
+- [load-balancer] Fix density mode failing to shutdown hosts (PR [#6253](https://github.com/vatesfr/xen-orchestra/pull/6253))
+- [Health] Make "Too many snapshots" table sortable by number of snapshots (PR [#6255](https://github.com/vatesfr/xen-orchestra/pull/6255))
+- [Remote] Show complete errors instead of only a potentially missing message (PR [#6216](https://github.com/vatesfr/xen-orchestra/pull/6216))
+
+### Released packages
+
+- @xen-orchestra/self-signed 0.1.3
+- vhd-lib 3.2.0
+- @xen-orchestra/fs 1.0.3
+- vhd-cli 0.7.2
+- xo-vmdk-to-vhd 2.4.0
+- @xen-orchestra/upload-ova 0.1.5
+- @xen-orchestra/xapi 1.1.0
+- @xen-orchestra/backups 0.24.0
+- @xen-orchestra/backups-cli 0.7.2
+- @xen-orchestra/emit-async 1.0.0
+- @xen-orchestra/mixins 0.5.0
+- @xen-orchestra/proxy 0.23.1
+- xo-server 5.95.0
+- xo-web 5.97.1
+- xo-server-backup-reports 0.17.0
+
 ## 5.70.2 (2022-05-16)
 
 ### Bug fixes
@@ -35,8 +183,6 @@
 
 ## 5.70.0 (2022-04-29)
 
-<img id="latest" src="https://badgen.net/badge/channel/latest/yellow" alt="Channel: latest" />
-
 ### Highlights
 
 - [VM export] Feat export to `ova` format (PR [#6006](https://github.com/vatesfr/xen-orchestra/pull/6006))
@@ -73,8 +219,6 @@
 
 ## **5.69.2** (2022-04-13)
 
-<img id="stable" src="https://badgen.net/badge/channel/stable/green" alt="Channel: stable" />
-
 ### Enhancements
 
 - [Rolling Pool Update] New algorithm for XCP-ng updates (PR [#6188](https://github.com/vatesfr/xen-orchestra/pull/6188))

CHANGELOG.unreleased.md

@@ -7,51 +7,29 @@
 
 > Users must be able to say: “Nice enhancement, I'm eager to test it”
 
-- [Backup] Merge multiple VHDs at once which will speed up the merging ĥase after reducing the retention of a backup job(PR [#6184](https://github.com/vatesfr/xen-orchestra/pull/6184))
-- [Backup] Implement file cache for listing the backups of a VM (PR [#6220](https://github.com/vatesfr/xen-orchestra/pull/6220))
-- [Backup] Add setting `backups.metadata.defaultSettings.unconditionalSnapshot` in `xo-server`'s configuration file to force a snapshot even when not required by the backup, this is useful to avoid locking the VM halted during the backup (PR [#6221](https://github.com/vatesfr/xen-orchestra/pull/6221))
-- [XO Web] Add ability to configure a default filter for Storage [#6236](https://github.com/vatesfr/xen-orchestra/issues/6236) (PR [#6237](https://github.com/vatesfr/xen-orchestra/pull/6237))
-
 ### Bug fixes
 
 > Users must be able to say: “I had this issue, happy to know it's fixed”
 
-- [S3] Fix S3 remote with empty directory not showing anything to restore (PR [#6218](https://github.com/vatesfr/xen-orchestra/pull/6218))
-- [S3] remote fom did not save the `https` and `allow unatuhorized`during remote creation (PR [#6219](https://github.com/vatesfr/xen-orchestra/pull/6219))
-
 ### Packages to release
 
-> Packages will be released in the order they are here, therefore, they should
-> be listed by inverse order of dependency.
+> When modifying a package, add it here with its release type.
 >
-> Rule of thumb: add packages on top.
+> The format is the following: - `$packageName` `$releaseType`
 >
-> The format is the following: - `$packageName` `$version`
->
-> Where `$version` is
+> Where `$releaseType` is
 >
 > - patch: if the change is a bug fix or a simple code improvement
 > - minor: if the change is a new feature
 > - major: if the change breaks compatibility
 >
-> In case of conflict, the highest (lowest in previous list) `$version` wins.
->
-> The `gen-deps-list` script can be used to generate this list of dependencies
-> Run `scripts/gen-deps-list.js --help` for usage
+> Keep this list alphabetically ordered to avoid merge conflicts
 
 <!--packages-start-->
 
-- vhd-lib patch
-- @xen-orchestra/fs patch
-- vhd-cli patch
-- xo-vmdk-to-vhd minor
-- @xen-orchestra/upload-ova patch
-- @xen-orchestra/backups minor
-- @xen-orchestra/backups-cli patch
-- @xen-orchestra/emit-async major
+- @vates/async-each major
 - @xen-orchestra/mixins minor
-- @xen-orchestra/proxy minor
-- xo-server minor
-- xo-web minor
+- @xen-orchestra/proxy patch
+- @xen-orchestra/xo-server patch
 
 <!--packages-end-->

docs/backup_troubleshooting.md

@@ -99,3 +99,38 @@ To solve this issue, we recommend that you:
 
 - wait until the other backup job is completed/the merge process is done
 - make sure your remote storage is not being overworked
+
+## Error: HTTP connection has timed out
+
+This error occurs when XO tries to fetch data from a host, via the HTTP GET method. This error essentially means that the host (dom0 specifically) isn't responding anymore, after we asked it to expose the disk to be exported. This could be a symptom of having an overloaded dom0 that couldn't respond fast enough. It can also be caused by dom0 having trouble attaching the disk in question to expose it for fetching via HTTP, or just not having enough resources to answer our GET request.
+
+::: warning
+As a temporary workaround you can increase the timeout higher than the default value, to allow the host more time to respond. But you will need to eventually diagnose the root cause of the slow host response or else you risk the issue returning.
+:::
+
+Create the following file:
+```
+/etc/xo-server/config.httpInactivityTimeout.toml
+```
+Add the following lines:
+```
+# XOA Support - Work-around HTTP timeout issue during backups
+[xapiOptions]
+httpInactivityTimeout = 1800000 # 30 mins
+```
+
+## Error: Expected values to be strictly equal
+
+This error occurs at the end of the transfer. XO checks the exported VM disk integrity, to ensure it's a valid VHD file (we check the VHD header as well as the footer of the received file). This error means the header and footage did not match, so the file is incomplete (likely the export from dom0 failed at some point and we only received a partial HD/VM disk).
+
+## Error: the job is already running
+
+This means the same job is still running, typically from the last scheduled run. This happens when you have a backup job scheduled too often. It can also occur if you have a long timeout configured for the job, and a slow VM export or slow transfer to your remote. In either case, you need to adjust your backup schedule to allow time for the job to finish or timeout before the next scheduled run. We consider this an error to ensure you'll be notified that the planned schedule won't run this time because the previous one isn't finished.
+
+## Error: VDI_IO_ERROR
+
+This error comes directly from your host/dom0, and not XO. Essentially, XO asked the host to expose a VM disk to export via HTTP (as usual), XO managed to make the HTTP GET connection, and even start the transfer. But then at some point the host couldn't read the VM disk any further, causing this error on the host side. This might happen if the VDI is corrupted on the storage, or if there's a race condition during snapshots. More rarely, this can also occur if your SR is just too slow to keep up with the export as well as live VM traffic.
+
+## Error: no XAPI associated to <UUID>
+
+This message means that XO had a UUID of a VM to backup, but when the job ran it couldn't find any object matching it. This could be caused by the pool where this VM lived no longer being connected to XO. Double-check that the pool hosting the VM is currently connected under Settings > Servers. You can also search for the VM UUID in the Home > VMs search bar. If you can see it, run the backup job again and it will work. If you cannot, either the VM was removed or the pool is not connected.

docs/configuration.md

@@ -66,12 +66,13 @@ You shouldn't have to change this. It's the path where `xo-web` files are served
 
 ## Custom certificate authority
 
-If you use certificates signed by an in-house CA for your XenServer hosts, and want to have Xen Orchestra connect to them without rejection, you need to add the `--use-openssl-ca` option in Node, but also add the CA to your trust store (`/etc/ssl/certs` via `update-ca-certificates` in your XOA).
+If you use certificates signed by an in-house CA for your XCP-ng or XenServer hosts, and want to have Xen Orchestra connect to them without rejection, you can use the [`NODE_EXTRA_CA_CERTS`](https://nodejs.org/api/cli.html#cli_node_extra_ca_certs_file) environment variable.
 
-To enable this option in your XOA, edit the `/etc/systemd/system/xo-server.service` file and add this:
+To enable this option in your XOA, create `/etc/systemd/system/xo-server.service.d/ca.conf` with the following content:
 
 ```
-Environment=NODE_OPTIONS=--use-openssl-ca
+[Service]
+Environment=NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/my-cert.crt
 ```
 
 Don't forget to reload `systemd` conf and restart `xo-server`:
@@ -81,9 +82,7 @@ Don't forget to reload `systemd` conf and restart `xo-server`:
 # systemctl restart xo-server.service
 ```
 
-:::tip
-The `--use-openssl-ca` option is ignored by Node if Xen-Orchestra is run with Linux capabilities. Capabilities are commonly used to bind applications to privileged ports (<1024) (i.e. `CAP_NET_BIND_SERVICE`). Local NAT rules (`iptables`) or a reverse proxy would be required to use privileged ports and a custom certficate authority.
-:::
+> For XO Proxy, the process is almost the same except the file to create is `/etc/systemd/system/xo-proxy.service.d/ca.conf` and the service to restart is `xo-proxy.service`.
 
 ## Redis server
 

docs/continuous_replication.md

@@ -18,6 +18,8 @@ If you lose your main pool, you can start the copy on the other side, with very
 
 :::warning
 It is normal that you can't boot the copied VM directly: we protect it. The normal workflow is to make a clone and then work on it.
+
+This also affects VMs with "Auto Power On" enabled, because of our protections you can ensure these won't start on your CR destination if you happen to reboot it.
 :::
 
 ## Configure it

docs/contributing.md

@@ -24,16 +24,15 @@ Please, do explain:
 The best way to propose a change to the documentation or code is
 to create a [GitHub pull request](https://help.github.com/articles/using-pull-requests/).
 
-:::tip
-Your pull request should always be against the `master` branch and not against `stable` which is the stable branch!
-:::
-
-1. Create a branch for your work
-2. Add a summary of your changes to `CHANGELOG.md` under the `next` section, if your changes do not relate to an existing changelog item
-3. Create a pull request for this branch against the `master` branch
-4. Push into the branch until the pull request is ready to merge
-5. Avoid unnecessary merges: keep you branch up to date by regularly rebasing `git rebase origin/master`
-6. When ready to merge, clean up the history (reorder commits, squash some of them together, rephrase messages): `git rebase -i origin/master`
+1. Fork the [Xen Orchestra repository](https://github.com/vatesfr/xen-orchestra) using the Fork button
+2. Follow [the documentation](installation.md#from-the-sources) to install and run Xen Orchestra from the sources
+3. Create a branch for your work
+4. Edit the source files
+5. Add a summary of your changes to `CHANGELOG.unreleased.md`, if your changes do not relate to an existing changelog item and update the list of packages that must be released to take your changes into account
+6. [Create a pull request](https://github.com/vatesfr/xen-orchestra/compare) for this branch against the `master` branch
+7. Push into the branch until the pull request is ready to merge
+8. Avoid unnecessary merges: keep you branch up to date by regularly rebasing `git rebase origin/master`
+9. When ready to merge, clean up the history (reorder commits, squash some of them together, rephrase messages): `git rebase -i origin/master`
 
 ### Issue triage
 

docs/disaster_recovery.md

@@ -35,3 +35,7 @@ A higher retention number will lead to huge space occupation on your SR.
 If you boot a copy of your production VM, be careful: if they share the same static IP, you'll have troubles.
 
 A good way to avoid this kind of problem is to remove the network interface on the DR VM and check if the export is correctly done.
+
+:::warning
+For each DR replicated VM, we add "start" as a blocked operation, meaning even VMs with "Auto power on" enabled will not be started on your DR destination if it reboots.
+:::

docs/restapi.md

@@ -141,6 +141,28 @@ curl \
   > myDisk.vhd
 ```
 
+## VDI Import
+
+A VHD can be imported on an SR to create a VDI at `/rest/v0/srs/<sr uuid>/vdis`.
+
+```bash
+curl \
+  -X POST \
+  -b authenticationToken=KQxQdm2vMiv7jBIK0hgkmgxKzemd8wSJ7ugFGKFkTbs \
+  -T myDisk.vhd \
+  'https://xo.example.org/rest/v0/srs/357bd56c-71f9-4b2a-83b8-3451dec04b8f/vdis?name_label=my_imported_VDI' \
+  | cat
+```
+
+> Note: the final `| cat` ensures cURL's standard output is not a TTY, which is necessary for upload stats to be dislayed.
+
+This request returns the UUID of the created VDI.
+
+The following query parameters are supported to customize the created VDI:
+
+- `name_label`
+- `name_description`
+
 ## The future
 
 We are adding features and improving the REST API step by step. If you have interesting use cases or feedback, please ask directly at <https://xcp-ng.org/forum/category/12/xen-orchestra>

package.json

@@ -3,9 +3,8 @@
     "@babel/core": "^7.0.0",
     "@babel/eslint-parser": "^7.13.8",
     "@babel/register": "^7.0.0",
-    "babel-jest": "^27.3.1",
+    "babel-jest": "^28.1.2",
     "benchmark": "^2.1.4",
-    "commander": "^9.2.0",
     "deptree": "^1.0.0",
     "eslint": "^8.7.0",
     "eslint-config-prettier": "^8.1.0",
@@ -20,7 +19,7 @@
     "globby": "^13.1.1",
     "handlebars": "^4.7.6",
     "husky": "^4.2.5",
-    "jest": "^27.3.1",
+    "jest": "^28.1.2",
     "lint-staged": "^12.0.3",
     "lodash": "^4.17.4",
     "prettier": "^2.0.5",
@@ -61,6 +60,7 @@
     "testEnvironment": "node",
     "testPathIgnorePatterns": [
       "/@vates/decorate-with/",
+      "/@vates/event-listeners-manager/",
       "/@vates/predicates/",
       "/@xen-orchestra/audit-core/",
       "/dist/",

packages/complex-matcher/.babelrc.js

@@ -1,3 +0,0 @@
-'use strict'
-
-module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))

packages/complex-matcher/.eslintrc.js

@@ -1 +0,0 @@
-../../scripts/babel-eslintrc.js

packages/complex-matcher/index.bench.js

@@ -0,0 +1,14 @@
+'use strict'
+
+const { parse } = require('./')
+const { ast, pattern } = require('./index.fixtures')
+
+module.exports = ({ benchmark }) => {
+  benchmark('parse', () => {
+    parse(pattern)
+  })
+
+  benchmark('toString', () => {
+    ast.toString()
+  })
+}

packages/complex-matcher/index.fixtures.js

@@ -1,8 +1,10 @@
-import * as CM from './'
+'use strict'
 
-export const pattern = 'foo !"\\\\ \\"" name:|(wonderwoman batman) hasCape? age:32 chi*go /^foo\\/bar\\./i'
+const CM = require('./')
 
-export const ast = new CM.And([
+exports.pattern = 'foo !"\\\\ \\"" name:|(wonderwoman batman) hasCape? age:32 chi*go /^foo\\/bar\\./i'
+
+exports.ast = new CM.And([
   new CM.String('foo'),
   new CM.Not(new CM.String('\\ "')),
   new CM.Property('name', new CM.Or([new CM.String('wonderwoman'), new CM.String('batman')])),

packages/complex-matcher/index.js

@@ -1,4 +1,6 @@
-import { escapeRegExp, isPlainObject, some } from 'lodash'
+'use strict'
+
+const { escapeRegExp, isPlainObject, some } = require('lodash')
 
 // ===================================================================
 
@@ -23,7 +25,7 @@ class Node {
   }
 }
 
-export class Null extends Node {
+class Null extends Node {
   match() {
     return true
   }
@@ -32,10 +34,11 @@ export class Null extends Node {
     return ''
   }
 }
+exports.Null = Null
 
 const formatTerms = terms => terms.map(term => term.toString(true)).join(' ')
 
-export class And extends Node {
+class And extends Node {
   constructor(children) {
     super()
 
@@ -54,8 +57,9 @@ export class And extends Node {
     return isNested ? `(${terms})` : terms
   }
 }
+exports.And = And
 
-export class Comparison extends Node {
+class Comparison extends Node {
   constructor(operator, value) {
     super()
     this._comparator = Comparison.comparators[operator]
@@ -71,6 +75,7 @@ export class Comparison extends Node {
     return this._operator + String(this._value)
   }
 }
+exports.Comparison = Comparison
 Comparison.comparators = {
   '>': (a, b) => a > b,
   '>=': (a, b) => a >= b,
@@ -78,7 +83,7 @@ Comparison.comparators = {
   '<=': (a, b) => a <= b,
 }
 
-export class Or extends Node {
+class Or extends Node {
   constructor(children) {
     super()
 
@@ -96,8 +101,9 @@ export class Or extends Node {
     return `|(${formatTerms(this.children)})`
   }
 }
+exports.Or = Or
 
-export class Not extends Node {
+class Not extends Node {
   constructor(child) {
     super()
 
@@ -112,8 +118,9 @@ export class Not extends Node {
     return '!' + this.child.toString(true)
   }
 }
+exports.Not = Not
 
-export class NumberNode extends Node {
+exports.Number = exports.NumberNode = class NumberNode extends Node {
   constructor(value) {
     super()
 
@@ -133,9 +140,8 @@ export class NumberNode extends Node {
     return String(this.value)
   }
 }
-export { NumberNode as Number }
 
-export class NumberOrStringNode extends Node {
+class NumberOrStringNode extends Node {
   constructor(value) {
     super()
 
@@ -160,9 +166,9 @@ export class NumberOrStringNode extends Node {
     return this.value
   }
 }
-export { NumberOrStringNode as NumberOrString }
+exports.NumberOrString = exports.NumberOrStringNode = NumberOrStringNode
 
-export class Property extends Node {
+class Property extends Node {
   constructor(name, child) {
     super()
 
@@ -178,12 +184,13 @@ export class Property extends Node {
     return `${formatString(this.name)}:${this.child.toString(true)}`
   }
 }
+exports.Property = Property
 
 const escapeChar = char => '\\' + char
 const formatString = value =>
   Number.isNaN(+value) ? (isRawString(value) ? value : `"${value.replace(/\\|"/g, escapeChar)}"`) : `"${value}"`
 
-export class GlobPattern extends Node {
+class GlobPattern extends Node {
   constructor(value) {
     // fallback to string node if no wildcard
     if (value.indexOf('*') === -1) {
@@ -216,8 +223,9 @@ export class GlobPattern extends Node {
     return this.value
   }
 }
+exports.GlobPattern = GlobPattern
 
-export class RegExpNode extends Node {
+class RegExpNode extends Node {
   constructor(pattern, flags) {
     super()
 
@@ -245,9 +253,9 @@ export class RegExpNode extends Node {
     return this.re.toString()
   }
 }
-export { RegExpNode as RegExp }
+exports.RegExp = RegExpNode
 
-export class StringNode extends Node {
+class StringNode extends Node {
   constructor(value) {
     super()
 
@@ -275,9 +283,9 @@ export class StringNode extends Node {
     return formatString(this.value)
   }
 }
-export { StringNode as String }
+exports.String = exports.StringNode = StringNode
 
-export class TruthyProperty extends Node {
+class TruthyProperty extends Node {
   constructor(name) {
     super()
 
@@ -292,6 +300,7 @@ export class TruthyProperty extends Node {
     return formatString(this.name) + '?'
   }
 }
+exports.TruthyProperty = TruthyProperty
 
 // -------------------------------------------------------------------
 
@@ -531,7 +540,7 @@ const parser = P.grammar({
     ),
   ws: P.regex(/\s*/),
 }).default
-export const parse = parser.parse.bind(parser)
+exports.parse = parser.parse.bind(parser)
 
 // -------------------------------------------------------------------
 
@@ -573,7 +582,7 @@ const _getPropertyClauseStrings = ({ child }) => {
 }
 
 // Find possible values for property clauses in a and clause.
-export const getPropertyClausesStrings = node => {
+exports.getPropertyClausesStrings = function getPropertyClausesStrings(node) {
   if (!node) {
     return {}
   }
@@ -605,7 +614,7 @@ export const getPropertyClausesStrings = node => {
 
 // -------------------------------------------------------------------
 
-export const setPropertyClause = (node, name, child) => {
+exports.setPropertyClause = function setPropertyClause(node, name, child) {
   const property = child && new Property(name, typeof child === 'string' ? new StringNode(child) : child)
 
   if (node === undefined) {

packages/complex-matcher/index.spec.js

@@ -1,7 +1,9 @@
 /* eslint-env jest */
 
-import { ast, pattern } from './index.fixtures'
-import {
+'use strict'
+
+const { ast, pattern } = require('./index.fixtures')
+const {
   getPropertyClausesStrings,
   GlobPattern,
   Null,
@@ -11,7 +13,7 @@ import {
   Property,
   setPropertyClause,
   StringNode,
-} from './'
+} = require('./')
 
 it('getPropertyClausesStrings', () => {
   const tmp = getPropertyClausesStrings(parse('foo bar:baz baz:|(foo bar /^boo$/ /^far$/) foo:/^bar$/'))

packages/complex-matcher/package.json

@@ -16,7 +16,6 @@
     "url": "https://vates.fr"
   },
   "preferGlobal": false,
-  "main": "dist/",
   "browserslist": [
     ">2%"
   ],
@@ -26,21 +25,7 @@
   "dependencies": {
     "lodash": "^4.17.4"
   },
-  "devDependencies": {
-    "@babel/cli": "^7.0.0",
-    "@babel/core": "^7.0.0",
-    "@babel/preset-env": "^7.0.0",
-    "babel-plugin-lodash": "^3.3.2",
-    "cross-env": "^7.0.2",
-    "rimraf": "^3.0.0"
-  },
   "scripts": {
-    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
-    "clean": "rimraf dist/",
-    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
-    "prebuild": "yarn run clean",
-    "predev": "yarn run prebuild",
-    "prepublishOnly": "yarn run build",
     "postversion": "npm publish"
   }
 }

packages/complex-matcher/src/index.bench.js

@@ -1,12 +0,0 @@
-import { parse } from './'
-import { ast, pattern } from './index.fixtures'
-
-export default ({ benchmark }) => {
-  benchmark('parse', () => {
-    parse(pattern)
-  })
-
-  benchmark('toString', () => {
-    ast.toString()
-  })
-}

packages/value-matcher/.babelrc.js

@@ -1,3 +0,0 @@
-'use strict'
-
-module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))

packages/value-matcher/.eslintrc.js

@@ -1 +0,0 @@
-../../scripts/babel-eslintrc.js

packages/value-matcher/index.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const match = (pattern, value) => {
   if (Array.isArray(pattern)) {
     return (
@@ -43,4 +45,6 @@ const match = (pattern, value) => {
   return pattern === value
 }
 
-export const createPredicate = pattern => value => match(pattern, value)
+exports.createPredicate = function createPredicate(pattern) {
+  return value => match(pattern, value)
+}

packages/value-matcher/package.json

@@ -16,27 +16,13 @@
     "url": "https://vates.fr"
   },
   "preferGlobal": false,
-  "main": "dist/",
   "browserslist": [
     ">2%"
   ],
   "engines": {
     "node": ">=6"
   },
-  "devDependencies": {
-    "@babel/cli": "^7.0.0",
-    "@babel/core": "^7.0.0",
-    "@babel/preset-env": "^7.0.0",
-    "cross-env": "^7.0.2",
-    "rimraf": "^3.0.0"
-  },
   "scripts": {
-    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
-    "clean": "rimraf dist/",
-    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
-    "prebuild": "yarn run clean",
-    "predev": "yarn run prebuild",
-    "prepublishOnly": "yarn run build",
     "postversion": "npm publish"
   }
 }

packages/vhd-cli/.babelrc.js

@@ -1,3 +0,0 @@
-'use strict'
-
-module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))

packages/vhd-cli/.eslintrc.js

@@ -1 +0,0 @@
-../../scripts/babel-eslintrc.js

packages/vhd-cli/_utils.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { createWriteStream } = require('fs')
 const { PassThrough } = require('stream')
 
@@ -12,7 +14,7 @@ const createOutputStream = path => {
   return stream
 }
 
-export const writeStream = (input, path) => {
+exports.writeStream = function writeStream(input, path) {
   const output = createOutputStream(path)
 
   return new Promise((resolve, reject) =>

packages/vhd-cli/commands/check.js

@@ -1,11 +1,13 @@
-import { VhdFile, checkVhdChain } from 'vhd-lib'
-import getopts from 'getopts'
-import { getHandler } from '@xen-orchestra/fs'
-import { resolve } from 'path'
+'use strict'
+
+const { VhdFile, checkVhdChain } = require('vhd-lib')
+const getopts = require('getopts')
+const { getHandler } = require('@xen-orchestra/fs')
+const { resolve } = require('path')
 
 const checkVhd = (handler, path) => new VhdFile(handler, path).readHeaderAndFooter()
 
-export default async rawArgs => {
+module.exports = async function check(rawArgs) {
   const { chain, _: args } = getopts(rawArgs, {
     boolean: ['chain'],
     default: {

packages/vhd-cli/commands/compare.js

@@ -1,9 +1,11 @@
-import { getSyncedHandler } from '@xen-orchestra/fs'
-import { openVhd, Constants } from 'vhd-lib'
-import Disposable from 'promise-toolbox/Disposable'
-import omit from 'lodash/omit'
+'use strict'
 
-const deepCompareObjects = function (src, dest, path) {
+const { getSyncedHandler } = require('@xen-orchestra/fs')
+const { openVhd, Constants } = require('vhd-lib')
+const Disposable = require('promise-toolbox/Disposable')
+const omit = require('lodash/omit')
+
+function deepCompareObjects(src, dest, path) {
   for (const key of Object.keys(src)) {
     const srcValue = src[key]
     const destValue = dest[key]
@@ -29,7 +31,7 @@ const deepCompareObjects = function (src, dest, path) {
   }
 }
 
-export default async args => {
+module.exports = async function compare(args) {
   if (args.length < 4 || args.some(_ => _ === '-h' || _ === '--help')) {
     return `Usage: compare <sourceRemoteUrl> <source VHD> <destionationRemoteUrl> <destination> `
   }

packages/vhd-cli/commands/copy.js

@@ -1,9 +1,11 @@
-import { getSyncedHandler } from '@xen-orchestra/fs'
-import { openVhd, VhdFile, VhdDirectory } from 'vhd-lib'
-import Disposable from 'promise-toolbox/Disposable'
-import getopts from 'getopts'
+'use strict'
 
-export default async rawArgs => {
+const { getSyncedHandler } = require('@xen-orchestra/fs')
+const { openVhd, VhdFile, VhdDirectory } = require('vhd-lib')
+const Disposable = require('promise-toolbox/Disposable')
+const getopts = require('getopts')
+
+module.exports = async function copy(rawArgs) {
   const {
     directory,
     help,

packages/vhd-cli/commands/index.js

@@ -0,0 +1,43 @@
+//
+// This file has been generated by [index-modules](https://npmjs.com/index-modules)
+//
+
+var d = Object.defineProperty
+function de(o, n, v) {
+  d(o, n, { enumerable: true, value: v })
+  return v
+}
+function dl(o, n, g, a) {
+  d(o, n, {
+    configurable: true,
+    enumerable: true,
+    get: function () {
+      return de(o, n, g(a))
+    },
+  })
+}
+function r(p) {
+  var v = require(p)
+  return v && v.__esModule
+    ? v
+    : typeof v === 'object' || typeof v === 'function'
+    ? Object.create(v, { default: { enumerable: true, value: v } })
+    : { default: v }
+}
+function e(p, i) {
+  dl(defaults, i, function () {
+    return exports[i].default
+  })
+  dl(exports, i, r, p)
+}
+
+d(exports, '__esModule', { value: true })
+var defaults = de(exports, 'default', {})
+e('./check.js', 'check')
+e('./compare.js', 'compare')
+e('./copy.js', 'copy')
+e('./info.js', 'info')
+e('./merge.js', 'merge')
+e('./raw.js', 'raw')
+e('./repl.js', 'repl')
+e('./synthetize.js', 'synthetize')

packages/vhd-cli/commands/info.js

@@ -1,9 +1,13 @@
-import { Constants, VhdFile } from 'vhd-lib'
-import { getHandler } from '@xen-orchestra/fs'
-import { resolve } from 'path'
-import * as UUID from 'uuid'
-import humanFormat from 'human-format'
-import invert from 'lodash/invert.js'
+'use strict'
+
+const { Constants, VhdFile } = require('vhd-lib')
+const { getHandler } = require('@xen-orchestra/fs')
+const { openVhd } = require('vhd-lib/openVhd')
+const { resolve } = require('path')
+const Disposable = require('promise-toolbox/Disposable')
+const humanFormat = require('human-format')
+const invert = require('lodash/invert.js')
+const UUID = require('uuid')
 
 const { PLATFORMS } = Constants
 
@@ -32,8 +36,8 @@ function mapProperties(object, mapping) {
   return result
 }
 
-export default async args => {
-  const vhd = new VhdFile(getHandler({ url: 'file:///' }), resolve(args[0]))
+async function showDetails(handler, path) {
+  const vhd = new VhdFile(handler, resolve(path))
 
   try {
     await vhd.readHeaderAndFooter()
@@ -67,3 +71,29 @@ export default async args => {
     })
   )
 }
+
+async function showList(handler, paths) {
+  let previousUuid
+  for (const path of paths) {
+    await Disposable.use(openVhd(handler, resolve(path)), async vhd => {
+      const uuid = MAPPERS.uuid(vhd.footer.uuid)
+      const fields = [path, MAPPERS.bytes(vhd.footer.currentSize), uuid, MAPPERS.diskType(vhd.footer.diskType)]
+      if (vhd.footer.diskType === Constants.DISK_TYPES.DIFFERENCING) {
+        const parentUuid = MAPPERS.uuid(vhd.header.parentUuid)
+        fields.push(parentUuid === previousUuid ? '<above VHD>' : parentUuid)
+      }
+      previousUuid = uuid
+      console.log(fields.join(' | '))
+    })
+  }
+}
+
+module.exports = async function info(args) {
+  const handler = getHandler({ url: 'file:///' })
+
+  if (args.length === 1) {
+    return showDetails(handler, args[0])
+  }
+
+  return showList(handler, args)
+}

packages/vhd-cli/commands/merge.js

@@ -1,9 +1,11 @@
-import { Bar } from 'cli-progress'
-import { mergeVhd } from 'vhd-lib'
-import { getHandler } from '@xen-orchestra/fs'
-import { resolve } from 'path'
+'use strict'
 
-export default async function main(args) {
+const { Bar } = require('cli-progress')
+const { mergeVhd } = require('vhd-lib')
+const { getHandler } = require('@xen-orchestra/fs')
+const { resolve } = require('path')
+
+module.exports = async function merge(args) {
   if (args.length < 2 || args.some(_ => _ === '-h' || _ === '--help')) {
     return `Usage: ${this.command} <child VHD> <parent VHD>`
   }

packages/vhd-cli/commands/raw.js

@@ -1,11 +1,13 @@
-import { openVhd } from 'vhd-lib'
-import { getSyncedHandler } from '@xen-orchestra/fs'
-import { resolve } from 'path'
+'use strict'
 
-import { writeStream } from '../_utils'
-import { Disposable } from 'promise-toolbox'
+const { openVhd } = require('vhd-lib')
+const { getSyncedHandler } = require('@xen-orchestra/fs')
+const { resolve } = require('path')
 
-export default async args => {
+const { writeStream } = require('../_utils')
+const { Disposable } = require('promise-toolbox')
+
+module.exports = async function raw(args) {
   if (args.length < 2 || args.some(_ => _ === '-h' || _ === '--help')) {
     return `Usage: ${this.command} <input VHD> [<output raw>]`
   }

packages/vhd-cli/commands/repl.js

@@ -1,10 +1,12 @@
-import { asCallback, fromCallback, fromEvent } from 'promise-toolbox'
-import { getHandler } from '@xen-orchestra/fs'
-import { relative } from 'path'
-import { start as createRepl } from 'repl'
-import * as vhdLib from 'vhd-lib'
+'use strict'
 
-export default async args => {
+const { asCallback, fromCallback, fromEvent } = require('promise-toolbox')
+const { getHandler } = require('@xen-orchestra/fs')
+const { relative } = require('path')
+const { start: createRepl } = require('repl')
+const vhdLib = require('vhd-lib')
+
+module.exports = async function repl(args) {
   const cwd = process.cwd()
   const handler = getHandler({ url: 'file://' + cwd })
   await handler.sync()

packages/vhd-cli/commands/synthetize.js

@@ -1,9 +1,11 @@
-import path from 'path'
-import { createSyntheticStream } from 'vhd-lib'
-import { createWriteStream } from 'fs'
-import { getHandler } from '@xen-orchestra/fs'
+'use strict'
 
-export default async function main(args) {
+const path = require('path')
+const { createSyntheticStream } = require('vhd-lib')
+const { createWriteStream } = require('fs')
+const { getHandler } = require('@xen-orchestra/fs')
+
+module.exports = async function synthetize(args) {
   if (args.length < 2 || args.some(_ => _ === '-h' || _ === '--help')) {
     return `Usage: ${this.command} <input VHD> <output VHD>`
   }