WiP: feat(self-signed): genSignedCert

.github/ISSUE_TEMPLATE/bug_report.md

@@ -4,6 +4,7 @@ about: Create a report to help us improve
 title: ''
 labels: 'status: triaging :triangular_flag_on_post:, type: bug :bug:'
 assignees: ''
+
 ---
 
 **Describe the bug**
@@ -11,7 +12,6 @@ A clear and concise description of what the bug is.
 
 **To Reproduce**
 Steps to reproduce the behavior:
-
 1. Go to '...'
 2. Click on '....'
 3. Scroll down to '....'
@@ -24,11 +24,10 @@ A clear and concise description of what you expected to happen.
 If applicable, add screenshots to help explain your problem.
 
 **Desktop (please complete the following information):**
-
-- Node: [e.g. 16.12.1]
-- xo-server: [e.g. 5.82.3]
-- xo-web: [e.g. 5.87.0]
-- hypervisor: [e.g. XCP-ng 8.2.0]
+ - Node: [e.g. 16.12.1]
+ - xo-server: [e.g. 5.82.3]
+ - xo-web: [e.g. 5.87.0]
+ - hypervisor: [e.g. XCP-ng 8.2.0]
 
 **Additional context**
 Add any other context about the problem here.

@vates/decorate-with/README.md

@@ -70,25 +70,6 @@ decorateMethodsWith(Foo, {
 })
 ```
 
-### `perInstance(fn, ...args)`
-
-Helper to decorate the method by instance instead of for the whole class.
-
-This is often necessary for caching or deduplicating calls.
-
-```js
-import { perInstance } from '@vates/decorateWith'
-
-class Foo {
-  @decorateWith(perInstance, lodash.memoize)
-  bar() {
-    // body
-  }
-}
-```
-
-Because it's a normal function, it can also be used with `decorateMethodsWith`, with `compose` or even by itself.
-
 ## Contributions
 
 Contributions are _very_ welcomed, either on the documentation or on

@vates/decorate-with/USAGE.md

@@ -51,22 +51,3 @@ decorateMethodsWith(Foo, {
   ])
 })
 ```
-
-### `perInstance(fn, ...args)`
-
-Helper to decorate the method by instance instead of for the whole class.
-
-This is often necessary for caching or deduplicating calls.
-
-```js
-import { perInstance } from '@vates/decorateWith'
-
-class Foo {
-  @decorateWith(perInstance, lodash.memoize)
-  bar() {
-    // body
-  }
-}
-```
-
-Because it's a normal function, it can also be used with `decorateMethodsWith`, with `compose` or even by itself.

@vates/decorate-with/index.js

@@ -19,15 +19,3 @@ exports.decorateMethodsWith = function decorateMethodsWith(klass, map) {
   }
   return klass
 }
-
-exports.perInstance = function perInstance(fn, decorator, ...args) {
-  const map = new WeakMap()
-  return function () {
-    let decorated = map.get(this)
-    if (decorated === undefined) {
-      decorated = decorator(fn, ...args)
-      map.set(this, decorated)
-    }
-    return decorated.apply(this, arguments)
-  }
-}

@vates/decorate-with/package.json

@@ -20,7 +20,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "1.0.0",
+  "version": "0.1.0",
   "engines": {
     "node": ">=8.10"
   },

@xen-orchestra/audit-core/package.json

@@ -30,7 +30,7 @@
     "rimraf": "^3.0.0"
   },
   "dependencies": {
-    "@vates/decorate-with": "^1.0.0",
+    "@vates/decorate-with": "^0.1.0",
     "@xen-orchestra/log": "^0.3.0",
     "golike-defer": "^0.5.1",
     "object-hash": "^2.0.1"

@xen-orchestra/backups-cli/package.json

@@ -7,8 +7,8 @@
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.18.3",
-    "@xen-orchestra/fs": "^0.19.3",
+    "@xen-orchestra/backups": "^0.16.2",
+    "@xen-orchestra/fs": "^0.19.2",
     "filenamify": "^4.1.0",
     "getopts": "^2.2.5",
     "lodash": "^4.17.15",

@xen-orchestra/backups/RemoteAdapter.js

@@ -3,10 +3,9 @@ const Disposable = require('promise-toolbox/Disposable.js')
 const fromCallback = require('promise-toolbox/fromCallback.js')
 const fromEvent = require('promise-toolbox/fromEvent.js')
 const pDefer = require('promise-toolbox/defer.js')
-const groupBy = require('lodash/groupBy.js')
 const { dirname, join, normalize, resolve } = require('path')
 const { createLogger } = require('@xen-orchestra/log')
-const { Constants, createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } = require('vhd-lib')
+const { Constants, createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdSynthetic } = require('vhd-lib')
 const { deduped } = require('@vates/disposable/deduped.js')
 const { execFile } = require('child_process')
 const { readdir, stat } = require('fs-extra')
@@ -68,11 +67,10 @@ const debounceResourceFactory = factory =>
   }
 
 class RemoteAdapter {
-  constructor(handler, { debounceResource = res => res, dirMode, vhdDirectoryCompression } = {}) {
+  constructor(handler, { debounceResource = res => res, dirMode } = {}) {
     this._debounceResource = debounceResource
     this._dirMode = dirMode
     this._handler = handler
-    this._vhdDirectoryCompression = vhdDirectoryCompression
   }
 
   get handler() {
@@ -192,22 +190,6 @@ class RemoteAdapter {
     return files
   }
 
-  // check if we will be allowed to merge a a vhd created in this adapter
-  // with the vhd at path `path`
-  async isMergeableParent(packedParentUid, path) {
-    return await Disposable.use(openVhd(this.handler, path), vhd => {
-      // this baseUuid is not linked with this vhd
-      if (!vhd.footer.uuid.equals(packedParentUid)) {
-        return false
-      }
-
-      const isVhdDirectory = vhd instanceof VhdDirectory
-      return isVhdDirectory
-        ? this.#useVhdDirectory() && this.#getCompressionType() === vhd.compressionType
-        : !this.#useVhdDirectory()
-    })
-  }
-
   fetchPartitionFiles(diskId, partitionId, paths) {
     const { promise, reject, resolve } = pDefer()
     Disposable.use(
@@ -261,34 +243,17 @@ class RemoteAdapter {
     )
   }
 
-  deleteVmBackup(file) {
-    return this.deleteVmBackups([file])
-  }
+  async deleteVmBackup(filename) {
+    const metadata = JSON.parse(String(await this._handler.readFile(filename)))
+    metadata._filename = filename
 
-  async deleteVmBackups(files) {
-    const { delta, full, ...others } = groupBy(await asyncMap(files, file => this.readVmBackupMetadata(file)), 'mode')
-
-    const unsupportedModes = Object.keys(others)
-    if (unsupportedModes.length !== 0) {
-      throw new Error('no deleter for backup modes: ' + unsupportedModes.join(', '))
+    if (metadata.mode === 'delta') {
+      await this.deleteDeltaVmBackups([metadata])
+    } else if (metadata.mode === 'full') {
+      await this.deleteFullVmBackups([metadata])
+    } else {
+      throw new Error(`no deleter for backup mode ${metadata.mode}`)
     }
-
-    await Promise.all([
-      delta !== undefined && this.deleteDeltaVmBackups(delta),
-      full !== undefined && this.deleteFullVmBackups(full),
-    ])
-  }
-
-  #getCompressionType() {
-    return this._vhdDirectoryCompression
-  }
-
-  #useVhdDirectory() {
-    return this.handler.type === 's3'
-  }
-
-  #useAlias() {
-    return this.#useVhdDirectory()
   }
 
   getDisk = Disposable.factory(this.getDisk)
@@ -347,10 +312,13 @@ class RemoteAdapter {
     return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId))
   }
 
-  // if we use alias on this remote, we have to name the file alias.vhd
+  // this function will be the one where we plug the logic of the storage format by fs type/user settings
+
+  // if the file is named .vhd => vhd
+  // if the file is named alias.vhd => alias to a vhd
   getVhdFileName(baseName) {
-    if (this.#useAlias()) {
-      return `${baseName}.alias.vhd`
+    if (this._handler.type === 's3') {
+      return `${baseName}.alias.vhd` // we want an alias to a vhddirectory
     }
     return `${baseName}.vhd`
   }
@@ -502,11 +470,10 @@ class RemoteAdapter {
   async writeVhd(path, input, { checksum = true, validator = noop } = {}) {
     const handler = this._handler
 
-    if (this.#useVhdDirectory()) {
+    if (path.endsWith('.alias.vhd')) {
       const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
       await createVhdDirectoryFromStream(handler, dataPath, input, {
         concurrency: 16,
-        compression: this.#getCompressionType(),
         async validator() {
           await input.task
           return validator.apply(this, arguments)

@xen-orchestra/backups/_backupWorker.js

@@ -70,7 +70,6 @@ class BackupWorker {
       yield new RemoteAdapter(handler, {
         debounceResource: this.debounceResource,
         dirMode: this.#config.dirMode,
-        vhdDirectoryCompression: this.#config.vhdDirectoryCompression,
       })
     } finally {
       await handler.forget()

@xen-orchestra/backups/_cleanVm.integ.spec.js

@@ -178,7 +178,7 @@ test('it merges delta of non destroyed chain', async () => {
     `metadata.json`,
     JSON.stringify({
       mode: 'delta',
-      size: 12000, // a size too small
+      size: 209920,
       vhds: [
         `${basePath}/grandchild.vhd`, // grand child should not be merged
         `${basePath}/child.vhd`,
@@ -204,25 +204,20 @@ test('it merges delta of non destroyed chain', async () => {
     },
   })
 
-  let loggued = []
+  let loggued = ''
   const onLog = message => {
-    loggued.push(message)
+    loggued += message + '\n'
   }
   await adapter.cleanVm('/', { remove: true, onLog })
-  expect(loggued[0]).toEqual(`the parent /${basePath}/orphan.vhd of the child /${basePath}/child.vhd is unused`)
-  expect(loggued[1]).toEqual(`incorrect size in metadata: 12000 instead of 209920`)
-
-  loggued = []
+  expect(loggued).toEqual(`the parent /${basePath}/orphan.vhd of the child /${basePath}/child.vhd is unused\n`)
+  loggued = ''
   await adapter.cleanVm('/', { remove: true, merge: true, onLog })
-  const [unused, merging] = loggued
+  const [unused, merging] = loggued.split('\n')
   expect(unused).toEqual(`the parent /${basePath}/orphan.vhd of the child /${basePath}/child.vhd is unused`)
   expect(merging).toEqual(`merging /${basePath}/child.vhd into /${basePath}/orphan.vhd`)
 
-  const metadata = JSON.parse(await handler.readFile(`metadata.json`))
-  // size should be the size of children + grand children after the merge
-  expect(metadata.size).toEqual(209920)
-
   // merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
+
   // only check deletion
   const remainingVhds = await handler.list(basePath)
   expect(remainingVhds.length).toEqual(2)
@@ -235,7 +230,7 @@ test('it finish unterminated merge ', async () => {
     `metadata.json`,
     JSON.stringify({
       mode: 'delta',
-      size: undefined,
+      size: 209920,
       vhds: [
         `${basePath}/orphan.vhd`, // grand child should not be merged
         `${basePath}/child.vhd`,
@@ -369,10 +364,6 @@ describe('tests mulitple combination ', () => {
         )
         await adapter.cleanVm('/', { remove: true, merge: true })
 
-        const metadata = JSON.parse(await handler.readFile(`metadata.json`))
-        // size should be the size of children + grand children + clean after the merge
-        expect(metadata.size).toEqual(vhdMode === 'file' ? 314880 : undefined)
-
         // broken vhd, non referenced, abandonned should be deleted ( alias and data)
         // ancestor and child should be merged
         // grand child and clean vhd should not have changed
@@ -397,11 +388,3 @@ describe('tests mulitple combination ', () => {
     }
   }
 })
-
-test('it cleans orphan merge states ', async () => {
-  await handler.writeFile(`${basePath}/.orphan.vhd.merge.json`, '')
-
-  await adapter.cleanVm('/', { remove: true })
-
-  expect(await handler.list(basePath)).toEqual([])
-})

@xen-orchestra/backups/_cleanVm.js

@@ -10,24 +10,6 @@ const { limitConcurrency } = require('limit-concurrency-decorator')
 const { Task } = require('./Task.js')
 const { Disposable } = require('promise-toolbox')
 
-// checking the size of a vhd directory is costly
-// 1 Http Query per 1000 blocks
-// we only check size of all the vhd are VhdFiles
-function shouldComputeVhdsSize(vhds) {
-  return vhds.every(vhd => vhd instanceof VhdFile)
-}
-
-const computeVhdsSize = (handler, vhdPaths) =>
-  Disposable.use(
-    vhdPaths.map(vhdPath => openVhd(handler, vhdPath)),
-    async vhds => {
-      if (shouldComputeVhdsSize(vhds)) {
-        const sizes = await asyncMap(vhds, vhd => vhd.getSize())
-        return sum(sizes)
-      }
-    }
-  )
-
 // chain is an array of VHDs from child to parent
 //
 // the whole chain will be merged into parent, parent will be renamed to child
@@ -100,10 +82,10 @@ async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
 
 const noop = Function.prototype
 
-const INTERRUPTED_VHDS_REG = /^\.(.+)\.merge.json$/
+const INTERRUPTED_VHDS_REG = /^(?:(.+)\/)?\.(.+)\.merge.json$/
 const listVhds = async (handler, vmDir) => {
-  const vhds = new Set()
-  const interruptedVhds = new Map()
+  const vhds = []
+  const interruptedVhds = new Set()
 
   await asyncMap(
     await handler.list(`${vmDir}/vdis`, {
@@ -118,14 +100,16 @@ const listVhds = async (handler, vmDir) => {
         async vdiDir => {
           const list = await handler.list(vdiDir, {
             filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file),
+            prependDir: true,
           })
 
           list.forEach(file => {
             const res = INTERRUPTED_VHDS_REG.exec(file)
             if (res === null) {
-              vhds.add(`${vdiDir}/${file}`)
+              vhds.push(file)
             } else {
-              interruptedVhds.set(`${vdiDir}/${res[1]}`, `${vdiDir}/${file}`)
+              const [, dir, file] = res
+              interruptedVhds.add(`${dir}/${file}`)
             }
           })
         }
@@ -145,16 +129,17 @@ exports.cleanVm = async function cleanVm(
 
   const handler = this._handler
 
-  const vhdsToJSons = new Set()
+  const vhds = new Set()
   const vhdParents = { __proto__: null }
   const vhdChildren = { __proto__: null }
 
-  const { vhds, interruptedVhds } = await listVhds(handler, vmDir)
+  const vhdsList = await listVhds(handler, vmDir)
 
   // remove broken VHDs
-  await asyncMap(vhds, async path => {
+  await asyncMap(vhdsList.vhds, async path => {
     try {
-      await Disposable.use(openVhd(handler, path, { checkSecondFooter: !interruptedVhds.has(path) }), vhd => {
+      await Disposable.use(openVhd(handler, path, { checkSecondFooter: !vhdsList.interruptedVhds.has(path) }), vhd => {
+        vhds.add(path)
         if (vhd.footer.diskType === DISK_TYPES.DIFFERENCING) {
           const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
           vhdParents[path] = parent
@@ -169,7 +154,6 @@ exports.cleanVm = async function cleanVm(
         }
       })
     } catch (error) {
-      vhds.delete(path)
       onLog(`error while checking the VHD with path ${path}`, { error })
       if (error?.code === 'ERR_ASSERTION' && remove) {
         onLog(`deleting broken ${path}`)
@@ -178,23 +162,6 @@ exports.cleanVm = async function cleanVm(
     }
   })
 
-  // remove interrupted merge states for missing VHDs
-  for (const interruptedVhd of interruptedVhds.keys()) {
-    if (!vhds.has(interruptedVhd)) {
-      const statePath = interruptedVhds.get(interruptedVhd)
-      interruptedVhds.delete(interruptedVhd)
-
-      onLog('orphan merge state', {
-        mergeStatePath: statePath,
-        missingVhdPath: interruptedVhd,
-      })
-      if (remove) {
-        onLog(`deleting orphan merge state ${statePath}`)
-        await handler.unlink(statePath)
-      }
-    }
-  }
-
   // @todo : add check for data folder of alias not referenced in a valid alias
 
   // remove VHDs with missing ancestors
@@ -235,7 +202,7 @@ exports.cleanVm = async function cleanVm(
     await Promise.all(deletions)
   }
 
-  const jsons = new Set()
+  const jsons = []
   const xvas = new Set()
   const xvaSums = []
   const entries = await handler.list(vmDir, {
@@ -243,7 +210,7 @@ exports.cleanVm = async function cleanVm(
   })
   entries.forEach(path => {
     if (isMetadataFile(path)) {
-      jsons.add(path)
+      jsons.push(path)
     } else if (isXvaFile(path)) {
       xvas.add(path)
     } else if (isXvaSumFile(path)) {
@@ -265,25 +232,22 @@ exports.cleanVm = async function cleanVm(
   // compile the list of unused XVAs and VHDs, and remove backup metadata which
   // reference a missing XVA/VHD
   await asyncMap(jsons, async json => {
-    let metadata
-    try {
-      metadata = JSON.parse(await handler.readFile(json))
-    } catch (error) {
-      onLog(`failed to read metadata file ${json}`, { error })
-      jsons.delete(json)
-      return
-    }
-
+    const metadata = JSON.parse(await handler.readFile(json))
     const { mode } = metadata
+    let size
     if (mode === 'full') {
       const linkedXva = resolve('/', vmDir, metadata.xva)
+
       if (xvas.has(linkedXva)) {
         unusedXvas.delete(linkedXva)
+
+        size = await handler.getSize(linkedXva).catch(error => {
+          onLog(`failed to get size of ${json}`, { error })
+        })
       } else {
         onLog(`the XVA linked to the metadata ${json} is missing`)
         if (remove) {
           onLog(`deleting incomplete backup ${json}`)
-          jsons.delete(json)
           await handler.unlink(json)
         }
       }
@@ -299,18 +263,46 @@ exports.cleanVm = async function cleanVm(
       // possible (existing disks) even if one disk is missing
       if (missingVhds.length === 0) {
         linkedVhds.forEach(_ => unusedVhds.delete(_))
-        linkedVhds.forEach(path => {
-          vhdsToJSons[path] = json
-        })
+
+        // checking the size of a vhd directory is costly
+        // 1 Http Query per 1000 blocks
+        // we only check size of all the vhd are VhdFiles
+
+        const shouldComputeSize = linkedVhds.every(vhd => vhd instanceof VhdFile)
+        if (shouldComputeSize) {
+          try {
+            await Disposable.use(Disposable.all(linkedVhds.map(vhdPath => openVhd(handler, vhdPath))), async vhds => {
+              const sizes = await asyncMap(vhds, vhd => vhd.getSize())
+              size = sum(sizes)
+            })
+          } catch (error) {
+            onLog(`failed to get size of ${json}`, { error })
+          }
+        }
       } else {
         onLog(`Some VHDs linked to the metadata ${json} are missing`, { missingVhds })
         if (remove) {
           onLog(`deleting incomplete backup ${json}`)
-          jsons.delete(json)
           await handler.unlink(json)
         }
       }
     }
+
+    const metadataSize = metadata.size
+    if (size !== undefined && metadataSize !== size) {
+      onLog(`incorrect size in metadata: ${metadataSize ?? 'none'} instead of ${size}`)
+
+      // don't update if the the stored size is greater than found files,
+      // it can indicates a problem
+      if (fixMetadata && (metadataSize === undefined || metadataSize < size)) {
+        try {
+          metadata.size = size
+          await handler.writeFile(json, JSON.stringify(metadata), { flags: 'w' })
+        } catch (error) {
+          onLog(`failed to update size in backup metadata ${json}`, { error })
+        }
+      }
+    }
   })
 
   // TODO: parallelize by vm/job/vdi
@@ -358,9 +350,9 @@ exports.cleanVm = async function cleanVm(
     })
 
     // merge interrupted VHDs
-    for (const parent of interruptedVhds.keys()) {
+    vhdsList.interruptedVhds.forEach(parent => {
       vhdChainsToMerge[parent] = [vhdChildren[parent], parent]
-    }
+    })
 
     Object.values(vhdChainsToMerge).forEach(chain => {
       if (chain !== undefined) {
@@ -369,15 +361,9 @@ exports.cleanVm = async function cleanVm(
     })
   }
 
-  const metadataWithMergedVhd = {}
-  const doMerge = async () => {
-    await asyncMap(toMerge, async chain => {
-      const merged = await limitedMergeVhdChain(chain, { handler, onLog, remove, merge })
-      if (merged !== undefined) {
-        const metadataPath = vhdsToJSons[chain[0]] // all the chain should have the same metada file
-        metadataWithMergedVhd[metadataPath] = true
-      }
-    })
+  const doMerge = () => {
+    const promise = asyncMap(toMerge, async chain => limitedMergeVhdChain(chain, { handler, onLog, remove, merge }))
+    return merge ? promise.then(sizes => ({ size: sum(sizes) })) : promise
   }
 
   await Promise.all([
@@ -402,52 +388,6 @@ exports.cleanVm = async function cleanVm(
     }),
   ])
 
-  // update size for delta metadata with merged VHD
-  // check for the other that the size is the same as the real file size
-
-  await asyncMap(jsons, async metadataPath => {
-    const metadata = JSON.parse(await handler.readFile(metadataPath))
-
-    let fileSystemSize
-    const merged = metadataWithMergedVhd[metadataPath] !== undefined
-
-    const { mode, size, vhds, xva } = metadata
-
-    try {
-      if (mode === 'full') {
-        // a full backup : check size
-        const linkedXva = resolve('/', vmDir, xva)
-        fileSystemSize = await handler.getSize(linkedXva)
-      } else if (mode === 'delta') {
-        const linkedVhds = Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
-        fileSystemSize = await computeVhdsSize(handler, linkedVhds)
-
-        // the size is not computed in some cases (e.g. VhdDirectory)
-        if (fileSystemSize === undefined) {
-          return
-        }
-
-        // don't warn if the size has changed after a merge
-        if (!merged && fileSystemSize !== size) {
-          onLog(`incorrect size in metadata: ${size ?? 'none'} instead of ${fileSystemSize}`)
-        }
-      }
-    } catch (error) {
-      onLog(`failed to get size of ${metadataPath}`, { error })
-      return
-    }
-
-    // systematically update size after a merge
-    if ((merged || fixMetadata) && size !== fileSystemSize) {
-      metadata.size = fileSystemSize
-      try {
-        await handler.writeFile(metadataPath, JSON.stringify(metadata), { flags: 'w' })
-      } catch (error) {
-        onLog(`failed to update size in backup metadata ${metadataPath} after merge`, { error })
-      }
-    }
-  })
-
   return {
     // boolean whether some VHDs were merged (or should be merged)
     merge: toMerge.length !== 0,

@xen-orchestra/backups/_isValidXva.js

@@ -1,24 +1,11 @@
 const assert = require('assert')
 
-const COMPRESSED_MAGIC_NUMBERS = [
+const isGzipFile = async (handler, fd) => {
   // https://tools.ietf.org/html/rfc1952.html#page-5
-  Buffer.from('1F8B', 'hex'),
+  const magicNumber = Buffer.allocUnsafe(2)
 
-  // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#zstandard-frames
-  Buffer.from('28B52FFD', 'hex'),
-]
-const MAGIC_NUMBER_MAX_LENGTH = Math.max(...COMPRESSED_MAGIC_NUMBERS.map(_ => _.length))
-
-const isCompressedFile = async (handler, fd) => {
-  const header = Buffer.allocUnsafe(MAGIC_NUMBER_MAX_LENGTH)
-  assert.strictEqual((await handler.read(fd, header, 0)).bytesRead, header.length)
-
-  for (const magicNumber of COMPRESSED_MAGIC_NUMBERS) {
-    if (magicNumber.compare(header, 0, magicNumber.length) === 0) {
-      return true
-    }
-  }
-  return false
+  assert.strictEqual((await handler.read(fd, magicNumber, 0)).bytesRead, magicNumber.length)
+  return magicNumber[0] === 31 && magicNumber[1] === 139
 }
 
 // TODO: better check?
@@ -56,8 +43,8 @@ async function isValidXva(path) {
         return false
       }
 
-      return (await isCompressedFile(handler, fd))
-        ? true // compressed files cannot be validated at this time
+      return (await isGzipFile(handler, fd))
+        ? true // gzip files cannot be validated at this time
         : await isValidTar(handler, size, fd)
     } finally {
       handler.closeFile(fd).catch(noop)

@xen-orchestra/backups/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.18.3",
+  "version": "0.16.2",
   "engines": {
     "node": ">=14.6"
   },
@@ -20,7 +20,7 @@
     "@vates/disposable": "^0.1.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^0.19.3",
+    "@xen-orchestra/fs": "^0.19.2",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/template": "^0.1.0",
     "compare-versions": "^4.0.1",
@@ -36,11 +36,11 @@
     "proper-lockfile": "^4.1.2",
     "pump": "^3.0.0",
     "uuid": "^8.3.2",
-    "vhd-lib": "^3.0.0",
+    "vhd-lib": "^2.0.3",
     "yazl": "^2.5.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^0.8.5"
+    "@xen-orchestra/xapi": "^0.8.4"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

@xen-orchestra/backups/writers/DeltaBackupWriter.js

@@ -24,7 +24,6 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
   async checkBaseVdis(baseUuidToSrcVdi) {
     const { handler } = this._adapter
     const backup = this._backup
-    const adapter = this._adapter
 
     const backupDir = getVmBackupDir(backup.vm.uuid)
     const vdisDir = `${backupDir}/vdis/${backup.job.id}`
@@ -36,18 +35,13 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           filter: _ => _[0] !== '.' && _.endsWith('.vhd'),
           prependDir: true,
         })
-        const packedBaseUuid = packUuid(baseUuid)
         await asyncMap(vhds, async path => {
           try {
             await checkVhdChain(handler, path)
-            // Warning, this should not be written as found = found || await adapter.isMergeableParent(packedBaseUuid, path)
-            //
-            // since all the checks of a path are done in parallel, found would be containing
-            // only the last answer of isMergeableParent which is probably not the right one
-            // this led to the support tickets  https://help.vates.fr/#ticket/zoom/4751 , 4729, 4665 and 4300
-
-            const isMergeable = await adapter.isMergeableParent(packedBaseUuid, path)
-            found = found || isMergeable
+            await Disposable.use(
+              openVhd(handler, path),
+              vhd => (found = found || vhd.footer.uuid.equals(packUuid(baseUuid)))
+            )
           } catch (error) {
             warn('checkBaseVdis', { error })
             await ignoreErrors.call(VhdAbstract.unlink(handler, path))

@xen-orchestra/fs/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "@xen-orchestra/fs",
-  "version": "0.19.3",
+  "version": "0.19.2",
   "license": "AGPL-3.0-or-later",
   "description": "The File System for Xen Orchestra backups.",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",

@xen-orchestra/fs/src/s3.js

@@ -246,7 +246,7 @@ export default class S3Handler extends RemoteHandlerAbstract {
         Prefix: this._dir + path + '/',
         ContinuationToken: NextContinuationToken,
       })
-      NextContinuationToken = result.isTruncated ? result.NextContinuationToken : undefined
+      NextContinuationToken = result.isTruncated ? null : result.NextContinuationToken
       for (const { Key } of result.Contents) {
         // _unlink will add the prefix, but Key contains everything
         // also we don't need to check if we delete a directory, since the list only return files
@@ -255,7 +255,7 @@ export default class S3Handler extends RemoteHandlerAbstract {
           Key,
         })
       }
-    } while (NextContinuationToken !== undefined)
+    } while (NextContinuationToken !== null)
   }
 
   async _write(file, buffer, position) {

@xen-orchestra/mixin/package.json

@@ -19,7 +19,7 @@
     "node": ">=6"
   },
   "dependencies": {
-    "bind-property-descriptor": "^2.0.0",
+    "bind-property-descriptor": "^1.0.0",
     "lodash": "^4.17.21"
   },
   "scripts": {

@xen-orchestra/mixins/Config.js

@@ -32,7 +32,7 @@ module.exports = class Config {
   get(path) {
     const value = get(this._config, path)
     if (value === undefined) {
-      throw new TypeError('missing config entry: ' + path)
+      throw new TypeError('missing config entry: ' + value)
     }
     return value
   }

@xen-orchestra/mixins/package.json

@@ -14,7 +14,7 @@
     "url": "https://vates.fr"
   },
   "license": "AGPL-3.0-or-later",
-  "version": "0.1.2",
+  "version": "0.1.1",
   "engines": {
     "node": ">=12"
   },
@@ -22,7 +22,7 @@
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/emit-async": "^0.1.0",
     "@xen-orchestra/log": "^0.3.0",
-    "app-conf": "^1.0.0",
+    "app-conf": "^0.9.0",
     "lodash": "^4.17.21"
   },
   "scripts": {

@xen-orchestra/proxy-cli/package.json

@@ -29,7 +29,7 @@
     "@iarna/toml": "^2.2.0",
     "@vates/read-chunk": "^0.1.2",
     "ansi-colors": "^4.1.1",
-    "app-conf": "^1.0.0",
+    "app-conf": "^0.9.0",
     "content-type": "^1.0.4",
     "cson-parser": "^4.0.7",
     "getopts": "^2.2.3",

@xen-orchestra/proxy/config.toml

@@ -20,7 +20,6 @@ keepAliveInterval = 10e3
 dirMode = 0o700
 disableMergeWorker = false
 snapshotNameLabelTpl = '[XO Backup {job.name}] {vm.name_label}'
-vhdDirectoryCompression = 'brotli'
 
 [backups.defaultSettings]
 reportWhen = 'failure'
@@ -88,20 +87,3 @@ ignoreNobakVdis = false
 
 maxUncoalescedVdis = 1
 watchEvents = ['network', 'PIF', 'pool', 'SR', 'task', 'VBD', 'VDI', 'VIF', 'VM']
-
-
-
-#compact mode
-[reverseProxies]
-# '/http/' = 'http://localhost:8081/'
-#The target can have a path ( like `http://target/sub/directory/`),
-# parameters (`?param=one`) and hash (`#jwt:32154`) that are automatically added to all queries transfered by the proxy.
-# If a parameter is present in the configuration and in the query, only the config parameter is transferred.
-# '/another' = http://hiddenServer:8765/path/
-
-# And use the extended mode when required
-# The additionnal options of a proxy's configuraiton's section are used to instantiate the `https` Agent(respectively the `http`).
-# A notable option is `rejectUnauthorized` which allow to connect to a HTTPS backend  with an invalid/ self signed certificate
-#[reverseProxies.'/https/']
-# target = 'https://localhost:8080/'
-# rejectUnauthorized = false

@xen-orchestra/proxy/package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "@xen-orchestra/proxy",
-  "version": "0.17.3",
+  "version": "0.15.5",
   "license": "AGPL-3.0-or-later",
   "description": "XO Proxy used to remotely execute backup jobs",
   "keywords": [
@@ -22,31 +22,30 @@
     "xo-proxy": "dist/index.mjs"
   },
   "engines": {
-    "node": ">=14.18"
+    "node": ">=14.13"
   },
   "dependencies": {
     "@iarna/toml": "^2.2.0",
     "@koa/router": "^10.0.0",
     "@vates/compose": "^2.1.0",
-    "@vates/decorate-with": "^1.0.0",
+    "@vates/decorate-with": "^0.1.0",
     "@vates/disposable": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.18.3",
-    "@xen-orchestra/fs": "^0.19.3",
+    "@xen-orchestra/backups": "^0.16.2",
+    "@xen-orchestra/fs": "^0.19.2",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/mixin": "^0.1.0",
-    "@xen-orchestra/mixins": "^0.1.2",
+    "@xen-orchestra/mixins": "^0.1.1",
     "@xen-orchestra/self-signed": "^0.1.0",
-    "@xen-orchestra/xapi": "^0.8.5",
+    "@xen-orchestra/xapi": "^0.8.4",
     "ajv": "^8.0.3",
-    "app-conf": "^1.0.0",
+    "app-conf": "^0.9.0",
     "async-iterator-to-stream": "^1.1.0",
     "fs-extra": "^10.0.0",
     "get-stream": "^6.0.0",
     "getopts": "^2.2.3",
     "golike-defer": "^0.5.1",
     "http-server-plus": "^0.11.0",
-    "http2-proxy": "^5.0.53",
     "json-rpc-protocol": "^0.13.1",
     "jsonrpc-websocket-client": "^0.7.2",
     "koa": "^2.5.1",
@@ -58,7 +57,7 @@
     "promise-toolbox": "^0.20.0",
     "source-map-support": "^0.5.16",
     "stoppable": "^1.0.6",
-    "xdg-basedir": "^5.1.0",
+    "xdg-basedir": "^4.0.0",
     "xen-api": "^0.35.1",
     "xo-common": "^0.7.0"
   },

@xen-orchestra/proxy/src/app/mixins/api.mjs

@@ -14,7 +14,7 @@ import { createLogger } from '@xen-orchestra/log'
 
 const { debug, warn } = createLogger('xo:proxy:api')
 
-const ndJsonStream = asyncIteratorToStream(async function* (responseId, iterable) {
+const ndJsonStream = asyncIteratorToStream(async function*(responseId, iterable) {
   try {
     let cursor, iterator
     try {
@@ -45,14 +45,14 @@ export default class Api {
   constructor(app, { appVersion, httpServer }) {
     this._ajv = new Ajv({ allErrors: true })
     this._methods = { __proto__: null }
-    const PREFIX = '/api/v1'
-    const router = new Router({ prefix: PREFIX }).post('/', async ctx => {
+
+    const router = new Router({ prefix: '/api/v1' }).post('/', async ctx => {
       // Before Node 13.0 there was an inactivity timeout of 2 mins, which may
       // not be enough for the API.
       ctx.req.setTimeout(0)
 
       const profile = await app.authentication.findProfile({
-        authenticationToken: ctx.cookies.get('authenticationToken'),
+        authenticationToken: ctx.cookies.get('authenticationToken')
       })
       if (profile === undefined) {
         ctx.status = 401
@@ -102,7 +102,6 @@ export default class Api {
           // breaks, send some data every 10s to keep it opened.
           const stopTimer = clearInterval.bind(
             undefined,
-            // @to check : can this add space inside binary data ?
             setInterval(() => stream.push(' '), keepAliveInterval)
           )
           stream.on('end', stopTimer).on('error', stopTimer)
@@ -119,19 +118,12 @@ export default class Api {
       .use(router.routes())
       .use(router.allowedMethods())
 
-    const callback = koa.callback()
-    httpServer.on('request', (req, res) => {
-      // only answers to query to the root url of this mixin
-      // do it before giving the request to Koa to ensure it's not modified
-      if (req.url.startsWith(PREFIX)) {
-        callback(req, res)
-      }
-    })
+    httpServer.on('request', koa.callback())
 
     this.addMethods({
       system: {
         getMethodsInfo: [
-          function* () {
+          function*() {
             const methods = this._methods
             for (const name in methods) {
               const { description, params = {} } = methods[name]
@@ -139,25 +131,25 @@ export default class Api {
             }
           }.bind(this),
           {
-            description: 'returns the signatures of all available API methods',
-          },
+            description: 'returns the signatures of all available API methods'
+          }
         ],
         getServerVersion: [
           () => appVersion,
           {
-            description: 'returns the version of xo-server',
-          },
+            description: 'returns the version of xo-server'
+          }
         ],
         listMethods: [
-          function* () {
+          function*() {
             const methods = this._methods
             for (const name in methods) {
               yield name
             }
           }.bind(this),
           {
-            description: 'returns the name of all available API methods',
-          },
+            description: 'returns the name of all available API methods'
+          }
         ],
         methodSignature: [
           ({ method: name }) => {
@@ -172,14 +164,14 @@ export default class Api {
           {
             description: 'returns the signature of an API method',
             params: {
-              method: { type: 'string' },
-            },
-          },
-        ],
+              method: { type: 'string' }
+            }
+          }
+        ]
       },
       test: {
         range: [
-          function* ({ start = 0, stop, step }) {
+          function*({ start = 0, stop, step }) {
             if (step === undefined) {
               step = start > stop ? -1 : 1
             }
@@ -197,11 +189,11 @@ export default class Api {
             params: {
               start: { optional: true, type: 'number' },
               step: { optional: true, type: 'number' },
-              stop: { type: 'number' },
-            },
-          },
-        ],
-      },
+              stop: { type: 'number' }
+            }
+          }
+        ]
+      }
     })
   }
 
@@ -228,7 +220,7 @@ export default class Api {
         return required
       }),
 
-      type: 'object',
+      type: 'object'
     })
 
     const m = params => {

@xen-orchestra/proxy/src/app/mixins/authentication.mjs

@@ -1,6 +1,6 @@
 import assert from 'assert'
 import fse from 'fs-extra'
-import { xdgConfig } from 'xdg-basedir'
+import xdg from 'xdg-basedir'
 import { createLogger } from '@xen-orchestra/log'
 import { execFileSync } from 'child_process'
 
@@ -20,7 +20,7 @@ export default class Authentication {
       // save this token in the automatically handled conf file
       fse.outputFileSync(
         // this file must take precedence over normal user config
-        `${xdgConfig}/${appName}/config.z-auto.json`,
+        `${xdg.config}/${appName}/config.z-auto.json`,
         JSON.stringify({ authenticationToken: token }),
         { mode: 0o600 }
       )

@xen-orchestra/proxy/src/app/mixins/backups.mjs

@@ -164,17 +164,6 @@ export default class Backups {
             },
           },
         ],
-        deleteVmBackups: [
-          ({ filenames, remote }) =>
-            Disposable.use(this.getAdapter(remote), adapter => adapter.deleteVmBackups(filenames)),
-          {
-            description: 'delete VM backups',
-            params: {
-              filenames: { type: 'array', items: { type: 'string' } },
-              remote: { type: 'object' },
-            },
-          },
-        ],
         fetchPartitionFiles: [
           ({ disk: diskId, remote, partition: partitionId, paths }) =>
             Disposable.use(this.getAdapter(remote), adapter => adapter.fetchPartitionFiles(diskId, partitionId, paths)),
@@ -414,7 +403,6 @@ export default class Backups {
     return new RemoteAdapter(yield app.remotes.getHandler(remote), {
       debounceResource: app.debounceResource.bind(app),
       dirMode: app.config.get('backups.dirMode'),
-      vhdDirectoryCompression: app.config.get('backups.vhdDirectoryCompression'),
     })
   }
 

@xen-orchestra/proxy/src/app/mixins/reverseProxy.mjs

@@ -1,120 +0,0 @@
-import { urlToHttpOptions } from 'url'
-import proxy from 'http2-proxy'
-
-function removeSlash(str) {
-  return str.replace(/^\/|\/$/g, '')
-}
-
-function mergeUrl(relative, base) {
-  const res = new URL(base)
-  const relativeUrl = new URL(relative, base)
-  res.pathname = relativeUrl.pathname
-  relativeUrl.searchParams.forEach((value, name) => {
-    // we do not allow to modify params already specified by config
-    if (!res.searchParams.has(name)) {
-      res.searchParams.append(name, value)
-    }
-  })
-  res.hash = relativeUrl.hash.length > 0 ? relativeUrl.hash : res.hash
-  return res
-}
-
-export function backendToLocalPath(basePath, target, backendUrl) {
-  // keep redirect url relative to local server
-  const localPath = `${basePath}/${backendUrl.pathname.substring(target.pathname.length)}${backendUrl.search}${
-    backendUrl.hash
-  }`
-  return localPath
-}
-
-export function localToBackendUrl(basePath, target, localPath) {
-  let localPathWithoutBase = removeSlash(localPath).substring(basePath.length)
-  localPathWithoutBase = './' + removeSlash(localPathWithoutBase)
-  const url = mergeUrl(localPathWithoutBase, target)
-  return url
-}
-
-export default class ReverseProxy {
-  constructor(app, { httpServer }) {
-    app.config.watch('reverseProxies', proxies => {
-      this._proxies = Object.keys(proxies)
-        .sort((a, b) => b.length - a.length)
-        .map(path => {
-          let config = proxies[path]
-          if (typeof config === 'string') {
-            config = { target: config }
-          }
-          config.path = '/proxy/v1/' + removeSlash(path) + '/'
-
-          return config
-        })
-    })
-
-    httpServer.on('request', (req, res) => this._proxy(req, res))
-    httpServer.on('upgrade', (req, socket, head) => this._upgrade(req, socket, head))
-  }
-
-  _getConfigFromRequest(req) {
-    return this._proxies.find(({ path }) => req.url.startsWith(path))
-  }
-
-  _proxy(req, res) {
-    const config = this._getConfigFromRequest(req)
-
-    if (config === undefined) {
-      res.writeHead(404)
-      res.end('404')
-      return
-    }
-
-    const url = new URL(config.target)
-    const targetUrl = localToBackendUrl(config.path, url, req.originalUrl || req.url)
-    proxy.web(req, res, {
-      ...urlToHttpOptions(targetUrl),
-      ...config.options,
-      onReq: (req, { headers }) => {
-        headers['x-forwarded-for'] = req.socket.remoteAddress
-        headers['x-forwarded-proto'] = req.socket.encrypted ? 'https' : 'http'
-        if (req.headers.host !== undefined) {
-          headers['x-forwarded-host'] = req.headers.host
-        }
-      },
-      onRes: (req, res, proxyRes) => {
-        // rewrite redirect to pass through this proxy
-        if (proxyRes.statusCode === 301 || proxyRes.statusCode === 302) {
-          // handle relative/ absolute location
-          const redirectTargetLocation = new URL(proxyRes.headers.location, url)
-
-          // this proxy should only allow communication between known hosts. Don't open it too much
-          if (redirectTargetLocation.hostname !== url.hostname || redirectTargetLocation.protocol !== url.protocol) {
-            throw new Error(`Can't redirect from ${url.hostname} to ${redirectTargetLocation.hostname} `)
-          }
-          res.writeHead(proxyRes.statusCode, {
-            ...proxyRes.headers,
-            location: backendToLocalPath(config.path, url, redirectTargetLocation),
-          })
-          res.end()
-          return
-        }
-        // pass through the answer of the remote server
-        res.writeHead(proxyRes.statusCode, proxyRes.headers)
-        // pass through content
-        proxyRes.pipe(res)
-      },
-    })
-  }
-
-  _upgrade(req, socket, head) {
-    const config = this._getConfigFromRequest(req)
-    if (config === undefined) {
-      return
-    }
-
-    const { path, target, options } = config
-    const targetUrl = localToBackendUrl(path, target, req.originalUrl || req.url)
-    proxy.ws(req, socket, head, {
-      ...urlToHttpOptions(targetUrl),
-      ...options,
-    })
-  }
-}

@xen-orchestra/proxy/tests/cert.pem

@@ -1,19 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDETCCAfkCFHXO1U7YJHI61bPNhYDvyBNJYH4LMA0GCSqGSIb3DQEBCwUAMEUx
-CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
-cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjIwMTEwMTI0MTU4WhcNNDkwNTI3MTI0
-MTU4WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE
-CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC
-AQ8AMIIBCgKCAQEA1jMLdHuZu2R1fETyB2iRect1alwv76clp/7A8tx4zNaVA9qB
-BcHbI83mkozuyrXpsEUblTvvcWkheBPAvWD4gj0eWSDSiuf0edcIS6aky+Lr/n0T
-W/vL5kVNrgPTlsO8OyQcXjDeuUOR1xDWIa8G71Ynd6wtATB7oXe7kaV/Z6b2fENr
-4wlW0YEDnMHik59c9jXDshhQYDlErwZsSyHuLwkC7xuYO26SUW9fPcHJA3uOfxeG
-BrCxMuSMOJtdmslRWhLCjbk0PT12OYCCRlvuTvPHa8N57GEQbi4xAu+XATgO1DUm
-Dq/oCSj0TcWUXXOykN/PAC2cjIyqkU2e7orGaQIDAQABMA0GCSqGSIb3DQEBCwUA
-A4IBAQCTshhF3V5WVhnpFGHd+tPfeHmUVrUnbC+xW7fSeWpamNmTjHb7XB6uDR0O
-DGswhEitbbSOsCiwz4/zpfE3/3+X07O8NPbdHTVHCei6D0uyegEeWQ2HoocfZs3X
-8CORe8TItuvQAevV17D0WkGRoJGVAOiKo+izpjI55QXQ+FjkJ0bfl1iksnUJk0+I
-ZNmRRNjNyOxo7NAzomSBHfJ5rDE+E440F2uvXIE9OIwHRiq6FGvQmvGijPeeP5J0
-LzcSK98jfINFSsA/Wn5vWE+gfH9ySD2G3r2cDTS904T77PNiYH+cNSP6ujtmNzvK
-Bgoa3jXZPRBi82TUOb2jj5DB33bg
------END CERTIFICATE-----

@xen-orchestra/proxy/tests/key.pem

@@ -1,27 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEpAIBAAKCAQEA1jMLdHuZu2R1fETyB2iRect1alwv76clp/7A8tx4zNaVA9qB
-BcHbI83mkozuyrXpsEUblTvvcWkheBPAvWD4gj0eWSDSiuf0edcIS6aky+Lr/n0T
-W/vL5kVNrgPTlsO8OyQcXjDeuUOR1xDWIa8G71Ynd6wtATB7oXe7kaV/Z6b2fENr
-4wlW0YEDnMHik59c9jXDshhQYDlErwZsSyHuLwkC7xuYO26SUW9fPcHJA3uOfxeG
-BrCxMuSMOJtdmslRWhLCjbk0PT12OYCCRlvuTvPHa8N57GEQbi4xAu+XATgO1DUm
-Dq/oCSj0TcWUXXOykN/PAC2cjIyqkU2e7orGaQIDAQABAoIBAQC65uVq6WLWGa1O
-FtbdUggGL1svyGrngYChGvB/uZMKoX57U1DbljDCCCrV23WNmbfkYBjWWervmZ1j
-qlC2roOJGQ1/Fd3A6O7w1YnegPUxFrt3XunijE55iiVi3uHknryDGlpKcfgVzfjW
-oVFHKPMzKYjcqnbGn+hwlwoq5y7JYFTOa57/dZbyommbodRyy9Dpn0OES0grQqwR
-VD1amrQ7XJhukcxQgYPuDc/jM3CuowoBsv9f+Q2zsPgr6CpHxxLLIs+kt8NQJT9v
-neg/pm8ojcwOa9qoILdtu6ue7ee3VE9cFnB1vutxS1+MPeI5wgTJjaYrgPCMxXBM
-2LdJJEmBAoGBAPA6LpuU1vv5R3x66hzenSk4LS1fj24K0WuBdTwFvzQmCr70oKdo
-Yywxt+ZkBw5aEtzQlB8GewolHobDJrzxMorU+qEXX3jP2BIPDVQl2orfjr03Yyus
-s5mYS/Qa6Zf1yObrjulTNm8oTn1WaG3TIvi8c5DyG2OK28N/9oMI1XGRAoGBAORD
-YKyII/S66gZsJSf45qmrhq1hHuVt1xae5LUPP6lVD+MCCAmuoJnReV8fc9h7Dvgd
-YPMINkWUTePFr3o4p1mh2ZC7ldczgDn6X4TldY2J3Zg47xJa5hL0L6JL4NiCGRIE
-FV5rLJxkGh/DDBfmC9hQQ6Yg6cHvyewso5xVnBtZAoGAI+OdWPMIl0ZrrqYyWbPM
-aP8SiMfRBtCo7tW9bQUyxpi0XEjxw3Dt+AlJfysMftFoJgMnTedK9H4NLHb1T579
-PQ6KjwyN39+1WSVUiXDKUJsLmSswLrMzdcvx9PscUO6QYCdrB2K+LCcqasFBAr9b
-ZyvIXCw/eUSihneUnYjxUnECgYAoPgCzKiU8ph9QFozOaUExNH4/3tl1lVHQOR8V
-FKUik06DtP35xwGlXJrLPF5OEhPnhjZrYk0/IxBAUb/ICmjmknQq4gdes0Ot9QgW
-A+Yfl+irR45ObBwXx1kGgd4YDYeh93pU9QweXj+Ezfw50mLQNgZXKYJMoJu2uX/2
-tdkZsQKBgQCTfDcW8qBntI6V+3Gh+sIThz+fjdv5+qT54heO4EHadc98ykEZX0M1
-sCWJiAQWM/zWXcsTndQDgDsvo23jpoulVPDitSEISp5gSe9FEN2njsVVID9h1OIM
-f30s5kwcJoiV9kUCya/BFtuS7kbuQfAyPU0v3I+lUey6VCW6A83OTg==
------END RSA PRIVATE KEY-----

@xen-orchestra/proxy/tests/localServer.mjs

@@ -1,60 +0,0 @@
-import { createServer as creatServerHttps } from 'https'
-import { createServer as creatServerHttp } from 'http'
-
-import { WebSocketServer } from 'ws'
-import fs from 'fs'
-
-const httpsServer = creatServerHttps({
-  key: fs.readFileSync('key.pem'),
-  cert: fs.readFileSync('cert.pem'),
-})
-const httpServer = creatServerHttp()
-
-const wss = new WebSocketServer({ noServer: true, perMessageDeflate: false })
-
-function upgrade(request, socket, head) {
-  const { pathname } = new URL(request.url)
-  // web socket server only on /foo url
-  if (pathname === '/foo') {
-    wss.handleUpgrade(request, socket, head, function done(ws) {
-      wss.emit('connection', ws, request)
-      ws.on('message', function message(data) {
-        ws.send(data)
-      })
-    })
-  } else {
-    socket.destroy()
-  }
-}
-
-function httpHandler(req, res) {
-  switch (req.url) {
-    case '/index.html':
-      res.end('hi')
-      return
-    case '/redirect':
-      res.writeHead(301, {
-        Location: 'index.html',
-      })
-      res.end()
-      return
-    case '/chainRedirect':
-      res.writeHead(301, {
-        Location: '/redirect',
-      })
-      res.end()
-      return
-    default:
-      res.writeHead(404)
-      res.end()
-  }
-}
-
-httpsServer.on('upgrade', upgrade)
-httpServer.on('upgrade', upgrade)
-
-httpsServer.on('request', httpHandler)
-httpServer.on('request', httpHandler)
-
-httpsServer.listen(8080)
-httpServer.listen(8081)

@xen-orchestra/proxy/tests/reverseProxy.unit.spec.mjs

@@ -1,123 +0,0 @@
-import ReverseProxy, { backendToLocalPath, localToBackendUrl } from '../dist/app/mixins/reverseProxy.mjs'
-import { deepEqual, strictEqual } from 'assert'
-
-function makeApp(reverseProxies) {
-  return {
-    config: {
-      get: () => reverseProxies,
-    },
-  }
-}
-
-const app = makeApp({
-  https: {
-    target: 'https://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
-    oneOption: true,
-  },
-  http: 'http://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
-})
-
-// test localToBackendUrl
-const expectedLocalToRemote = {
-  https: [
-    {
-      local: '/proxy/v1/https/',
-      remote: 'https://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
-    },
-    {
-      local: '/proxy/v1/https/sub',
-      remote: 'https://localhost:8080/remotePath/sub?baseParm=1#one=2&another=3',
-    },
-    {
-      local: '/proxy/v1/https/sub/index.html',
-      remote: 'https://localhost:8080/remotePath/sub/index.html?baseParm=1#one=2&another=3',
-    },
-    {
-      local: '/proxy/v1/https/sub?param=1',
-      remote: 'https://localhost:8080/remotePath/sub?baseParm=1&param=1#one=2&another=3',
-    },
-    {
-      local: '/proxy/v1/https/sub?baseParm=willbeoverwritten&param=willstay',
-      remote: 'https://localhost:8080/remotePath/sub?baseParm=1&param=willstay#one=2&another=3',
-    },
-    {
-      local: '/proxy/v1/https/sub?param=1#another=willoverwrite',
-      remote: 'https://localhost:8080/remotePath/sub?baseParm=1&param=1#another=willoverwrite',
-    },
-  ],
-}
-const proxy = new ReverseProxy(app, { httpServer: { on: () => {} } })
-for (const proxyId in expectedLocalToRemote) {
-  for (const { local, remote } of expectedLocalToRemote[proxyId]) {
-    const config = proxy._getConfigFromRequest({ url: local })
-    const url = new URL(config.target)
-    strictEqual(localToBackendUrl(config.path, url, local).href, remote, 'error converting to backend')
-  }
-}
-
-// test backendToLocalPath
-const expectedRemoteToLocal = {
-  https: [
-    {
-      local: '/proxy/v1/https/',
-      remote: 'https://localhost:8080/remotePath/',
-    },
-    {
-      local: '/proxy/v1/https/sub/index.html',
-      remote: '/remotePath/sub/index.html',
-    },
-    {
-      local: '/proxy/v1/https/?baseParm=1#one=2&another=3',
-      remote: '?baseParm=1#one=2&another=3',
-    },
-    {
-      local: '/proxy/v1/https/sub?baseParm=1#one=2&another=3',
-      remote: 'https://localhost:8080/remotePath/sub?baseParm=1#one=2&another=3',
-    },
-  ],
-}
-
-for (const proxyId in expectedRemoteToLocal) {
-  for (const { local, remote } of expectedRemoteToLocal[proxyId]) {
-    const config = proxy._getConfigFromRequest({ url: local })
-    const targetUrl = new URL('https://localhost:8080/remotePath/?baseParm=1#one=2&another=3')
-    const remoteUrl = new URL(remote, targetUrl)
-    strictEqual(backendToLocalPath(config.path, targetUrl, remoteUrl), local, 'error converting to local')
-  }
-}
-
-// test _getConfigFromRequest
-
-const expectedConfig = [
-  {
-    local: '/proxy/v1/http/other',
-    config: {
-      target: 'http://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
-      options: {},
-      path: '/proxy/v1/http',
-    },
-  },
-  {
-    local: '/proxy/v1/http',
-    config: undefined,
-  },
-
-  {
-    local: '/proxy/v1/other',
-    config: undefined,
-  },
-  {
-    local: '/proxy/v1/https/',
-    config: {
-      target: 'https://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
-      options: {
-        oneOption: true,
-      },
-      path: '/proxy/v1/https',
-    },
-  },
-]
-
-for (const { local, config } of expectedConfig) {
-  deepEqual(proxy._getConfigFromRequest({ url: local }), config)
-}

@xen-orchestra/self-signed/index.js

@@ -1,4 +1,7 @@
 const { execFile } = require('child_process')
+const { promisify } = require('util')
+
+const randomBytes = promisify(require('crypto').randomBytes)
 
 const openssl = (cmd, args, { input, ...opts } = {}) =>
   new Promise((resolve, reject) => {
@@ -10,12 +13,35 @@ const openssl = (cmd, args, { input, ...opts } = {}) =>
     }
   })
 
-exports.genSelfSignedCert = async ({ days = 360 } = {}) => {
+const req = (key, selfSigned, { days = 360 } = {}) => {
+  const args = ['-batch', '-new', '-key', '-', '-nodes']
+  if (selfSigned) {
+    args.push('-x509', '-days', String(days))
+  }
+  return openssl('req', args, { input: key })
+}
+
+exports.genSelfSignedCert = async opts => {
   const key = await openssl('genrsa', ['2048'])
   return {
-    cert: await openssl('req', ['-batch', '-new', '-key', '-', '-x509', '-days', String(days), '-nodes'], {
-      input: key,
-    }),
+    cert: await req(key, true, opts),
+    key,
+  }
+}
+
+exports.genSignedCert = async (ca, { days = 360 } = {}) => {
+  const key = await openssl('genrsa', ['2048'])
+  const csr = await req(key, false)
+  const serial = '0x' + (await randomBytes(40)).toString('hex')
+  const input = [csr, ca.cert, ca.key].join('\n')
+  return {
+    cert: await openssl(
+      'x509',
+      ['-req', '-in', '-', '-CA', '-', '-CAkey', '-', '-days', String(days), '-set_serial', serial],
+      {
+        input,
+      }
+    ),
     key,
   }
 }

@xen-orchestra/upload-ova/package.json

@@ -45,7 +45,7 @@
     "strip-indent": "^3.0.0",
     "xdg-basedir": "^4.0.0",
     "xo-lib": "^0.11.1",
-    "xo-vmdk-to-vhd": "^2.0.3"
+    "xo-vmdk-to-vhd": "^2.0.1"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

@xen-orchestra/xapi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xen-orchestra/xapi",
-  "version": "0.8.5",
+  "version": "0.8.4",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/xapi",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
@@ -38,7 +38,7 @@
     "prepublishOnly": "yarn run build"
   },
   "dependencies": {
-    "@vates/decorate-with": "^1.0.0",
+    "@vates/decorate-with": "^0.1.0",
     "@xen-orchestra/async-map": "^0.1.2",
     "@xen-orchestra/log": "^0.3.0",
     "d3-time-format": "^3.0.0",

@xen-orchestra/xapi/src/index.js

@@ -15,18 +15,16 @@ exports.VDI_FORMAT_VHD = 'vhd'
 // xapi.call('host.get_servertime', host.$ref) for example
 exports.formatDateTime = utcFormat('%Y%m%dT%H:%M:%SZ')
 
-const dateTimeParsers = ['%Y%m%dT%H:%M:%SZ', '%Y-%m-%dT%H:%M:%S.%LZ'].map(utcParse)
+const parseDateTimeHelper = utcParse('%Y%m%dT%H:%M:%SZ')
 exports.parseDateTime = function (str, defaultValue) {
-  for (const parser of dateTimeParsers) {
-    const date = parser(str)
-    if (date !== null) {
-      return date.getTime()
+  const date = parseDateTimeHelper(str)
+  if (date === null) {
+    if (arguments.length > 1) {
+      return defaultValue
     }
+    throw new RangeError(`unable to parse XAPI datetime ${JSON.stringify(str)}`)
   }
-  if (arguments.length > 1) {
-    return defaultValue
-  }
-  throw new RangeError(`unable to parse XAPI datetime ${JSON.stringify(str)}`)
+  return date.getTime()
 }
 
 const hasProps = o => {

@xen-orchestra/xapi/src/index.spec.js

@@ -1,17 +0,0 @@
-/* eslint-env jest */
-
-const { parseDateTime } = require('./')
-
-describe('parseDateTime()', () => {
-  it('parses legacy XAPI format', () => {
-    expect(parseDateTime('20220106T21:25:15Z')).toBe(1641504315000)
-  })
-
-  it('parses ISO 8601 with millisconds format', () => {
-    expect(parseDateTime('2022-01-06T17:32:35.000Z')).toBe(1641490355000)
-  })
-
-  it('throws when value cannot be parsed', () => {
-    expect(() => parseDateTime('foo bar')).toThrow('unable to parse XAPI datetime "foo bar"')
-  })
-})

@xen-orchestra/xapi/src/vm.js

@@ -60,7 +60,7 @@ module.exports = class Vm {
       try {
         vdi = await this[vdiRefOrUuid.startsWith('OpaqueRef:') ? 'getRecord' : 'getRecordByUuid']('VDI', vdiRefOrUuid)
       } catch (error) {
-        warn('_assertHealthyVdiChain, could not fetch VDI', { error })
+        warn(error)
         return
       }
       cache[vdi.$ref] = vdi
@@ -81,7 +81,7 @@ module.exports = class Vm {
                   try {
                     vdi = await this.getRecord('VDI', vdiRef)
                   } catch (error) {
-                    warn('_assertHealthyVdiChain, could not fetch VDI', { error })
+                    warn(error)
                     return
                   }
                   cache[vdiRef] = vdi
@@ -167,7 +167,7 @@ module.exports = class Vm {
       memory_static_min,
       name_description,
       name_label,
-      NVRAM,
+      // NVRAM, // experimental
       order,
       other_config = {},
       PCI_bus = '',
@@ -256,7 +256,6 @@ module.exports = class Vm {
       is_vmss_snapshot,
       name_description,
       name_label,
-      NVRAM,
       order,
       reference_label,
       shutdown_delay,

CHANGELOG.md

@@ -1,91 +1,14 @@
 # ChangeLog
 
-## **5.66.2** (2022-01-05)
+## **5.65.2** (2021-12-10)
 
 <img id="latest" src="https://badgen.net/badge/channel/latest/yellow" alt="Channel: latest" />
 
 ### Bug fixes
 
-- [Import/Disk] Fix `JSON.parse` and `createReadableSparseStream is not a function` errors [#6068](https://github.com/vatesfr/xen-orchestra/issues/6068)
-- [Backup] Fix delta backup are almost always full backup instead of differentials [Forum#5256](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it/69) [Forum#5371](https://xcp-ng.org/forum/topic/5371/delta-backup-changes-in-5-66) (PR [#6075](https://github.com/vatesfr/xen-orchestra/pull/6075))
-
-### Released packages
-
-- vhd-lib 3.0.0
-- xo-vmdk-to-vhd 2.0.3
-- @xen-orchestra/backups 0.18.3
-- @xen-orchestra/proxy 0.17.3
-- xo-server 5.86.3
-- xo-web 5.91.2
-
-## **5.66.1** (2021-12-23)
-
-### Bug fixes
-
-- [Dashboard/Health] Fix `error has occured` when a pool has no default SR
-- [Delta Backup] Fix unnecessary full backup when not using S3 [Forum #5371](https://xcp-ng.org/forum/topic/5371/delta-backup-changes-in-5-66)d (PR [#6070](https://github.com/vatesfr/xen-orchestra/pull/6070))
-- [Backup] Fix incorrect warnings `incorrect size [...] instead of undefined`
-
-### Released packages
-
-- @xen-orchestra/backups 0.18.2
-- @xen-orchestra/proxy 0.17.2
-- xo-server 5.86.2
-- xo-web 5.91.1
-
-## **5.66.0** (2021-12-21)
-
-### Enhancements
-
-- [About] Show commit instead of version numbers for source users (PR [#6045](https://github.com/vatesfr/xen-orchestra/pull/6045))
-- [Health] Display default SRs that aren't shared [#5871](https://github.com/vatesfr/xen-orchestra/issues/5871) (PR [#6033](https://github.com/vatesfr/xen-orchestra/pull/6033))
-- [Pool,VM/advanced] Ability to change the suspend SR [#4163](https://github.com/vatesfr/xen-orchestra/issues/4163) (PR [#6044](https://github.com/vatesfr/xen-orchestra/pull/6044))
-- [Home/VMs/Backup filter] Filter out VMs in disabled backup jobs (PR [#6037](https://github.com/vatesfr/xen-orchestra/pull/6037))
-- [Rolling Pool Update] Automatically disable High Availability during the update [#5711](https://github.com/vatesfr/xen-orchestra/issues/5711) (PR [#6057](https://github.com/vatesfr/xen-orchestra/pull/6057))
-- [Delta Backup on S3] Compress blocks by default ([Brotli](https://en.wikipedia.org/wiki/Brotli)) which reduces remote usage and increase backup speed (PR [#5932](https://github.com/vatesfr/xen-orchestra/pull/5932))
-
-### Bug fixes
-
-- [Tables/actions] Fix collapsed actions being clickable despite being disabled (PR [#6023](https://github.com/vatesfr/xen-orchestra/pull/6023))
-- [Backup] Remove incorrect size warning following a merge [Forum #5727](https://xcp-ng.org/forum/topic/4769/warnings-showing-in-system-logs-following-each-backup-job/4) (PR [#6010](https://github.com/vatesfr/xen-orchestra/pull/6010))
-- [Delta Backup] Preserve UEFI boot parameters [#6054](https://github.com/vatesfr/xen-orchestra/issues/6054) [Forum #5319](https://xcp-ng.org/forum/topic/5319/bug-uefi-boot-parameters-not-preserved-with-delta-backups)
-
-### Released packages
-
-- @xen-orchestra/mixins 0.1.2
-- @xen-orchestra/xapi 0.8.5
-- vhd-lib 2.1.0
-- xo-vmdk-to-vhd 2.0.2
-- @xen-orchestra/backups 0.18.1
-- @xen-orchestra/proxy 0.17.1
-- xo-server 5.86.1
-- xo-web 5.91.0
-
-## **5.65.3** (2021-12-20)
-
-<img id="stable" src="https://badgen.net/badge/channel/stable/green" alt="Channel: stable" />
-
-### Bug fixes
-
-- [Continuous Replication] Fix `could not find the base VM`
-- [Backup/Smart mode] Always ignore replicated VMs created by the current job
-- [Backup] Fix `Unexpected end of JSON input` during merge step
-- [Backup] Fix stuck jobs when using S3 remotes (PR [#6067](https://github.com/vatesfr/xen-orchestra/pull/6067))
-
-### Released packages
-
-- @xen-orchestra/fs 0.19.3
-- vhd-lib 2.0.4
-- @xen-orchestra/backups 0.17.1
-- xo-server 5.85.1
-
-## **5.65.2** (2021-12-10)
-
-### Bug fixes
-
 - [Backup] Fix `handler.rmTree` is not a function (Forum [5256](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it/29) PR [#6041](https://github.com/vatesfr/xen-orchestra/pull/6041) )
 - [Backup] Fix `EEXIST` in logs when multiple merge tasks are created at the same time ([Forum #5301](https://xcp-ng.org/forum/topic/5301/warnings-errors-in-journalctl))
-- [Backup] Fix missing backup on restore (Forum [5256](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it/29) (PR [#6048](https://github.com/vatesfr/xen-orchestra/pull/6048))
+- [Backup] Fix missing backup on restore (Forum [5256](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it/29) (PR [#6048](https://github.com/vatesfr/xen-orchestra/pull/6048))
 
 ### Released packages
 
@@ -119,7 +42,7 @@
 ### Highlights
 
 - [VM] Ability to export a snapshot's memory (PR [#6015](https://github.com/vatesfr/xen-orchestra/pull/6015))
-- [Cloud config] Ability to create a network cloud config template and reuse it in the VM creation [#5931](https://github.com/vatesfr/xen-orchestra/issues/5931) (PR [#5979](https://github.com/vatesfr/xen-orchestra/pull/5979))
+- [Cloud config] Ability to create a network cloud config template and reuse it in the VM creation [#5931](https://github.com/vatesfr/xen-orchestra/issues/5931) (PR [#5979](https://github.com/vatesfr/xen-orchestra/pull/5979))
 - [Backup/logs] identify XAPI errors (PR [#6001](https://github.com/vatesfr/xen-orchestra/pull/6001))
 - [lite] Highlight selected VM (PR [#5939](https://github.com/vatesfr/xen-orchestra/pull/5939))
 
@@ -147,6 +70,8 @@
 
 ## **5.64.0** (2021-10-29)
 
+<img id="stable" src="https://badgen.net/badge/channel/stable/green" alt="Channel: stable" />
+
 ## Highlights
 
 - [Netbox] Support older versions of Netbox and prevent "active is not a valid choice" error [#5898](https://github.com/vatesfr/xen-orchestra/issues/5898) (PR [#5946](https://github.com/vatesfr/xen-orchestra/pull/5946))
@@ -154,8 +79,8 @@
 - [Host] Handle evacuation failure during host shutdown (PR [#5966](https://github.com/vatesfr/xen-orchestra/pull/#5966))
 - [Menu] Notify user when proxies need to be upgraded (PR [#5930](https://github.com/vatesfr/xen-orchestra/pull/5930))
 - [Servers] Ability to use an HTTP proxy between XO and a server (PR [#5958](https://github.com/vatesfr/xen-orchestra/pull/5958))
-- [VM/export] Ability to copy the export URL (PR [#5948](https://github.com/vatesfr/xen-orchestra/pull/5948))
-- [Pool/advanced] Ability to define network for importing/exporting VMs/VDIs (PR [#5957](https://github.com/vatesfr/xen-orchestra/pull/5957))
+- [VM/export] Ability to copy the export URL (PR [#5948](https://github.com/vatesfr/xen-orchestra/pull/5948))
+- [Pool/advanced] Ability to define network for importing/exporting VMs/VDIs (PR [#5957](https://github.com/vatesfr/xen-orchestra/pull/5957))
 - [Host/advanced] Add button to enable/disable the host (PR [#5952](https://github.com/vatesfr/xen-orchestra/pull/5952))
 - [Backups] Enable merge worker by default
 
@@ -191,8 +116,8 @@
 ### Highlights
 
 - [Backup] Go back to previous page instead of going to the overview after editing a job: keeps current filters and page (PR [#5913](https://github.com/vatesfr/xen-orchestra/pull/5913))
-- [Health] Do not take into consideration duplicated MAC addresses from CR VMs (PR [#5916](https://github.com/vatesfr/xen-orchestra/pull/5916))
-- [Health] Ability to filter duplicated MAC addresses by running VMs (PR [#5917](https://github.com/vatesfr/xen-orchestra/pull/5917))
+- [Health] Do not take into consideration duplicated MAC addresses from CR VMs (PR [#5916](https://github.com/vatesfr/xen-orchestra/pull/5916))
+- [Health] Ability to filter duplicated MAC addresses by running VMs (PR [#5917](https://github.com/vatesfr/xen-orchestra/pull/5917))
 - [Tables] Move the search bar and pagination to the top of the table (PR [#5914](https://github.com/vatesfr/xen-orchestra/pull/5914))
 - [Netbox] Handle nested prefixes by always assigning an IP to the smallest prefix it matches (PR [#5908](https://github.com/vatesfr/xen-orchestra/pull/5908))
 
@@ -459,7 +384,7 @@
 - [Proxy] _Redeploy_ now works when the bound VM is missing
 - [VM template] Fix confirmation modal doesn't appear on deleting a default template (PR [#5644](https://github.com/vatesfr/xen-orchestra/pull/5644))
 - [OVA VM Import] Fix imported VMs all having the same MAC addresses
-- [Disk import] Fix `an error has occurred` when importing wrong format or corrupted files [#5663](https://github.com/vatesfr/xen-orchestra/issues/5663) (PR [#5683](https://github.com/vatesfr/xen-orchestra/pull/5683))
+- [Disk import] Fix `an error has occurred` when importing wrong format or corrupted files [#5663](https://github.com/vatesfr/xen-orchestra/issues/5663) (PR [#5683](https://github.com/vatesfr/xen-orchestra/pull/5683))
 
 ### Released packages
 

CHANGELOG.unreleased.md

@@ -7,15 +7,17 @@
 
 > Users must be able to say: “Nice enhancement, I'm eager to test it”
 
-- Limit number of concurrent VM migrations per pool to `3` [#6065](https://github.com/vatesfr/xen-orchestra/issues/6065) (PR [#6076](https://github.com/vatesfr/xen-orchestra/pull/6076))
-  Can be changed in `xo-server`'s configuration file: `xapiOptions.vmMigrationConcurrency`
-- [Proxy] Now ships a reverse proxy [PR#6072](https://github.com/vatesfr/xen-orchestra/pull/6072)
+- [About] Show commit instead of version numbers for source users (PR [#6045](https://github.com/vatesfr/xen-orchestra/pull/6045))
+- [Health] Display default SRs that aren't shared [#5871](https://github.com/vatesfr/xen-orchestra/issues/5871) (PR [#6033](https://github.com/vatesfr/xen-orchestra/pull/6033))
+- [Pool,VM/advanced] Ability to change the suspend SR [#4163](https://github.com/vatesfr/xen-orchestra/issues/4163) (PR [#6044](https://github.com/vatesfr/xen-orchestra/pull/6044))
 
 ### Bug fixes
 
 > Users must be able to say: “I had this issue, happy to know it's fixed”
 
-- [Backup] Detect and clear orphan merge states, fix `ENOENT` errors (PR [#6087](https://github.com/vatesfr/xen-orchestra/pull/6087))
+- [Tables/actions] Fix collapsed actions being clickable despite being disabled (PR [#6023](https://github.com/vatesfr/xen-orchestra/pull/6023))
+- [Continuous Replication] Fix `could not find the base VM`
+- [Backup/Smart mode] Always ignore replicated VMs created by the current job
 
 ### Packages to release
 
@@ -35,6 +37,6 @@
 > In case of conflict, the highest (lowest in previous list) `$version` wins.
 
 - @xen-orchestra/backups minor
-- @xen-orchestra/backups-cli minor
 - @xen-orchestra/proxy minor
 - xo-server minor
+- xo-web minor

package.json

@@ -46,6 +46,7 @@
       ],
       "^(value-matcher)$": "$1/src",
       "^(vhd-cli)$": "$1/src",
+      "^(vhd-lib)$": "$1/src",
       "^(xo-[^/]+)$": [
         "$1/src",
         "$1"

packages/vhd-cli/package.json

@@ -24,12 +24,12 @@
     "node": ">=8.10"
   },
   "dependencies": {
-    "@xen-orchestra/fs": "^0.19.3",
+    "@xen-orchestra/fs": "^0.19.2",
     "cli-progress": "^3.1.0",
     "exec-promise": "^0.7.0",
     "getopts": "^2.2.3",
     "human-format": "^0.11.0",
-    "vhd-lib": "^3.0.0"
+    "vhd-lib": "^2.0.3"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/vhd-lib/.babelrc.js

@@ -0,0 +1 @@
+module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))

packages/vhd-lib/Vhd/_utils.js

@@ -1,67 +0,0 @@
-const assert = require('assert')
-const { BLOCK_UNUSED, SECTOR_SIZE } = require('../_constants')
-const { fuFooter, fuHeader, checksumStruct, unpackField } = require('../_structs')
-const checkFooter = require('../checkFooter')
-const checkHeader = require('../_checkHeader')
-
-const computeBatSize = entries => sectorsToBytes(sectorsRoundUpNoZero(entries * 4))
-exports.computeBatSize = computeBatSize
-
-const computeSectorsPerBlock = blockSize => blockSize / SECTOR_SIZE
-exports.computeSectorsPerBlock = computeSectorsPerBlock
-// one bit per sector
-const computeBlockBitmapSize = blockSize => computeSectorsPerBlock(blockSize) >>> 3
-exports.computeBlockBitmapSize = computeBlockBitmapSize
-const computeFullBlockSize = blockSize => blockSize + SECTOR_SIZE * computeSectorOfBitmap(blockSize)
-exports.computeFullBlockSize = computeFullBlockSize
-const computeSectorOfBitmap = blockSize => sectorsRoundUpNoZero(computeBlockBitmapSize(blockSize))
-exports.computeSectorOfBitmap = computeSectorOfBitmap
-
-// Sectors conversions.
-const sectorsRoundUpNoZero = bytes => Math.ceil(bytes / SECTOR_SIZE) || 1
-exports.sectorsRoundUpNoZero = sectorsRoundUpNoZero
-const sectorsToBytes = sectors => sectors * SECTOR_SIZE
-exports.sectorsToBytes = sectorsToBytes
-
-const assertChecksum = (name, buf, struct) => {
-  const actual = unpackField(struct.fields.checksum, buf)
-  const expected = checksumStruct(buf, struct)
-  assert.strictEqual(actual, expected, `invalid ${name} checksum ${actual}, expected ${expected}`)
-}
-exports.assertChecksum = assertChecksum
-
-// unused block as buffer containing a uint32BE
-const BUF_BLOCK_UNUSED = Buffer.allocUnsafe(4)
-BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0)
-exports.BUF_BLOCK_UNUSED = BUF_BLOCK_UNUSED
-
-/**
- * Check and parse the header buffer to build an header object
- *
- * @param {Buffer} bufHeader
- * @param {Object} footer
- * @returns {Object} the parsed header
- */
-exports.unpackHeader = (bufHeader, footer) => {
-  assertChecksum('header', bufHeader, fuHeader)
-
-  const header = fuHeader.unpack(bufHeader)
-  checkHeader(header, footer)
-  return header
-}
-
-/**
- * Check and parse the footer buffer to build a footer object
- *
- * @param {Buffer} bufHeader
- * @param {Object} footer
- * @returns {Object} the parsed footer
- */
-
-exports.unpackFooter = bufFooter => {
-  assertChecksum('footer', bufFooter, fuFooter)
-
-  const footer = fuFooter.unpack(bufFooter)
-  checkFooter(footer)
-  return footer
-}

packages/vhd-lib/_bitmap.js

@@ -1,7 +0,0 @@
-const MASK = 0x80
-
-exports.set = (map, bit) => {
-  map[bit >> 3] |= MASK >> (bit & 7)
-}
-
-exports.test = (map, bit) => ((map[bit >> 3] << (bit & 7)) & MASK) !== 0

packages/vhd-lib/_constants.js

@@ -1,40 +0,0 @@
-exports.BLOCK_UNUSED = 0xffffffff
-
-// This lib has been extracted from the Xen Orchestra project.
-exports.CREATOR_APPLICATION = 'xo  '
-
-// Sizes in bytes.
-exports.FOOTER_SIZE = 512
-exports.HEADER_SIZE = 1024
-exports.SECTOR_SIZE = 512
-exports.DEFAULT_BLOCK_SIZE = 0x00200000 // from the spec
-
-exports.FOOTER_COOKIE = 'conectix'
-exports.HEADER_COOKIE = 'cxsparse'
-
-exports.DISK_TYPES = {
-  __proto__: null,
-
-  FIXED: 2,
-  DYNAMIC: 3,
-  DIFFERENCING: 4,
-}
-
-exports.PARENT_LOCATOR_ENTRIES = 8
-
-exports.PLATFORMS = {
-  __proto__: null,
-
-  NONE: 0,
-  WI2R: 0x57693272,
-  WI2K: 0x5769326b,
-  W2RU: 0x57327275,
-  W2KU: 0x57326b75,
-  MAC: 0x4d616320,
-  MACX: 0x4d616358,
-}
-
-exports.FILE_FORMAT_VERSION = 1 << 16
-exports.HEADER_VERSION = 1 << 16
-
-exports.ALIAS_MAX_PATH_LENGTH = 1024

packages/vhd-lib/_noop.js

@@ -1 +0,0 @@
-module.exports = Function.prototype

packages/vhd-lib/checkChain.js

@@ -1,14 +0,0 @@
-const { openVhd } = require('./openVhd')
-const resolveRelativeFromFile = require('./_resolveRelativeFromFile')
-const { DISK_TYPES } = require('./_constants')
-const { Disposable } = require('promise-toolbox')
-
-module.exports = async function checkChain(handler, path) {
-  await Disposable.use(function* () {
-    let vhd
-    do {
-      vhd = yield openVhd(handler, path)
-      path = resolveRelativeFromFile(path, vhd.header.parentUnicodeName)
-    } while (vhd.footer.diskType !== DISK_TYPES.DYNAMIC)
-  })
-}

packages/vhd-lib/createVhdDirectoryFromStream.js

@@ -1,54 +0,0 @@
-const { parseVhdStream } = require('./parseVhdStream.js')
-const { VhdDirectory } = require('./Vhd/VhdDirectory.js')
-const { Disposable } = require('promise-toolbox')
-const { asyncEach } = require('@vates/async-each')
-
-const buildVhd = Disposable.wrap(async function* (handler, path, inputStream, { concurrency, compression }) {
-  const vhd = yield VhdDirectory.create(handler, path, { compression })
-  await asyncEach(
-    parseVhdStream(inputStream),
-    async function (item) {
-      switch (item.type) {
-        case 'footer':
-          vhd.footer = item.footer
-          break
-        case 'header':
-          vhd.header = item.header
-          break
-        case 'parentLocator':
-          await vhd.writeParentLocator({ ...item, data: item.buffer })
-          break
-        case 'block':
-          await vhd.writeEntireBlock(item)
-          break
-        case 'bat':
-          // it exists but  I don't care
-          break
-        default:
-          throw new Error(`unhandled type of block generated by parser : ${item.type} while generating ${path}`)
-      }
-    },
-    {
-      concurrency,
-    }
-  )
-  await Promise.all([vhd.writeFooter(), vhd.writeHeader(), vhd.writeBlockAllocationTable()])
-})
-
-exports.createVhdDirectoryFromStream = async function createVhdDirectoryFromStream(
-  handler,
-  path,
-  inputStream,
-  { validator, concurrency = 16, compression } = {}
-) {
-  try {
-    await buildVhd(handler, path, inputStream, { concurrency, compression })
-    if (validator !== undefined) {
-      await validator.call(this, path)
-    }
-  } catch (error) {
-    // cleanup on error
-    await handler.rmtree(path)
-    throw error
-  }
-}

packages/vhd-lib/index.js

@@ -1,16 +0,0 @@
-exports.chainVhd = require('./chain')
-exports.checkFooter = require('./checkFooter')
-exports.checkVhdChain = require('./checkChain')
-exports.createReadableRawStream = require('./createReadableRawStream')
-exports.createReadableSparseStream = require('./createReadableSparseStream')
-exports.createVhdStreamWithLength = require('./createVhdStreamWithLength')
-exports.createVhdDirectoryFromStream = require('./createVhdDirectoryFromStream').createVhdDirectoryFromStream
-exports.mergeVhd = require('./merge')
-exports.peekFooterFromVhdStream = require('./peekFooterFromVhdStream')
-exports.openVhd = require('./openVhd').openVhd
-exports.parseVhdStream = require('./parseVhdStream').parseVhdStream
-exports.VhdAbstract = require('./Vhd/VhdAbstract').VhdAbstract
-exports.VhdDirectory = require('./Vhd/VhdDirectory').VhdDirectory
-exports.VhdFile = require('./Vhd/VhdFile').VhdFile
-exports.VhdSynthetic = require('./Vhd/VhdSynthetic').VhdSynthetic
-exports.Constants = require('./_constants')

packages/vhd-lib/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "vhd-lib",
-  "version": "3.0.0",
+  "version": "2.0.3",
   "license": "AGPL-3.0-or-later",
   "description": "Primitives for VHD file handling",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-lib",
@@ -11,8 +11,9 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
+  "main": "dist/",
   "engines": {
-    "node": ">=12"
+    "node": ">=10"
   },
   "dependencies": {
     "@vates/async-each": "^0.1.0",
@@ -27,12 +28,25 @@
     "uuid": "^8.3.1"
   },
   "devDependencies": {
-    "@xen-orchestra/fs": "^0.19.3",
+    "@babel/cli": "^7.0.0",
+    "@babel/core": "^7.0.0",
+    "@babel/preset-env": "^7.0.0",
+    "@xen-orchestra/fs": "^0.19.2",
+    "babel-plugin-lodash": "^3.3.2",
+    "cross-env": "^7.0.2",
+    "execa": "^5.0.0",
     "get-stream": "^6.0.0",
     "readable-stream": "^3.0.6",
+    "rimraf": "^3.0.0",
     "tmp": "^0.2.1"
   },
   "scripts": {
+    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
+    "clean": "rimraf dist/",
+    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
+    "prebuild": "yarn run clean",
+    "predev": "yarn run clean",
+    "prepublishOnly": "yarn run build",
     "postversion": "npm publish"
   },
   "author": {

packages/vhd-lib/peekFooterFromVhdStream.js

@@ -1,11 +0,0 @@
-const { readChunk } = require('@vates/read-chunk')
-
-const { FOOTER_SIZE } = require('./_constants')
-const { fuFooter } = require('./_structs')
-
-module.exports = async function peekFooterFromStream(stream) {
-  const footerBuffer = await readChunk(stream, FOOTER_SIZE)
-  const footer = fuFooter.unpack(footerBuffer)
-  stream.unshift(footerBuffer)
-  return footer
-}

packages/vhd-lib/src/Vhd/VhdAbstract.integ.spec.js

@@ -1,16 +1,16 @@
 /* eslint-env jest */
 
-const rimraf = require('rimraf')
-const tmp = require('tmp')
-const fs = require('fs-extra')
-const { getSyncedHandler } = require('@xen-orchestra/fs')
-const { Disposable, pFromCallback } = require('promise-toolbox')
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import fs from 'fs-extra'
+import { getSyncedHandler } from '@xen-orchestra/fs'
+import { Disposable, pFromCallback } from 'promise-toolbox'
 
-const { openVhd } = require('../index')
-const { checkFile, createRandomFile, convertFromRawToVhd, createRandomVhdDirectory } = require('../tests/utils')
-const { VhdAbstract } = require('./VhdAbstract')
-const { BLOCK_UNUSED, FOOTER_SIZE, HEADER_SIZE, PLATFORMS, SECTOR_SIZE } = require('../_constants')
-const { unpackHeader, unpackFooter } = require('./_utils')
+import { openVhd } from '../index'
+import { checkFile, createRandomFile, convertFromRawToVhd, createRandomVhdDirectory } from '../tests/utils'
+import { VhdAbstract } from './VhdAbstract'
+import { BLOCK_UNUSED, FOOTER_SIZE, HEADER_SIZE, PLATFORMS, SECTOR_SIZE } from '../_constants'
+import { unpackHeader, unpackFooter } from './_utils'
 
 let tempDir
 

packages/vhd-lib/src/Vhd/VhdAbstract.js

@@ -1,11 +1,5 @@
-const {
-  computeBatSize,
-  computeFullBlockSize,
-  computeSectorOfBitmap,
-  computeSectorsPerBlock,
-  sectorsToBytes,
-} = require('./_utils')
-const {
+import { computeBatSize, computeSectorOfBitmap, computeSectorsPerBlock, sectorsToBytes } from './_utils'
+import {
   ALIAS_MAX_PATH_LENGTH,
   PLATFORMS,
   SECTOR_SIZE,
@@ -13,20 +7,20 @@ const {
   FOOTER_SIZE,
   HEADER_SIZE,
   BLOCK_UNUSED,
-} = require('../_constants')
-const assert = require('assert')
-const path = require('path')
-const asyncIteratorToStream = require('async-iterator-to-stream')
-const { checksumStruct, fuFooter, fuHeader } = require('../_structs')
-const { isVhdAlias, resolveAlias } = require('../_resolveAlias')
+} from '../_constants'
+import assert from 'assert'
+import path from 'path'
+import asyncIteratorToStream from 'async-iterator-to-stream'
+import { checksumStruct, fuFooter, fuHeader } from '../_structs'
+import { isVhdAlias, resolveAlias } from '../_resolveAlias'
 
-exports.VhdAbstract = class VhdAbstract {
+export class VhdAbstract {
   get bitmapSize() {
     return sectorsToBytes(this.sectorsOfBitmap)
   }
 
   get fullBlockSize() {
-    return computeFullBlockSize(this.header.blockSize)
+    return sectorsToBytes(this.sectorsOfBitmap + this.sectorsPerBlock)
   }
 
   get sectorsOfBitmap() {

packages/vhd-lib/src/Vhd/VhdDirectory.integ.spec.js

@@ -1,13 +1,12 @@
 /* eslint-env jest */
 
-const rimraf = require('rimraf')
-const tmp = require('tmp')
-const fs = require('fs-extra')
-const { getHandler, getSyncedHandler } = require('@xen-orchestra/fs')
-const { Disposable, pFromCallback } = require('promise-toolbox')
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { getHandler } from '@xen-orchestra/fs'
+import { Disposable, pFromCallback } from 'promise-toolbox'
 
-const { openVhd, VhdDirectory } = require('../')
-const { createRandomFile, convertFromRawToVhd, convertToVhdDirectory } = require('../tests/utils')
+import { openVhd } from '../openVhd'
+import { createRandomFile, convertFromRawToVhd, convertToVhdDirectory } from '../tests/utils'
 
 let tempDir = null
 
@@ -66,49 +65,3 @@ test('Can coalesce block', async () => {
     expect(parentBlockData).toEqual(childBlockData)
   })
 })
-
-test('compressed blocks and metadata works', async () => {
-  const initalSize = 4
-  const rawFileName = `${tempDir}/randomfile`
-  const vhdName = `${tempDir}/parent.vhd`
-
-  await createRandomFile(rawFileName, initalSize)
-  await convertFromRawToVhd(rawFileName, vhdName)
-  await Disposable.use(async function* () {
-    const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
-    const vhd = yield openVhd(handler, 'parent.vhd')
-    await vhd.readBlockAllocationTable()
-    const compressedVhd = yield VhdDirectory.create(handler, 'compressed.vhd', { compression: 'gzip' })
-    compressedVhd.header = vhd.header
-    compressedVhd.footer = vhd.footer
-    for await (const block of vhd.blocks()) {
-      await compressedVhd.writeEntireBlock(block)
-    }
-    await Promise
-      .all[(await compressedVhd.writeHeader(), await compressedVhd.writeFooter(), await compressedVhd.writeBlockAllocationTable())]
-
-    // compressed vhd have a metadata file
-    expect(await fs.exists(`${tempDir}/compressed.vhd/metadata.json`)).toEqual(true)
-    const metada = JSON.parse(await handler.readFile('compressed.vhd/metadata.json'))
-    expect(metada.compression.type).toEqual('gzip')
-    expect(metada.compression.options.level).toEqual(1)
-
-    // compressed vhd should not be broken
-    await compressedVhd.readHeaderAndFooter()
-    await compressedVhd.readBlockAllocationTable()
-
-    // check that footer and header are not modified
-    expect(compressedVhd.footer).toEqual(vhd.footer)
-    expect(compressedVhd.header).toEqual(vhd.header)
-
-    // their block content should not have changed
-    let counter = 0
-    for await (const block of compressedVhd.blocks()) {
-      const source = await vhd.readBlock(block.id)
-      expect(source.data.equals(block.data)).toEqual(true)
-      counter++
-    }
-    // neither the number of blocks
-    expect(counter).toEqual(2)
-  })
-})

packages/vhd-lib/src/Vhd/VhdDirectory.js

@@ -1,67 +1,15 @@
-const { unpackHeader, unpackFooter, sectorsToBytes } = require('./_utils')
-const { createLogger } = require('@xen-orchestra/log')
-const { fuFooter, fuHeader, checksumStruct } = require('../_structs')
-const { test, set: setBitmap } = require('../_bitmap')
-const { VhdAbstract } = require('./VhdAbstract')
-const assert = require('assert')
-const promisify = require('promise-toolbox/promisify')
-const zlib = require('zlib')
+import { unpackHeader, unpackFooter, sectorsToBytes } from './_utils'
+import { createLogger } from '@xen-orchestra/log'
+import { fuFooter, fuHeader, checksumStruct } from '../_structs'
+import { test, set as setBitmap } from '../_bitmap'
+import { VhdAbstract } from './VhdAbstract'
+import assert from 'assert'
 
 const { debug } = createLogger('vhd-lib:VhdDirectory')
 
-const NULL_COMPRESSOR = {
-  compress: buffer => buffer,
-  decompress: buffer => buffer,
-  baseOptions: {},
-}
-
-const COMPRESSORS = {
-  gzip: {
-    compress: (
-      gzip => buffer =>
-        gzip(buffer, { level: zlib.constants.Z_BEST_SPEED })
-    )(promisify(zlib.gzip)),
-    decompress: promisify(zlib.gunzip),
-  },
-  brotli: {
-    compress: (
-      brotliCompress => buffer =>
-        brotliCompress(buffer, {
-          params: {
-            [zlib.constants.BROTLI_PARAM_QUALITY]: zlib.constants.BROTLI_MIN_QUALITY,
-          },
-        })
-    )(promisify(zlib.brotliCompress)),
-    decompress: promisify(zlib.brotliDecompress),
-  },
-}
-
-// inject identifiers
-for (const id of Object.keys(COMPRESSORS)) {
-  COMPRESSORS[id].id = id
-}
-
-function getCompressor(compressorType) {
-  if (compressorType === undefined) {
-    return NULL_COMPRESSOR
-  }
-
-  const compressor = COMPRESSORS[compressorType]
-
-  if (compressor === undefined) {
-    throw new Error(`Compression type ${compressorType} is not supported`)
-  }
-
-  return compressor
-}
-
 // ===================================================================
 // Directory format
 // <path>
-// ├─ chunk-filters.json
-// │    Ordered array of filters that have been applied before writing chunks.
-// │    These filters needs to be applied in reverse order to read them.
-// │
 // ├─ header // raw content of the header
 // ├─ footer // raw content of the footer
 // ├─ bat // bit array. A zero bit indicates at a position that this block is not present
@@ -70,15 +18,10 @@ function getCompressor(compressorType) {
 //    └─ <the first to  {blockId.length -3} numbers of blockId >
 //         └─ <the three last numbers  of blockID >  // block content.
 
-exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
+export class VhdDirectory extends VhdAbstract {
   #uncheckedBlockTable
   #header
   footer
-  #compressor
-
-  get compressionType() {
-    return this.#compressor.id
-  }
 
   set header(header) {
     this.#header = header
@@ -114,9 +57,9 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
     }
   }
 
-  static async create(handler, path, { flags = 'wx+', compression } = {}) {
+  static async create(handler, path, { flags = 'wx+' } = {}) {
     await handler.mkdir(path)
-    const vhd = new VhdDirectory(handler, path, { flags, compression })
+    const vhd = new VhdDirectory(handler, path, { flags })
     return {
       dispose: () => {},
       value: vhd,
@@ -128,7 +71,6 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
     this._handler = handler
     this._path = path
     this._opts = opts
-    this.#compressor = getCompressor(opts?.compression)
   }
 
   async readBlockAllocationTable() {
@@ -148,9 +90,8 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
     // here we can implement compression and / or crypto
     const buffer = await this._handler.readFile(this._getChunkPath(partName))
 
-    const uncompressed = await this.#compressor.decompress(buffer)
     return {
-      buffer: uncompressed,
+      buffer: Buffer.from(buffer),
     }
   }
 
@@ -160,9 +101,9 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
       'r',
       `Can't write a chunk ${partName} in ${this._path} with read permission`
     )
+    // here we can implement compression and / or crypto
 
-    const compressed = await this.#compressor.compress(buffer)
-    return this._handler.outputFile(this._getChunkPath(partName), compressed, this._opts)
+    return this._handler.outputFile(this._getChunkPath(partName), buffer, this._opts)
   }
 
   // put block in subdirectories to limit impact when doing directory listing
@@ -173,8 +114,6 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
   }
 
   async readHeaderAndFooter() {
-    // we need to know if thre is compression before reading headers
-    await this.#readChunkFilters()
     const { buffer: bufHeader } = await this._readChunk('header')
     const { buffer: bufFooter } = await this._readChunk('footer')
     const footer = unpackFooter(bufFooter)
@@ -211,13 +150,12 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
     await this._writeChunk('footer', rawFooter)
   }
 
-  async writeHeader() {
+  writeHeader() {
     const { header } = this
     const rawHeader = fuHeader.pack(header)
     header.checksum = checksumStruct(rawHeader, fuHeader)
     debug(`Write header  (checksum=${header.checksum}). (data=${rawHeader.toString('hex')})`)
-    await this._writeChunk('header', rawHeader)
-    await this.#writeChunkFilters()
+    return this._writeChunk('header', rawHeader)
   }
 
   writeBlockAllocationTable() {
@@ -229,13 +167,9 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
 
   // only works if data are in the same handler
   // and if the full block is modified in child ( which is the case whit xcp)
-  // and if the compression type is same on both sides
+
   async coalesceBlock(child, blockId) {
-    if (
-      !(child instanceof VhdDirectory) ||
-      this._handler !== child._handler ||
-      child.compressionType !== this.compressionType
-    ) {
+    if (!(child instanceof VhdDirectory) || this._handler !== child._handler) {
       return super.coalesceBlock(child, blockId)
     }
     await this._handler.copy(
@@ -258,24 +192,4 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
     await this._writeChunk('parentLocatorEntry' + id, data)
     this.header.parentLocatorEntry[id].platformDataOffset = 0
   }
-
-  async #writeChunkFilters() {
-    const compressionType = this.compressionType
-    const path = this._path + '/chunk-filters.json'
-    if (compressionType === undefined) {
-      await this._handler.unlink(path)
-    } else {
-      await this._handler.writeFile(path, JSON.stringify([compressionType]))
-    }
-  }
-
-  async #readChunkFilters() {
-    const chunkFilters = await this._handler.readFile(this._path + '/chunk-filters.json').then(JSON.parse, error => {
-      if (error.code === 'ENOENT') {
-        return []
-      }
-      throw error
-    })
-    this.#compressor = getCompressor(chunkFilters[0])
-  }
 }

packages/vhd-lib/src/Vhd/VhdFile.integ.spec.js

@@ -1,25 +1,25 @@
 /* eslint-env jest */
 
-const execa = require('execa')
-const fs = require('fs-extra')
-const getStream = require('get-stream')
-const rimraf = require('rimraf')
-const tmp = require('tmp')
-const { getHandler } = require('@xen-orchestra/fs')
-const { Disposable, pFromCallback } = require('promise-toolbox')
-const { randomBytes } = require('crypto')
+import execa from 'execa'
+import fs from 'fs-extra'
+import getStream from 'get-stream'
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { getHandler } from '@xen-orchestra/fs'
+import { Disposable, pFromCallback } from 'promise-toolbox'
+import { randomBytes } from 'crypto'
 
-const { VhdFile } = require('./VhdFile')
-const { openVhd } = require('../openVhd')
+import { VhdFile } from './VhdFile'
+import { openVhd } from '../openVhd'
 
-const { SECTOR_SIZE } = require('../_constants')
-const {
+import { SECTOR_SIZE } from '../_constants'
+import {
   checkFile,
   createRandomFile,
   convertFromRawToVhd,
   convertToVhdDirectory,
   recoverRawContent,
-} = require('../tests/utils')
+} from '../tests/utils'
 
 let tempDir = null
 

packages/vhd-lib/src/Vhd/VhdFile.js

@@ -1,18 +1,11 @@
-const {
-  BLOCK_UNUSED,
-  FOOTER_SIZE,
-  HEADER_SIZE,
-  PLATFORMS,
-  SECTOR_SIZE,
-  PARENT_LOCATOR_ENTRIES,
-} = require('../_constants')
-const { computeBatSize, sectorsToBytes, unpackHeader, unpackFooter, BUF_BLOCK_UNUSED } = require('./_utils')
-const { createLogger } = require('@xen-orchestra/log')
-const { fuFooter, fuHeader, checksumStruct } = require('../_structs')
-const { set: mapSetBit } = require('../_bitmap')
-const { VhdAbstract } = require('./VhdAbstract')
-const assert = require('assert')
-const getFirstAndLastBlocks = require('../_getFirstAndLastBlocks')
+import { BLOCK_UNUSED, FOOTER_SIZE, HEADER_SIZE, PLATFORMS, SECTOR_SIZE, PARENT_LOCATOR_ENTRIES } from '../_constants'
+import { computeBatSize, sectorsToBytes, unpackHeader, unpackFooter, BUF_BLOCK_UNUSED } from './_utils'
+import { createLogger } from '@xen-orchestra/log'
+import { fuFooter, fuHeader, checksumStruct } from '../_structs'
+import { set as mapSetBit } from '../_bitmap'
+import { VhdAbstract } from './VhdAbstract'
+import assert from 'assert'
+import getFirstAndLastBlocks from '../_getFirstAndLastBlocks'
 
 const { debug } = createLogger('vhd-lib:VhdFile')
 
@@ -50,7 +43,7 @@ const { debug } = createLogger('vhd-lib:VhdFile')
 // - parentLocatorSize(i) = header.parentLocatorEntry[i].platformDataSpace * sectorSize
 // - sectorSize = 512
 
-exports.VhdFile = class VhdFile extends VhdAbstract {
+export class VhdFile extends VhdAbstract {
   #uncheckedBlockTable
   #header
   footer

packages/vhd-lib/src/Vhd/VhdSynthetic.integ.spec.js

@@ -1,14 +1,14 @@
 /* eslint-env jest */
 
-const rimraf = require('rimraf')
-const tmp = require('tmp')
-const { Disposable, pFromCallback } = require('promise-toolbox')
-const { getSyncedHandler } = require('@xen-orchestra/fs')
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { Disposable, pFromCallback } from 'promise-toolbox'
+import { getSyncedHandler } from '@xen-orchestra/fs'
 
-const { SECTOR_SIZE, PLATFORMS } = require('../_constants')
-const { createRandomFile, convertFromRawToVhd } = require('../tests/utils')
-const { openVhd, chainVhd } = require('..')
-const { VhdSynthetic } = require('./VhdSynthetic')
+import { SECTOR_SIZE, PLATFORMS } from '../_constants'
+import { createRandomFile, convertFromRawToVhd } from '../tests/utils'
+import { openVhd, chainVhd } from '..'
+import { VhdSynthetic } from './VhdSynthetic'
 
 let tempDir = null
 

packages/vhd-lib/src/Vhd/VhdSynthetic.js

@@ -1,12 +1,12 @@
-const UUID = require('uuid')
-const cloneDeep = require('lodash/cloneDeep.js')
-const { asyncMap } = require('@xen-orchestra/async-map')
-const { VhdAbstract } = require('./VhdAbstract')
-const { DISK_TYPES, FOOTER_SIZE, HEADER_SIZE } = require('../_constants')
+import * as UUID from 'uuid'
+import cloneDeep from 'lodash/cloneDeep.js'
+import { asyncMap } from '@xen-orchestra/async-map'
+import { VhdAbstract } from './VhdAbstract'
+import { DISK_TYPES, FOOTER_SIZE, HEADER_SIZE } from '../_constants'
 
-const assert = require('assert')
+import assert from 'assert'
 
-exports.VhdSynthetic = class VhdSynthetic extends VhdAbstract {
+export class VhdSynthetic extends VhdAbstract {
   #vhds = []
 
   get header() {

packages/vhd-lib/src/Vhd/_utils.js

@@ -0,0 +1,57 @@
+import assert from 'assert'
+import { BLOCK_UNUSED, SECTOR_SIZE } from '../_constants'
+import { fuFooter, fuHeader, checksumStruct, unpackField } from '../_structs'
+import checkFooter from '../checkFooter'
+import checkHeader from '../_checkHeader'
+
+export const computeBatSize = entries => sectorsToBytes(sectorsRoundUpNoZero(entries * 4))
+
+export const computeSectorsPerBlock = blockSize => blockSize / SECTOR_SIZE
+// one bit per sector
+export const computeBlockBitmapSize = blockSize => computeSectorsPerBlock(blockSize) >>> 3
+export const computeSectorOfBitmap = blockSize => sectorsRoundUpNoZero(computeBlockBitmapSize(blockSize))
+
+// Sectors conversions.
+export const sectorsRoundUpNoZero = bytes => Math.ceil(bytes / SECTOR_SIZE) || 1
+export const sectorsToBytes = sectors => sectors * SECTOR_SIZE
+
+export const assertChecksum = (name, buf, struct) => {
+  const actual = unpackField(struct.fields.checksum, buf)
+  const expected = checksumStruct(buf, struct)
+  assert.strictEqual(actual, expected, `invalid ${name} checksum ${actual}, expected ${expected}`)
+}
+
+// unused block as buffer containing a uint32BE
+export const BUF_BLOCK_UNUSED = Buffer.allocUnsafe(4)
+BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0)
+
+/**
+ * Check and parse the header buffer to build an header object
+ *
+ * @param {Buffer} bufHeader
+ * @param {Object} footer
+ * @returns {Object} the parsed header
+ */
+export const unpackHeader = (bufHeader, footer) => {
+  assertChecksum('header', bufHeader, fuHeader)
+
+  const header = fuHeader.unpack(bufHeader)
+  checkHeader(header, footer)
+  return header
+}
+
+/**
+ * Check and parse the footer buffer to build a footer object
+ *
+ * @param {Buffer} bufHeader
+ * @param {Object} footer
+ * @returns {Object} the parsed footer
+ */
+
+export const unpackFooter = bufFooter => {
+  assertChecksum('footer', bufFooter, fuFooter)
+
+  const footer = fuFooter.unpack(bufFooter)
+  checkFooter(footer)
+  return footer
+}

packages/vhd-lib/src/_bitmap.js

@@ -0,0 +1,7 @@
+const MASK = 0x80
+
+export const set = (map, bit) => {
+  map[bit >> 3] |= MASK >> (bit & 7)
+}
+
+export const test = (map, bit) => ((map[bit >> 3] << (bit & 7)) & MASK) !== 0

packages/vhd-lib/src/_checkHeader.js

@@ -1,8 +1,8 @@
-const assert = require('assert')
+import assert from 'assert'
 
-const { HEADER_COOKIE, HEADER_VERSION, SECTOR_SIZE } = require('./_constants')
+import { HEADER_COOKIE, HEADER_VERSION, SECTOR_SIZE } from './_constants'
 
-module.exports = (header, footer) => {
+export default (header, footer) => {
   assert.strictEqual(header.cookie, HEADER_COOKIE)
   assert.strictEqual(header.dataOffset, undefined)
   assert.strictEqual(header.headerVersion, HEADER_VERSION)

packages/vhd-lib/src/_computeGeometryForSize.js

@@ -1,6 +1,6 @@
-const { SECTOR_SIZE } = require('./_constants')
+import { SECTOR_SIZE } from './_constants'
 
-module.exports = function computeGeometryForSize(size) {
+export default function computeGeometryForSize(size) {
   const totalSectors = Math.min(Math.ceil(size / 512), 65535 * 16 * 255)
   let sectorsPerTrackCylinder
   let heads

packages/vhd-lib/src/_constants.js

@@ -0,0 +1,40 @@
+export const BLOCK_UNUSED = 0xffffffff
+
+// This lib has been extracted from the Xen Orchestra project.
+export const CREATOR_APPLICATION = 'xo  '
+
+// Sizes in bytes.
+export const FOOTER_SIZE = 512
+export const HEADER_SIZE = 1024
+export const SECTOR_SIZE = 512
+export const DEFAULT_BLOCK_SIZE = 0x00200000 // from the spec
+
+export const FOOTER_COOKIE = 'conectix'
+export const HEADER_COOKIE = 'cxsparse'
+
+export const DISK_TYPES = {
+  __proto__: null,
+
+  FIXED: 2,
+  DYNAMIC: 3,
+  DIFFERENCING: 4,
+}
+
+export const PARENT_LOCATOR_ENTRIES = 8
+
+export const PLATFORMS = {
+  __proto__: null,
+
+  NONE: 0,
+  WI2R: 0x57693272,
+  WI2K: 0x5769326b,
+  W2RU: 0x57327275,
+  W2KU: 0x57326b75,
+  MAC: 0x4d616320,
+  MACX: 0x4d616358,
+}
+
+export const FILE_FORMAT_VERSION = 1 << 16
+export const HEADER_VERSION = 1 << 16
+
+export const ALIAS_MAX_PATH_LENGTH = 1024

packages/vhd-lib/src/_createFooterHeader.integ.spec.js

@@ -1,5 +1,5 @@
 /* eslint-env jest */
-const { createFooter } = require('./_createFooterHeader')
+import { createFooter } from './_createFooterHeader'
 
 test('createFooter() does not crash', () => {
   createFooter(104448, Math.floor(Date.now() / 1000), {

packages/vhd-lib/src/_createFooterHeader.js

@@ -1,9 +1,9 @@
-const { v4: generateUuid } = require('uuid')
+import { v4 as generateUuid } from 'uuid'
 
-const { checksumStruct, fuFooter, fuHeader } = require('./_structs')
-const {
+import { checksumStruct, fuFooter, fuHeader } from './_structs'
+import {
   CREATOR_APPLICATION,
-  DEFAULT_BLOCK_SIZE: VHD_BLOCK_SIZE_BYTES,
+  DEFAULT_BLOCK_SIZE as VHD_BLOCK_SIZE_BYTES,
   DISK_TYPES,
   FILE_FORMAT_VERSION,
   FOOTER_COOKIE,
@@ -12,9 +12,9 @@ const {
   HEADER_SIZE,
   HEADER_VERSION,
   PLATFORMS,
-} = require('./_constants')
+} from './_constants'
 
-exports.createFooter = function createFooter(size, timestamp, geometry, dataOffset, diskType = DISK_TYPES.FIXED) {
+export function createFooter(size, timestamp, geometry, dataOffset, diskType = DISK_TYPES.FIXED) {
   const footer = fuFooter.pack({
     cookie: FOOTER_COOKIE,
     features: 2,
@@ -33,7 +33,7 @@ exports.createFooter = function createFooter(size, timestamp, geometry, dataOffs
   return footer
 }
 
-exports.createHeader = function createHeader(
+export function createHeader(
   maxTableEntries,
   tableOffset = HEADER_SIZE + FOOTER_SIZE,
   blockSize = VHD_BLOCK_SIZE_BYTES

packages/vhd-lib/src/_getFirstAndLastBlocks.js

@@ -1,10 +1,10 @@
-const assert = require('assert')
+import assert from 'assert'
 
-const { BLOCK_UNUSED } = require('./_constants')
+import { BLOCK_UNUSED } from './_constants'
 
 // get the identifiers and first sectors of the first and last block
 // in the file
-module.exports = bat => {
+export default bat => {
   const n = bat.length
   if (n === 0) {
     return

packages/vhd-lib/src/_noop.js

@@ -0,0 +1 @@
+export default Function.prototype

packages/vhd-lib/src/_resolveAlias.integ.spec.js

@@ -1,12 +1,12 @@
 /* eslint-env jest */
 
-const rimraf = require('rimraf')
-const tmp = require('tmp')
-const { getSyncedHandler } = require('@xen-orchestra/fs')
-const { Disposable, pFromCallback } = require('promise-toolbox')
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { getSyncedHandler } from '@xen-orchestra/fs'
+import { Disposable, pFromCallback } from 'promise-toolbox'
 
-const { isVhdAlias, resolveAlias } = require('./_resolveAlias')
-const { ALIAS_MAX_PATH_LENGTH } = require('./_constants')
+import { isVhdAlias, resolveAlias } from './_resolveAlias'
+import { ALIAS_MAX_PATH_LENGTH } from './_constants'
 
 let tempDir
 

packages/vhd-lib/src/_resolveAlias.js

@@ -1,12 +1,11 @@
-const { ALIAS_MAX_PATH_LENGTH } = require('./_constants')
-const resolveRelativeFromFile = require('./_resolveRelativeFromFile')
+import { ALIAS_MAX_PATH_LENGTH } from './_constants'
+import resolveRelativeFromFile from './_resolveRelativeFromFile'
 
-function isVhdAlias(filename) {
+export function isVhdAlias(filename) {
   return filename.endsWith('.alias.vhd')
 }
-exports.isVhdAlias = isVhdAlias
 
-exports.resolveAlias = async function resolveAlias(handler, filename) {
+export async function resolveAlias(handler, filename) {
   if (!isVhdAlias(filename)) {
     return filename
   }

packages/vhd-lib/src/_resolveRelativeFromFile.js

@@ -1,5 +1,5 @@
-const { dirname, resolve } = require('path')
+import { dirname, resolve } from 'path'
 
 const resolveRelativeFromFile = (file, path) => resolve('/', dirname(file), path).slice(1)
 
-module.exports = resolveRelativeFromFile
+export { resolveRelativeFromFile as default }

packages/vhd-lib/src/_structs.js

@@ -1,7 +1,7 @@
-const assert = require('assert')
-const fu = require('struct-fu')
+import assert from 'assert'
+import fu from 'struct-fu'
 
-const { FOOTER_SIZE, HEADER_SIZE, PARENT_LOCATOR_ENTRIES } = require('./_constants')
+import { FOOTER_SIZE, HEADER_SIZE, PARENT_LOCATOR_ENTRIES } from './_constants'
 
 const SIZE_OF_32_BITS = Math.pow(2, 32)
 
@@ -17,7 +17,7 @@ const uint64Undefinable = fu.derive(
   _ => (_[0] === 0xffffffff && _[1] === 0xffffffff ? undefined : _[0] * SIZE_OF_32_BITS + _[1])
 )
 
-const fuFooter = fu.struct([
+export const fuFooter = fu.struct([
   fu.char('cookie', 8), // 0
   fu.uint32('features'), // 8
   fu.uint32('fileFormatVersion'), // 12
@@ -40,10 +40,9 @@ const fuFooter = fu.struct([
   fu.char('hidden'), // 85 TODO: should probably be merged in reserved
   fu.char('reserved', 426), // 86
 ])
-exports.fuFooter = fuFooter
 assert.strictEqual(fuFooter.size, FOOTER_SIZE)
 
-const fuHeader = fu.struct([
+export const fuHeader = fu.struct([
   fu.char('cookie', 8),
   uint64Undefinable('dataOffset'),
   uint64('tableOffset'),
@@ -68,18 +67,15 @@ const fuHeader = fu.struct([
   ),
   fu.char('reserved2', 256),
 ])
-exports.fuHeader = fuHeader
-
 assert.strictEqual(fuHeader.size, HEADER_SIZE)
 
-const packField = (field, value, buf) => {
+export const packField = (field, value, buf) => {
   const { offset } = field
 
   field.pack(value, buf, typeof offset !== 'object' ? { bytes: offset, bits: 0 } : offset)
 }
-exports.packField = packField
 
-exports.unpackField = (field, buf) => {
+export const unpackField = (field, buf) => {
   const { offset } = field
 
   return field.unpack(buf, typeof offset !== 'object' ? { bytes: offset, bits: 0 } : offset)
@@ -87,7 +83,7 @@ exports.unpackField = (field, buf) => {
 
 // Returns the checksum of a raw struct.
 // The raw struct (footer or header) is altered with the new sum.
-exports.checksumStruct = function checksumStruct(buf, struct) {
+export function checksumStruct(buf, struct) {
   const checksumField = struct.fields.checksum
   let sum = 0
 

packages/vhd-lib/src/chain.js

@@ -1,10 +1,10 @@
-const { dirname, relative } = require('path')
+import { dirname, relative } from 'path'
 
-const { openVhd } = require('./openVhd')
-const { DISK_TYPES } = require('./_constants')
-const { Disposable } = require('promise-toolbox')
+import { openVhd } from './'
+import { DISK_TYPES } from './_constants'
+import { Disposable } from 'promise-toolbox'
 
-module.exports = async function chain(parentHandler, parentPath, childHandler, childPath, force = false) {
+export default async function chain(parentHandler, parentPath, childHandler, childPath, force = false) {
   await Disposable.use(
     [openVhd(parentHandler, parentPath), openVhd(childHandler, childPath)],
     async ([parentVhd, childVhd]) => {

packages/vhd-lib/src/checkChain.js

@@ -0,0 +1,14 @@
+import { openVhd } from '.'
+import resolveRelativeFromFile from './_resolveRelativeFromFile'
+import { DISK_TYPES } from './_constants'
+import { Disposable } from 'promise-toolbox'
+
+export default async function checkChain(handler, path) {
+  await Disposable.use(function* () {
+    let vhd
+    do {
+      vhd = yield openVhd(handler, path)
+      path = resolveRelativeFromFile(path, vhd.header.parentUnicodeName)
+    } while (vhd.footer.diskType !== DISK_TYPES.DYNAMIC)
+  })
+}

packages/vhd-lib/src/checkFooter.js

@@ -1,8 +1,8 @@
-const assert = require('assert')
+import assert from 'assert'
 
-const { DISK_TYPES, FILE_FORMAT_VERSION, FOOTER_COOKIE, FOOTER_SIZE } = require('./_constants')
+import { DISK_TYPES, FILE_FORMAT_VERSION, FOOTER_COOKIE, FOOTER_SIZE } from './_constants'
 
-module.exports = footer => {
+export default footer => {
   assert.strictEqual(footer.cookie, FOOTER_COOKIE)
   assert.strictEqual(footer.dataOffset, FOOTER_SIZE)
   assert.strictEqual(footer.fileFormatVersion, FILE_FORMAT_VERSION)

packages/vhd-lib/src/createReadableRawStream.integ.spec.js

@@ -1,15 +1,14 @@
 /* eslint-env jest */
-const execa = require('execa')
-const rimraf = require('rimraf')
-const tmp = require('tmp')
-const { createWriteStream, readFile } = require('fs-extra')
-const { fromEvent, pFromCallback } = require('promise-toolbox')
-const { pipeline } = require('readable-stream')
+import execa from 'execa'
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { createWriteStream, readFile } from 'fs-extra'
+import { fromEvent, pFromCallback } from 'promise-toolbox'
+import { pipeline } from 'readable-stream'
 
-const { createReadableRawStream } = require('./createReadableRawStream.js')
-const { createReadableSparseStream } = require('./createReadableSparseStream.js')
+import { createReadableRawStream, createReadableSparseStream } from './'
 
-const { checkFile, convertFromVhdToRaw } = require('./tests/utils')
+import { checkFile, convertFromVhdToRaw } from './tests/utils'
 
 let tempDir = null
 

packages/vhd-lib/src/createReadableRawStream.js

@@ -1,9 +1,9 @@
-const asyncIteratorToStream = require('async-iterator-to-stream')
+import asyncIteratorToStream from 'async-iterator-to-stream'
 
-const computeGeometryForSize = require('./_computeGeometryForSize')
-const { createFooter } = require('./_createFooterHeader')
+import computeGeometryForSize from './_computeGeometryForSize'
+import { createFooter } from './_createFooterHeader'
 
-module.exports = asyncIteratorToStream(async function* (size, blockParser) {
+export default asyncIteratorToStream(async function* (size, blockParser) {
   const geometry = computeGeometryForSize(size)
   const actualSize = geometry.actualSize
   const footer = createFooter(actualSize, Math.floor(Date.now() / 1000), geometry)

packages/vhd-lib/src/createReadableSparseStream.js

@@ -1,19 +1,19 @@
-const assert = require('assert')
-const asyncIteratorToStream = require('async-iterator-to-stream')
-const { forEachRight } = require('lodash')
+import assert from 'assert'
+import asyncIteratorToStream from 'async-iterator-to-stream'
+import { forEachRight } from 'lodash'
 
-const computeGeometryForSize = require('./_computeGeometryForSize')
-const { createFooter, createHeader } = require('./_createFooterHeader')
-const {
+import computeGeometryForSize from './_computeGeometryForSize'
+import { createFooter, createHeader } from './_createFooterHeader'
+import {
   BLOCK_UNUSED,
-  DEFAULT_BLOCK_SIZE: VHD_BLOCK_SIZE_BYTES,
+  DEFAULT_BLOCK_SIZE as VHD_BLOCK_SIZE_BYTES,
   DISK_TYPES,
   FOOTER_SIZE,
   HEADER_SIZE,
   SECTOR_SIZE,
-} = require('./_constants')
+} from './_constants'
 
-const { set: setBitmap } = require('./_bitmap')
+import { set as setBitmap } from './_bitmap'
 
 const VHD_BLOCK_SIZE_SECTORS = VHD_BLOCK_SIZE_BYTES / SECTOR_SIZE
 
@@ -55,12 +55,7 @@ function createBAT({ firstBlockPosition, fragmentLogicAddressList, fragmentSize,
  * @returns {Promise<Function>}
  */
 
-module.exports = async function createReadableStream(
-  diskSize,
-  fragmentSize,
-  fragmentLogicAddressList,
-  fragmentIterator
-) {
+export default async function createReadableStream(diskSize, fragmentSize, fragmentLogicAddressList, fragmentIterator) {
   const ratio = VHD_BLOCK_SIZE_BYTES / fragmentSize
   if (ratio % 1 !== 0) {
     throw new Error(

packages/vhd-lib/src/createVhdDirectoryFromStream.js

@@ -1,7 +1,10 @@
-const { BLOCK_UNUSED, FOOTER_SIZE, HEADER_SIZE, SECTOR_SIZE } = require('./_constants')
-const { readChunk } = require('@vates/read-chunk')
-const assert = require('assert')
-const { unpackFooter, unpackHeader, computeFullBlockSize } = require('./Vhd/_utils')
+import { VhdDirectory } from './'
+import { BLOCK_UNUSED, FOOTER_SIZE, HEADER_SIZE, SECTOR_SIZE } from './_constants'
+import { readChunk } from '@vates/read-chunk'
+import assert from 'assert'
+import { Disposable } from 'promise-toolbox'
+import { unpackFooter, unpackHeader, computeBlockBitmapSize } from './Vhd/_utils'
+import { asyncEach } from '@vates/async-each'
 
 const cappedBufferConcat = (buffers, maxSize) => {
   let buffer = Buffer.concat(buffers)
@@ -11,7 +14,7 @@ const cappedBufferConcat = (buffers, maxSize) => {
   return buffer
 }
 
-exports.parseVhdStream = async function* parseVhdStream(stream) {
+async function* parse(stream) {
   let bytesRead = 0
 
   // handle empty space between elements
@@ -40,9 +43,8 @@ exports.parseVhdStream = async function* parseVhdStream(stream) {
   const blockSize = header.blockSize
   assert.strictEqual(blockSize % SECTOR_SIZE, 0)
 
-  const fullBlockSize = computeFullBlockSize(blockSize)
-
-  const bitmapSize = fullBlockSize - blockSize
+  const blockBitmapSize = computeBlockBitmapSize(blockSize)
+  const blockAndBitmapSize = blockBitmapSize + blockSize
 
   const index = []
 
@@ -75,13 +77,8 @@ exports.parseVhdStream = async function* parseVhdStream(stream) {
   while (index.length > 0) {
     const item = index.shift()
     const buffer = await read(item.offset, item.size)
-    item.buffer = buffer
-
-    const { type } = item
-    if (type === 'bat') {
-      // found the BAT : read it and add block to index
-
-      let blockCount = 0
+    if (item.type === 'bat') {
+      // found the BAT : read it and ad block to index
       for (let blockCounter = 0; blockCounter < header.maxTableEntries; blockCounter++) {
         const batEntrySector = buffer.readUInt32BE(blockCounter * 4)
         // unallocated block, no need to export it
@@ -93,20 +90,15 @@ exports.parseVhdStream = async function* parseVhdStream(stream) {
             type: 'block',
             id: blockCounter,
             offset: batEntryBytes,
-            size: fullBlockSize,
+            size: blockAndBitmapSize,
           })
-          blockCount++
         }
       }
       // sort again index to ensure block and parent locator are in the right order
       index.sort((a, b) => a.offset - b.offset)
-      item.blockCount = blockCount
-    } else if (type === 'block') {
-      item.bitmap = buffer.slice(0, bitmapSize)
-      item.data = buffer.slice(bitmapSize)
+    } else {
+      yield { ...item, buffer }
     }
-
-    yield item
   }
 
   /**
@@ -132,3 +124,45 @@ function readLastSector(stream) {
     stream.on('error', reject)
   })
 }
+
+const buildVhd = Disposable.wrap(async function* (handler, path, inputStream, { concurrency }) {
+  const vhd = yield VhdDirectory.create(handler, path)
+  await asyncEach(
+    parse(inputStream),
+    async function (item) {
+      switch (item.type) {
+        case 'footer':
+          vhd.footer = item.footer
+          break
+        case 'header':
+          vhd.header = item.header
+          break
+        case 'parentLocator':
+          await vhd.writeParentLocator({ ...item, data: item.buffer })
+          break
+        case 'block':
+          await vhd.writeEntireBlock(item)
+          break
+        default:
+          throw new Error(`unhandled type of block generated by parser : ${item.type} while generating ${path}`)
+      }
+    },
+    {
+      concurrency,
+    }
+  )
+  await Promise.all([vhd.writeFooter(), vhd.writeHeader(), vhd.writeBlockAllocationTable()])
+})
+
+export async function createVhdDirectoryFromStream(handler, path, inputStream, { validator, concurrency = 16 } = {}) {
+  try {
+    await buildVhd(handler, path, inputStream, { concurrency })
+    if (validator !== undefined) {
+      await validator.call(this, path)
+    }
+  } catch (error) {
+    // cleanup on error
+    await handler.rmtree(path)
+    throw error
+  }
+}

packages/vhd-lib/src/createVhdStreamWithLength.integ.spec.js

@@ -1,17 +1,17 @@
 /* eslint-env jest */
 
-const execa = require('execa')
-const fs = require('fs-extra')
-const rimraf = require('rimraf')
-const getStream = require('get-stream')
-const tmp = require('tmp')
-const { createReadStream, createWriteStream } = require('fs')
-const { pFromCallback } = require('promise-toolbox')
-const { pipeline } = require('readable-stream')
+import execa from 'execa'
+import fs from 'fs-extra'
+import rimraf from 'rimraf'
+import getStream from 'get-stream'
+import tmp from 'tmp'
+import { createReadStream, createWriteStream } from 'fs'
+import { pFromCallback } from 'promise-toolbox'
+import { pipeline } from 'readable-stream'
 
-const { createVhdStreamWithLength } = require('./createVhdStreamWithLength.js')
-const { FOOTER_SIZE } = require('./_constants')
-const { createRandomFile, convertFromRawToVhd, convertFromVhdToRaw } = require('./tests/utils')
+import { createVhdStreamWithLength } from '.'
+import { FOOTER_SIZE } from './_constants'
+import { createRandomFile, convertFromRawToVhd, convertFromVhdToRaw } from './tests/utils'
 
 let tempDir = null
 

packages/vhd-lib/src/createVhdStreamWithLength.js

@@ -1,13 +1,13 @@
-const assert = require('assert')
-const { pipeline, Transform } = require('readable-stream')
-const { readChunk } = require('@vates/read-chunk')
+import assert from 'assert'
+import { pipeline, Transform } from 'readable-stream'
+import { readChunk } from '@vates/read-chunk'
 
-const checkFooter = require('./checkFooter')
-const checkHeader = require('./_checkHeader')
-const noop = require('./_noop')
-const getFirstAndLastBlocks = require('./_getFirstAndLastBlocks')
-const { FOOTER_SIZE, HEADER_SIZE, SECTOR_SIZE } = require('./_constants')
-const { fuFooter, fuHeader } = require('./_structs')
+import checkFooter from './checkFooter'
+import checkHeader from './_checkHeader'
+import noop from './_noop'
+import getFirstAndLastBlocks from './_getFirstAndLastBlocks'
+import { FOOTER_SIZE, HEADER_SIZE, SECTOR_SIZE } from './_constants'
+import { fuFooter, fuHeader } from './_structs'
 
 class EndCutterStream extends Transform {
   constructor(footerOffset, footerBuffer) {
@@ -35,7 +35,7 @@ class EndCutterStream extends Transform {
   }
 }
 
-module.exports = async function createVhdStreamWithLength(stream) {
+export default async function createVhdStreamWithLength(stream) {
   const readBuffers = []
   let streamPosition = 0
 

packages/vhd-lib/src/index.js

@@ -0,0 +1,15 @@
+export { default as chainVhd } from './chain'
+export { default as checkFooter } from './checkFooter'
+export { default as checkVhdChain } from './checkChain'
+export { default as createReadableRawStream } from './createReadableRawStream'
+export { default as createReadableSparseStream } from './createReadableSparseStream'
+export { default as createVhdStreamWithLength } from './createVhdStreamWithLength'
+export { createVhdDirectoryFromStream } from './createVhdDirectoryFromStream'
+export { default as mergeVhd } from './merge'
+export { default as peekFooterFromVhdStream } from './peekFooterFromVhdStream'
+export { openVhd } from './openVhd'
+export { VhdAbstract } from './Vhd/VhdAbstract'
+export { VhdDirectory } from './Vhd/VhdDirectory'
+export { VhdFile } from './Vhd/VhdFile'
+export { VhdSynthetic } from './Vhd/VhdSynthetic'
+export * as Constants from './_constants'

packages/vhd-lib/src/merge.integ.spec.js

@@ -1,14 +1,14 @@
 /* eslint-env jest */
 
-const fs = require('fs-extra')
-const rimraf = require('rimraf')
-const tmp = require('tmp')
-const { getHandler } = require('@xen-orchestra/fs')
-const { pFromCallback } = require('promise-toolbox')
+import fs from 'fs-extra'
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { getHandler } from '@xen-orchestra/fs'
+import { pFromCallback } from 'promise-toolbox'
 
-const { VhdFile, chainVhd, mergeVhd: vhdMerge } = require('./index')
+import { VhdFile, chainVhd, mergeVhd as vhdMerge } from './index'
 
-const { checkFile, createRandomFile, convertFromRawToVhd } = require('./tests/utils')
+import { checkFile, createRandomFile, convertFromRawToVhd } from './tests/utils'
 
 let tempDir = null
 

packages/vhd-lib/src/merge.js

@@ -1,21 +1,21 @@
 // TODO: remove once completely merged in vhd.js
 
-const assert = require('assert')
-const noop = require('./_noop')
-const { createLogger } = require('@xen-orchestra/log')
-const { limitConcurrency } = require('limit-concurrency-decorator')
+import assert from 'assert'
+import noop from './_noop'
+import { createLogger } from '@xen-orchestra/log'
+import { limitConcurrency } from 'limit-concurrency-decorator'
 
-const { openVhd } = require('./openVhd')
-const { basename, dirname } = require('path')
-const { DISK_TYPES } = require('./_constants')
-const { Disposable } = require('promise-toolbox')
+import { openVhd } from '.'
+import { basename, dirname } from 'path'
+import { DISK_TYPES } from './_constants'
+import { Disposable } from 'promise-toolbox'
 
 const { warn } = createLogger('vhd-lib:merge')
 
 // Merge vhd child into vhd parent.
 //
 // TODO: rename the VHD file during the merge
-module.exports = limitConcurrency(2)(async function merge(
+export default limitConcurrency(2)(async function merge(
   parentHandler,
   parentPath,
   childHandler,
@@ -25,23 +25,12 @@ module.exports = limitConcurrency(2)(async function merge(
   const mergeStatePath = dirname(parentPath) + '/' + '.' + basename(parentPath) + '.merge.json'
 
   return await Disposable.use(async function* () {
-    let mergeState = await parentHandler
-      .readFile(mergeStatePath)
-      .then(content => {
-        const state = JSON.parse(content)
-
-        // ensure the correct merge will be continued
-        assert.strictEqual(parentVhd.header.checksum, state.parent.header)
-        assert.strictEqual(childVhd.header.checksum, state.child.header)
-
-        return state
-      })
-      .catch(error => {
-        if (error.code !== 'ENOENT') {
-          warn('problem while checking the merge state', { error })
-        }
-      })
-
+    let mergeState = await parentHandler.readFile(mergeStatePath).catch(error => {
+      if (error.code !== 'ENOENT') {
+        throw error
+      }
+      // no merge state in case of missing file
+    })
     // during merging, the end footer of the parent can be overwritten by new blocks
     // we should use it as a way to check vhd health
     const parentVhd = yield openVhd(parentHandler, parentPath, {
@@ -49,7 +38,13 @@ module.exports = limitConcurrency(2)(async function merge(
       checkSecondFooter: mergeState === undefined,
     })
     const childVhd = yield openVhd(childHandler, childPath)
-    if (mergeState === undefined) {
+    if (mergeState !== undefined) {
+      mergeState = JSON.parse(mergeState)
+
+      // ensure the correct merge will be continued
+      assert.strictEqual(parentVhd.header.checksum, mergeState.parent.header)
+      assert.strictEqual(childVhd.header.checksum, mergeState.child.header)
+    } else {
       assert.strictEqual(childVhd.header.blockSize, parentVhd.header.blockSize)
 
       const parentDiskType = parentVhd.footer.diskType

packages/vhd-lib/src/openVhd.integ.spec.js

@@ -1,14 +1,14 @@
 /* eslint-env jest */
 
-const rimraf = require('rimraf')
-const tmp = require('tmp')
-const { getSyncedHandler } = require('@xen-orchestra/fs')
-const { Disposable, pFromCallback } = require('promise-toolbox')
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { getSyncedHandler } from '@xen-orchestra/fs'
+import { Disposable, pFromCallback } from 'promise-toolbox'
 
-const { openVhd } = require('./index')
-const { createRandomFile, convertFromRawToVhd, createRandomVhdDirectory } = require('./tests/utils')
+import { openVhd } from './index'
+import { createRandomFile, convertFromRawToVhd, createRandomVhdDirectory } from './tests/utils'
 
-const { VhdAbstract } = require('./Vhd/VhdAbstract')
+import { VhdAbstract } from './Vhd/VhdAbstract'
 
 let tempDir
 

packages/vhd-lib/src/openVhd.js

@@ -1,8 +1,7 @@
-const { resolveAlias } = require('./_resolveAlias')
-const { VhdDirectory } = require('./Vhd/VhdDirectory.js')
-const { VhdFile } = require('./Vhd/VhdFile.js')
+import { resolveAlias } from './_resolveAlias'
+import { VhdFile, VhdDirectory } from './'
 
-exports.openVhd = async function openVhd(handler, path, opts) {
+export async function openVhd(handler, path, opts) {
   const resolved = await resolveAlias(handler, path)
   try {
     return await VhdFile.open(handler, resolved, opts)

packages/vhd-lib/src/peekFooterFromVhdStream.js

@@ -0,0 +1,11 @@
+import { readChunk } from '@vates/read-chunk'
+
+import { FOOTER_SIZE } from './_constants'
+import { fuFooter } from './_structs'
+
+export default async function peekFooterFromStream(stream) {
+  const footerBuffer = await readChunk(stream, FOOTER_SIZE)
+  const footer = fuFooter.unpack(footerBuffer)
+  stream.unshift(footerBuffer)
+  return footer
+}

packages/vhd-lib/src/tests/utils.js

@@ -1,9 +1,9 @@
-const { pFromCallback } = require('promise-toolbox')
-const { pipeline } = require('readable-stream')
-const asyncIteratorToStream = require('async-iterator-to-stream')
-const execa = require('execa')
-const fs = require('fs-extra')
-const { randomBytes } = require('crypto')
+import { pFromCallback } from 'promise-toolbox'
+import { pipeline } from 'readable-stream'
+import asyncIteratorToStream from 'async-iterator-to-stream'
+import execa from 'execa'
+import fs from 'fs-extra'
+import { randomBytes } from 'crypto'
 
 const createRandomStream = asyncIteratorToStream(function* (size) {
   while (size > 0) {
@@ -12,16 +12,14 @@ const createRandomStream = asyncIteratorToStream(function* (size) {
   }
 })
 
-async function createRandomFile(name, sizeMB) {
+export async function createRandomFile(name, sizeMB) {
   const input = createRandomStream(sizeMB * 1024 * 1024)
   await pFromCallback(cb => pipeline(input, fs.createWriteStream(name), cb))
 }
-exports.createRandomFile = createRandomFile
 
-async function checkFile(vhdName) {
+export async function checkFile(vhdName) {
   await execa('vhd-util', ['check', '-p', '-b', '-t', '-n', vhdName])
 }
-exports.checkFile = checkFile
 
 const RAW = 'raw'
 const VHD = 'vpc'
@@ -31,21 +29,19 @@ async function convert(inputFormat, inputFile, outputFormat, outputFile) {
   await execa('qemu-img', ['convert', `-f${inputFormat}`, '-O', outputFormat, inputFile, outputFile])
 }
 
-async function convertFromRawToVhd(rawName, vhdName) {
+export async function convertFromRawToVhd(rawName, vhdName) {
   await convert(RAW, rawName, VHD, vhdName)
 }
-exports.convertFromRawToVhd = convertFromRawToVhd
 
-async function convertFromVhdToRaw(vhdName, rawName) {
+export async function convertFromVhdToRaw(vhdName, rawName) {
   await convert(VHD, vhdName, RAW, rawName)
 }
-exports.convertFromVhdToRaw = convertFromVhdToRaw
 
-exports.convertFromVmdkToRaw = async function convertFromVmdkToRaw(vmdkName, rawName) {
+export async function convertFromVmdkToRaw(vmdkName, rawName) {
   await convert(VMDK, vmdkName, RAW, rawName)
 }
 
-exports.recoverRawContent = async function recoverRawContent(vhdName, rawName, originalSize) {
+export async function recoverRawContent(vhdName, rawName, originalSize) {
   // todo should use createContentStream
   await checkFile(vhdName)
   await convertFromVhdToRaw(vhdName, rawName)
@@ -55,7 +51,7 @@ exports.recoverRawContent = async function recoverRawContent(vhdName, rawName, o
 }
 
 // @ todo how can I call vhd-cli copy from here
-async function convertToVhdDirectory(rawFileName, vhdFileName, path) {
+export async function convertToVhdDirectory(rawFileName, vhdFileName, path) {
   fs.mkdirp(path)
 
   const srcVhd = await fs.open(vhdFileName, 'r')
@@ -91,9 +87,8 @@ async function convertToVhdDirectory(rawFileName, vhdFileName, path) {
   }
   await fs.close(srcRaw)
 }
-exports.convertToVhdDirectory = convertToVhdDirectory
 
-exports.createRandomVhdDirectory = async function createRandomVhdDirectory(path, sizeMB) {
+export async function createRandomVhdDirectory(path, sizeMB) {
   fs.mkdirp(path)
   const rawFileName = `${path}/temp.raw`
   await createRandomFile(rawFileName, sizeMB)

packages/xen-api/examples/package.json

@@ -8,6 +8,6 @@
     "promise-toolbox": "^0.19.2",
     "readable-stream": "^3.1.1",
     "throttle": "^1.0.3",
-    "vhd-lib": "^3.0.0"
+    "vhd-lib": "^2.0.3"
   }
 }

packages/xen-api/package.json

@@ -31,8 +31,7 @@
     "node": ">=7.6"
   },
   "dependencies": {
-    "@vates/coalesce-calls": "^0.1.0",
-    "bind-property-descriptor": "^2.0.0",
+    "bind-property-descriptor": "^1.0.0",
     "blocked": "^1.2.1",
     "debug": "^4.0.1",
     "http-request-plus": "^0.13.0",

packages/xen-api/src/_coalesceCalls.js

@@ -0,0 +1,15 @@
+// decorates fn so that more than one concurrent calls will be coalesced
+export default function coalesceCalls(fn) {
+  let promise
+  const clean = () => {
+    promise = undefined
+  }
+  return function () {
+    if (promise !== undefined) {
+      return promise
+    }
+    promise = fn.apply(this, arguments)
+    promise.then(clean, clean)
+    return promise
+  }
+}

packages/xen-api/src/_coalesceCalls.spec.js

@@ -0,0 +1,26 @@
+/* eslint-env jest */
+
+import pDefer from 'promise-toolbox/defer'
+
+import coalesceCalls from './_coalesceCalls'
+
+describe('coalesceCalls', () => {
+  it('decorates an async function', async () => {
+    const fn = coalesceCalls(promise => promise)
+
+    const defer1 = pDefer()
+    const promise1 = fn(defer1.promise)
+    const defer2 = pDefer()
+    const promise2 = fn(defer2.promise)
+
+    defer1.resolve('foo')
+    expect(await promise1).toBe('foo')
+    expect(await promise2).toBe('foo')
+
+    const defer3 = pDefer()
+    const promise3 = fn(defer3.promise)
+
+    defer3.resolve('bar')
+    expect(await promise3).toBe('bar')
+  })
+})

packages/xen-api/src/index.js

@@ -3,7 +3,6 @@ import dns from 'dns'
 import kindOf from 'kindof'
 import ms from 'ms'
 import httpRequest from 'http-request-plus'
-import { coalesceCalls } from '@vates/coalesce-calls'
 import { Collection } from 'xo-collection'
 import { EventEmitter } from 'events'
 import { map, noop, omit } from 'lodash'
@@ -11,6 +10,7 @@ import { cancelable, defer, fromCallback, fromEvents, ignoreErrors, pDelay, pRet
 import { limitConcurrency } from 'limit-concurrency-decorator'
 
 import autoTransport from './transports/auto'
+import coalesceCalls from './_coalesceCalls'
 import debug from './_debug'
 import getTaskResult from './_getTaskResult'
 import isGetAllRecordsMethod from './_isGetAllRecordsMethod'

packages/xo-server-auth-saml/package.json

@@ -26,10 +26,10 @@
   "preferGlobal": false,
   "main": "dist/",
   "engines": {
-    "node": ">=12"
+    "node": ">=6"
   },
   "dependencies": {
-    "passport-saml": "^3.2.0"
+    "passport-saml": "^2.0.2"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/xo-server-test/package.json

@@ -26,9 +26,9 @@
     "@babel/plugin-proposal-decorators": "^7.4.0",
     "@babel/preset-env": "^7.1.6",
     "@iarna/toml": "^2.2.1",
-    "@vates/decorate-with": "^1.0.0",
+    "@vates/decorate-with": "^0.1.0",
     "@vates/parse-duration": "^0.1.1",
-    "app-conf": "^1.0.0",
+    "app-conf": "^0.9.0",
     "babel-plugin-lodash": "^3.2.11",
     "golike-defer": "^0.5.1",
     "jest": "^27.3.1",

packages/xo-server/config.toml

@@ -87,8 +87,6 @@ snapshotNameLabelTpl = '[XO Backup {job.name}] {vm.name_label}'
 # Delay for which backups listing on a remote is cached
 listingDebounce = '1 min'
 
-vhdDirectoryCompression = 'brotli'
-
 [backups.defaultSettings]
 reportWhen = 'failure'
 

packages/xo-server/package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "xo-server",
-  "version": "5.86.3",
+  "version": "5.84.3",
   "license": "AGPL-3.0-or-later",
   "description": "Server part of Xen-Orchestra",
   "keywords": [
@@ -29,28 +29,28 @@
   "dependencies": {
     "@iarna/toml": "^2.2.1",
     "@vates/compose": "^2.1.0",
-    "@vates/decorate-with": "^1.0.0",
+    "@vates/decorate-with": "^0.1.0",
     "@vates/disposable": "^0.1.1",
     "@vates/multi-key-map": "^0.1.0",
     "@vates/parse-duration": "^0.1.1",
     "@vates/read-chunk": "^0.1.2",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.18.3",
+    "@xen-orchestra/backups": "^0.16.2",
     "@xen-orchestra/cron": "^1.0.6",
     "@xen-orchestra/defined": "^0.0.1",
     "@xen-orchestra/emit-async": "^0.1.0",
-    "@xen-orchestra/fs": "^0.19.3",
+    "@xen-orchestra/fs": "^0.19.2",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/mixin": "^0.1.0",
-    "@xen-orchestra/mixins": "^0.1.2",
+    "@xen-orchestra/mixins": "^0.1.1",
     "@xen-orchestra/self-signed": "^0.1.0",
     "@xen-orchestra/template": "^0.1.0",
-    "@xen-orchestra/xapi": "^0.8.5",
+    "@xen-orchestra/xapi": "^0.8.4",
     "ajv": "^8.0.3",
-    "app-conf": "^1.0.0",
+    "app-conf": "^0.9.0",
     "async-iterator-to-stream": "^1.0.1",
     "base64url": "^3.0.0",
-    "bind-property-descriptor": "^2.0.0",
+    "bind-property-descriptor": "^1.0.0",
     "blocked-at": "^1.2.0",
     "bluebird": "^3.5.1",
     "body-parser": "^1.18.2",
@@ -60,14 +60,14 @@
     "content-type": "^1.0.4",
     "cookie": "^0.4.0",
     "cookie-parser": "^1.4.3",
-    "d3-time-format": "^4.1.0",
+    "d3-time-format": "^3.0.0",
     "decorator-synchronized": "^0.6.0",
     "deptree": "^1.0.0",
     "exec-promise": "^0.7.0",
-    "execa": "^6.0.0",
+    "execa": "^5.0.0",
     "express": "^4.16.2",
     "express-session": "^1.15.6",
-    "fast-xml-parser": "^4.0.0",
+    "fast-xml-parser": "^3.17.4",
     "fatfs": "^0.10.4",
     "fs-extra": "^10.0.0",
     "get-stream": "^6.0.0",
@@ -117,21 +117,21 @@
     "source-map-support": "^0.5.16",
     "split2": "^4.1.0",
     "stoppable": "^1.0.5",
-    "subleveldown": "^6.0.1",
+    "subleveldown": "^5.0.1",
     "tar-stream": "^2.0.1",
     "tmp": "^0.2.1",
     "unzipper": "^0.10.5",
     "uuid": "^8.3.1",
     "value-matcher": "^0.2.0",
-    "vhd-lib": "^3.0.0",
+    "vhd-lib": "^2.0.3",
     "ws": "^8.2.3",
-    "xdg-basedir": "^5.1.0",
+    "xdg-basedir": "^4.0.0",
     "xen-api": "^0.35.1",
     "xo-acl-resolver": "^0.4.1",
     "xo-collection": "^0.5.0",
     "xo-common": "^0.7.0",
     "xo-remote-parser": "^0.8.0",
-    "xo-vmdk-to-vhd": "^2.0.3"
+    "xo-vmdk-to-vhd": "^2.0.1"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/xo-server/src/api/backup-ng.mjs

@@ -221,19 +221,6 @@ deleteVmBackup.params = {
   },
 }
 
-export function deleteVmBackups({ ids }) {
-  return this.deleteVmBackupsNg(ids)
-}
-
-deleteVmBackups.permission = 'admin'
-
-deleteVmBackups.params = {
-  ids: {
-    type: 'array',
-    items: { type: 'string' },
-  },
-}
-
 export function listVmBackups({ remotes, _forceRefresh }) {
   return this.listVmBackupsNg(remotes, _forceRefresh)
 }

packages/xo-server/src/api/xosan.mjs

@@ -1,6 +1,6 @@
 import assert from 'assert'
 import asyncMapSettled from '@xen-orchestra/async-map/legacy.js'
-import { execa } from 'execa'
+import execa from 'execa'
 import filter from 'lodash/filter.js'
 import find from 'lodash/find.js'
 import fs from 'fs-extra'

packages/xo-server/src/index.mjs

@@ -22,7 +22,7 @@ import serveStatic from 'serve-static'
 import stoppable from 'stoppable'
 import WebServer from 'http-server-plus'
 import WebSocket, { WebSocketServer } from 'ws'
-import { xdgConfig } from 'xdg-basedir'
+import xdg from 'xdg-basedir'
 import { createLogger } from '@xen-orchestra/log'
 import { createRequire } from 'module'
 import { genSelfSignedCert } from '@xen-orchestra/self-signed'
@@ -99,7 +99,7 @@ async function loadConfiguration() {
   return config
 }
 
-const LOCAL_CONFIG_FILE = `${xdgConfig}/${APP_NAME}/config.z-auto.json`
+const LOCAL_CONFIG_FILE = `${xdg.config}/${APP_NAME}/config.z-auto.json`
 async function updateLocalConfig(diff) {
   // TODO lock file
   const localConfig = await fse.readFile(LOCAL_CONFIG_FILE).then(JSON.parse, () => ({}))
@@ -733,6 +733,27 @@ export default async function main(args) {
     log.warn('Failed to change user/group:', { error })
   }
 
+  const safeMode = includes(args, '--safe-mode')
+
+  // Creates main object.
+  const xo = new Xo({
+    appDir: APP_DIR,
+    appName: APP_NAME,
+    appVersion: APP_VERSION,
+    config,
+    httpServer: webServer,
+    safeMode,
+  })
+
+  // Register web server close on XO stop.
+  xo.hooks.on('stop', () => fromCallback.call(webServer, 'stop'))
+
+  // Connects to all registered servers.
+  await xo.hooks.start()
+
+  // Trigger a clean job.
+  await xo.hooks.clean()
+
   // Express is used to manage non WebSocket connections.
   const express = await createExpressApp(config)
 
@@ -758,32 +779,6 @@ export default async function main(args) {
     }
   }
 
-  // Attaches express to the web server.
-  webServer.on('request', express)
-  webServer.on('upgrade', (req, socket, head) => {
-    express.emit('upgrade', req, socket, head)
-  })
-
-  const safeMode = includes(args, '--safe-mode')
-
-  // Creates main object.
-  const xo = new Xo({
-    appDir: APP_DIR,
-    appName: APP_NAME,
-    appVersion: APP_VERSION,
-    config,
-    safeMode,
-  })
-
-  // Register web server close on XO stop.
-  xo.hooks.on('stop', () => fromCallback.call(webServer, 'stop'))
-
-  // Connects to all registered servers.
-  await xo.hooks.start()
-
-  // Trigger a clean job.
-  await xo.hooks.clean()
-
   // Must be set up before the API.
   setUpConsoleProxy(webServer, xo)
 
@@ -794,6 +789,12 @@ export default async function main(args) {
   // to work properly.
   await setUpPassport(express, xo, config)
 
+  // Attaches express to the web server.
+  webServer.on('request', express)
+  webServer.on('upgrade', (req, socket, head) => {
+    express.emit('upgrade', req, socket, head)
+  })
+
   // Must be set up before the static files.
   setUpApi(webServer, xo, config)