feat(xo-web-6): initial config

- more efficient than creating a function at each call
- better logging

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,33 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: 'status: triaging :triangular_flag_on_post:, type: bug :bug:'
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - Node: [e.g. 16.12.1]
+ - xo-server: [e.g. 5.82.3]
+ - xo-web: [e.g. 5.87.0]
+ - hypervisor: [e.g. XCP-ng 8.2.0]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

@vates/disposable/package.json

@@ -24,7 +24,7 @@
   "dependencies": {
     "@vates/multi-key-map": "^0.1.0",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/log": "^0.2.1",
+    "@xen-orchestra/log": "^0.3.0",
     "ensure-array": "^1.0.0"
   }
 }

@xen-orchestra/audit-core/package.json

@@ -31,7 +31,7 @@
   },
   "dependencies": {
     "@vates/decorate-with": "^0.1.0",
-    "@xen-orchestra/log": "^0.2.1",
+    "@xen-orchestra/log": "^0.3.0",
     "golike-defer": "^0.5.1",
     "object-hash": "^2.0.1"
   },

@xen-orchestra/audit-core/src/specification.ts

@@ -17,10 +17,10 @@ interface Record {
 }
 
 export class AuditCore {
-  constructor(storage: Storage) { }
-  public add(subject: any, event: string, data: any): Promise<Record> { }
-  public checkIntegrity(oldest: string, newest: string): Promise<number> { }
-  public getFrom(newest?: string): AsyncIterator { }
-  public deleteFrom(newest: string): Promise<void> { }
-  public deleteRangeAndRewrite(newest: string, oldest: string): Promise<void> { }
+  constructor(storage: Storage) {}
+  public add(subject: any, event: string, data: any): Promise<Record> {}
+  public checkIntegrity(oldest: string, newest: string): Promise<number> {}
+  public getFrom(newest?: string): AsyncIterator {}
+  public deleteFrom(newest: string): Promise<void> {}
+  public deleteRangeAndRewrite(newest: string, oldest: string): Promise<void> {}
 }

@xen-orchestra/babel-config/index.js

@@ -46,7 +46,7 @@ module.exports = function (pkg, configs = {}) {
 
   return {
     comments: !__PROD__,
-    ignore: __PROD__ ? [/\.spec\.js$/] : undefined,
+    ignore: __PROD__ ? [/\btests?\//, /\.spec\.js$/] : undefined,
     plugins: Object.keys(plugins)
       .map(plugin => [plugin, plugins[plugin]])
       .sort(([a], [b]) => {

@xen-orchestra/backups-cli/commands/clean-vms.js

@@ -10,12 +10,13 @@ const { resolve } = require('path')
 const adapter = new RemoteAdapter(require('@xen-orchestra/fs').getHandler({ url: 'file://' }))
 
 module.exports = async function main(args) {
-  const { _, remove, merge } = getopts(args, {
+  const { _, fix, remove, merge } = getopts(args, {
     alias: {
+      fix: 'f',
       remove: 'r',
       merge: 'm',
     },
-    boolean: ['merge', 'remove'],
+    boolean: ['fix', 'merge', 'remove'],
     default: {
       merge: false,
       remove: false,
@@ -25,7 +26,7 @@ module.exports = async function main(args) {
   await asyncMap(_, async vmDir => {
     vmDir = resolve(vmDir)
     try {
-      await adapter.cleanVm(vmDir, { remove, merge, onLog: log => console.warn(log) })
+      await adapter.cleanVm(vmDir, { fixMetadata: fix, remove, merge, onLog: (...args) => console.warn(...args) })
     } catch (error) {
       console.error('adapter.cleanVm', vmDir, error)
     }

@xen-orchestra/backups-cli/index.js

@@ -5,11 +5,12 @@ require('./_composeCommands')({
     get main() {
       return require('./commands/clean-vms')
     },
-    usage: `[--merge] [--remove] xo-vm-backups/*
+    usage: `[--fix] [--merge] [--remove] xo-vm-backups/*
 
       Detects and repair issues with VM backups.
 
       Options:
+        -f, --fix       Fix metadata issues (like size)
         -m, --merge     Merge (or continue merging) VHD files that are unused
         -r, --remove    Remove unused, incomplete, orphan, or corrupted files
 `,

@xen-orchestra/backups-cli/package.json

@@ -7,12 +7,12 @@
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.11.0",
-    "@xen-orchestra/fs": "^0.17.0",
+    "@xen-orchestra/backups": "^0.15.1",
+    "@xen-orchestra/fs": "^0.18.0",
     "filenamify": "^4.1.0",
     "getopts": "^2.2.5",
     "lodash": "^4.17.15",
-    "promise-toolbox": "^0.19.2"
+    "promise-toolbox": "^0.20.0"
   },
   "engines": {
     "node": ">=7.10.1"

@xen-orchestra/backups/RemoteAdapter.js

@@ -6,7 +6,7 @@ const pDefer = require('promise-toolbox/defer.js')
 const pump = require('pump')
 const { basename, dirname, join, normalize, resolve } = require('path')
 const { createLogger } = require('@xen-orchestra/log')
-const { createSyntheticStream, mergeVhd, default: Vhd } = require('vhd-lib')
+const { createSyntheticStream, mergeVhd, VhdFile } = require('vhd-lib')
 const { deduped } = require('@vates/disposable/deduped.js')
 const { execFile } = require('child_process')
 const { readdir, stat } = require('fs-extra')
@@ -86,7 +86,7 @@ class RemoteAdapter {
       }),
       async path => {
         try {
-          const vhd = new Vhd(handler, path)
+          const vhd = new VhdFile(handler, path)
           await vhd.readHeaderAndFooter()
           return {
             footer: vhd.footer,
@@ -253,16 +253,9 @@ class RemoteAdapter {
 
   async deleteDeltaVmBackups(backups) {
     const handler = this._handler
-    let mergedDataSize = 0
-    await asyncMapSettled(backups, ({ _filename, vhds }) =>
-      Promise.all([
-        handler.unlink(_filename),
-        asyncMap(Object.values(vhds), async _ => {
-          mergedDataSize += await this._deleteVhd(resolveRelativeFromFile(_filename, _))
-        }),
-      ])
-    )
-    return mergedDataSize
+
+    // unused VHDs will be detected by `cleanVm`
+    await asyncMapSettled(backups, ({ _filename }) => handler.unlink(_filename))
   }
 
   async deleteMetadataBackup(backupId) {

@xen-orchestra/backups/_VmBackup.js

@@ -1,9 +1,10 @@
 const assert = require('assert')
 const findLast = require('lodash/findLast.js')
+const groupBy = require('lodash/groupBy.js')
 const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
 const keyBy = require('lodash/keyBy.js')
 const mapValues = require('lodash/mapValues.js')
-const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
+const { asyncMap } = require('@xen-orchestra/async-map')
 const { createLogger } = require('@xen-orchestra/log')
 const { defer } = require('golike-defer')
 const { formatDateTime } = require('@xen-orchestra/xapi')
@@ -103,9 +104,21 @@ exports.VmBackup = class VmBackup {
   // calls fn for each function, warns of any errors, and throws only if there are no writers left
   async _callWriters(fn, warnMessage, parallel = true) {
     const writers = this._writers
-    if (writers.size === 0) {
+    const n = writers.size
+    if (n === 0) {
       return
     }
+    if (n === 1) {
+      const [writer] = writers
+      try {
+        await fn(writer)
+      } catch (error) {
+        writers.delete(writer)
+        throw error
+      }
+      return
+    }
+
     await (parallel ? asyncMap : asyncEach)(writers, async function (writer) {
       try {
         await fn(writer)
@@ -272,17 +285,28 @@ exports.VmBackup = class VmBackup {
   }
 
   async _removeUnusedSnapshots() {
-    // TODO: handle all schedules (no longer existing schedules default to 0 retention)
-
-    const { scheduleId } = this
-    const scheduleSnapshots = this._jobSnapshots.filter(_ => _.other_config['xo:backup:schedule'] === scheduleId)
-
+    const jobSettings = this.job.settings
     const baseVmRef = this._baseVm?.$ref
+    const { config } = this
+    const baseSettings = {
+      ...config.defaultSettings,
+      ...config.metadata.defaultSettings,
+      ...jobSettings[''],
+    }
+
+    const snapshotsPerSchedule = groupBy(this._jobSnapshots, _ => _.other_config['xo:backup:schedule'])
     const xapi = this._xapi
-    await asyncMap(getOldEntries(this._settings.snapshotRetention, scheduleSnapshots), ({ $ref }) => {
-      if ($ref !== baseVmRef) {
-        return xapi.VM_destroy($ref)
+    await asyncMap(Object.entries(snapshotsPerSchedule), ([scheduleId, snapshots]) => {
+      const settings = {
+        ...baseSettings,
+        ...jobSettings[scheduleId],
+        ...jobSettings[this.vm.uuid],
       }
+      return asyncMap(getOldEntries(settings.snapshotRetention, snapshots), ({ $ref }) => {
+        if ($ref !== baseVmRef) {
+          return xapi.VM_destroy($ref)
+        }
+      })
     })
   }
 
@@ -291,12 +315,14 @@ exports.VmBackup = class VmBackup {
 
     let baseVm = findLast(this._jobSnapshots, _ => 'xo:backup:exported' in _.other_config)
     if (baseVm === undefined) {
+      debug('no base VM found')
       return
     }
 
     const fullInterval = this._settings.fullInterval
     const deltaChainLength = +(baseVm.other_config['xo:backup:deltaChainLength'] ?? 0) + 1
     if (!(fullInterval === 0 || fullInterval > deltaChainLength)) {
+      debug('not using base VM becaust fullInterval reached')
       return
     }
 
@@ -311,6 +337,10 @@ exports.VmBackup = class VmBackup {
       const srcVdi = srcVdis[snapshotOf]
       if (srcVdi !== undefined) {
         baseUuidToSrcVdi.set(await xapi.getField('VDI', baseRef, 'uuid'), srcVdi)
+      } else {
+        debug('no base VDI found', {
+          vdi: srcVdi.uuid,
+        })
       }
     })
 
@@ -323,7 +353,16 @@ exports.VmBackup = class VmBackup {
 
     const fullVdisRequired = new Set()
     baseUuidToSrcVdi.forEach((srcVdi, baseUuid) => {
-      if (!presentBaseVdis.has(baseUuid)) {
+      if (presentBaseVdis.has(baseUuid)) {
+        debug('found base VDI', {
+          base: baseUuid,
+          vdi: srcVdi.uuid,
+        })
+      } else {
+        debug('missing base VDI', {
+          base: baseUuid,
+          vdi: srcVdi.uuid,
+        })
         fullVdisRequired.add(srcVdi.uuid)
       }
     })

@xen-orchestra/backups/_cleanVm.js

@@ -1,16 +1,19 @@
 const assert = require('assert')
+const sum = require('lodash/sum')
 const { asyncMap } = require('@xen-orchestra/async-map')
-const { default: Vhd, mergeVhd } = require('vhd-lib')
+const { VhdFile, mergeVhd } = require('vhd-lib')
 const { dirname, resolve } = require('path')
 const { DISK_TYPE_DIFFERENCING } = require('vhd-lib/dist/_constants.js')
 const { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } = require('./_backupType.js')
 const { limitConcurrency } = require('limit-concurrency-decorator')
 
+const { Task } = require('./Task.js')
+
 // chain is an array of VHDs from child to parent
 //
 // the whole chain will be merged into parent, parent will be renamed to child
 // and all the others will deleted
-const mergeVhdChain = limitConcurrency(1)(async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
+async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
   assert(chain.length >= 2)
 
   let child = chain[0]
@@ -43,7 +46,7 @@ const mergeVhdChain = limitConcurrency(1)(async function mergeVhdChain(chain, {
       }
     }, 10e3)
 
-    await mergeVhd(
+    const mergedSize = await mergeVhd(
       handler,
       parent,
       handler,
@@ -71,8 +74,10 @@ const mergeVhdChain = limitConcurrency(1)(async function mergeVhdChain(chain, {
         }
       }),
     ])
+
+    return mergedSize
   }
-})
+}
 
 const noop = Function.prototype
 
@@ -113,7 +118,14 @@ const listVhds = async (handler, vmDir) => {
   return { vhds, interruptedVhds }
 }
 
-exports.cleanVm = async function cleanVm(vmDir, { remove, merge, onLog = noop }) {
+const defaultMergeLimiter = limitConcurrency(1)
+
+exports.cleanVm = async function cleanVm(
+  vmDir,
+  { fixMetadata, remove, merge, mergeLimiter = defaultMergeLimiter, onLog = noop }
+) {
+  const limitedMergeVhdChain = mergeLimiter(mergeVhdChain)
+
   const handler = this._handler
 
   const vhds = new Set()
@@ -125,7 +137,7 @@ exports.cleanVm = async function cleanVm(vmDir, { remove, merge, onLog = noop })
   // remove broken VHDs
   await asyncMap(vhdsList.vhds, async path => {
     try {
-      const vhd = new Vhd(handler, path)
+      const vhd = new VhdFile(handler, path)
       await vhd.readHeaderAndFooter(!vhdsList.interruptedVhds.has(path))
       vhds.add(path)
       if (vhd.footer.diskType === DISK_TYPE_DIFFERENCING) {
@@ -219,11 +231,16 @@ exports.cleanVm = async function cleanVm(vmDir, { remove, merge, onLog = noop })
   await asyncMap(jsons, async json => {
     const metadata = JSON.parse(await handler.readFile(json))
     const { mode } = metadata
+    let size
     if (mode === 'full') {
       const linkedXva = resolve('/', vmDir, metadata.xva)
 
       if (xvas.has(linkedXva)) {
         unusedXvas.delete(linkedXva)
+
+        size = await handler.getSize(linkedXva).catch(error => {
+          onLog(`failed to get size of ${json}`, { error })
+        })
       } else {
         onLog(`the XVA linked to the metadata ${json} is missing`)
         if (remove) {
@@ -241,6 +258,10 @@ exports.cleanVm = async function cleanVm(vmDir, { remove, merge, onLog = noop })
       // possible (existing disks) even if one disk is missing
       if (linkedVhds.every(_ => vhds.has(_))) {
         linkedVhds.forEach(_ => unusedVhds.delete(_))
+
+        size = await asyncMap(linkedVhds, vhd => handler.getSize(vhd)).then(sum, error => {
+          onLog(`failed to get size of ${json}`, { error })
+        })
       } else {
         onLog(`Some VHDs linked to the metadata ${json} are missing`)
         if (remove) {
@@ -249,10 +270,27 @@ exports.cleanVm = async function cleanVm(vmDir, { remove, merge, onLog = noop })
         }
       }
     }
+
+    const metadataSize = metadata.size
+    if (size !== undefined && metadataSize !== size) {
+      onLog(`incorrect size in metadata: ${metadataSize ?? 'none'} instead of ${size}`)
+
+      // don't update if the the stored size is greater than found files,
+      // it can indicates a problem
+      if (fixMetadata && (metadataSize === undefined || metadataSize < size)) {
+        try {
+          metadata.size = size
+          await handler.writeFile(json, JSON.stringify(metadata), { flags: 'w' })
+        } catch (error) {
+          onLog(`failed to update size in backup metadata ${json}`, { error })
+        }
+      }
+    }
   })
 
   // TODO: parallelize by vm/job/vdi
   const unusedVhdsDeletion = []
+  const toMerge = []
   {
     // VHD chains (as list from child to ancestor) to merge indexed by last
     // ancestor
@@ -295,22 +333,25 @@ exports.cleanVm = async function cleanVm(vmDir, { remove, merge, onLog = noop })
     })
 
     // merge interrupted VHDs
-    if (merge) {
-      vhdsList.interruptedVhds.forEach(parent => {
-        vhdChainsToMerge[parent] = [vhdChildren[parent], parent]
-      })
-    }
+    vhdsList.interruptedVhds.forEach(parent => {
+      vhdChainsToMerge[parent] = [vhdChildren[parent], parent]
+    })
 
-    Object.keys(vhdChainsToMerge).forEach(key => {
-      const chain = vhdChainsToMerge[key]
+    Object.values(vhdChainsToMerge).forEach(chain => {
       if (chain !== undefined) {
-        unusedVhdsDeletion.push(mergeVhdChain(chain, { handler, onLog, remove, merge }))
+        toMerge.push(chain)
       }
     })
   }
 
+  const doMerge = () => {
+    const promise = asyncMap(toMerge, async chain => limitedMergeVhdChain(chain, { handler, onLog, remove, merge }))
+    return merge ? promise.then(sizes => ({ size: sum(sizes) })) : promise
+  }
+
   await Promise.all([
     ...unusedVhdsDeletion,
+    toMerge.length !== 0 && (merge ? Task.run({ name: 'merge' }, doMerge) : doMerge()),
     asyncMap(unusedXvas, path => {
       onLog(`the XVA ${path} is unused`)
       if (remove) {
@@ -329,4 +370,9 @@ exports.cleanVm = async function cleanVm(vmDir, { remove, merge, onLog = noop })
       }
     }),
   ])
+
+  return {
+    // boolean whether some VHDs were merged (or should be merged)
+    merge: toMerge.length !== 0,
+  }
 }

@xen-orchestra/backups/_deltaVm.js

@@ -202,6 +202,7 @@ exports.importDeltaVm = defer(async function importDeltaVm(
       blocked_operations: {
         ...vmRecord.blocked_operations,
         start: 'Importing…',
+        start_on: 'Importing…',
       },
       ha_always_run: false,
       is_a_template: false,
@@ -305,9 +306,6 @@ exports.importDeltaVm = defer(async function importDeltaVm(
       }
     }),
 
-    // Wait for VDI export tasks (if any) termination.
-    Promise.all(Object.values(streams).map(stream => stream.task)),
-
     // Create VIFs.
     asyncMap(Object.values(deltaVm.vifs), vif => {
       let network = vif.$network$uuid && xapi.getObjectByUuid(vif.$network$uuid, undefined)

@xen-orchestra/backups/_lvm.js

@@ -7,23 +7,25 @@ const { execFile } = require('child_process')
 const parse = createParser({
   keyTransform: key => key.slice(5).toLowerCase(),
 })
-const makeFunction = command => async (fields, ...args) => {
-  const info = await fromCallback(execFile, command, [
-    '--noheading',
-    '--nosuffix',
-    '--nameprefixes',
-    '--unbuffered',
-    '--units',
-    'b',
-    '-o',
-    String(fields),
-    ...args,
-  ])
-  return info
-    .trim()
-    .split(/\r?\n/)
-    .map(Array.isArray(fields) ? parse : line => parse(line)[fields])
-}
+const makeFunction =
+  command =>
+  async (fields, ...args) => {
+    const info = await fromCallback(execFile, command, [
+      '--noheading',
+      '--nosuffix',
+      '--nameprefixes',
+      '--unbuffered',
+      '--units',
+      'b',
+      '-o',
+      String(fields),
+      ...args,
+    ])
+    return info
+      .trim()
+      .split(/\r?\n/)
+      .map(Array.isArray(fields) ? parse : line => parse(line)[fields])
+  }
 
 exports.lvs = makeFunction('lvs')
 exports.pvs = makeFunction('pvs')

@xen-orchestra/backups/merge-worker/cli.js

@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+
+const { catchGlobalErrors } = require('@xen-orchestra/log/configure.js')
+const { createLogger } = require('@xen-orchestra/log')
+const { getSyncedHandler } = require('@xen-orchestra/fs')
+const { join } = require('path')
+const Disposable = require('promise-toolbox/Disposable')
+const min = require('lodash/min')
+
+const { getVmBackupDir } = require('../_getVmBackupDir.js')
+const { RemoteAdapter } = require('../RemoteAdapter.js')
+
+const { CLEAN_VM_QUEUE } = require('./index.js')
+
+// -------------------------------------------------------------------
+
+catchGlobalErrors(createLogger('xo:backups:mergeWorker'))
+
+const { fatal, info, warn } = createLogger('xo:backups:mergeWorker')
+
+// -------------------------------------------------------------------
+
+const main = Disposable.wrap(async function* main(args) {
+  const handler = yield getSyncedHandler({ url: 'file://' + process.cwd() })
+
+  yield handler.lock(CLEAN_VM_QUEUE)
+
+  const adapter = new RemoteAdapter(handler)
+
+  const listRetry = async () => {
+    const timeoutResolver = resolve => setTimeout(resolve, 10e3)
+    for (let i = 0; i < 10; ++i) {
+      const entries = await handler.list(CLEAN_VM_QUEUE)
+      if (entries.length !== 0) {
+        return entries
+      }
+      await new Promise(timeoutResolver)
+    }
+  }
+
+  let taskFiles
+  while ((taskFiles = await listRetry()) !== undefined) {
+    const taskFileBasename = min(taskFiles)
+    const taskFile = join(CLEAN_VM_QUEUE, '_' + taskFileBasename)
+
+    // move this task to the end
+    await handler.rename(join(CLEAN_VM_QUEUE, taskFileBasename), taskFile)
+    try {
+      const vmDir = getVmBackupDir(String(await handler.readFile(taskFile)))
+      await adapter.cleanVm(vmDir, { merge: true, onLog: info, remove: true })
+
+      handler.unlink(taskFile).catch(error => warn('deleting task failure', { error }))
+    } catch (error) {
+      warn('failure handling task', { error })
+    }
+  }
+})
+
+info('starting')
+main(process.argv.slice(2)).then(
+  () => {
+    info('bye :-)')
+  },
+  error => {
+    fatal(error)
+
+    process.exit(1)
+  }
+)

@xen-orchestra/backups/merge-worker/index.js

@@ -0,0 +1,25 @@
+const { join, resolve } = require('path')
+const { spawn } = require('child_process')
+const { check } = require('proper-lockfile')
+
+const CLEAN_VM_QUEUE = (exports.CLEAN_VM_QUEUE = '/xo-vm-backups/.queue/clean-vm/')
+
+const CLI_PATH = resolve(__dirname, 'cli.js')
+exports.run = async function runMergeWorker(remotePath) {
+  try {
+    // TODO: find a way to pass the acquire the lock and then pass it down the worker
+    if (await check(join(remotePath, CLEAN_VM_QUEUE))) {
+      // already locked, don't start another worker
+      return
+    }
+
+    spawn(CLI_PATH, {
+      cwd: remotePath,
+      detached: true,
+      stdio: 'inherit',
+    }).unref()
+  } catch (error) {
+    // we usually don't want to throw if the merge worker failed to start
+    return error
+  }
+}

@xen-orchestra/backups/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.11.0",
+  "version": "0.15.1",
   "engines": {
     "node": ">=14.6"
   },
@@ -20,25 +20,26 @@
     "@vates/disposable": "^0.1.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^0.17.0",
-    "@xen-orchestra/log": "^0.2.1",
+    "@xen-orchestra/fs": "^0.18.0",
+    "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/template": "^0.1.0",
-    "compare-versions": "^3.6.0",
+    "compare-versions": "^4.0.1",
     "d3-time-format": "^3.0.0",
     "end-of-stream": "^1.4.4",
-    "fs-extra": "^9.0.0",
+    "fs-extra": "^10.0.0",
     "golike-defer": "^0.5.1",
     "limit-concurrency-decorator": "^0.5.0",
     "lodash": "^4.17.20",
     "node-zone": "^0.4.0",
     "parse-pairs": "^1.1.0",
+    "promise-toolbox": "^0.20.0",
+    "proper-lockfile": "^4.1.2",
     "pump": "^3.0.0",
-    "promise-toolbox": "^0.19.2",
-    "vhd-lib": "^1.0.0",
+    "vhd-lib": "^1.3.0",
     "yazl": "^2.5.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^0.6.3"
+    "@xen-orchestra/xapi": "^0.8.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

@xen-orchestra/backups/writers/DeltaBackupWriter.js

@@ -3,7 +3,7 @@ const map = require('lodash/map.js')
 const mapValues = require('lodash/mapValues.js')
 const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
 const { asyncMap } = require('@xen-orchestra/async-map')
-const { chainVhd, checkVhdChain, default: Vhd } = require('vhd-lib')
+const { chainVhd, checkVhdChain, VhdFile } = require('vhd-lib')
 const { createLogger } = require('@xen-orchestra/log')
 const { dirname } = require('path')
 
@@ -38,7 +38,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           try {
             await checkVhdChain(handler, path)
 
-            const vhd = new Vhd(handler, path)
+            const vhd = new VhdFile(handler, path)
             await vhd.readHeaderAndFooter()
             found = found || vhd.footer.uuid.equals(packUuid(baseUuid))
           } catch (error) {
@@ -113,19 +113,13 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
   }
 
   async _deleteOldEntries() {
-    return Task.run({ name: 'merge' }, async () => {
-      const adapter = this._adapter
-      const oldEntries = this._oldEntries
+    const adapter = this._adapter
+    const oldEntries = this._oldEntries
 
-      let size = 0
-      // delete sequentially from newest to oldest to avoid unnecessary merges
-      for (let i = oldEntries.length; i-- > 0; ) {
-        size += await adapter.deleteDeltaVmBackups([oldEntries[i]])
-      }
-      return {
-        size,
-      }
-    })
+    // delete sequentially from newest to oldest to avoid unnecessary merges
+    for (let i = oldEntries.length; i-- > 0; ) {
+      await adapter.deleteDeltaVmBackups([oldEntries[i]])
+    }
   }
 
   async _transfer({ timestamp, deltaExport, sizeContainers }) {
@@ -206,7 +200,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           }
 
           // set the correct UUID in the VHD
-          const vhd = new Vhd(handler, path)
+          const vhd = new VhdFile(handler, path)
           await vhd.readHeaderAndFooter()
           vhd.footer.uuid = packUuid(vdi.uuid)
           await vhd.readBlockAllocationTable() // required by writeFooter()

@xen-orchestra/backups/writers/DeltaReplicationWriter.js

@@ -106,9 +106,11 @@ exports.DeltaReplicationWriter = class DeltaReplicationWriter extends MixinRepli
       targetVm.ha_restart_priority !== '' &&
         Promise.all([targetVm.set_ha_restart_priority(''), targetVm.add_tags('HA disabled')]),
       targetVm.set_name_label(`${vm.name_label} - ${job.name} - (${formatFilenameDate(timestamp)})`),
-      targetVm.update_blocked_operations(
-        'start',
-        'Start operation for this vm is blocked, clone it if you want to use it.'
+      asyncMap(['start', 'start_on'], op =>
+        targetVm.update_blocked_operations(
+          op,
+          'Start operation for this vm is blocked, clone it if you want to use it.'
+        )
       ),
       targetVm.update_other_config({
         'xo:backup:sr': srUuid,

@xen-orchestra/backups/writers/FullReplicationWriter.js

@@ -1,5 +1,5 @@
 const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
-const { asyncMapSettled } = require('@xen-orchestra/async-map')
+const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
 const { formatDateTime } = require('@xen-orchestra/xapi')
 
 const { formatFilenameDate } = require('../_filenameDate.js')
@@ -64,9 +64,11 @@ exports.FullReplicationWriter = class FullReplicationWriter extends MixinReplica
     const targetVm = await xapi.getRecord('VM', targetVmRef)
 
     await Promise.all([
-      targetVm.update_blocked_operations(
-        'start',
-        'Start operation for this vm is blocked, clone it if you want to use it.'
+      asyncMap(['start', 'start_on'], op =>
+        targetVm.update_blocked_operations(
+          op,
+          'Start operation for this vm is blocked, clone it if you want to use it.'
+        )
       ),
       targetVm.update_other_config({
         'xo:backup:sr': srUuid,

@xen-orchestra/backups/writers/_MixinBackupWriter.js

@@ -1,34 +1,51 @@
 const { createLogger } = require('@xen-orchestra/log')
+const { join } = require('path')
 
-const { getVmBackupDir } = require('../_getVmBackupDir.js')
+const { BACKUP_DIR, getVmBackupDir } = require('../_getVmBackupDir.js')
+const MergeWorker = require('../merge-worker/index.js')
+const { formatFilenameDate } = require('../_filenameDate.js')
 
 const { warn } = createLogger('xo:backups:MixinBackupWriter')
 
 exports.MixinBackupWriter = (BaseClass = Object) =>
   class MixinBackupWriter extends BaseClass {
+    #lock
+    #vmBackupDir
+
     constructor({ remoteId, ...rest }) {
       super(rest)
 
       this._adapter = rest.backup.remoteAdapters[remoteId]
       this._remoteId = remoteId
-      this._lock = undefined
+
+      this.#vmBackupDir = getVmBackupDir(this._backup.vm.uuid)
     }
 
     _cleanVm(options) {
       return this._adapter
-        .cleanVm(getVmBackupDir(this._backup.vm.uuid), { ...options, onLog: warn, lock: false })
+        .cleanVm(this.#vmBackupDir, { ...options, fixMetadata: true, onLog: warn, lock: false })
         .catch(warn)
     }
 
     async beforeBackup() {
       const { handler } = this._adapter
-      const vmBackupDir = getVmBackupDir(this._backup.vm.uuid)
+      const vmBackupDir = this.#vmBackupDir
       await handler.mktree(vmBackupDir)
-      this._lock = await handler.lock(vmBackupDir)
+      this.#lock = await handler.lock(vmBackupDir)
     }
 
     async afterBackup() {
-      await this._cleanVm({ remove: true, merge: true })
-      await this._lock.dispose()
+      const { disableMergeWorker } = this._backup.config
+
+      const { merge } = await this._cleanVm({ remove: true, merge: disableMergeWorker })
+      await this.#lock.dispose()
+
+      // merge worker only compatible with local remotes
+      const { handler } = this._adapter
+      if (merge && !disableMergeWorker && typeof handler._getRealPath === 'function') {
+        await handler.outputFile(join(MergeWorker.CLEAN_VM_QUEUE, formatFilenameDate(new Date())), this._backup.vm.uuid)
+        const remotePath = handler._getRealPath()
+        await MergeWorker.run(remotePath)
+      }
     }
   }

@xen-orchestra/backups/writers/_checkVhd.js

@@ -1,4 +1,4 @@
-const Vhd = require('vhd-lib').default
+const Vhd = require('vhd-lib').VhdFile
 
 exports.checkVhd = async function checkVhd(handler, path) {
   await new Vhd(handler, path).readHeaderAndFooter()

@xen-orchestra/cr-seed-cli/index.js

@@ -77,7 +77,11 @@ ${cliName} v${pkg.version}
       'xo:backup:sr': tgtSr.uuid,
       'xo:copy_of': srcSnapshotUuid,
     }),
-    tgtVm.update_blocked_operations('start', 'Start operation for this vm is blocked, clone it if you want to use it.'),
+    Promise.all(
+      ['start', 'start_on'].map(op =>
+        tgtVm.update_blocked_operations(op, 'Start operation for this vm is blocked, clone it if you want to use it.')
+      )
+    ),
     Promise.all(
       userDevices.map(userDevice => {
         const srcDisk = srcDisks[userDevice]

@xen-orchestra/cr-seed-cli/package.json

@@ -18,7 +18,7 @@
   "preferGlobal": true,
   "dependencies": {
     "golike-defer": "^0.5.1",
-    "xen-api": "^0.33.0"
+    "xen-api": "^0.35.1"
   },
   "scripts": {
     "postversion": "npm publish"

@xen-orchestra/fs/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "@xen-orchestra/fs",
-  "version": "0.17.0",
+  "version": "0.18.0",
   "license": "AGPL-3.0-or-later",
   "description": "The File System for Xen Orchestra backups.",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
@@ -17,19 +17,19 @@
     "node": ">=14"
   },
   "dependencies": {
-    "@marsaud/smb2": "^0.17.2",
+    "@marsaud/smb2": "^0.18.0",
     "@sindresorhus/df": "^3.1.1",
     "@sullux/aws-sdk": "^1.0.5",
     "@vates/coalesce-calls": "^0.1.0",
     "@xen-orchestra/async-map": "^0.1.2",
     "aws-sdk": "^2.686.0",
-    "decorator-synchronized": "^0.5.0",
+    "decorator-synchronized": "^0.6.0",
     "execa": "^5.0.0",
-    "fs-extra": "^9.0.0",
+    "fs-extra": "^10.0.0",
     "get-stream": "^6.0.0",
     "limit-concurrency-decorator": "^0.5.0",
     "lodash": "^4.17.4",
-    "promise-toolbox": "^0.19.2",
+    "promise-toolbox": "^0.20.0",
     "proper-lockfile": "^4.1.2",
     "readable-stream": "^3.0.6",
     "through2": "^4.0.2",
@@ -45,7 +45,7 @@
     "async-iterator-to-stream": "^1.1.0",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^7.0.2",
-    "dotenv": "^8.0.0",
+    "dotenv": "^10.0.0",
     "rimraf": "^3.0.0"
   },
   "scripts": {

@xen-orchestra/fs/src/abstract.js

@@ -1,13 +1,13 @@
 import asyncMapSettled from '@xen-orchestra/async-map/legacy'
 import getStream from 'get-stream'
 import path, { basename } from 'path'
-import synchronized from 'decorator-synchronized'
 import { coalesceCalls } from '@vates/coalesce-calls'
 import { fromCallback, fromEvent, ignoreErrors, timeout } from 'promise-toolbox'
 import { limitConcurrency } from 'limit-concurrency-decorator'
 import { parse } from 'xo-remote-parser'
 import { pipeline } from 'stream'
 import { randomBytes } from 'crypto'
+import { synchronized } from 'decorator-synchronized'
 
 import normalizePath from './_normalizePath'
 import { createChecksumStream, validChecksumOfReadStream } from './checksum'

@xen-orchestra/fs/src/index.js

@@ -27,3 +27,12 @@ export const getHandler = (remote, ...rest) => {
   }
   return new Handler(remote, ...rest)
 }
+
+export const getSyncedHandler = async (...opts) => {
+  const handler = getHandler(...opts)
+  await handler.sync()
+  return {
+    dispose: () => handler.forget(),
+    value: handler,
+  }
+}

@xen-orchestra/fs/src/s3.js

@@ -183,9 +183,21 @@ export default class S3Handler extends RemoteHandlerAbstract {
     }
     const params = this._createParams(file)
     params.Range = `bytes=${position}-${position + buffer.length - 1}`
-    const result = await this._s3.getObject(params)
-    result.Body.copy(buffer)
-    return { bytesRead: result.Body.length, buffer }
+    try {
+      const result = await this._s3.getObject(params)
+      result.Body.copy(buffer)
+      return { bytesRead: result.Body.length, buffer }
+    } catch (e) {
+      if (e.code === 'NoSuchKey') {
+        if (await this._isNotEmptyDir(file)) {
+          const error = new Error(`${file} is a directory`)
+          error.code = 'EISDIR'
+          error.path = file
+          throw error
+        }
+      }
+      throw e
+    }
   }
 
   async _rmdir(path) {

@xen-orchestra/log/README.md

@@ -66,6 +66,10 @@ configure([
     // if filter is a string, then it is pattern
     // (https://github.com/visionmedia/debug#wildcards) which is
     // matched against the namespace of the logs
+    //
+    // If it's an array, it will be handled as an array of filters
+    // and the transport will be used if any one of them match the
+    // current log
     filter: process.env.DEBUG,
 
     transport: transportConsole(),

@xen-orchestra/log/configure.js

@@ -4,6 +4,42 @@ const { compileGlobPattern } = require('./utils')
 
 // ===================================================================
 
+const compileFilter = filter => {
+  if (filter === undefined) {
+    return
+  }
+
+  const type = typeof filter
+  if (type === 'function') {
+    return filter
+  }
+  if (type === 'string') {
+    const re = compileGlobPattern(filter)
+    return log => re.test(log.namespace)
+  }
+
+  if (Array.isArray(filter)) {
+    const filters = filter.map(compileFilter).filter(_ => _ !== undefined)
+    const { length } = filters
+    if (length === 0) {
+      return
+    }
+    if (length === 1) {
+      return filters[0]
+    }
+    return log => {
+      for (let i = 0; i < length; ++i) {
+        if (filters[i](log)) {
+          return true
+        }
+      }
+      return false
+    }
+  }
+
+  throw new TypeError('unsupported `filter`')
+}
+
 const createTransport = config => {
   if (typeof config === 'function') {
     return config
@@ -19,26 +55,15 @@ const createTransport = config => {
     }
   }
 
-  let { filter } = config
-  let transport = createTransport(config.transport)
   const level = resolve(config.level)
+  const filter = compileFilter([config.filter, level === undefined ? undefined : log => log.level >= level])
+
+  let transport = createTransport(config.transport)
 
   if (filter !== undefined) {
-    if (typeof filter === 'string') {
-      const re = compileGlobPattern(filter)
-      filter = log => re.test(log.namespace)
-    }
-
     const orig = transport
     transport = function (log) {
-      if ((level !== undefined && log.level >= level) || filter(log)) {
-        return orig.apply(this, arguments)
-      }
-    }
-  } else if (level !== undefined) {
-    const orig = transport
-    transport = function (log) {
-      if (log.level >= level) {
+      if (filter(log)) {
         return orig.apply(this, arguments)
       }
     }

@xen-orchestra/log/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "@xen-orchestra/log",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "license": "ISC",
   "description": "Logging system with decoupled producers/consumer",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/log",
@@ -24,7 +24,7 @@
   },
   "dependencies": {
     "lodash": "^4.17.4",
-    "promise-toolbox": "^0.19.2"
+    "promise-toolbox": "^0.20.0"
   },
   "scripts": {
     "postversion": "npm publish"

@xen-orchestra/log/transports/console.js

@@ -20,36 +20,8 @@ if (process.stdout !== undefined && process.stdout.isTTY && process.stderr !== u
   }
 
   const NAMESPACE_COLORS = [
-    196,
-    202,
-    208,
-    214,
-    220,
-    226,
-    190,
-    154,
-    118,
-    82,
-    46,
-    47,
-    48,
-    49,
-    50,
-    51,
-    45,
-    39,
-    33,
-    27,
-    21,
-    57,
-    93,
-    129,
-    165,
-    201,
-    200,
-    199,
-    198,
-    197,
+    196, 202, 208, 214, 220, 226, 190, 154, 118, 82, 46, 47, 48, 49, 50, 51, 45, 39, 33, 27, 21, 57, 93, 129, 165, 201,
+    200, 199, 198, 197,
   ]
   formatNamespace = namespace => {
     // https://werxltd.com/wp/2010/05/13/javascript-implementation-of-javas-string-hashcode-method/

@xen-orchestra/mixins/Config.js

@@ -1,5 +1,6 @@
 const get = require('lodash/get')
 const identity = require('lodash/identity')
+const isEqual = require('lodash/isEqual')
 const { createLogger } = require('@xen-orchestra/log')
 const { parseDuration } = require('@vates/parse-duration')
 const { watch } = require('app-conf')
@@ -48,7 +49,7 @@ module.exports = class Config {
     const watcher = config => {
       try {
         const value = processor(get(config, path))
-        if (value !== prev) {
+        if (!isEqual(value, prev)) {
           prev = value
           cb(value)
         }

@xen-orchestra/mixins/package.json

@@ -14,14 +14,14 @@
     "url": "https://vates.fr"
   },
   "license": "AGPL-3.0-or-later",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "engines": {
     "node": ">=12"
   },
   "dependencies": {
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/emit-async": "^0.1.0",
-    "@xen-orchestra/log": "^0.2.1",
+    "@xen-orchestra/log": "^0.3.0",
     "app-conf": "^0.9.0",
     "lodash": "^4.17.21"
   },

@xen-orchestra/openflow/src/util/addrress-parser.js

@@ -28,9 +28,10 @@ export default {
     buffer.toString('hex', offset + 5, offset + 6),
 
   stringToEth: (string, buffer, offset) => {
-    const eth = /^([0-9A-Fa-f]{2}):([0-9A-Fa-f]{2}):([0-9A-Fa-f]{2}):([0-9A-Fa-f]{2}):([0-9A-Fa-f]{2}):([0-9A-Fa-f]{2})$/.exec(
-      string
-    )
+    const eth =
+      /^([0-9A-Fa-f]{2}):([0-9A-Fa-f]{2}):([0-9A-Fa-f]{2}):([0-9A-Fa-f]{2}):([0-9A-Fa-f]{2}):([0-9A-Fa-f]{2})$/.exec(
+        string
+      )
     assert(eth !== null)
     buffer.writeUInt8(parseInt(eth[1], 16), offset)
     buffer.writeUInt8(parseInt(eth[2], 16), offset + 1)
@@ -50,9 +51,10 @@ export default {
     ),
 
   stringToip4: (string, buffer, offset) => {
-    const ip = /^([1-9]?\d|1\d\d|2[0-4]\d|25[0-5])\.([1-9]?\d|1\d\d|2[0-4]\d|25[0-5])\.([1-9]?\d|1\d\d|2[0-4]\d|25[0-5])\.([1-9]?\d|1\d\d|2[0-4]\d|25[0-5])$/.exec(
-      string
-    )
+    const ip =
+      /^([1-9]?\d|1\d\d|2[0-4]\d|25[0-5])\.([1-9]?\d|1\d\d|2[0-4]\d|25[0-5])\.([1-9]?\d|1\d\d|2[0-4]\d|25[0-5])\.([1-9]?\d|1\d\d|2[0-4]\d|25[0-5])$/.exec(
+        string
+      )
     assert(ip !== null)
     buffer.writeUInt8(parseInt(ip[1], 10), offset)
     buffer.writeUInt8(parseInt(ip[2], 10), offset + 1)

@xen-orchestra/proxy-cli/package.json

@@ -23,7 +23,7 @@
     "xo-proxy-cli": "dist/index.js"
   },
   "engines": {
-    "node": ">=8.10"
+    "node": ">=12"
   },
   "dependencies": {
     "@iarna/toml": "^2.2.0",
@@ -33,12 +33,12 @@
     "content-type": "^1.0.4",
     "cson-parser": "^4.0.7",
     "getopts": "^2.2.3",
-    "http-request-plus": "^0.10.0",
+    "http-request-plus": "^0.13.0",
     "json-rpc-protocol": "^0.13.1",
-    "promise-toolbox": "^0.19.2",
+    "promise-toolbox": "^0.20.0",
     "pump": "^3.0.0",
     "pumpify": "^2.0.1",
-    "split2": "^3.1.1"
+    "split2": "^4.1.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

@xen-orchestra/proxy-cli/src/index.js

@@ -36,7 +36,14 @@ async function main(argv) {
 
   const { hostname = 'localhost', port } = config?.http?.listen?.https ?? {}
 
-  const { _: args, file, help, host, raw, token } = getopts(argv, {
+  const {
+    _: args,
+    file,
+    help,
+    host,
+    raw,
+    token,
+  } = getopts(argv, {
     alias: { file: 'f', help: 'h' },
     boolean: ['help', 'raw'],
     default: {

@xen-orchestra/proxy/config.toml

@@ -18,6 +18,7 @@ keepAliveInterval = 10e3
 #
 # https://en.wikipedia.org/wiki/File-system_permissions#Numeric_notation
 dirMode = 0o700
+disableMergeWorker = false
 snapshotNameLabelTpl = '[XO Backup {job.name}] {vm.name_label}'
 
 [backups.defaultSettings]
@@ -59,6 +60,13 @@ cert = '/var/lib/xo-proxy/certificate.pem'
 key = '/var/lib/xo-proxy/key.pem'
 port = 443
 
+[logs]
+# Display all logs matching this filter, regardless of their level
+#filter = 'xo:backups:*'
+
+# Display all logs with level >=, regardless of their namespace
+level = 'info'
+
 [remoteOptions]
 mountsDir = '/run/xo-proxy/mounts'
 

@xen-orchestra/proxy/docs/api.md

@@ -93,10 +93,7 @@ declare namespace event {
 declare namespace backup {
   type SimpleIdPattern = { id: string | { __or: string[] } }
 
-declare namespace backup {
-  type SimpleIdPattern = { id: string | { __or: string[] } }
-
-  interface BackupJob {
+  interface BackupJob {
     id: string
     type: 'backup'
     compression?: 'native' | 'zstd' | ''
@@ -146,13 +143,13 @@ declare namespace backup {
   }
 
   function listXoMetadataBackups(_: { remotes: { [id: string]: Remote } }): { [remoteId: string]: object[] }
-
-  function run(_: {
-    job: BackupJob | MetadataBackupJob
+
+  function run(_: {
+    job: BackupJob | MetadataBackupJob
     remotes: { [id: string]: Remote }
-    schedule: Schedule
-    xapis?: { [id: string]: Xapi }
-    recordToXapi?: { [recordUuid: string]: string }
+    schedule: Schedule
+    xapis?: { [id: string]: Xapi }
+    recordToXapi?: { [recordUuid: string]: string }
     streamLogs: boolean = false
   }): string
 

@xen-orchestra/proxy/package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "@xen-orchestra/proxy",
-  "version": "0.14.1",
+  "version": "0.15.2",
   "license": "AGPL-3.0-or-later",
   "description": "XO Proxy used to remotely execute backup jobs",
   "keywords": [
@@ -31,17 +31,17 @@
     "@vates/decorate-with": "^0.1.0",
     "@vates/disposable": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.11.0",
-    "@xen-orchestra/fs": "^0.17.0",
-    "@xen-orchestra/log": "^0.2.1",
+    "@xen-orchestra/backups": "^0.15.1",
+    "@xen-orchestra/fs": "^0.18.0",
+    "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/mixin": "^0.1.0",
-    "@xen-orchestra/mixins": "^0.1.0",
+    "@xen-orchestra/mixins": "^0.1.1",
     "@xen-orchestra/self-signed": "^0.1.0",
-    "@xen-orchestra/xapi": "^0.6.3",
+    "@xen-orchestra/xapi": "^0.8.0",
     "ajv": "^8.0.3",
     "app-conf": "^0.9.0",
     "async-iterator-to-stream": "^1.1.0",
-    "fs-extra": "^9.1.0",
+    "fs-extra": "^10.0.0",
     "get-stream": "^6.0.0",
     "getopts": "^2.2.3",
     "golike-defer": "^0.5.1",
@@ -54,11 +54,11 @@
     "lodash": "^4.17.10",
     "node-zone": "^0.4.0",
     "parse-pairs": "^1.0.0",
-    "promise-toolbox": "^0.19.2",
+    "promise-toolbox": "^0.20.0",
     "source-map-support": "^0.5.16",
     "stoppable": "^1.0.6",
     "xdg-basedir": "^4.0.0",
-    "xen-api": "^0.33.0",
+    "xen-api": "^0.35.1",
     "xo-common": "^0.7.0"
   },
   "devDependencies": {
@@ -72,7 +72,7 @@
     "@vates/toggle-scripts": "^1.0.0",
     "babel-plugin-transform-dev": "^2.0.1",
     "cross-env": "^7.0.2",
-    "index-modules": "^0.4.0"
+    "index-modules": "^0.4.3"
   },
   "scripts": {
     "_build": "index-modules --index-file index.mjs src/app/mixins && babel --delete-dir-on-start --keep-file-extension --source-maps --out-dir=dist/ src/",

@xen-orchestra/proxy/src/app/mixins/api.mjs

@@ -14,14 +14,30 @@ import { createLogger } from '@xen-orchestra/log'
 
 const { debug, warn } = createLogger('xo:proxy:api')
 
-const ndJsonStream = asyncIteratorToStream(async function* (responseId, iterable) {
-  yield format.response(responseId, { $responseType: 'ndjson' }) + '\n'
-  for await (const data of iterable) {
+const ndJsonStream = asyncIteratorToStream(async function*(responseId, iterable) {
+  try {
+    let cursor, iterator
     try {
-      yield JSON.stringify(data) + '\n'
+      const getIterator = iterable[Symbol.iterator] ?? iterable[Symbol.asyncIterator]
+      iterator = getIterator.call(iterable)
+
+      cursor = await iterator.next()
+      yield format.response(responseId, { $responseType: 'ndjson' }) + '\n'
     } catch (error) {
-      warn('ndJsonStream', { error })
+      yield format.error(responseId, error)
+      throw error
     }
+
+    while (!cursor.done) {
+      try {
+        yield JSON.stringify(cursor.value) + '\n'
+      } catch (error) {
+        warn('ndJsonStream, item error', { error })
+      }
+      cursor = await iterator.next()
+    }
+  } catch (error) {
+    warn('ndJsonStream, fatal error', { error })
   }
 })
 
@@ -36,7 +52,7 @@ export default class Api {
       ctx.req.setTimeout(0)
 
       const profile = await app.authentication.findProfile({
-        authenticationToken: ctx.cookies.get('authenticationToken'),
+        authenticationToken: ctx.cookies.get('authenticationToken')
       })
       if (profile === undefined) {
         ctx.status = 401
@@ -107,7 +123,7 @@ export default class Api {
     this.addMethods({
       system: {
         getMethodsInfo: [
-          function* () {
+          function*() {
             const methods = this._methods
             for (const name in methods) {
               const { description, params = {} } = methods[name]
@@ -115,25 +131,25 @@ export default class Api {
             }
           }.bind(this),
           {
-            description: 'returns the signatures of all available API methods',
-          },
+            description: 'returns the signatures of all available API methods'
+          }
         ],
         getServerVersion: [
           () => appVersion,
           {
-            description: 'returns the version of xo-server',
-          },
+            description: 'returns the version of xo-server'
+          }
         ],
         listMethods: [
-          function* () {
+          function*() {
             const methods = this._methods
             for (const name in methods) {
               yield name
             }
           }.bind(this),
           {
-            description: 'returns the name of all available API methods',
-          },
+            description: 'returns the name of all available API methods'
+          }
         ],
         methodSignature: [
           ({ method: name }) => {
@@ -148,14 +164,14 @@ export default class Api {
           {
             description: 'returns the signature of an API method',
             params: {
-              method: { type: 'string' },
-            },
-          },
-        ],
+              method: { type: 'string' }
+            }
+          }
+        ]
       },
       test: {
         range: [
-          function* ({ start = 0, stop, step }) {
+          function*({ start = 0, stop, step }) {
             if (step === undefined) {
               step = start > stop ? -1 : 1
             }
@@ -173,11 +189,11 @@ export default class Api {
             params: {
               start: { optional: true, type: 'number' },
               step: { optional: true, type: 'number' },
-              stop: { type: 'number' },
-            },
-          },
-        ],
-      },
+              stop: { type: 'number' }
+            }
+          }
+        ]
+      }
     })
   }
 
@@ -204,7 +220,7 @@ export default class Api {
         return required
       }),
 
-      type: 'object',
+      type: 'object'
     })
 
     const m = params => {

@xen-orchestra/proxy/src/app/mixins/backups.mjs

@@ -11,6 +11,7 @@ import { DurablePartition } from '@xen-orchestra/backups/DurablePartition.js'
 import { execFile } from 'child_process'
 import { formatVmBackups } from '@xen-orchestra/backups/formatVmBackups.js'
 import { ImportVmBackup } from '@xen-orchestra/backups/ImportVmBackup.js'
+import { JsonRpcError } from 'json-rpc-protocol'
 import { Readable } from 'stream'
 import { RemoteAdapter } from '@xen-orchestra/backups/RemoteAdapter.js'
 import { RestoreMetadataBackup } from '@xen-orchestra/backups/RestoreMetadataBackup.js'
@@ -108,7 +109,7 @@ export default class Backups {
         if (!__DEV__) {
           const license = await app.appliance.getSelfLicense()
           if (license === undefined) {
-            throw new Error('no valid proxy license')
+            throw new JsonRpcError('no valid proxy license')
           }
         }
         return run.apply(this, arguments)

@xen-orchestra/proxy/src/app/mixins/logs.mjs

@@ -0,0 +1,17 @@
+import transportConsole from '@xen-orchestra/log/transports/console.js'
+import { configure } from '@xen-orchestra/log/configure.js'
+
+export default class Logs {
+  constructor(app) {
+    const transport = transportConsole()
+    app.config.watch('logs', ({ filter, level }) => {
+      configure([
+        {
+          filter: [process.env.DEBUG, filter],
+          level,
+          transport,
+        },
+      ])
+    })
+  }
+}

@xen-orchestra/upload-ova/package.json

@@ -33,9 +33,9 @@
     "chalk": "^4.1.0",
     "exec-promise": "^0.7.0",
     "form-data": "^4.0.0",
-    "fs-extra": "^9.0.0",
+    "fs-extra": "^10.0.0",
     "get-stream": "^6.0.0",
-    "http-request-plus": "^0.10.0",
+    "http-request-plus": "^0.13.0",
     "human-format": "^0.11.0",
     "l33teral": "^3.0.3",
     "lodash": "^4.17.4",

@xen-orchestra/xapi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xen-orchestra/xapi",
-  "version": "0.6.3",
+  "version": "0.8.0",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/xapi",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
@@ -25,7 +25,7 @@
     "xo-common": "^0.7.0"
   },
   "peerDependencies": {
-    "xen-api": "^0.33.0"
+    "xen-api": "^0.35.1"
   },
   "scripts": {
     "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
@@ -40,11 +40,11 @@
   "dependencies": {
     "@vates/decorate-with": "^0.1.0",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/log": "^0.2.1",
+    "@xen-orchestra/log": "^0.3.0",
     "d3-time-format": "^3.0.0",
     "golike-defer": "^0.5.1",
     "lodash": "^4.17.15",
-    "promise-toolbox": "^0.19.2"
+    "promise-toolbox": "^0.20.0"
   },
   "private": false,
   "license": "AGPL-3.0-or-later",

@xen-orchestra/xapi/src/vdi.js

@@ -1,6 +1,7 @@
 const CancelToken = require('promise-toolbox/CancelToken.js')
 const pCatch = require('promise-toolbox/catch.js')
 const pRetry = require('promise-toolbox/retry.js')
+const { decorateWith } = require('@vates/decorate-with')
 
 const extractOpaqueRef = require('./_extractOpaqueRef.js')
 
@@ -11,10 +12,13 @@ module.exports = class Vdi {
     return extractOpaqueRef(await this.callAsync('VDI.clone', vdiRef))
   }
 
+  // work around a race condition in XCP-ng/XenServer where the disk is not fully unmounted yet
+  @decorateWith(pRetry.wrap, function () {
+    return this._vdiDestroyRetryWhenInUse
+  })
   async destroy(vdiRef) {
     await pCatch.call(
-      // work around a race condition in XCP-ng/XenServer where the disk is not fully unmounted yet
-      pRetry(() => this.callAsync('VDI.destroy', vdiRef), this._vdiDestroyRetryWhenInUse),
+      this.callAsync('VDI.destroy', vdiRef),
       // if this VDI is not found, consider it destroyed
       { code: 'HANDLE_INVALID' },
       noop

CHANGELOG.md

@@ -1,20 +1,197 @@
+## **5.64.0** (2021-10-29)
+
 # ChangeLog
 
-## **next**
+<img id="latest" src="https://badgen.net/badge/channel/latest/yellow" alt="Channel: latest" />
+
+## Highlights
+
+- [Netbox] Support older versions of Netbox and prevent "active is not a valid choice" error [#5898](https://github.com/vatesfr/xen-orchestra/issues/5898) (PR [#5946](https://github.com/vatesfr/xen-orchestra/pull/5946))
+- [Tasks] Filter out short tasks using a default filter (PR [#5921](https://github.com/vatesfr/xen-orchestra/pull/5921))
+- [Host] Handle evacuation failure during host shutdown (PR [#5966](https://github.com/vatesfr/xen-orchestra/pull/#5966))
+- [Menu] Notify user when proxies need to be upgraded (PR [#5930](https://github.com/vatesfr/xen-orchestra/pull/5930))
+- [Servers] Ability to use an HTTP proxy between XO and a server (PR [#5958](https://github.com/vatesfr/xen-orchestra/pull/5958))
+- [VM/export] Ability to copy the export URL (PR [#5948](https://github.com/vatesfr/xen-orchestra/pull/5948))
+- [Pool/advanced] Ability to define network for importing/exporting VMs/VDIs (PR [#5957](https://github.com/vatesfr/xen-orchestra/pull/5957))
+- [Host/advanced] Add button to enable/disable the host (PR [#5952](https://github.com/vatesfr/xen-orchestra/pull/5952))
+- [Backups] Enable merge worker by default
+
+### Enhancements
+
+- [Jobs] Ability to copy a job ID (PR [#5951](https://github.com/vatesfr/xen-orchestra/pull/5951))
+
+### Bug fixes
+
+- [Backups] Delete unused snapshots related to other schedules (even no longer existing) (PR [#5949](https://github.com/vatesfr/xen-orchestra/pull/5949))
+- [Jobs] Fix `job.runSequence` method (PR [#5944](https://github.com/vatesfr/xen-orchestra/pull/5944))
+- [Netbox] Fix error when testing plugin on versions older than 2.10 (PR [#5963](https://github.com/vatesfr/xen-orchestra/pull/5963))
+- [Snapshot] Fix "Create VM from snapshot" creating a template instead of a VM (PR [#5955](https://github.com/vatesfr/xen-orchestra/pull/5955))
+- [Host/Logs] Improve the display of log content (PR [#5943](https://github.com/vatesfr/xen-orchestra/pull/5943))
+- [XOA licenses] Fix expiration date displaying "Invalid date" in some rare cases (PR [#5967](https://github.com/vatesfr/xen-orchestra/pull/5967))
+- [API/pool.listPoolsMatchingCriteria] Fix `checkSrName`/`checkPoolName` `is not a function` error
+
+### Released packages
+
+- xo-server-netbox 0.3.3
+- vhd-lib 1.3.0
+- xen-api 0.35.1
+- @xen-orchestra/xapi 0.8.0
+- @xen-orchestra/backups 0.15.1
+- @xen-orchestra/proxy 0.15.2
+- vhd-cli 0.5.0
+- xapi-explore-sr 0.4.0
+- xo-server 5.83.0
+- xo-web 5.89.0
+
+## **5.63.0** (2021-09-30)
+
+<img id="stable" src="https://badgen.net/badge/channel/stable/green" alt="Channel: stable" />
+
+### Highlights
+
+- [Backup] Go back to previous page instead of going to the overview after editing a job: keeps current filters and page (PR [#5913](https://github.com/vatesfr/xen-orchestra/pull/5913))
+- [Health] Do not take into consideration duplicated MAC addresses from CR VMs (PR [#5916](https://github.com/vatesfr/xen-orchestra/pull/5916))
+- [Health] Ability to filter duplicated MAC addresses by running VMs (PR [#5917](https://github.com/vatesfr/xen-orchestra/pull/5917))
+- [Tables] Move the search bar and pagination to the top of the table (PR [#5914](https://github.com/vatesfr/xen-orchestra/pull/5914))
+- [Netbox] Handle nested prefixes by always assigning an IP to the smallest prefix it matches (PR [#5908](https://github.com/vatesfr/xen-orchestra/pull/5908))
+
+### Bug fixes
+
+- [SSH keys] Allow SSH key to be broken anywhere to avoid breaking page formatting (Thanks [@tstivers1990](https://github.com/tstivers1990)!) [#5891](https://github.com/vatesfr/xen-orchestra/issues/5891) (PR [#5892](https://github.com/vatesfr/xen-orchestra/pull/5892))
+- [Netbox] Better handling and error messages when encountering issues due to UUID custom field not being configured correctly [#5905](https://github.com/vatesfr/xen-orchestra/issues/5905) [#5806](https://github.com/vatesfr/xen-orchestra/issues/5806) [#5834](https://github.com/vatesfr/xen-orchestra/issues/5834) (PR [#5909](https://github.com/vatesfr/xen-orchestra/pull/5909))
+- [New VM] Don't send network config if untouched as all commented config can make Cloud-init fail [#5918](https://github.com/vatesfr/xen-orchestra/issues/5918) (PR [#5923](https://github.com/vatesfr/xen-orchestra/pull/5923))
+
+### Released packages
+
+- xen-api 0.34.3
+- vhd-lib 1.2.0
+- xo-server-netbox 0.3.1
+- @xen-orchestra/proxy 0.14.7
+- xo-server 5.82.3
+- xo-web 5.88.0
+
+## **5.62.1** (2021-09-17)
+
+### Bug fixes
+
+- [VM/Advanced] Fix conversion from UEFI to BIOS boot firmware (PR [#5895](https://github.com/vatesfr/xen-orchestra/pull/5895))
+- [VM/network] Support newline-delimited IP addresses reported by some guest tools
+- Fix VM/host stats, VM creation with Cloud-init, and VM backups, with NATted hosts [#5896](https://github.com/vatesfr/xen-orchestra/issues/5896)
+- [VM/import] Very small VMDK and OVA files were mangled upon import (PR [#5903](https://github.com/vatesfr/xen-orchestra/pull/5903))
+
+### Released packages
+
+- xen-api 0.34.2
+- @xen-orchestra/proxy 0.14.6
+- xo-server 5.82.2
+
+## **5.62.0** (2021-08-31)
+
+### Highlights
+
+- [Host] Add warning in case of unmaintained host version [#5840](https://github.com/vatesfr/xen-orchestra/issues/5840) (PR [#5847](https://github.com/vatesfr/xen-orchestra/pull/5847))
+- [Backup] Use default migration network if set when importing/exporting VMs/VDIs (PR [#5883](https://github.com/vatesfr/xen-orchestra/pull/5883))
+
+### Enhancements
+
+- [New network] Ability for pool's admin to create a new network within the pool (PR [#5873](https://github.com/vatesfr/xen-orchestra/pull/5873))
+- [Netbox] Synchronize primary IPv4 and IPv6 addresses [#5633](https://github.com/vatesfr/xen-orchestra/issues/5633) (PR [#5879](https://github.com/vatesfr/xen-orchestra/pull/5879))
+
+### Bug fixes
+
+- [VM/network] Fix an issue where multiple IPs would be displayed in the same tag when using old Xen tools. This also fixes Netbox's IP synchronization for the affected VMs. (PR [#5860](https://github.com/vatesfr/xen-orchestra/pull/5860))
+- [LDAP] Handle groups with no members (PR [#5862](https://github.com/vatesfr/xen-orchestra/pull/5862))
+- Fix empty button on small size screen (PR [#5874](https://github.com/vatesfr/xen-orchestra/pull/5874))
+- [Host] Fix `Cannot read property 'other_config' of undefined` error when enabling maintenance mode (PR [#5875](https://github.com/vatesfr/xen-orchestra/pull/5875))
+
+### Released packages
+
+- xen-api 0.34.1
+- @xen-orchestra/xapi 0.7.0
+- @xen-orchestra/backups 0.13.0
+- @xen-orchestra/fs 0.18.0
+- @xen-orchestra/log 0.3.0
+- @xen-orchestra/mixins 0.1.1
+- xo-server-auth-ldap 0.10.4
+- xo-server-netbox 0.3.0
+- xo-server 5.82.1
+- xo-web 5.87.0
+
+## **5.61.0** (2021-07-30)
+
+### Highlights
+
+- [SR/disks] Display base copies' active VDIs (PR [#5826](https://github.com/vatesfr/xen-orchestra/pull/5826))
+- [Netbox] Optionally allow self-signed certificates (PR [#5850](https://github.com/vatesfr/xen-orchestra/pull/5850))
+- [Host] When supported, use pool's default migration network to evacuate host [#5802](https://github.com/vatesfr/xen-orchestra/issues/5802) (PR [#5851](https://github.com/vatesfr/xen-orchestra/pull/5851))
+- [VM] shutdown/reboot: offer to force shutdown/reboot the VM if no Xen tools were detected [#5838](https://github.com/vatesfr/xen-orchestra/issues/5838) (PR [#5855](https://github.com/vatesfr/xen-orchestra/pull/5855))
+
+### Enhancements
+
+- [Netbox] Add information about a failed request to the error log to help better understand what happened [#5834](https://github.com/vatesfr/xen-orchestra/issues/5834) (PR [#5842](https://github.com/vatesfr/xen-orchestra/pull/5842))
+- [VM/console] Ability to rescan ISO SRs (PR [#5841](https://github.com/vatesfr/xen-orchestra/pull/5841))
+
+### Bug fixes
+
+- [VM/disks] Fix `an error has occured` when self service user was on VM disk view (PR [#5841](https://github.com/vatesfr/xen-orchestra/pull/5841))
+- [Backup] Protect replicated VMs from being started on specific hosts (PR [#5852](https://github.com/vatesfr/xen-orchestra/pull/5852))
+
+### Released packages
+
+- @xen-orchestra/backups 0.12.2
+- @xen-orchestra/proxy 0.14.4
+- xo-server-netbox 0.2.0
+- xo-web 5.86.0
+- xo-server 5.81.2
+
+## **5.60.0** (2021-06-30)
+
+### Highlights
+
+- [VM/disks] Ability to rescan ISO SRs (PR [#5814](https://github.com/vatesfr/xen-orchestra/pull/5814))
+- [VM/snapshots] Identify VM's current snapshot with an icon next to the snapshot's name (PR [#5824](https://github.com/vatesfr/xen-orchestra/pull/5824))
+
+### Enhancements
+
+- [OVA import] improve OVA import error reporting (PR [#5797](https://github.com/vatesfr/xen-orchestra/pull/5797))
+- [Backup] Distinguish error messages between cancelation and interrupted HTTP connection
+- [Jobs] Add `host.emergencyShutdownHost` to the list of methods that jobs can call (PR [#5818](https://github.com/vatesfr/xen-orchestra/pull/5818))
+- [Host/Load-balancer] Log VM and host names when a VM is migrated + category (density, performance, ...) (PR [#5808](https://github.com/vatesfr/xen-orchestra/pull/5808))
+- [VM/new disk] Auto-fill disk name input with generated unique name (PR [#5828](https://github.com/vatesfr/xen-orchestra/pull/5828))
+
+### Bug fixes
+
+- [IPs] Handle space-delimited IP address format provided by outdated guest tools [5801](https://github.com/vatesfr/xen-orchestra/issues/5801) (PR [5805](https://github.com/vatesfr/xen-orchestra/pull/5805))
+- [API/pool.listPoolsMatchingCriteria] fix `unknown error from the peer` error (PR [5807](https://github.com/vatesfr/xen-orchestra/pull/5807))
+- [Backup] Limit number of connections to hosts, which should reduce the occurences of `ECONNRESET`
+- [Plugins/perf-alert] All mode: only selects running hosts and VMs (PR [5811](https://github.com/vatesfr/xen-orchestra/pull/5811))
+- [New VM] Fix summary section always showing "0 B" for RAM (PR [#5817](https://github.com/vatesfr/xen-orchestra/pull/5817))
+- [Backup/Restore] Fix _start VM after restore_ [5820](https://github.com/vatesfr/xen-orchestra/issues/5820)
+- [Netbox] Fix a bug where some devices' IPs would get deleted from Netbox (PR [#5821](https://github.com/vatesfr/xen-orchestra/pull/5821))
+- [Netbox] Fix an issue where some IPv6 would be deleted just to be immediately created again (PR [#5822](https://github.com/vatesfr/xen-orchestra/pull/5822))
 
 ### Released packages
 
 - @vates/decorate-with 0.1.0
+- xen-api 0.33.1
+- @xen-orchestra/xapi 0.6.4
+- @xen-orchestra/backups 0.12.0
+- @xen-orchestra/proxy 0.14.3
+- vhd-lib 1.1.0
+- vhd-cli 0.4.0
+- xo-server-netbox 0.1.2
+- xo-server-perf-alert 0.3.2
+- xo-server-load-balancer 0.7.0
+- xo-server 5.80.0
+- xo-web 5.84.0
 
 ## **5.59.0** (2021-05-31)
 
-<img id="latest" src="https://badgen.net/badge/channel/latest/yellow" alt="Channel: latest" />
-
 ### Highlights
 
 - [Smart backup] Report missing pools [#2844](https://github.com/vatesfr/xen-orchestra/issues/2844) (PR [#5768](https://github.com/vatesfr/xen-orchestra/pull/5768))
 - [Metadata Backup] Add a warning on restoring a metadata backup (PR [#5769](https://github.com/vatesfr/xen-orchestra/pull/5769))
-- [Netbox] [Plugin](https://xen-orchestra.com/docs/advanced.html#netbox) to synchronize pools, VMs and IPs with [Netbox](https://netbox.readthedocs.io/en/stable/) (PR [#5783](https://github.com/vatesfr/xen-orchestra/pull/5783))
+- [Netbox][plugin](https://xen-orchestra.com/docs/advanced.html#netbox) to synchronize pools, VMs and IPs with [Netbox](https://netbox.readthedocs.io/en/stable/) (PR [#5783](https://github.com/vatesfr/xen-orchestra/pull/5783))
 
 ### Enhancements
 
@@ -41,8 +218,6 @@
 
 ## **5.58.1** (2021-05-06)
 
-<img id="stable" src="https://badgen.net/badge/channel/stable/green" alt="Channel: stable" />
-
 ### Bug fixes
 
 - [Backups] Better handling of errors in remotes, fix `task has already ended`

CHANGELOG.unreleased.md

@@ -11,6 +11,8 @@
 
 > Users must be able to say: “I had this issue, happy to know it's fixed”
 
+[Import/VM] Fix the import of OVA files (PR [#5976](https://github.com/vatesfr/xen-orchestra/pull/5976))
+
 ### Packages to release
 
 > Packages will be released in the order they are here, therefore, they should
@@ -27,3 +29,7 @@
 > - major: if the change breaks compatibility
 >
 > In case of conflict, the highest (lowest in previous list) `$version` wins.
+
+- @xen-orchestra/fs minor
+- xo-server patch
+- vhd-cli minor

ISSUE_TEMPLATE.md

@@ -1,27 +0,0 @@
-<!--
-Welcome to the issue section of Xen Orchestra!
-
-Here you can:
-- report an issue
-- propose an enhancement
-- ask a question
-
-Please, respect this template as much as possible, it helps us sort
-the issues :)
--->
-
-### Context
-
-- **XO origin**: the sources / XO Appliance
-- **Versions**:
-  - Node: **FILL HERE**
-  - xo-web: **FILL HERE**
-  - xo-server: **FILL HERE**
-
-### Expected behavior
-
-<!-- What you expect to happen -->
-
-### Current behavior
-
-<!-- What is actually happening -->

docs/advanced.md

@@ -114,17 +114,18 @@ We need your feedback on this feature!
 
 The plugin "web-hooks" needs to be installed and loaded for this feature to work.
 
-You can trigger an HTTP POST request to a URL when a Xen Orchestra API method is called.
+You can trigger an HTTP POST request to a URL when a Xen Orchestra API method is called or when a backup job runs.
 
 - Go to Settings > Plugins > Web hooks
 - Add new hooks
 - For each hook, configure:
-  - Method: the XO API method that will trigger the HTTP request when called
+  - Method: the XO API method that will trigger the HTTP request when called. For backup jobs, choose `backupNg.runJob`.
   - Type:
     - pre: the request will be sent when the method is called
     - post: the request will be sent after the method action is completed
     - pre/post: both
   - URL: the full URL which the requests will be sent to
+  - Wait for response: you can choose to wait for the web hook response before the method is actually called ("pre" hooks only). This can be useful if you need to automatically run some tasks before a certain method is called.
 - Save the plugin configuration
 
 From now on, a request will be sent to the corresponding URLs when a configured method is called by an XO client.
@@ -340,13 +341,14 @@ XO will try to find the right prefix for each IP address. If it can't find a pre
   - Create a token with "Write enabled"
 - Add a UUID custom field:
   - Got to Admin > Custom fields > Add custom field
-  - Create a custom field called "uuid"
+  - Create a custom field called "uuid" (lower case!)
   - Assign it to object types `virtualization > cluster` and `virtualization > virtual machine`
 
 ![](./assets/customfield.png)
 
 - Go to Xen Orchestra > Settings > Plugins > Netbox and fill out the configuration:
   - Endpoint: the URL of your Netbox instance (e.g.: `https://netbox.company.net`)
+  - Unauthorized certificate: only for HTTPS, enable this option if your Netbox instance uses a self-signed SSL certificate
   - Token: the token you generated earlier
   - Pools: the pools you wish to automatically synchronize with Netbox
   - Interval: the time interval (in hours) between 2 auto-synchronizations. Leave empty if you don't want to synchronize automatically.

docs/backup_reports.md

@@ -87,3 +87,7 @@ You need to be an admin:
 ![Mattermost configuration](./assets/DocImg8.png)
 
 ![Mattermost](./assets/DocImg9.png)
+
+## Web hooks
+
+You can also configure web hooks to be sent to a custom server before and/or after a backup job runs. This won't send a formatted report but raw JSON data that you can use in custom scripts on your side. Follow the [web-hooks plugin documentation](./advanced.html#web-hooks) to configure it.

docs/backups.md

@@ -283,39 +283,42 @@ When it's done exporting, we'll remove the snapshot. Note: this operation will t
 
 ### Concurrency
 
+Concurrency is a parameter that let you define how many VMs your backup job will manage simultaneously.
+
+:::tip
+
+- Default concurrency value is 2 if left empty.
+  :::
+
 Let's say you want to backup 50 VMs (each with 1x disk) at 3:00 AM. There are **2 different strategies**:
 
 1. backup VM #1 (snapshot, export, delete snapshots) **then** backup VM #2 -> _fully sequential strategy_
 2. snapshot all VMs, **then** export all snapshots, **then** delete all snapshots for finished exports -> _fully parallel strategy_
 
-The first purely sequential strategy will lead to a big problem: **you can't predict when a snapshot of your data will occur**. Because you can't predict the first VM export time (let's say 3 hours), then your second VM will have its snapshot taken 3 hours later, at 6 AM. We assume that's not what you meant when you specified "backup everything at 3 AM". You would end up with data from 6 AM (and later) for other VMs.
-
-Strategy number 2 is better in this aspect: all the snapshots will be taken at 3 AM. However **it's risky without limits**: it means potentially doing 50 snapshots or more at once on the same storage. **Since XenServer doesn't have a queue**, it will try to do all of them at once. This is also prone to race conditions and could cause crashes on your storage.
-
-So what's the best choice? Continue below to learn how to best configure concurrency for your needs.
-
-#### Best choice
-
-By default the _parallel strategy_ is, on paper, the most logical one. But we need to give it some limits on concurrency.
+The first purely sequential strategy will lead to the fact that: **you can't predict when a snapshot of your data will occur**. Because you can't predict the first VM export time (let's say 3 hours), then your second VM will have its snapshot taken 3 hours later, at 6 AM.
 
 :::tip
-Xen Orchestra can be connected to multiple pools at once. So the concurrency number applies **per pool**.
+If you need your backup to be done at a specific time you should consider creating a specific backup task for this VM.
 :::
 
-Each step has its own concurrency to fit its requirements:
+Strategy number 2 is to parallelise: all the snapshots will be taken at 3 AM. However **it's risky without limits**: it means potentially doing 50 snapshots or more at once on the same storage. **Since XenServer doesn't have a queue**, it will try to do all of them at once. This is also prone to race conditions and could cause crashes on your storage.
 
-- **snapshot process** needs to be performed with the lowest concurrency possible. 2 is a good compromise: one snapshot is fast, but a stuck snapshot won't block the whole job. That's why a concurrency of 2 is not too bad on your storage. Basically, at 3 AM, we'll do all the VM snapshots needed, 2 at a time.
-- **disk export process** is bottlenecked by XCP-ng/XenServer - so to get the most of it, you can use up to 12 in parallel. As soon a snapshot is done, the export process will start, until reaching 12 at once. Then as soon as one in those 12 is finished, another one will appear until there is nothing more to export.
-- **VM export process:** the 12 disk export limit mentioned above applies to VDI exports, which happen during delta exports. For full VM exports (for example, for full backup job types), there is a built in limit of 2. This means if you have a full backup job of 6 VMs, only 2 will be exported at once.
-- **snapshot deletion** can't happen all at once because the previous step durations are random - no need to implement concurrency on this one.
+By default the _parallel strategy_ is, on paper, the most logical one. But you need to be careful and give it some limits on concurrency.
 
-This is how it currently works in Xen Orchestra. But sometimes, you also want to have _sequential_ backups combined with the _parallel strategy_. That's why we introduced a sequential option in the advanced section of backup-ng:
-
-:::tip
-0 means it will be fully **parallel** for all VMs.
+:::danger
+High concurrency could impact your dom0 and network performances.
 :::
 
-If you job contains 50 VMs for example, you could specify a sequential backup with a limit of "25 at once" (enter 25 in the concurrency field). This means at 3 AM, we'll do 25 snapshots (2 at a time), then exports. As soon as the first VM backup is completely finished (snapshot removed), then we'll start the 26th and so on, to always keep a max of 25x VM backups going in parallel.
+You should be aware of your hardware limitation when defining the best concurrency for your XCP-ng infrastructure, never put concurrency too high or you could impact your VMs performances.
+The best way to define the best concurrency for you is by increasing it slowly and watching the result on backup time.
+
+So to summarize, if you set your concurrency at 6 and you have 20 Vms to backup the process will be the following:
+
+- We start the backup of the first 6 VMs.
+- When one VM backup as ended we will launch the next VM backup.
+- We're keep launching new VM backup until the 20 VMs are finished, keeping 6 backups running.
+
+Removing the snapshot will trigger the coalesce process for the first VM, this is an automated action not triggered directly by the backup job.
 
 ## Backup modifier tags
 

docs/from_the_sources.md

@@ -46,7 +46,7 @@ apt-get install build-essential redis-server libpng-dev git python-minimal libvh
 You need to use the `git` source code manager to fetch the code. Ideally, you should run XO as a non-root user, and if you choose to, you need to set up `sudo` to be able to mount NFS remotes. As your chosen non-root (or root) user, run the following:
 
 ```
-git clone -b master http://github.com/vatesfr/xen-orchestra
+git clone -b master https://github.com/vatesfr/xen-orchestra
 ```
 
 > Note: xo-server and xo-web have been migrated to the [xen-orchestra](https://github.com/vatesfr/xen-orchestra) mono-repository - so you only need the single clone command above

docs/full_backups.md

@@ -1,6 +1,6 @@
 # Full backups
 
-You can schedule full backups of your VMs, by exporting them to the local XOA file-system, or directly to an NFS or SMB share. The "rentention" parameter allows you to modify how many backups are retained (by removing the oldest one).
+You can schedule full backups of your VMs, by exporting them to the local XOA file-system, or directly to an NFS or SMB share. The "retention" parameter allows you to modify how many backups are retained (by removing the oldest one).
 
 [![](./assets/backupexample.png)](https://xen-orchestra.com/blog/backup-your-xenserver-vms-with-xen-orchestra/)
 

docs/installation.md

@@ -20,7 +20,7 @@ Once you have started the VM, you can access the web UI by putting the IP you co
 :::tip
 
 - Default Web UI credentials are `admin@admin.net` / `admin`
-- Default console/SSH credentials are `xoa` / `xoa` (first login)
+- Default console/SSH credentials are not set, you need to set them [as described here](troubleshooting.md#set-or-recover-xoa-vm-password).
   :::
 
 ### Registration

docs/load_balancing.md

@@ -94,3 +94,21 @@ The global situation (resource usage) is examined **every minute**.
 :::tip
 TODO: more details to come here
 :::
+
+## VM anti-affinity
+
+VM anti-affinity is a feature that prevents VMs with the same user tags from running on the same host. This functionality is available directly in the load-balancer plugin.
+This way, you can avoid having pairs of redundant VMs or similar running on the same host.
+
+Let's look at a simple example: you have multiple VMs running MySQL and PostgreSQL with high availability/replication. Obviously, you don't want to lose the replicated database inside the VMs on the same physical host. Just create your plan like this:
+
+![](./assets/antiaffinity.png)
+
+- Simple plan: means no active load balancing mechanism used
+- Anti-affinity: we added our 2x tags, meaning any VMs with one of these tags will never run on the same host (if possible) with another VM having the same tag
+
+You can also use the performance plan with the anti-affinity mode activated to continue to migrate non-tagged VMs.
+
+:::tip
+This feature is not limited by the number of VMs using the same tag, i.e. if you have 6 VMs with the same anti-affinity tag and 2 hosts, the plugin will always try to place 3 VMs on each host. It will distribute as much as possible the VMs fairly and it takes precedence (in the majority of the cases) over the performance algorithm.
+:::

docs/manage_infrastructure.md

@@ -320,6 +320,7 @@ You can learn more about XenServer [resource management on the Citrix Website](h
 :::tip
 XCP-ng doesn't limit VMs to 32 vCPU
 :::
+
 ### VDI live migration
 
 Thanks to Xen Storage Motion, it's easy to move a VM disk from one storage location to another, while the VM is running! This feature can help you migrate from your local storage to a SAN, or just upgrade your SAN without any downtime.
@@ -491,10 +492,12 @@ If you are behind a proxy, please update your `xo-server` configuration to add a
 ::: danger
 As specified in the [documentation](https://xcp-ng.org/docs/requirements.html#pool-requirements) your pool shouldn't consist of hosts from different CPU vendors.
 :::
+
 ::: warning
 - Even with matching CPU vendors, in the case of different CPU models XCP-ng will scale the pool CPU ability to the CPU having the least instructions.
 - All the hosts in a pool must run the same XCP-ng version.
 :::
+
 ### Creating a pool
 
 First you should add your new host to XOA by going to New > Server as described in [the relevant chapter](manage_infrastructure.md#add-a-host).

docs/proxy.md

@@ -59,9 +59,11 @@ While creating a standard backup job from your main Xen Orchestra appliance, you
 
 Login is disabled by default on proxy appliances.
 If you need to login for some reason, you need to set a password for the xoa user via the XenStore of the VM. The following is to be ran on your XCP-ng host:
+
 ```
 xe vm-param-set uuid=<UUID> xenstore-data:vm-data/system-account-xoa-password=<password>
 ```
+
 Where UUID is the uuid of your proxy VM.
 
 Then you need to restart the proxy VM.
@@ -74,15 +76,19 @@ First you will need to add a second VIF to your Proxy VM. This can be done in th
 After adding the VIF you will need to set an IP for the new NIC, for that you will first need to SSH to the VM [as describe before](/proxy.md#enabling-login-to-proxy-appliance).
 
 Then set the new IP:
+
 ```
 $ xoa network static eth1
 ? Static IP for this machine 192.168.100.120
 ? Network mask (eg 255.255.255.0) 255.255.255.0
 ```
+
 If you want to set a static address.
+
 ```
 $ xoa network dhcp eth1
 ```
+
 If you prefer using DHCP.
 :::tip
 As XOA uses the first IP address reported by XAPI to contact the proxy appliance, you may have to switch the network card order if you want your proxy to be connected through a specific IP address.

docs/troubleshooting.md

@@ -16,6 +16,18 @@ It means you don't have a default SR set on the pool you are importing XOA on. T
 
 XOA uses HVM mode. If your physical host doesn't support virtualization extensions, XOA won't work. To check if your XenServer supports hardware assisted virtualization (HVM), you can enter this command in your host: `grep --color vmx /proc/cpuinfo`. If you don't have any result, it means XOA won't work on this hardware.
 
+## Set or recover XOA VM password
+
+As no password is set for the xoa system user by default, you will need to set your own. This can be done via the XenStore data of the VM. The following is to be ran on your XCP-ng host:
+
+```
+xe vm-param-set uuid=<UUID> xenstore-data:vm-data/system-account-xoa-password=<password>
+```
+
+Where UUID is the uuid of your XOA VM.
+
+Then you need to restart the VM.
+
 ## Recover web login password
 
 If you have lost your password to log in to the XOA webpage, you can reset it. From the XOA CLI (for login/access info for the CLI, [see here](xoa.md#first-console-connection)), use the following command and insert the email/account you wish to recover:
@@ -162,9 +174,9 @@ Connect to your appliance via SSH, then as root execute these commands:
 
 ```
 $ cd /etc/ssl
-$ cp server.crt server.crt.old
-$ cp server.key server.key.old
-$ openssl req -x509 -newkey rsa:2048 -keyout server.key -out server.crt -nodes -days 360
+$ cp cert.pem cert.pem-old
+$ cp key.pem key.pem-old
+$ openssl req -x509 -newkey rsa:2048 -keyout key.pem -out cert.pem -nodes -days 360
 $ systemctl restart xo-server.service
 ```
 

docs/updater.md

@@ -10,7 +10,7 @@ By design, the updater is only available in XOA. If you are using XO from the so
 
 ## Requirements
 
-In order to work, the updater needs access to `xen-orchestra.com` (port 443).
+In order to work, the updater needs access to `xen-orchestra.com` (port 443) and `nodejs.org` (port 443).
 
 ## Usage
 

docs/xoa.md

@@ -97,59 +97,26 @@ After the VM is imported, you just need to start it with `xe vm-start vm="XOA"`
 
 ## First console connection
 
-If you connect via SSH or console, the default credentials are:
+### Deployed with the [web deploy form](https://xen-orchestra.com/#!/xoa)
 
-- user: xoa
-- password: xoa
+In that case, you already set the password for `xoa` user. If you forgot it, see below.
 
-During your first connection, the system will ask you to:
+### Manually deployed
 
-- enter the current password again (`xoa`)
-- enter your new password
-- retype your new password
+If you connect via SSH or console for the first time without using our [web deploy form](https://xen-orchestra.com/#!/xoa), be aware **there's NO default password set for security reasons**. To set it, you need to connect to your host to find the XOA VM UUID (eg via `xe vm-list`).
 
-When it's done, you'll be disconnected, so reconnect again with your new password.
-
-Here is an example when you connect via SSH for the first time:
+Then replace `<UUID>` with the previously find UUID, and `<password>` with your password:
 
 ```
-$ ssh xoa@192.168.100.146
-Warning: Permanently added '192.168.100.146' (ECDSA) to the list of known hosts.
-xoa@192.168.100.146's password:
-You are required to change your password immediately (root enforced)
- __   __             ____           _               _
- \ \ / /            / __ \         | |             | |
-  \ V / ___ _ __   | |  | |_ __ ___| |__   ___  ___| |_ _ __ __ _
-   > < / _ \ '_ \  | |  | | '__/ __| '_ \ / _ \/ __| __| '__/ _` |
-  / . \  __/ | | | | |__| | | | (__| | | |  __/\__ \ |_| | | (_| |
- /_/ \_\___|_| |_|  \____/|_|  \___|_| |_|\___||___/\__|_|  \__,_|
-
-Welcome to XOA Unified Edition, with Pro Support.
-
-* Restart XO: sudo systemctl restart xo-server.service
-* Display logs: sudo systemctl status xo-server.service
-* Register your XOA: sudo xoa-updater --register
-* Update your XOA: sudo xoa-updater --upgrade
-
-OFFICIAL XOA DOCUMENTATION HERE: https://xen-orchestra.com/docs/xoa.html
-
-Support available at https://xen-orchestra.com/#!/member/support
-
-Build number: 16.10.24
-
-Based on Debian GNU/Linux 8 (Stable) 64bits in PVHVM mode
-
-WARNING: Your password has expired.
-You must change your password now and login again!
-Changing password for xoa.
-(current) UNIX password:
-Enter new UNIX password:
-Retype new UNIX password:
-passwd: password updated successfully
-Connection to 192.168.100.146 closed.
-$
+xe vm-param-set uuid=<UUID> xenstore-data:vm-data/system-account-xoa-password=<password>
 ```
 
+:::tip
+Don't forget to use quotes for your password, eg: `xenstore-data:vm-data/system-account-xoa-password='MyPassW0rd!'`
+:::
+
+Then, you could connect with `xoa` username and the password you defined in the previous command, eg with `ssh xoa@<XOA IP ADDRESS>`.
+
 ### Using sudo
 
 To avoid typing `sudo` for any admin command, you can have a root shell with `sudo -s`:

package.json

@@ -3,7 +3,7 @@
     "@babel/core": "^7.0.0",
     "@babel/eslint-parser": "^7.13.8",
     "@babel/register": "^7.0.0",
-    "babel-jest": "^26.0.1",
+    "babel-jest": "^27.3.1",
     "benchmark": "^2.1.4",
     "eslint": "^7.6.0",
     "eslint-config-prettier": "^8.1.0",
@@ -12,17 +12,17 @@
     "eslint-plugin-eslint-comments": "^3.2.0",
     "eslint-plugin-import": "^2.22.1",
     "eslint-plugin-node": "^11.1.0",
-    "eslint-plugin-promise": "^4.2.1",
+    "eslint-plugin-promise": "^5.1.0",
     "eslint-plugin-react": "^7.21.5",
     "exec-promise": "^0.7.0",
     "globby": "^11.0.1",
     "handlebars": "^4.7.6",
     "husky": "^4.2.5",
-    "jest": "^26.0.1",
-    "lint-staged": "^10.2.7",
+    "jest": "^27.3.1",
+    "lint-staged": "^11.1.2",
     "lodash": "^4.17.4",
     "prettier": "^2.0.5",
-    "promise-toolbox": "^0.19.2",
+    "promise-toolbox": "^0.20.0",
     "sorted-object": "^2.0.1",
     "vuepress": "^1.4.1"
   },

packages/vhd-cli/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "vhd-cli",
-  "version": "0.3.1",
+  "version": "0.5.0",
   "license": "ISC",
   "description": "Tools to read/create and merge VHD files",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-cli",
@@ -24,11 +24,11 @@
     "node": ">=8.10"
   },
   "dependencies": {
-    "@xen-orchestra/fs": "^0.17.0",
+    "@xen-orchestra/fs": "^0.18.0",
     "cli-progress": "^3.1.0",
     "exec-promise": "^0.7.0",
     "getopts": "^2.2.3",
-    "vhd-lib": "^1.0.0"
+    "vhd-lib": "^1.3.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",
@@ -36,8 +36,8 @@
     "@babel/preset-env": "^7.0.0",
     "cross-env": "^7.0.2",
     "execa": "^5.0.0",
-    "index-modules": "^0.3.0",
-    "promise-toolbox": "^0.19.2",
+    "index-modules": "^0.4.3",
+    "promise-toolbox": "^0.20.0",
     "rimraf": "^3.0.0",
     "tmp": "^0.2.1"
   },

packages/vhd-cli/src/commands/check.js

@@ -1,9 +1,9 @@
-import Vhd, { checkVhdChain } from 'vhd-lib'
+import { VhdFile, checkVhdChain } from 'vhd-lib'
 import getopts from 'getopts'
 import { getHandler } from '@xen-orchestra/fs'
 import { resolve } from 'path'
 
-const checkVhd = (handler, path) => new Vhd(handler, path).readHeaderAndFooter()
+const checkVhd = (handler, path) => new VhdFile(handler, path).readHeaderAndFooter()
 
 export default async rawArgs => {
   const { chain, _: args } = getopts(rawArgs, {

packages/vhd-cli/src/commands/compare.js

@@ -0,0 +1,81 @@
+import { getSyncedHandler } from '@xen-orchestra/fs'
+import { openVhd, Constants } from 'vhd-lib'
+import Disposable from 'promise-toolbox/Disposable'
+import omit from 'lodash/omit'
+
+const deepCompareObjects = function (src, dest, path) {
+  for (const key of Object.keys(src)) {
+    const srcValue = src[key]
+    const destValue = dest[key]
+    if (srcValue !== destValue) {
+      const srcType = typeof srcValue
+      const destType = typeof destValue
+      if (srcType !== destType) {
+        throw new Error(`key ${path + '/' + key} is of type *${srcType}* in source and *${destType}* in dest`)
+      }
+
+      if (srcType !== 'object') {
+        throw new Error(`key ${path + '/' + key} is *${srcValue}* in source and *${destValue}* in dest`)
+      }
+
+      if (Buffer.isBuffer(srcValue)) {
+        if (!(Buffer.isBuffer(destValue) && srcValue.equals(destValue))) {
+          throw new Error(`key ${path + '/' + key} is buffer in source that does not equal dest`)
+        }
+      } else {
+        deepCompareObjects(src[key], dest[key], path + '/' + key)
+      }
+    }
+  }
+}
+
+export default async args => {
+  if (args.length < 4 || args.some(_ => _ === '-h' || _ === '--help')) {
+    return `Usage: compare <sourceRemoteUrl> <source VHD> <destionationRemoteUrl> <destination> `
+  }
+  const [sourceRemoteUrl, sourcePath, destRemoteUrl, destPath] = args
+
+  await Disposable.use(async function* () {
+    const sourceHandler = yield getSyncedHandler({ url: sourceRemoteUrl })
+    const src = yield openVhd(sourceHandler, sourcePath)
+    const destHandler = yield getSyncedHandler({ url: destRemoteUrl })
+    const dest = yield openVhd(destHandler, destPath)
+
+    // parent locator entries contains offset that can be different without impacting the vhd
+    // we'll compare them later
+    // table offset and checksum are also implementation specific
+    const ignoredEntries = ['checksum', 'parentLocatorEntry', 'tableOffset']
+    deepCompareObjects(omit(src.header, ignoredEntries), omit(dest.header, ignoredEntries), 'header')
+    deepCompareObjects(src.footer, dest.footer, 'footer')
+
+    await src.readBlockAllocationTable()
+    await dest.readBlockAllocationTable()
+
+    for (let i = 0; i < src.header.maxTableEntries; i++) {
+      if (src.containsBlock(i)) {
+        if (dest.containsBlock(i)) {
+          const srcBlock = await src.readBlock(i)
+          const destBlock = await dest.readBlock(i)
+
+          if (!srcBlock.buffer.equals(destBlock.buffer)) {
+            throw new Error(`Block  ${i} has different data in src and dest`)
+          }
+        } else {
+          throw new Error(`Block  ${i} is present in source but not in dest `)
+        }
+      } else if (dest.containsBlock(i)) {
+        throw new Error(`Block  ${i} is present in dest but not in source `)
+      }
+    }
+
+    for (let parentLocatorId = 0; parentLocatorId < Constants.PARENT_LOCATOR_ENTRIES; parentLocatorId++) {
+      const srcParentLocator = await src.readParentLocator(parentLocatorId)
+      const destParentLocator = await dest.readParentLocator(parentLocatorId)
+      if (!srcParentLocator.data || !srcParentLocator.data.equals(destParentLocator.data)) {
+        console.log(srcParentLocator, destParentLocator)
+        throw new Error(`Parent Locator  ${parentLocatorId} has different data in src and dest`)
+      }
+    }
+    console.log('there is no difference between theses vhd')
+  })
+}

packages/vhd-cli/src/commands/copy.js

@@ -0,0 +1,50 @@
+import { getSyncedHandler } from '@xen-orchestra/fs'
+import { openVhd, VhdFile, VhdDirectory } from 'vhd-lib'
+import Disposable from 'promise-toolbox/Disposable'
+import getopts from 'getopts'
+
+export default async rawArgs => {
+  const {
+    directory,
+    help,
+    _: args,
+  } = getopts(rawArgs, {
+    alias: {
+      directory: 'd',
+      help: 'h',
+    },
+    boolean: ['directory', 'force'],
+    default: {
+      directory: false,
+      help: false,
+    },
+  })
+  if (args.length < 4 || help) {
+    return `Usage: index.js copy <sourceRemoteUrl> <source VHD> <destionationRemoteUrl> <destination> --directory`
+  }
+  const [sourceRemoteUrl, sourcePath, destRemoteUrl, destPath] = args
+
+  await Disposable.use(async function* () {
+    const sourceHandler = yield getSyncedHandler({ url: sourceRemoteUrl })
+    const src = yield openVhd(sourceHandler, sourcePath)
+    await src.readBlockAllocationTable()
+    const destHandler = yield getSyncedHandler({ url: destRemoteUrl })
+    const dest = yield directory ? VhdDirectory.create(destHandler, destPath) : VhdFile.create(destHandler, destPath)
+    // copy data
+    dest.header = src.header
+    dest.footer = src.footer
+
+    for await (const block of src.blocks()) {
+      await dest.writeEntireBlock(block)
+    }
+
+    // copy parent locators
+    for (let parentLocatorId = 0; parentLocatorId < 8; parentLocatorId++) {
+      const parentLocator = await src.readParentLocator(parentLocatorId)
+      await dest.writeParentLocator(parentLocator)
+    }
+    await dest.writeFooter()
+    await dest.writeHeader()
+    await dest.writeBlockAllocationTable()
+  })
+}

packages/vhd-cli/src/commands/info.js

@@ -1,9 +1,9 @@
-import Vhd from 'vhd-lib'
+import { VhdFile } from 'vhd-lib'
 import { getHandler } from '@xen-orchestra/fs'
 import { resolve } from 'path'
 
 export default async args => {
-  const vhd = new Vhd(getHandler({ url: 'file:///' }), resolve(args[0]))
+  const vhd = new VhdFile(getHandler({ url: 'file:///' }), resolve(args[0]))
 
   try {
     await vhd.readHeaderAndFooter()

packages/vhd-cli/src/commands/repl.js

@@ -2,7 +2,7 @@ import { asCallback, fromCallback, fromEvent } from 'promise-toolbox'
 import { getHandler } from '@xen-orchestra/fs'
 import { relative } from 'path'
 import { start as createRepl } from 'repl'
-import Vhd, * as vhdLib from 'vhd-lib'
+import * as vhdLib from 'vhd-lib'
 
 export default async args => {
   const cwd = process.cwd()
@@ -14,7 +14,7 @@ export default async args => {
     })
     Object.assign(repl.context, vhdLib)
     repl.context.handler = handler
-    repl.context.open = path => new Vhd(handler, relative(cwd, path))
+    repl.context.open = path => new vhdLib.VhdFile(handler, relative(cwd, path))
 
     // Make the REPL waits for promise completion.
     repl.eval = (evaluate => (cmd, context, filename, cb) => {

packages/vhd-lib/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "vhd-lib",
-  "version": "1.0.0",
+  "version": "1.3.0",
   "license": "AGPL-3.0-or-later",
   "description": "Primitives for VHD file handling",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-lib",
@@ -17,12 +17,12 @@
   },
   "dependencies": {
     "@vates/read-chunk": "^0.1.2",
-    "@xen-orchestra/log": "^0.2.1",
+    "@xen-orchestra/log": "^0.3.0",
     "async-iterator-to-stream": "^1.0.2",
-    "fs-extra": "^9.0.0",
+    "fs-extra": "^10.0.0",
     "limit-concurrency-decorator": "^0.5.0",
     "lodash": "^4.17.4",
-    "promise-toolbox": "^0.19.2",
+    "promise-toolbox": "^0.20.0",
     "struct-fu": "^1.2.0",
     "uuid": "^8.3.1"
   },
@@ -30,7 +30,7 @@
     "@babel/cli": "^7.0.0",
     "@babel/core": "^7.0.0",
     "@babel/preset-env": "^7.0.0",
-    "@xen-orchestra/fs": "^0.17.0",
+    "@xen-orchestra/fs": "^0.18.0",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^7.0.2",
     "execa": "^5.0.0",

packages/vhd-lib/src/Vhd/VhdAbstract.js

@@ -0,0 +1,167 @@
+import { computeBatSize, sectorsRoundUpNoZero, sectorsToBytes } from './_utils'
+import { PLATFORM_NONE, SECTOR_SIZE, PLATFORM_W2KU, PARENT_LOCATOR_ENTRIES } from '../_constants'
+import assert from 'assert'
+
+export class VhdAbstract {
+  #header
+  bitmapSize
+  footer
+  fullBlockSize
+  sectorsOfBitmap
+  sectorsPerBlock
+
+  get header() {
+    assert.notStrictEqual(this.#header, undefined, `header must be read before it's used`)
+    return this.#header
+  }
+
+  set header(header) {
+    this.#header = header
+    this.sectorsPerBlock = header.blockSize / SECTOR_SIZE
+    this.sectorsOfBitmap = sectorsRoundUpNoZero(this.sectorsPerBlock >> 3)
+    this.fullBlockSize = sectorsToBytes(this.sectorsOfBitmap + this.sectorsPerBlock)
+    this.bitmapSize = sectorsToBytes(this.sectorsOfBitmap)
+  }
+
+  /**
+   * instantiate a Vhd
+   *
+   * @returns {AbstractVhd}
+   */
+  static async open() {
+    throw new Error('open not implemented')
+  }
+
+  /**
+   * Check if this vhd contains a block with id blockId
+   * Must be called after readBlockAllocationTable
+   *
+   * @param {number} blockId
+   * @returns {boolean}
+   *
+   */
+  containsBlock(blockId) {
+    throw new Error(`checking if this vhd contains the block ${blockId} is not implemented`)
+  }
+
+  /**
+   * Read the header and the footer
+   * check their integrity
+   * if checkSecondFooter also checks that the footer at the end is equal to the one at the beginning
+   *
+   * @param {boolean} checkSecondFooter
+   */
+  readHeaderAndFooter(checkSecondFooter = true) {
+    throw new Error(
+      `reading and checking footer, ${checkSecondFooter ? 'second footer,' : ''} and header is not implemented`
+    )
+  }
+
+  readBlockAllocationTable() {
+    throw new Error(`reading block allocation table is not implemented`)
+  }
+
+  /**
+   *
+   * @param {number} blockId
+   * @param {boolean} onlyBitmap
+   * @returns {Buffer}
+   */
+  readBlock(blockId, onlyBitmap = false) {
+    throw new Error(`reading  ${onlyBitmap ? 'bitmap of block' : 'block'} ${blockId} is not implemented`)
+  }
+
+  /**
+   * coalesce the block with id blockId from the child vhd into
+   * this vhd
+   *
+   * @param {AbstractVhd} child
+   * @param {number} blockId
+   *
+   * @returns {number} the merged data size
+   */
+  coalesceBlock(child, blockId) {
+    throw new Error(`coalescing the block ${blockId} from ${child} is not implemented`)
+  }
+
+  /**
+   * ensure the bat size can store at least entries block
+   * move blocks if needed
+   * @param {number} entries
+   */
+  ensureBatSize(entries) {
+    throw new Error(`ensuring batSize can store at least  ${entries} is not implemented`)
+  }
+
+  // Write a context footer. (At the end and beginning of a vhd file.)
+  writeFooter(onlyEndFooter = false) {
+    throw new Error(`writing footer   ${onlyEndFooter ? 'only at end' : 'on both side'} is not implemented`)
+  }
+
+  writeHeader() {
+    throw new Error(`writing header is not implemented`)
+  }
+
+  _writeParentLocatorData(parentLocatorId, platformDataOffset, data) {
+    throw new Error(`write Parent locator ${parentLocatorId} is not implemented`)
+  }
+
+  _readParentLocatorData(parentLocatorId, platformDataOffset, platformDataSpace) {
+    throw new Error(`read Parent locator ${parentLocatorId} is not implemented`)
+  }
+  // common
+  get batSize() {
+    return computeBatSize(this.header.maxTableEntries)
+  }
+
+  async writeParentLocator({ id, platformCode = PLATFORM_NONE, data = Buffer.alloc(0) }) {
+    assert(id >= 0, 'parent Locator id must be a positive number')
+    assert(id < PARENT_LOCATOR_ENTRIES, `parent Locator id  must be less than ${PARENT_LOCATOR_ENTRIES}`)
+
+    await this._writeParentLocatorData(id, data)
+
+    const entry = this.header.parentLocatorEntry[id]
+    const dataSpaceSectors = Math.ceil(data.length / SECTOR_SIZE)
+    entry.platformCode = platformCode
+    entry.platformDataSpace = dataSpaceSectors * SECTOR_SIZE
+    entry.platformDataLength = data.length
+  }
+
+  async readParentLocator(id) {
+    assert(id >= 0, 'parent Locator id must be a positive number')
+    assert(id < PARENT_LOCATOR_ENTRIES, `parent Locator id  must be less than ${PARENT_LOCATOR_ENTRIES}`)
+    const data = await this._readParentLocatorData(id)
+    // offset is storage specific, don't expose it
+    const { platformCode } = this.header.parentLocatorEntry[id]
+    return {
+      platformCode,
+      id,
+      data,
+    }
+  }
+
+  async setUniqueParentLocator(fileNameString) {
+    await this.writeParentLocator({
+      id: 0,
+      code: PLATFORM_W2KU,
+      data: Buffer.from(fileNameString, 'utf16le'),
+    })
+
+    for (let i = 1; i < PARENT_LOCATOR_ENTRIES; i++) {
+      await this.writeParentLocator({
+        id: i,
+        code: PLATFORM_NONE,
+        data: Buffer.alloc(0),
+      })
+    }
+  }
+
+  async *blocks() {
+    const nBlocks = this.header.maxTableEntries
+    for (let blockId = 0; blockId < nBlocks; ++blockId) {
+      if (await this.containsBlock(blockId)) {
+        yield await this.readBlock(blockId)
+      }
+    }
+  }
+}

packages/vhd-lib/src/Vhd/VhdDirectory.js

@@ -0,0 +1,190 @@
+import { buildHeader, buildFooter } from './_utils'
+import { createLogger } from '@xen-orchestra/log'
+import { fuFooter, fuHeader, checksumStruct } from '../_structs'
+import { test, set as setBitmap } from '../_bitmap'
+import { VhdAbstract } from './VhdAbstract'
+import assert from 'assert'
+
+const { debug } = createLogger('vhd-lib:VhdDirectory')
+
+// ===================================================================
+// Directory format
+// <path>
+// ├─ header // raw content of the header
+// ├─ footer // raw content of the footer
+// ├─ bat // bit array. A zero bit indicates at a position that this block is not present
+// ├─ parentLocatorEntry{0-7} // data of a parent locator
+// ├─ blocks // blockId is the position in the BAT
+//    └─ <the first to  {blockId.length -3} numbers of blockId >
+//         └─ <the three last numbers  of blockID >  // block content.
+
+export class VhdDirectory extends VhdAbstract {
+  #uncheckedBlockTable
+
+  set header(header) {
+    super.header = header
+    this.#blockTable = Buffer.alloc(header.maxTableEntries)
+  }
+
+  get header() {
+    return super.header
+  }
+
+  get #blockTable() {
+    assert.notStrictEqual(this.#uncheckedBlockTable, undefined, 'Block table must be initialized before access')
+    return this.#uncheckedBlockTable
+  }
+
+  set #blockTable(blockTable) {
+    this.#uncheckedBlockTable = blockTable
+  }
+
+  static async open(handler, path) {
+    const vhd = new VhdDirectory(handler, path)
+
+    // openning a file for reading does not trigger EISDIR as long as we don't really read from it :
+    // https://man7.org/linux/man-pages/man2/open.2.html
+    // EISDIR pathname refers to a directory and the access requested
+    // involved writing (that is, O_WRONLY or O_RDWR is set).
+    // reading the header ensure we have a well formed directory immediatly
+    await vhd.readHeaderAndFooter()
+    return {
+      dispose: () => {},
+      value: vhd,
+    }
+  }
+
+  static async create(handler, path) {
+    await handler.mkdir(path)
+    const vhd = new VhdDirectory(handler, path)
+    return {
+      dispose: () => {},
+      value: vhd,
+    }
+  }
+
+  constructor(handler, path) {
+    super()
+    this._handler = handler
+    this._path = path
+  }
+
+  async readBlockAllocationTable() {
+    const { buffer } = await this._readChunk('bat')
+    this.#blockTable = buffer
+  }
+
+  containsBlock(blockId) {
+    return test(this.#blockTable, blockId)
+  }
+
+  getChunkPath(partName) {
+    return this._path + '/' + partName
+  }
+
+  async _readChunk(partName) {
+    // here we can implement compression and / or crypto
+    const buffer = await this._handler.readFile(this.getChunkPath(partName))
+
+    return {
+      buffer: Buffer.from(buffer),
+    }
+  }
+
+  async _writeChunk(partName, buffer) {
+    assert(Buffer.isBuffer(buffer))
+    // here we can implement compression and / or crypto
+
+    // chunks can be in sub directories :  create direcotries if necessary
+    const pathParts = partName.split('/')
+    let currentPath = this._path
+
+    // the last one is the file name
+    for (let i = 0; i < pathParts.length - 1; i++) {
+      currentPath += '/' + pathParts[i]
+      await this._handler.mkdir(currentPath)
+    }
+
+    return this._handler.writeFile(this.getChunkPath(partName), buffer)
+  }
+
+  // put block in subdirectories to limit impact when doing directory listing
+  _getBlockPath(blockId) {
+    const blockPrefix = Math.floor(blockId / 1e3)
+    const blockSuffix = blockId - blockPrefix * 1e3
+    return `blocks/${blockPrefix}/${blockSuffix}`
+  }
+
+  async readHeaderAndFooter() {
+    const { buffer: bufHeader } = await this._readChunk('header')
+    const { buffer: bufFooter } = await this._readChunk('footer')
+    const footer = buildFooter(bufFooter)
+    const header = buildHeader(bufHeader, footer)
+
+    this.footer = footer
+    this.header = header
+  }
+
+  async readBlock(blockId, onlyBitmap = false) {
+    if (onlyBitmap) {
+      throw new Error(`reading 'bitmap of block' ${blockId} in a VhdDirectory is not implemented`)
+    }
+    const { buffer } = await this._readChunk(this._getBlockPath(blockId))
+    return {
+      id: blockId,
+      bitmap: buffer.slice(0, this.bitmapSize),
+      data: buffer.slice(this.bitmapSize),
+      buffer,
+    }
+  }
+  ensureBatSize() {
+    // nothing to do in directory mode
+  }
+
+  async writeFooter() {
+    const { footer } = this
+
+    const rawFooter = fuFooter.pack(footer)
+
+    footer.checksum = checksumStruct(rawFooter, fuFooter)
+    debug(`Write footer  (checksum=${footer.checksum}). (data=${rawFooter.toString('hex')})`)
+
+    await this._writeChunk('footer', rawFooter)
+  }
+
+  writeHeader() {
+    const { header } = this
+    const rawHeader = fuHeader.pack(header)
+    header.checksum = checksumStruct(rawHeader, fuHeader)
+    debug(`Write header  (checksum=${header.checksum}). (data=${rawHeader.toString('hex')})`)
+    return this._writeChunk('header', rawHeader)
+  }
+
+  writeBlockAllocationTable() {
+    assert.notStrictEqual(this.#blockTable, undefined, 'Block allocation table has not been read')
+    assert.notStrictEqual(this.#blockTable.length, 0, 'Block allocation table is empty')
+
+    return this._writeChunk('bat', this.#blockTable)
+  }
+
+  // only works if data are in the same bucket
+  // and if the full block is modified in child ( which is the case whit xcp)
+
+  coalesceBlock(child, blockId) {
+    this._handler.copy(child.getChunkPath(blockId), this.getChunkPath(blockId))
+  }
+
+  async writeEntireBlock(block) {
+    await this._writeChunk(this._getBlockPath(block.id), block.buffer)
+    setBitmap(this.#blockTable, block.id)
+  }
+
+  async _readParentLocatorData(id) {
+    return (await this._readChunk('parentLocatorEntry' + id)).buffer
+  }
+
+  async _writeParentLocatorData(id, data) {
+    await this._writeChunk('parentLocatorEntry' + id, data)
+    this.header.parentLocatorEntry[id].platformDataOffset = 0
+  }
+}

packages/vhd-lib/src/Vhd/VhdFile.js

@@ -1,22 +1,20 @@
-import assert from 'assert'
-import { createLogger } from '@xen-orchestra/log'
-
-import checkFooter from './_checkFooter'
-import checkHeader from './_checkHeader'
-import getFirstAndLastBlocks from './_getFirstAndLastBlocks'
-import { fuFooter, fuHeader, checksumStruct, unpackField } from './_structs'
-import { set as mapSetBit, test as mapTestBit } from './_bitmap'
 import {
   BLOCK_UNUSED,
   FOOTER_SIZE,
   HEADER_SIZE,
-  PARENT_LOCATOR_ENTRIES,
   PLATFORM_NONE,
-  PLATFORM_W2KU,
   SECTOR_SIZE,
-} from './_constants'
+  PARENT_LOCATOR_ENTRIES,
+} from '../_constants'
+import { computeBatSize, sectorsToBytes, buildHeader, buildFooter, BUF_BLOCK_UNUSED } from './_utils'
+import { createLogger } from '@xen-orchestra/log'
+import { fuFooter, fuHeader, checksumStruct } from '../_structs'
+import { set as mapSetBit, test as mapTestBit } from '../_bitmap'
+import { VhdAbstract } from './VhdAbstract'
+import assert from 'assert'
+import getFirstAndLastBlocks from '../_getFirstAndLastBlocks'
 
-const { debug } = createLogger('vhd-lib:Vhd')
+const { debug } = createLogger('vhd-lib:VhdFile')
 
 // ===================================================================
 //
@@ -28,22 +26,6 @@ const { debug } = createLogger('vhd-lib:Vhd')
 //
 // ===================================================================
 
-const computeBatSize = entries => sectorsToBytes(sectorsRoundUpNoZero(entries * 4))
-
-// Sectors conversions.
-const sectorsRoundUpNoZero = bytes => Math.ceil(bytes / SECTOR_SIZE) || 1
-const sectorsToBytes = sectors => sectors * SECTOR_SIZE
-
-const assertChecksum = (name, buf, struct) => {
-  const actual = unpackField(struct.fields.checksum, buf)
-  const expected = checksumStruct(buf, struct)
-  assert.strictEqual(actual, expected, `invalid ${name} checksum ${actual}, expected ${expected}`)
-}
-
-// unused block as buffer containing a uint32BE
-const BUF_BLOCK_UNUSED = Buffer.allocUnsafe(4)
-BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0)
-
 // ===================================================================
 
 // Format:
@@ -68,12 +50,60 @@ BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0)
 // - parentLocatorSize(i) = header.parentLocatorEntry[i].platformDataSpace * sectorSize
 // - sectorSize = 512
 
-export default class Vhd {
+export class VhdFile extends VhdAbstract {
+  #uncheckedBlockTable
+
+  get #blockTable() {
+    assert.notStrictEqual(this.#uncheckedBlockTable, undefined, 'Block table must be initialized before access')
+    return this.#uncheckedBlockTable
+  }
+
+  set #blockTable(blockTable) {
+    this.#uncheckedBlockTable = blockTable
+  }
+
   get batSize() {
     return computeBatSize(this.header.maxTableEntries)
   }
 
+  set header(header) {
+    super.header = header
+    const size = this.batSize
+    this.#blockTable = Buffer.alloc(size)
+    for (let i = 0; i < this.header.maxTableEntries; i++) {
+      this.#blockTable.writeUInt32BE(BLOCK_UNUSED, i * 4)
+    }
+  }
+  get header() {
+    return super.header
+  }
+
+  static async open(handler, path) {
+    const fd = await handler.openFile(path, 'r+')
+    const vhd = new VhdFile(handler, fd)
+    // openning a file for reading does not trigger EISDIR as long as we don't really read from it :
+    // https://man7.org/linux/man-pages/man2/open.2.html
+    // EISDIR pathname refers to a directory and the access requested
+    // involved writing (that is, O_WRONLY or O_RDWR is set).
+    // reading the header ensure we have a well formed file immediatly
+    await vhd.readHeaderAndFooter()
+    return {
+      dispose: () => handler.closeFile(fd),
+      value: vhd,
+    }
+  }
+
+  static async create(handler, path) {
+    const fd = await handler.openFile(path, 'wx')
+    const vhd = new VhdFile(handler, fd)
+    return {
+      dispose: () => handler.closeFile(fd),
+      value: vhd,
+    }
+  }
+
   constructor(handler, path) {
+    super()
     this._handler = handler
     this._path = path
   }
@@ -87,11 +117,6 @@ export default class Vhd {
     assert.strictEqual(bytesRead, n)
     return buffer
   }
-
-  containsBlock(id) {
-    return this._getBatEntry(id) !== BLOCK_UNUSED
-  }
-
   // Returns the first address after metadata. (In bytes)
   _getEndOfHeaders() {
     const { header } = this
@@ -114,17 +139,24 @@ export default class Vhd {
     return end
   }
 
+  // return the first sector (bitmap) of a block
+  _getBatEntry(blockId) {
+    const i = blockId * 4
+    const blockTable = this.#blockTable
+    return i < blockTable.length ? blockTable.readUInt32BE(i) : BLOCK_UNUSED
+  }
+
   // Returns the first sector after data.
   _getEndOfData() {
     let end = Math.ceil(this._getEndOfHeaders() / SECTOR_SIZE)
 
-    const fullBlockSize = this.sectorsOfBitmap + this.sectorsPerBlock
+    const sectorsOfFullBlock = this.sectorsOfBitmap + this.sectorsPerBlock
     const { maxTableEntries } = this.header
     for (let i = 0; i < maxTableEntries; i++) {
       const blockAddr = this._getBatEntry(i)
 
       if (blockAddr !== BLOCK_UNUSED) {
-        end = Math.max(end, blockAddr + fullBlockSize)
+        end = Math.max(end, blockAddr + sectorsOfFullBlock)
       }
     }
 
@@ -133,7 +165,11 @@ export default class Vhd {
     return sectorsToBytes(end)
   }
 
-  // TODO: extract the checks into reusable functions:
+  containsBlock(id) {
+    return this._getBatEntry(id) !== BLOCK_UNUSED
+  }
+
+  // TODO:
   // - better human reporting
   // - auto repair if possible
   async readHeaderAndFooter(checkSecondFooter = true) {
@@ -141,50 +177,25 @@ export default class Vhd {
     const bufFooter = buf.slice(0, FOOTER_SIZE)
     const bufHeader = buf.slice(FOOTER_SIZE)
 
-    assertChecksum('footer', bufFooter, fuFooter)
-    assertChecksum('header', bufHeader, fuHeader)
+    const footer = buildFooter(bufFooter)
+    const header = buildHeader(bufHeader, footer)
 
     if (checkSecondFooter) {
       const size = await this._handler.getSize(this._path)
       assert(bufFooter.equals(await this._read(size - FOOTER_SIZE, FOOTER_SIZE)), 'footer1 !== footer2')
     }
 
-    const footer = (this.footer = fuFooter.unpack(bufFooter))
-    checkFooter(footer)
-
-    const header = (this.header = fuHeader.unpack(bufHeader))
-    checkHeader(header, footer)
-
-    // Compute the number of sectors in one block.
-    // Default: One block contains 4096 sectors of 512 bytes.
-    const sectorsPerBlock = (this.sectorsPerBlock = header.blockSize / SECTOR_SIZE)
-
-    // Compute bitmap size in sectors.
-    // Default: 1.
-    const sectorsOfBitmap = (this.sectorsOfBitmap = sectorsRoundUpNoZero(sectorsPerBlock >> 3))
-
-    // Full block size => data block size + bitmap size.
-    this.fullBlockSize = sectorsToBytes(sectorsPerBlock + sectorsOfBitmap)
-
-    // In bytes.
-    // Default: 512.
-    this.bitmapSize = sectorsToBytes(sectorsOfBitmap)
+    this.footer = footer
+    this.header = header
   }
 
   // Returns a buffer that contains the block allocation table of a vhd file.
   async readBlockAllocationTable() {
     const { header } = this
-    this.blockTable = await this._read(header.tableOffset, header.maxTableEntries * 4)
+    this.#blockTable = await this._read(header.tableOffset, header.maxTableEntries * 4)
   }
 
-  // return the first sector (bitmap) of a block
-  _getBatEntry(blockId) {
-    const i = blockId * 4
-    const { blockTable } = this
-    return i < blockTable.length ? blockTable.readUInt32BE(i) : BLOCK_UNUSED
-  }
-
-  _readBlock(blockId, onlyBitmap = false) {
+  readBlock(blockId, onlyBitmap = false) {
     const blockAddr = this._getBatEntry(blockId)
     if (blockAddr === BLOCK_UNUSED) {
       throw new Error(`no such block ${blockId}`)
@@ -214,7 +225,7 @@ export default class Vhd {
   }
 
   async _freeFirstBlockSpace(spaceNeededBytes) {
-    const firstAndLastBlocks = getFirstAndLastBlocks(this.blockTable)
+    const firstAndLastBlocks = getFirstAndLastBlocks(this.#blockTable)
     if (firstAndLastBlocks === undefined) {
       return
     }
@@ -249,8 +260,8 @@ export default class Vhd {
     const newBatSize = computeBatSize(entries)
     await this._freeFirstBlockSpace(newBatSize - this.batSize)
     const maxTableEntries = (header.maxTableEntries = entries)
-    const prevBat = this.blockTable
-    const bat = (this.blockTable = Buffer.allocUnsafe(newBatSize))
+    const prevBat = this.#blockTable
+    const bat = (this.#blockTable = Buffer.allocUnsafe(newBatSize))
     prevBat.copy(bat)
     bat.fill(BUF_BLOCK_UNUSED, prevMaxTableEntries * 4)
     debug(`ensureBatSize: extend BAT ${prevMaxTableEntries} -> ${maxTableEntries}`)
@@ -264,7 +275,7 @@ export default class Vhd {
   // set the first sector (bitmap) of a block
   _setBatEntry(block, blockSector) {
     const i = block * 4
-    const { blockTable } = this
+    const blockTable = this.#blockTable
 
     blockTable.writeUInt32BE(blockSector, i)
 
@@ -298,7 +309,7 @@ export default class Vhd {
     await this._write(bitmap, sectorsToBytes(blockAddr))
   }
 
-  async _writeEntireBlock(block) {
+  async writeEntireBlock(block) {
     let blockAddr = this._getBatEntry(block.id)
 
     if (blockAddr === BLOCK_UNUSED) {
@@ -314,7 +325,7 @@ export default class Vhd {
       blockAddr = await this._createBlock(block.id)
       parentBitmap = Buffer.alloc(this.bitmapSize, 0)
     } else if (parentBitmap === undefined) {
-      parentBitmap = (await this._readBlock(block.id, true)).bitmap
+      parentBitmap = (await this.readBlock(block.id, true)).bitmap
     }
 
     const offset = blockAddr + this.sectorsOfBitmap + beginSectorId
@@ -333,7 +344,7 @@ export default class Vhd {
   }
 
   async coalesceBlock(child, blockId) {
-    const block = await child._readBlock(blockId)
+    const block = await child.readBlock(blockId)
     const { bitmap, data } = block
 
     debug(`coalesceBlock block=${blockId}`)
@@ -358,10 +369,10 @@ export default class Vhd {
 
       const isFullBlock = i === 0 && endSector === sectorsPerBlock
       if (isFullBlock) {
-        await this._writeEntireBlock(block)
+        await this.writeEntireBlock(block)
       } else {
         if (parentBitmap === null) {
-          parentBitmap = (await this._readBlock(blockId, true)).bitmap
+          parentBitmap = (await this.readBlock(blockId, true)).bitmap
         }
         await this._writeBlockSectors(block, i, endSector, parentBitmap)
       }
@@ -399,6 +410,13 @@ export default class Vhd {
     return this._write(rawHeader, offset)
   }
 
+  writeBlockAllocationTable() {
+    const header = this.header
+    const blockTable = this.#blockTable
+    debug(`Write BlockAllocationTable at: ${header.tableOffset} ). (data=${blockTable.toString('hex')})`)
+    return this._write(blockTable, header.tableOffset)
+  }
+
   async writeData(offsetSectors, buffer) {
     const bufferSizeSectors = Math.ceil(buffer.length / SECTOR_SIZE)
     const startBlock = Math.floor(offsetSectors / this.sectorsPerBlock)
@@ -436,26 +454,35 @@ export default class Vhd {
       const deltaSectors = neededSectors - currentSpace
       await this._freeFirstBlockSpace(sectorsToBytes(deltaSectors))
       this.header.tableOffset += sectorsToBytes(deltaSectors)
-      await this._write(this.blockTable, this.header.tableOffset)
+      await this._write(this.#blockTable, this.header.tableOffset)
     }
     return firstLocatorOffset
   }
 
-  async setUniqueParentLocator(fileNameString) {
+  async _readParentLocatorData(parentLocatorId) {
+    const { platformDataOffset, platformDataLength } = this.header.parentLocatorEntry[parentLocatorId]
+    if (platformDataLength > 0) {
+      return (await this._read(platformDataOffset, platformDataLength)).buffer
+    }
+    return Buffer.alloc(0)
+  }
+
+  async _writeParentLocatorData(parentLocatorId, data) {
+    let position
     const { header } = this
-    header.parentLocatorEntry[0].platformCode = PLATFORM_W2KU
-    const encodedFilename = Buffer.from(fileNameString, 'utf16le')
-    const dataSpaceSectors = Math.ceil(encodedFilename.length / SECTOR_SIZE)
-    const position = await this._ensureSpaceForParentLocators(dataSpaceSectors)
-    await this._write(encodedFilename, position)
-    header.parentLocatorEntry[0].platformDataSpace = dataSpaceSectors * SECTOR_SIZE
-    header.parentLocatorEntry[0].platformDataLength = encodedFilename.length
-    header.parentLocatorEntry[0].platformDataOffset = position
-    for (let i = 1; i < 8; i++) {
-      header.parentLocatorEntry[i].platformCode = PLATFORM_NONE
-      header.parentLocatorEntry[i].platformDataSpace = 0
-      header.parentLocatorEntry[i].platformDataLength = 0
-      header.parentLocatorEntry[i].platformDataOffset = 0
+    if (data.length === 0) {
+      // reset offset if data is empty
+      header.parentLocatorEntry[parentLocatorId].platformDataOffset = 0
+    } else {
+      if (data.length <= header.parentLocatorEntry[parentLocatorId].platformDataSpace) {
+        // new parent locator length is smaller than available space : keep it in place
+        position = header.parentLocatorEntry[parentLocatorId].platformDataOffset
+      } else {
+        // new parent locator length is bigger than available space : move it to the end
+        position = this._getEndOfData()
+      }
+      await this._write(data, position)
+      header.parentLocatorEntry[parentLocatorId].platformDataOffset = position
     }
   }
 }

packages/vhd-lib/src/Vhd/_utils.js

@@ -0,0 +1,52 @@
+import assert from 'assert'
+import { BLOCK_UNUSED, SECTOR_SIZE } from '../_constants'
+import { fuFooter, fuHeader, checksumStruct, unpackField } from '../_structs'
+import checkFooter from '../checkFooter'
+import checkHeader from '../_checkHeader'
+
+export const computeBatSize = entries => sectorsToBytes(sectorsRoundUpNoZero(entries * 4))
+
+// Sectors conversions.
+export const sectorsRoundUpNoZero = bytes => Math.ceil(bytes / SECTOR_SIZE) || 1
+export const sectorsToBytes = sectors => sectors * SECTOR_SIZE
+
+export const assertChecksum = (name, buf, struct) => {
+  const actual = unpackField(struct.fields.checksum, buf)
+  const expected = checksumStruct(buf, struct)
+  assert.strictEqual(actual, expected, `invalid ${name} checksum ${actual}, expected ${expected}`)
+}
+
+// unused block as buffer containing a uint32BE
+export const BUF_BLOCK_UNUSED = Buffer.allocUnsafe(4)
+BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0)
+
+/**
+ * Check and parse the header buffer to build an header object
+ *
+ * @param {Buffer} bufHeader
+ * @param {Object} footer
+ * @returns {Object} the parsed header
+ */
+export const buildHeader = (bufHeader, footer) => {
+  assertChecksum('header', bufHeader, fuHeader)
+
+  const header = fuHeader.unpack(bufHeader)
+  checkHeader(header, footer)
+  return header
+}
+
+/**
+ * Check and parse the footer buffer to build a footer object
+ *
+ * @param {Buffer} bufHeader
+ * @param {Object} footer
+ * @returns {Object} the parsed footer
+ */
+
+export const buildFooter = bufFooter => {
+  assertChecksum('footer', bufFooter, fuFooter)
+
+  const footer = fuFooter.unpack(bufFooter)
+  checkFooter(footer)
+  return footer
+}

packages/vhd-lib/src/_getFirstAndLastBlocks.js

@@ -6,7 +6,9 @@ import { BLOCK_UNUSED } from './_constants'
 // in the file
 export default bat => {
   const n = bat.length
-  assert.notStrictEqual(n, 0)
+  if (n === 0) {
+    return
+  }
   assert.strictEqual(n % 4, 0)
 
   let i = 0

packages/vhd-lib/src/chain.js

@@ -1,11 +1,11 @@
 import { dirname, relative } from 'path'
 
-import Vhd from './vhd'
+import { VhdFile } from './'
 import { DISK_TYPE_DIFFERENCING } from './_constants'
 
 export default async function chain(parentHandler, parentPath, childHandler, childPath, force = false) {
-  const parentVhd = new Vhd(parentHandler, parentPath)
-  const childVhd = new Vhd(childHandler, childPath)
+  const parentVhd = new VhdFile(parentHandler, parentPath)
+  const childVhd = new VhdFile(childHandler, childPath)
 
   await childVhd.readHeaderAndFooter()
   const { header, footer } = childVhd

packages/vhd-lib/src/checkChain.js

@@ -1,10 +1,10 @@
-import Vhd from './vhd'
+import { VhdFile } from '.'
 import resolveRelativeFromFile from './_resolveRelativeFromFile'
 import { DISK_TYPE_DYNAMIC } from './_constants'
 
 export default async function checkChain(handler, path) {
   while (true) {
-    const vhd = new Vhd(handler, path)
+    const vhd = new VhdFile(handler, path)
     await vhd.readHeaderAndFooter()
 
     if (vhd.footer.diskType === DISK_TYPE_DYNAMIC) {

packages/vhd-lib/src/createContentStream.js

@@ -1,11 +1,11 @@
 import asyncIteratorToStream from 'async-iterator-to-stream'
 
-import Vhd from './vhd'
+import { VhdFile } from '.'
 
 export default asyncIteratorToStream(async function* (handler, path) {
   const fd = await handler.openFile(path, 'r')
   try {
-    const vhd = new Vhd(handler, fd)
+    const vhd = new VhdFile(handler, fd)
     await vhd.readHeaderAndFooter()
     await vhd.readBlockAllocationTable()
     const {
@@ -17,10 +17,10 @@ export default asyncIteratorToStream(async function* (handler, path) {
 
     const emptyBlock = Buffer.alloc(blockSize)
     for (let i = 0; i < nFullBlocks; ++i) {
-      yield vhd.containsBlock(i) ? (await vhd._readBlock(i)).data : emptyBlock
+      yield vhd.containsBlock(i) ? (await vhd.readBlock(i)).data : emptyBlock
     }
     if (nLeftoverBytes !== 0) {
-      yield (vhd.containsBlock(nFullBlocks) ? (await vhd._readBlock(nFullBlocks)).data : emptyBlock).slice(
+      yield (vhd.containsBlock(nFullBlocks) ? (await vhd.readBlock(nFullBlocks)).data : emptyBlock).slice(
         0,
         nLeftoverBytes
       )

packages/vhd-lib/src/createSyntheticStream.js

@@ -3,7 +3,7 @@ import { createLogger } from '@xen-orchestra/log'
 
 import resolveRelativeFromFile from './_resolveRelativeFromFile'
 
-import Vhd from './vhd'
+import { VhdFile } from '.'
 import { BLOCK_UNUSED, DISK_TYPE_DYNAMIC, FOOTER_SIZE, HEADER_SIZE, SECTOR_SIZE } from './_constants'
 import { fuFooter, fuHeader, checksumStruct } from './_structs'
 import { test as mapTestBit } from './_bitmap'
@@ -27,7 +27,7 @@ export default async function createSyntheticStream(handler, paths) {
     const open = async path => {
       const fd = await handler.openFile(path, 'r')
       fds.push(fd)
-      const vhd = new Vhd(handler, fd)
+      const vhd = new VhdFile(handler, fd)
       vhds.push(vhd)
       await vhd.readHeaderAndFooter()
       await vhd.readBlockAllocationTable()
@@ -126,7 +126,7 @@ export default async function createSyntheticStream(handler, paths) {
             }
             let block = blocksByVhd.get(vhd)
             if (block === undefined) {
-              block = yield vhd._readBlock(iBlock)
+              block = yield vhd.readBlock(iBlock)
               blocksByVhd.set(vhd, block)
             }
             const { bitmap, data } = block

packages/vhd-lib/src/createVhdStreamWithLength.integ.spec.js

@@ -1,6 +1,5 @@
 /* eslint-env jest */
 
-import asyncIteratorToStream from 'async-iterator-to-stream'
 import execa from 'execa'
 import fs from 'fs-extra'
 import rimraf from 'rimraf'
@@ -12,6 +11,7 @@ import { pipeline } from 'readable-stream'
 
 import { createVhdStreamWithLength } from '.'
 import { FOOTER_SIZE } from './_constants'
+import { createRandomFile, convertFromRawToVhd, convertFromVhdToRaw } from './tests/utils'
 
 let tempDir = null
 
@@ -23,27 +23,6 @@ afterEach(async () => {
   await pFromCallback(cb => rimraf(tempDir, cb))
 })
 
-const RAW = 'raw'
-const VHD = 'vpc'
-const convert = (inputFormat, inputFile, outputFormat, outputFile) =>
-  execa('qemu-img', ['convert', '-f', inputFormat, '-O', outputFormat, inputFile, outputFile])
-
-const createRandomStream = asyncIteratorToStream(function* (size) {
-  let requested = Math.min(size, yield)
-  while (size > 0) {
-    const buf = Buffer.allocUnsafe(requested)
-    for (let i = 0; i < requested; ++i) {
-      buf[i] = Math.floor(Math.random() * 256)
-    }
-    requested = Math.min((size -= requested), yield buf)
-  }
-})
-
-async function createRandomFile(name, size) {
-  const input = await createRandomStream(size)
-  await pFromCallback(cb => pipeline(input, fs.createWriteStream(name), cb))
-}
-
 const forOwn = (object, cb) => Object.keys(object).forEach(key => cb(object[key], key, object))
 
 describe('createVhdStreamWithLength', () => {
@@ -58,10 +37,10 @@ describe('createVhdStreamWithLength', () => {
     (size, title) =>
       it(title, async () => {
         const inputRaw = `${tempDir}/input.raw`
-        await createRandomFile(inputRaw, size)
+        await createRandomFile(inputRaw, size / 1024 / 1024)
 
         const inputVhd = `${tempDir}/input.vhd`
-        await convert(RAW, inputRaw, VHD, inputVhd)
+        await convertFromRawToVhd(inputRaw, inputVhd)
 
         const result = await createVhdStreamWithLength(await createReadStream(inputVhd))
         const { length } = result
@@ -75,18 +54,18 @@ describe('createVhdStreamWithLength', () => {
 
         // ensure the generated VHD is correct and contains the same data
         const outputRaw = `${tempDir}/output.raw`
-        await convert(VHD, outputVhd, RAW, outputRaw)
+        await convertFromVhdToRaw(outputVhd, outputRaw)
         await execa('cmp', [inputRaw, outputRaw])
       })
   )
 
   it('can skip blank after the last block and before the footer', async () => {
-    const initialSize = 4 * 1024
+    const initialSize = 4
     const rawFileName = `${tempDir}/randomfile`
     const vhdName = `${tempDir}/randomfile.vhd`
     const outputVhdName = `${tempDir}/output.vhd`
     await createRandomFile(rawFileName, initialSize)
-    await convert(RAW, rawFileName, VHD, vhdName)
+    await convertFromRawToVhd(rawFileName, vhdName)
     const { size: vhdSize } = await fs.stat(vhdName)
     // read file footer
     const footer = await getStream.buffer(createReadStream(vhdName, { start: vhdSize - FOOTER_SIZE }))

packages/vhd-lib/src/createVhdStreamWithLength.js

@@ -2,7 +2,7 @@ import assert from 'assert'
 import { pipeline, Transform } from 'readable-stream'
 import { readChunk } from '@vates/read-chunk'
 
-import checkFooter from './_checkFooter'
+import checkFooter from './checkFooter'
 import checkHeader from './_checkHeader'
 import noop from './_noop'
 import getFirstAndLastBlocks from './_getFirstAndLastBlocks'

packages/vhd-lib/src/index.js

@@ -1,10 +1,14 @@
-export { default } from './vhd'
 export { default as chainVhd } from './chain'
+export { default as checkFooter } from './checkFooter'
 export { default as checkVhdChain } from './checkChain'
 export { default as createContentStream } from './createContentStream'
 export { default as createReadableRawStream } from './createReadableRawStream'
 export { default as createReadableSparseStream } from './createReadableSparseStream'
 export { default as createSyntheticStream } from './createSyntheticStream'
-export { default as mergeVhd } from './merge'
 export { default as createVhdStreamWithLength } from './createVhdStreamWithLength'
+export { default as mergeVhd } from './merge'
 export { default as peekFooterFromVhdStream } from './peekFooterFromVhdStream'
+export { openVhd } from './openVhd'
+export { VhdDirectory } from './Vhd/VhdDirectory'
+export { VhdFile } from './Vhd/VhdFile'
+export * as Constants from './_constants'

packages/vhd-lib/src/merge.integ.spec.js

@@ -1,6 +1,5 @@
 /* eslint-env jest */
 
-import asyncIteratorToStream from 'async-iterator-to-stream'
 import execa from 'execa'
 import fs from 'fs-extra'
 import getStream from 'get-stream'
@@ -11,9 +10,10 @@ import { pFromCallback } from 'promise-toolbox'
 import { pipeline } from 'readable-stream'
 import { randomBytes } from 'crypto'
 
-import Vhd, { chainVhd, createSyntheticStream, mergeVhd as vhdMerge } from './index'
+import { VhdFile, chainVhd, createSyntheticStream, mergeVhd as vhdMerge } from './index'
 
 import { SECTOR_SIZE } from './_constants'
+import { checkFile, createRandomFile, convertFromRawToVhd, recoverRawContent } from './tests/utils'
 
 let tempDir = null
 
@@ -27,32 +27,6 @@ afterEach(async () => {
   await pFromCallback(cb => rimraf(tempDir, cb))
 })
 
-async function createRandomFile(name, sizeMB) {
-  const createRandomStream = asyncIteratorToStream(function* (size) {
-    while (size-- > 0) {
-      yield Buffer.from([Math.floor(Math.random() * 256)])
-    }
-  })
-  const input = createRandomStream(sizeMB * 1024 * 1024)
-  await pFromCallback(cb => pipeline(input, fs.createWriteStream(name), cb))
-}
-
-async function checkFile(vhdName) {
-  await execa('vhd-util', ['check', '-p', '-b', '-t', '-n', vhdName])
-}
-
-async function recoverRawContent(vhdName, rawName, originalSize) {
-  await checkFile(vhdName)
-  await execa('qemu-img', ['convert', '-fvpc', '-Oraw', vhdName, rawName])
-  if (originalSize !== undefined) {
-    await execa('truncate', ['-s', originalSize, rawName])
-  }
-}
-
-async function convertFromRawToVhd(rawName, vhdName) {
-  await execa('qemu-img', ['convert', '-f', 'raw', '-Ovpc', rawName, vhdName])
-}
-
 test('blocks can be moved', async () => {
   const initalSize = 4
   const rawFileName = `${tempDir}/randomfile`
@@ -61,7 +35,7 @@ test('blocks can be moved', async () => {
   await convertFromRawToVhd(rawFileName, vhdFileName)
   const handler = getHandler({ url: 'file://' })
   const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new Vhd(handler, vhdFileName)
+  const newVhd = new VhdFile(handler, vhdFileName)
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   await newVhd._freeFirstBlockSpace(8000000)
@@ -75,7 +49,7 @@ test('the BAT MSB is not used for sign', async () => {
   const emptyFileName = `${tempDir}/empty.vhd`
   await execa('qemu-img', ['create', '-fvpc', emptyFileName, '1.8T'])
   const handler = getHandler({ url: 'file://' })
-  const vhd = new Vhd(handler, emptyFileName)
+  const vhd = new VhdFile(handler, emptyFileName)
   await vhd.readHeaderAndFooter()
   await vhd.readBlockAllocationTable()
   // we want the bit 31 to be on, to prove it's not been used for sign
@@ -92,13 +66,12 @@ test('the BAT MSB is not used for sign', async () => {
   const recoveredFileName = `${tempDir}/recovered`
   const recoveredFile = await fs.open(recoveredFileName, 'w')
   try {
-    const vhd2 = new Vhd(handler, emptyFileName)
+    const vhd2 = new VhdFile(handler, emptyFileName)
     await vhd2.readHeaderAndFooter()
     await vhd2.readBlockAllocationTable()
     for (let i = 0; i < vhd.header.maxTableEntries; i++) {
-      const entry = vhd._getBatEntry(i)
-      if (entry !== 0xffffffff) {
-        const block = (await vhd2._readBlock(i)).data
+      if (vhd.containsBlock(i)) {
+        const block = (await vhd2.readBlock(i)).data
         await fs.write(recoveredFile, block, 0, block.length, vhd2.header.blockSize * i)
       }
     }
@@ -123,7 +96,7 @@ test('writeData on empty file', async () => {
   const randomData = await fs.readFile(rawFileName)
   const handler = getHandler({ url: 'file://' })
   const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new Vhd(handler, emptyFileName)
+  const newVhd = new VhdFile(handler, emptyFileName)
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   await newVhd.writeData(0, randomData)
@@ -142,7 +115,7 @@ test('writeData in 2 non-overlaping operations', async () => {
   const randomData = await fs.readFile(rawFileName)
   const handler = getHandler({ url: 'file://' })
   const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new Vhd(handler, emptyFileName)
+  const newVhd = new VhdFile(handler, emptyFileName)
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   const splitPointSectors = 2
@@ -162,7 +135,7 @@ test('writeData in 2 overlaping operations', async () => {
   const randomData = await fs.readFile(rawFileName)
   const handler = getHandler({ url: 'file://' })
   const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new Vhd(handler, emptyFileName)
+  const newVhd = new VhdFile(handler, emptyFileName)
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   const endFirstWrite = 3
@@ -182,7 +155,7 @@ test('BAT can be extended and blocks moved', async () => {
   await convertFromRawToVhd(rawFileName, vhdFileName)
   const handler = getHandler({ url: 'file://' })
   const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new Vhd(handler, vhdFileName)
+  const newVhd = new VhdFile(handler, vhdFileName)
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   await newVhd.ensureBatSize(2000)
@@ -226,7 +199,7 @@ test('coalesce works in normal cases', async () => {
   await convertFromRawToVhd(randomFileName, child1FileName)
   const handler = getHandler({ url: 'file://' })
   await execa('vhd-util', ['snapshot', '-n', child2FileName, '-p', child1FileName])
-  const vhd = new Vhd(handler, child2FileName)
+  const vhd = new VhdFile(handler, child2FileName)
   await vhd.readHeaderAndFooter()
   await vhd.readBlockAllocationTable()
   vhd.footer.creatorApplication = 'xoa'
@@ -238,7 +211,7 @@ test('coalesce works in normal cases', async () => {
   await chainVhd(handler, child1FileName, handler, child2FileName, true)
   await execa('vhd-util', ['check', '-t', '-n', child2FileName])
   const smallRandom = await fs.readFile(smallRandomFileName)
-  const newVhd = new Vhd(handler, child2FileName)
+  const newVhd = new VhdFile(handler, child2FileName)
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   await newVhd.writeData(5, smallRandom)

packages/vhd-lib/src/merge.js

@@ -5,7 +5,7 @@ import noop from './_noop'
 import { createLogger } from '@xen-orchestra/log'
 import { limitConcurrency } from 'limit-concurrency-decorator'
 
-import Vhd from './vhd'
+import { VhdFile } from '.'
 import { basename, dirname } from 'path'
 import { DISK_TYPE_DIFFERENCING, DISK_TYPE_DYNAMIC } from './_constants'
 
@@ -25,10 +25,10 @@ export default limitConcurrency(2)(async function merge(
 
   const parentFd = await parentHandler.openFile(parentPath, 'r+')
   try {
-    const parentVhd = new Vhd(parentHandler, parentFd)
+    const parentVhd = new VhdFile(parentHandler, parentFd)
     const childFd = await childHandler.openFile(childPath, 'r')
     try {
-      const childVhd = new Vhd(childHandler, childFd)
+      const childVhd = new VhdFile(childHandler, childFd)
 
       let mergeState = await parentHandler.readFile(mergeStatePath).catch(error => {
         if (error.code !== 'ENOENT') {

packages/vhd-lib/src/openVhd.js

@@ -0,0 +1,12 @@
+import { VhdFile, VhdDirectory } from './'
+
+export async function openVhd(handler, path) {
+  try {
+    return await VhdFile.open(handler, path)
+  } catch (e) {
+    if (e.code !== 'EISDIR') {
+      throw e
+    }
+    return await VhdDirectory.open(handler, path)
+  }
+}

packages/vhd-lib/src/tests/utils.js

@@ -0,0 +1,50 @@
+import { pFromCallback } from 'promise-toolbox'
+import { pipeline } from 'readable-stream'
+import asyncIteratorToStream from 'async-iterator-to-stream'
+import execa from 'execa'
+import fs from 'fs-extra'
+import { randomBytes } from 'crypto'
+
+const createRandomStream = asyncIteratorToStream(function* (size) {
+  while (size > 0) {
+    yield randomBytes(Math.min(size, 1024))
+    size -= 1024
+  }
+})
+
+export async function createRandomFile(name, sizeMB) {
+  const input = createRandomStream(sizeMB * 1024 * 1024)
+  await pFromCallback(cb => pipeline(input, fs.createWriteStream(name), cb))
+}
+
+export async function checkFile(vhdName) {
+  await execa('vhd-util', ['check', '-p', '-b', '-t', '-n', vhdName])
+}
+
+const RAW = 'raw'
+const VHD = 'vpc'
+const VMDK = 'vmdk'
+
+async function convert(inputFormat, inputFile, outputFormat, outputFile) {
+  await execa('qemu-img', ['convert', `-f${inputFormat}`, '-O', outputFormat, inputFile, outputFile])
+}
+
+export async function convertFromRawToVhd(rawName, vhdName) {
+  await convert(RAW, rawName, VHD, vhdName)
+}
+
+export async function convertFromVhdToRaw(vhdName, rawName) {
+  await convert(VHD, vhdName, RAW, rawName)
+}
+
+export async function convertFromVmdkToRaw(vmdkName, rawName) {
+  await convert(VMDK, vmdkName, RAW, rawName)
+}
+
+export async function recoverRawContent(vhdName, rawName, originalSize) {
+  await checkFile(vhdName)
+  await convertFromVhdToRaw(vhdName, rawName)
+  if (originalSize !== undefined) {
+    await execa('truncate', ['-s', originalSize, rawName])
+  }
+}

packages/vhd-lib/src/vhd.integ.spec.js

@@ -9,6 +9,7 @@ import { pipeline } from 'readable-stream'
 import { createReadableRawStream, createReadableSparseStream } from './'
 
 import { createFooter } from './_createFooterHeader'
+import { checkFile, convertFromVhdToRaw } from './tests/utils'
 
 let tempDir = null
 
@@ -111,8 +112,8 @@ test('ReadableSparseVHDStream can handle a sparse file', async () => {
   expect(stream.length).toEqual(4197888)
   const pipe = stream.pipe(createWriteStream(`${tempDir}/output.vhd`))
   await fromEvent(pipe, 'finish')
-  await execa('vhd-util', ['check', '-t', '-i', '-n', `${tempDir}/output.vhd`])
-  await execa('qemu-img', ['convert', '-f', 'vpc', '-O', 'raw', `${tempDir}/output.vhd`, `${tempDir}/out1.raw`])
+  await checkFile(`${tempDir}/output.vhd`)
+  await convertFromVhdToRaw(`${tempDir}/output.vhd`, `${tempDir}/out1.raw`)
   const out1 = await readFile(`${tempDir}/out1.raw`)
   const expected = Buffer.alloc(fileSize)
   blocks.forEach(b => {

packages/xapi-explore-sr/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "xapi-explore-sr",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "license": "ISC",
   "description": "Display the list of VDIs (unmanaged and snapshots included) of a SR",
   "keywords": [
@@ -39,7 +39,7 @@
     "human-format": "^0.11.0",
     "lodash": "^4.17.4",
     "pw": "^0.0.4",
-    "xen-api": "^0.33.0"
+    "xen-api": "^0.35.1"
   },
   "devDependencies": {
     "@babel/cli": "^7.1.5",

packages/xapi-explore-sr/src/index.js

@@ -3,6 +3,7 @@
 import archy from 'archy'
 import chalk from 'chalk'
 import execPromise from 'exec-promise'
+import firstDefined from '@xen-orchestra/defined'
 import humanFormat from 'human-format'
 import pw from 'pw'
 import { createClient } from 'xen-api'
@@ -69,11 +70,13 @@ execPromise(async args => {
     url = required('Host URL'),
     user = required('Host user'),
     password = await askPassword('Host password'),
+    httpProxy = firstDefined(process.env.http_proxy, process.env.HTTP_PROXY),
   ] = args
 
   const xapi = createClient({
     allowUnauthorized: true,
     auth: { user, password },
+    httpProxy,
     readOnly: true,
     url,
     watchEvents: false,

packages/xen-api/README.md

@@ -52,6 +52,7 @@ Options:
 - `auth`: credentials used to sign in (can also be specified in the URL)
 - `readOnly = false`: if true, no methods with side-effects can be called
 - `callTimeout`: number of milliseconds after which a call is considered failed (can also be a map of timeouts by methods)
+- `httpProxy`: URL of the HTTP/HTTPS proxy used to reach the host, can include credentials
 
 ```js
 // Force connection.

packages/xen-api/examples/package.json

@@ -8,6 +8,6 @@
     "promise-toolbox": "^0.19.2",
     "readable-stream": "^3.1.1",
     "throttle": "^1.0.3",
-    "vhd-lib": "^1.0.0"
+    "vhd-lib": "^1.3.0"
   }
 }

packages/xen-api/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "xen-api",
-  "version": "0.33.0",
+  "version": "0.35.1",
   "license": "ISC",
   "description": "Connector to the Xen API",
   "keywords": [
@@ -34,15 +34,17 @@
     "bind-property-descriptor": "^1.0.0",
     "blocked": "^1.2.1",
     "debug": "^4.0.1",
-    "http-request-plus": "^0.10.0",
-    "jest-diff": "^26.4.2",
+    "http-request-plus": "^0.13.0",
+    "jest-diff": "^27.3.1",
     "json-rpc-protocol": "^0.13.1",
     "kindof": "^2.0.0",
+    "limit-concurrency-decorator": "^0.5.0",
     "lodash": "^4.17.4",
     "make-error": "^1.3.0",
     "minimist": "^1.2.0",
     "ms": "^2.1.1",
-    "promise-toolbox": "^0.19.2",
+    "promise-toolbox": "^0.20.0",
+    "proxy-agent": "^5.0.0",
     "pw": "0.0.4",
     "xmlrpc": "^1.3.2",
     "xo-collection": "^0.5.0"

packages/xen-api/src/cli.js

@@ -2,10 +2,10 @@
 
 import blocked from 'blocked'
 import createDebug from 'debug'
-import diff from 'jest-diff'
 import minimist from 'minimist'
 import pw from 'pw'
 import { asCallback, fromCallback, fromEvent } from 'promise-toolbox'
+import { diff } from 'jest-diff'
 import { filter, find } from 'lodash'
 import { getBoundPropertyDescriptor } from 'bind-property-descriptor'
 import { start as createRepl } from 'repl'

packages/xen-api/src/index.js

@@ -7,6 +7,7 @@ import { Collection } from 'xo-collection'
 import { EventEmitter } from 'events'
 import { map, noop, omit } from 'lodash'
 import { cancelable, defer, fromCallback, fromEvents, ignoreErrors, pDelay, pRetry, pTimeout } from 'promise-toolbox'
+import { limitConcurrency } from 'limit-concurrency-decorator'
 
 import autoTransport from './transports/auto'
 import coalesceCalls from './_coalesceCalls'
@@ -88,6 +89,8 @@ export class Xapi extends EventEmitter {
     this._RecordsByType = { __proto__: null }
     this._reverseHostIpAddresses = opts.reverseHostIpAddresses ?? false
 
+    this._call = limitConcurrency(opts.callConcurrency ?? 20)(this._call)
+
     this._roCallRetryOptions = {
       delay: 1e3,
       tries: 10,
@@ -112,6 +115,7 @@ export class Xapi extends EventEmitter {
     }
 
     this._allowUnauthorized = opts.allowUnauthorized
+    this._httpProxy = opts.httpProxy
     this._setUrl(url)
 
     this._connected = new Promise(resolve => {
@@ -356,22 +360,35 @@ export class Xapi extends EventEmitter {
       }
     }
 
-    const response = await httpRequest(
-      $cancelToken,
-      this._url,
-      host !== undefined && {
-        hostname: await this._getHostAddress(this.getObject(host)),
-      },
+    let url = new URL('http://localhost')
+    url.protocol = this._url.protocol
+    url.pathname = pathname
+    url.search = new URLSearchParams(query)
+    await this._setHostAddressInUrl(url, host)
+
+    const response = await pRetry(
+      async () =>
+        httpRequest($cancelToken, url.href, {
+          rejectUnauthorized: !this._allowUnauthorized,
+
+          // this is an inactivity timeout (unclear in Node doc)
+          timeout: this._httpInactivityTimeout,
+
+          maxRedirects: 0,
+
+          // Support XS <= 6.5 with Node => 12
+          minVersion: 'TLSv1',
+        }),
       {
-        pathname,
-        query,
-        rejectUnauthorized: !this._allowUnauthorized,
-
-        // this is an inactivity timeout (unclear in Node doc)
-        timeout: this._httpInactivityTimeout,
-
-        // Support XS <= 6.5 with Node => 12
-        minVersion: 'TLSv1',
+        when: { code: 302 },
+        onRetry: async error => {
+          const response = error.response
+          if (response === undefined) {
+            throw error
+          }
+          response.cancel()
+          url = await this._replaceHostAddressInUrl(new URL(response.headers.location, url))
+        },
       }
     )
 
@@ -418,32 +435,28 @@ export class Xapi extends EventEmitter {
       headers['content-length'] = '1125899906842624'
     }
 
-    const doRequest = httpRequest.put.bind(
-      undefined,
-      $cancelToken,
-      this._url,
-      host !== undefined && {
-        hostname: await this._getHostAddress(this.getObject(host)),
-      },
-      {
-        body,
-        headers,
-        pathname,
-        query,
-        rejectUnauthorized: !this._allowUnauthorized,
+    const url = new URL('http://localhost')
+    url.protocol = this._url.protocol
+    url.pathname = pathname
+    url.search = new URLSearchParams(query)
+    await this._setHostAddressInUrl(url, host)
 
-        // this is an inactivity timeout (unclear in Node doc)
-        timeout: this._httpInactivityTimeout,
+    const doRequest = httpRequest.put.bind(undefined, $cancelToken, {
+      body,
+      headers,
+      rejectUnauthorized: !this._allowUnauthorized,
 
-        // Support XS <= 6.5 with Node => 12
-        minVersion: 'TLSv1',
-      }
-    )
+      // this is an inactivity timeout (unclear in Node doc)
+      timeout: this._httpInactivityTimeout,
+
+      // Support XS <= 6.5 with Node => 12
+      minVersion: 'TLSv1',
+    })
 
     // if body is a stream, sends a dummy request to probe for a redirection
     // before consuming body
     const response = await (isStream
-      ? doRequest({
+      ? doRequest(url.href, {
           body: '',
 
           // omit task_id because this request will fail on purpose
@@ -453,9 +466,9 @@ export class Xapi extends EventEmitter {
         }).then(
           response => {
             response.cancel()
-            return doRequest()
+            return doRequest(url.href)
           },
-          error => {
+          async error => {
             let response
             if (error != null && (response = error.response) != null) {
               response.cancel()
@@ -466,14 +479,16 @@ export class Xapi extends EventEmitter {
               } = response
               if (statusCode === 302 && location !== undefined) {
                 // ensure the original query is sent
-                return doRequest(location, { query })
+                const newUrl = new URL(location, url)
+                newUrl.searchParams.set('task_id', query.task_id)
+                return doRequest((await this._replaceHostAddressInUrl(newUrl)).href)
               }
             }
 
             throw error
           }
         )
-      : doRequest())
+      : doRequest(url.href))
 
     if (pTaskResult !== undefined) {
       pTaskResult = pTaskResult.catch(error => {
@@ -789,7 +804,35 @@ export class Xapi extends EventEmitter {
     }
   }
 
-  async _getHostAddress({ address }) {
+  async _setHostAddressInUrl(url, host) {
+    const pool = this._pool
+
+    const poolBackupNetwork = pool.other_config['xo:backupNetwork']
+    if (host === undefined) {
+      if (poolBackupNetwork === undefined) {
+        const xapiUrl = this._url
+        url.hostname = xapiUrl.hostname
+        url.port = xapiUrl.port
+        return
+      }
+
+      host = await this.getRecord('host', pool.master)
+    }
+
+    let { address } = host
+    if (poolBackupNetwork !== undefined) {
+      const hostPifs = new Set(host.PIFs)
+      try {
+        const networkRef = await this._roCall('network.get_by_uuid', [poolBackupNetwork])
+        const networkPifs = await this.getField('network', networkRef, 'PIFs')
+
+        const backupNetworkPifRef = networkPifs.find(hostPifs.has, hostPifs)
+        address = await this.getField('PIF', backupNetworkPifRef, 'IP')
+      } catch (error) {
+        console.warn('unable to get the host address linked to the pool backup network', poolBackupNetwork, error)
+      }
+    }
+
     if (this._reverseHostIpAddresses) {
       try {
         ;[address] = await fromCallback(dns.reverse, address)
@@ -797,7 +840,8 @@ export class Xapi extends EventEmitter {
         console.warn('reversing host address', address, error)
       }
     }
-    return address
+
+    url.hostname = address
   }
 
   _setUrl(url) {
@@ -808,6 +852,7 @@ export class Xapi extends EventEmitter {
         rejectUnauthorized: !this._allowUnauthorized,
       },
       url,
+      httpProxy: this._httpProxy,
     })
     this._url = url
   }
@@ -859,6 +904,19 @@ export class Xapi extends EventEmitter {
     }
   }
 
+  async _replaceHostAddressInUrl(url) {
+    try {
+      // TODO: look for hostname in all addresses of this host (including all its PIFs)
+      const host = (await this.getAllRecords('host')).find(host => host.address === url.hostname)
+      if (host !== undefined) {
+        await this._setHostAddressInUrl(url, host)
+      }
+    } catch (error) {
+      console.warn('_replaceHostAddressInUrl', url, error)
+    }
+    return url
+  }
+
   _processEvents(events) {
     const flush = this._objects.bufferEvents()
     events.forEach(event => {

packages/xen-api/src/inject-events.js

@@ -1,31 +0,0 @@
-#!/usr/bin/env node
-
-import { pDelay } from 'promise-toolbox'
-
-import { createClient } from './'
-
-async function main([url]) {
-  const xapi = createClient({
-    allowUnauthorized: true,
-    url,
-    watchEvents: false,
-  })
-  await xapi.connect()
-
-  let loop = true
-  process.on('SIGINT', () => {
-    loop = false
-  })
-
-  const { pool } = xapi
-  // eslint-disable-next-line no-unmodified-loop-condition
-  while (loop) {
-    await pool.update_other_config('xo:injectEvents', Math.random().toString(36).slice(2))
-    await pDelay(1e2)
-  }
-
-  await pool.update_other_config('xo:injectEvents', null)
-  await xapi.disconnect()
-}
-
-main(process.argv.slice(2)).catch(console.error)

packages/xen-api/src/transports/json-rpc.js

@@ -1,4 +1,5 @@
 import httpRequestPlus from 'http-request-plus'
+import ProxyAgent from 'proxy-agent'
 import { format, parse } from 'json-rpc-protocol'
 
 import XapiError from '../_XapiError'
@@ -6,7 +7,11 @@ import XapiError from '../_XapiError'
 import UnsupportedTransport from './_UnsupportedTransport'
 
 // https://github.com/xenserver/xenadmin/blob/0df39a9d83cd82713f32d24704852a0fd57b8a64/XenModel/XenAPI/Session.cs#L403-L433
-export default ({ secureOptions, url }) => {
+export default ({ secureOptions, url, httpProxy }) => {
+  let agent
+  if (httpProxy !== undefined) {
+    agent = new ProxyAgent(httpProxy)
+  }
   return (method, args) =>
     httpRequestPlus
       .post(url, {
@@ -17,6 +22,7 @@ export default ({ secureOptions, url }) => {
           'Content-Type': 'application/json',
         },
         path: '/jsonrpc',
+        agent,
       })
       .readAll('utf8')
       .then(

packages/xen-api/src/transports/xml-rpc-json.js

@@ -1,5 +1,6 @@
 import { createClient, createSecureClient } from 'xmlrpc'
 import { promisify } from 'promise-toolbox'
+import ProxyAgent from 'proxy-agent'
 
 import XapiError from '../_XapiError'
 
@@ -70,10 +71,15 @@ const parseResult = result => {
   throw new UnsupportedTransport()
 }
 
-export default ({ secureOptions, url: { hostname, port, protocol } }) => {
+export default ({ secureOptions, url: { hostname, port, protocol }, httpProxy }) => {
   const secure = protocol === 'https:'
+  let agent
+  if (httpProxy !== undefined) {
+    agent = new ProxyAgent(httpProxy)
+  }
   const client = (secure ? createSecureClient : createClient)({
     ...(secure ? secureOptions : undefined),
+    agent,
     host: hostname,
     path: '/json',
     port,

packages/xen-api/src/transports/xml-rpc.js

@@ -1,5 +1,6 @@
 import { createClient, createSecureClient } from 'xmlrpc'
 import { promisify } from 'promise-toolbox'
+import ProxyAgent from 'proxy-agent'
 
 import XapiError from '../_XapiError'
 
@@ -30,10 +31,15 @@ const parseResult = result => {
   return result.Value
 }
 
-export default ({ secureOptions, url: { hostname, port, protocol } }) => {
+export default ({ secureOptions, url: { hostname, port, protocol, httpProxy } }) => {
   const secure = protocol === 'https:'
+  let agent
+  if (httpProxy !== undefined) {
+    agent = new ProxyAgent(httpProxy)
+  }
   const client = (secure ? createSecureClient : createClient)({
     ...(secure ? secureOptions : undefined),
+    agent,
     host: hostname,
     port,
   })

packages/xo-acl-resolver/index.js

@@ -15,24 +15,28 @@ const authorized = () => true // eslint-disable-line no-unused-vars
 const forbiddden = () => false // eslint-disable-line no-unused-vars
 
 // eslint-disable-next-line no-unused-vars
-const and = (...checkers) => (object, permission) => {
-  for (const checker of checkers) {
-    if (!checker(object, permission)) {
-      return false
+const and =
+  (...checkers) =>
+  (object, permission) => {
+    for (const checker of checkers) {
+      if (!checker(object, permission)) {
+        return false
+      }
     }
+    return true
   }
-  return true
-}
 
 // eslint-disable-next-line no-unused-vars
-const or = (...checkers) => (object, permission) => {
-  for (const checker of checkers) {
-    if (checker(object, permission)) {
-      return true
+const or =
+  (...checkers) =>
+  (object, permission) => {
+    for (const checker of checkers) {
+      if (checker(object, permission)) {
+        return true
+      }
     }
+    return false
   }
-  return false
-}
 
 // -------------------------------------------------------------------