fix: remove root need for openVhd.integ.spec.js and merge.integ.spec.js

The version of curl installed on XCP-ng 8.2.0, (curl 7.29.0) does not support any encryption algos available on https://xoa.io

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,33 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: 'status: triaging :triangular_flag_on_post:, type: bug :bug:'
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - Node: [e.g. 16.12.1]
+ - xo-server: [e.g. 5.82.3]
+ - xo-web: [e.g. 5.87.0]
+ - hypervisor: [e.g. XCP-ng 8.2.0]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

@vates/disposable/package.json

@@ -24,7 +24,7 @@
   "dependencies": {
     "@vates/multi-key-map": "^0.1.0",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/log": "^0.2.1",
+    "@xen-orchestra/log": "^0.3.0",
     "ensure-array": "^1.0.0"
   }
 }

@xen-orchestra/audit-core/package.json

@@ -31,7 +31,7 @@
   },
   "dependencies": {
     "@vates/decorate-with": "^0.1.0",
-    "@xen-orchestra/log": "^0.2.1",
+    "@xen-orchestra/log": "^0.3.0",
     "golike-defer": "^0.5.1",
     "object-hash": "^2.0.1"
   },

@xen-orchestra/babel-config/index.js

@@ -46,7 +46,7 @@ module.exports = function (pkg, configs = {}) {
 
   return {
     comments: !__PROD__,
-    ignore: __PROD__ ? [/\.spec\.js$/] : undefined,
+    ignore: __PROD__ ? [/\btests?\//, /\.spec\.js$/] : undefined,
     plugins: Object.keys(plugins)
       .map(plugin => [plugin, plugins[plugin]])
       .sort(([a], [b]) => {

@xen-orchestra/backups-cli/package.json

@@ -7,12 +7,12 @@
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.12.2",
-    "@xen-orchestra/fs": "^0.17.0",
+    "@xen-orchestra/backups": "^0.15.1",
+    "@xen-orchestra/fs": "^0.18.0",
     "filenamify": "^4.1.0",
     "getopts": "^2.2.5",
     "lodash": "^4.17.15",
-    "promise-toolbox": "^0.19.2"
+    "promise-toolbox": "^0.20.0"
   },
   "engines": {
     "node": ">=7.10.1"

@xen-orchestra/backups/RemoteAdapter.js

@@ -6,7 +6,7 @@ const pDefer = require('promise-toolbox/defer.js')
 const pump = require('pump')
 const { basename, dirname, join, normalize, resolve } = require('path')
 const { createLogger } = require('@xen-orchestra/log')
-const { createSyntheticStream, mergeVhd, default: Vhd } = require('vhd-lib')
+const { createSyntheticStream, mergeVhd, VhdFile } = require('vhd-lib')
 const { deduped } = require('@vates/disposable/deduped.js')
 const { execFile } = require('child_process')
 const { readdir, stat } = require('fs-extra')
@@ -86,7 +86,7 @@ class RemoteAdapter {
       }),
       async path => {
         try {
-          const vhd = new Vhd(handler, path)
+          const vhd = new VhdFile(handler, path)
           await vhd.readHeaderAndFooter()
           return {
             footer: vhd.footer,
@@ -253,16 +253,9 @@ class RemoteAdapter {
 
   async deleteDeltaVmBackups(backups) {
     const handler = this._handler
-    let mergedDataSize = 0
-    await asyncMapSettled(backups, ({ _filename, vhds }) =>
-      Promise.all([
-        handler.unlink(_filename),
-        asyncMap(Object.values(vhds), async _ => {
-          mergedDataSize += await this._deleteVhd(resolveRelativeFromFile(_filename, _))
-        }),
-      ])
-    )
-    return mergedDataSize
+
+    // unused VHDs will be detected by `cleanVm`
+    await asyncMapSettled(backups, ({ _filename }) => handler.unlink(_filename))
   }
 
   async deleteMetadataBackup(backupId) {

@xen-orchestra/backups/_VmBackup.js

@@ -1,9 +1,10 @@
 const assert = require('assert')
 const findLast = require('lodash/findLast.js')
+const groupBy = require('lodash/groupBy.js')
 const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
 const keyBy = require('lodash/keyBy.js')
 const mapValues = require('lodash/mapValues.js')
-const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
+const { asyncMap } = require('@xen-orchestra/async-map')
 const { createLogger } = require('@xen-orchestra/log')
 const { defer } = require('golike-defer')
 const { formatDateTime } = require('@xen-orchestra/xapi')
@@ -284,17 +285,28 @@ exports.VmBackup = class VmBackup {
   }
 
   async _removeUnusedSnapshots() {
-    // TODO: handle all schedules (no longer existing schedules default to 0 retention)
-
-    const { scheduleId } = this
-    const scheduleSnapshots = this._jobSnapshots.filter(_ => _.other_config['xo:backup:schedule'] === scheduleId)
-
+    const jobSettings = this.job.settings
     const baseVmRef = this._baseVm?.$ref
+    const { config } = this
+    const baseSettings = {
+      ...config.defaultSettings,
+      ...config.metadata.defaultSettings,
+      ...jobSettings[''],
+    }
+
+    const snapshotsPerSchedule = groupBy(this._jobSnapshots, _ => _.other_config['xo:backup:schedule'])
     const xapi = this._xapi
-    await asyncMap(getOldEntries(this._settings.snapshotRetention, scheduleSnapshots), ({ $ref }) => {
-      if ($ref !== baseVmRef) {
-        return xapi.VM_destroy($ref)
+    await asyncMap(Object.entries(snapshotsPerSchedule), ([scheduleId, snapshots]) => {
+      const settings = {
+        ...baseSettings,
+        ...jobSettings[scheduleId],
+        ...jobSettings[this.vm.uuid],
       }
+      return asyncMap(getOldEntries(settings.snapshotRetention, snapshots), ({ $ref }) => {
+        if ($ref !== baseVmRef) {
+          return xapi.VM_destroy($ref)
+        }
+      })
     })
   }
 
@@ -303,12 +315,14 @@ exports.VmBackup = class VmBackup {
 
     let baseVm = findLast(this._jobSnapshots, _ => 'xo:backup:exported' in _.other_config)
     if (baseVm === undefined) {
+      debug('no base VM found')
       return
     }
 
     const fullInterval = this._settings.fullInterval
     const deltaChainLength = +(baseVm.other_config['xo:backup:deltaChainLength'] ?? 0) + 1
     if (!(fullInterval === 0 || fullInterval > deltaChainLength)) {
+      debug('not using base VM becaust fullInterval reached')
       return
     }
 
@@ -323,6 +337,10 @@ exports.VmBackup = class VmBackup {
       const srcVdi = srcVdis[snapshotOf]
       if (srcVdi !== undefined) {
         baseUuidToSrcVdi.set(await xapi.getField('VDI', baseRef, 'uuid'), srcVdi)
+      } else {
+        debug('no base VDI found', {
+          vdi: srcVdi.uuid,
+        })
       }
     })
 
@@ -335,7 +353,16 @@ exports.VmBackup = class VmBackup {
 
     const fullVdisRequired = new Set()
     baseUuidToSrcVdi.forEach((srcVdi, baseUuid) => {
-      if (!presentBaseVdis.has(baseUuid)) {
+      if (presentBaseVdis.has(baseUuid)) {
+        debug('found base VDI', {
+          base: baseUuid,
+          vdi: srcVdi.uuid,
+        })
+      } else {
+        debug('missing base VDI', {
+          base: baseUuid,
+          vdi: srcVdi.uuid,
+        })
         fullVdisRequired.add(srcVdi.uuid)
       }
     })

@xen-orchestra/backups/_cleanVm.js

@@ -1,17 +1,19 @@
 const assert = require('assert')
 const sum = require('lodash/sum')
 const { asyncMap } = require('@xen-orchestra/async-map')
-const { default: Vhd, mergeVhd } = require('vhd-lib')
+const { VhdFile, mergeVhd } = require('vhd-lib')
 const { dirname, resolve } = require('path')
 const { DISK_TYPE_DIFFERENCING } = require('vhd-lib/dist/_constants.js')
 const { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } = require('./_backupType.js')
 const { limitConcurrency } = require('limit-concurrency-decorator')
 
+const { Task } = require('./Task.js')
+
 // chain is an array of VHDs from child to parent
 //
 // the whole chain will be merged into parent, parent will be renamed to child
 // and all the others will deleted
-const mergeVhdChain = limitConcurrency(1)(async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
+async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
   assert(chain.length >= 2)
 
   let child = chain[0]
@@ -44,7 +46,7 @@ const mergeVhdChain = limitConcurrency(1)(async function mergeVhdChain(chain, {
       }
     }, 10e3)
 
-    await mergeVhd(
+    const mergedSize = await mergeVhd(
       handler,
       parent,
       handler,
@@ -72,8 +74,10 @@ const mergeVhdChain = limitConcurrency(1)(async function mergeVhdChain(chain, {
         }
       }),
     ])
+
+    return mergedSize
   }
-})
+}
 
 const noop = Function.prototype
 
@@ -114,7 +118,14 @@ const listVhds = async (handler, vmDir) => {
   return { vhds, interruptedVhds }
 }
 
-exports.cleanVm = async function cleanVm(vmDir, { fixMetadata, remove, merge, onLog = noop }) {
+const defaultMergeLimiter = limitConcurrency(1)
+
+exports.cleanVm = async function cleanVm(
+  vmDir,
+  { fixMetadata, remove, merge, mergeLimiter = defaultMergeLimiter, onLog = noop }
+) {
+  const limitedMergeVhdChain = mergeLimiter(mergeVhdChain)
+
   const handler = this._handler
 
   const vhds = new Set()
@@ -126,7 +137,7 @@ exports.cleanVm = async function cleanVm(vmDir, { fixMetadata, remove, merge, on
   // remove broken VHDs
   await asyncMap(vhdsList.vhds, async path => {
     try {
-      const vhd = new Vhd(handler, path)
+      const vhd = new VhdFile(handler, path)
       await vhd.readHeaderAndFooter(!vhdsList.interruptedVhds.has(path))
       vhds.add(path)
       if (vhd.footer.diskType === DISK_TYPE_DIFFERENCING) {
@@ -279,6 +290,7 @@ exports.cleanVm = async function cleanVm(vmDir, { fixMetadata, remove, merge, on
 
   // TODO: parallelize by vm/job/vdi
   const unusedVhdsDeletion = []
+  const toMerge = []
   {
     // VHD chains (as list from child to ancestor) to merge indexed by last
     // ancestor
@@ -321,22 +333,25 @@ exports.cleanVm = async function cleanVm(vmDir, { fixMetadata, remove, merge, on
     })
 
     // merge interrupted VHDs
-    if (merge) {
-      vhdsList.interruptedVhds.forEach(parent => {
-        vhdChainsToMerge[parent] = [vhdChildren[parent], parent]
-      })
-    }
+    vhdsList.interruptedVhds.forEach(parent => {
+      vhdChainsToMerge[parent] = [vhdChildren[parent], parent]
+    })
 
-    Object.keys(vhdChainsToMerge).forEach(key => {
-      const chain = vhdChainsToMerge[key]
+    Object.values(vhdChainsToMerge).forEach(chain => {
       if (chain !== undefined) {
-        unusedVhdsDeletion.push(mergeVhdChain(chain, { handler, onLog, remove, merge }))
+        toMerge.push(chain)
       }
     })
   }
 
+  const doMerge = () => {
+    const promise = asyncMap(toMerge, async chain => limitedMergeVhdChain(chain, { handler, onLog, remove, merge }))
+    return merge ? promise.then(sizes => ({ size: sum(sizes) })) : promise
+  }
+
   await Promise.all([
     ...unusedVhdsDeletion,
+    toMerge.length !== 0 && (merge ? Task.run({ name: 'merge' }, doMerge) : doMerge()),
     asyncMap(unusedXvas, path => {
       onLog(`the XVA ${path} is unused`)
       if (remove) {
@@ -355,4 +370,9 @@ exports.cleanVm = async function cleanVm(vmDir, { fixMetadata, remove, merge, on
       }
     }),
   ])
+
+  return {
+    // boolean whether some VHDs were merged (or should be merged)
+    merge: toMerge.length !== 0,
+  }
 }

@xen-orchestra/backups/merge-worker/cli.js

@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+
+const { catchGlobalErrors } = require('@xen-orchestra/log/configure.js')
+const { createLogger } = require('@xen-orchestra/log')
+const { getSyncedHandler } = require('@xen-orchestra/fs')
+const { join } = require('path')
+const Disposable = require('promise-toolbox/Disposable')
+const min = require('lodash/min')
+
+const { getVmBackupDir } = require('../_getVmBackupDir.js')
+const { RemoteAdapter } = require('../RemoteAdapter.js')
+
+const { CLEAN_VM_QUEUE } = require('./index.js')
+
+// -------------------------------------------------------------------
+
+catchGlobalErrors(createLogger('xo:backups:mergeWorker'))
+
+const { fatal, info, warn } = createLogger('xo:backups:mergeWorker')
+
+// -------------------------------------------------------------------
+
+const main = Disposable.wrap(async function* main(args) {
+  const handler = yield getSyncedHandler({ url: 'file://' + process.cwd() })
+
+  yield handler.lock(CLEAN_VM_QUEUE)
+
+  const adapter = new RemoteAdapter(handler)
+
+  const listRetry = async () => {
+    const timeoutResolver = resolve => setTimeout(resolve, 10e3)
+    for (let i = 0; i < 10; ++i) {
+      const entries = await handler.list(CLEAN_VM_QUEUE)
+      if (entries.length !== 0) {
+        return entries
+      }
+      await new Promise(timeoutResolver)
+    }
+  }
+
+  let taskFiles
+  while ((taskFiles = await listRetry()) !== undefined) {
+    const taskFileBasename = min(taskFiles)
+    const taskFile = join(CLEAN_VM_QUEUE, '_' + taskFileBasename)
+
+    // move this task to the end
+    await handler.rename(join(CLEAN_VM_QUEUE, taskFileBasename), taskFile)
+    try {
+      const vmDir = getVmBackupDir(String(await handler.readFile(taskFile)))
+      await adapter.cleanVm(vmDir, { merge: true, onLog: info, remove: true })
+
+      handler.unlink(taskFile).catch(error => warn('deleting task failure', { error }))
+    } catch (error) {
+      warn('failure handling task', { error })
+    }
+  }
+})
+
+info('starting')
+main(process.argv.slice(2)).then(
+  () => {
+    info('bye :-)')
+  },
+  error => {
+    fatal(error)
+
+    process.exit(1)
+  }
+)

@xen-orchestra/backups/merge-worker/index.js

@@ -0,0 +1,25 @@
+const { join, resolve } = require('path')
+const { spawn } = require('child_process')
+const { check } = require('proper-lockfile')
+
+const CLEAN_VM_QUEUE = (exports.CLEAN_VM_QUEUE = '/xo-vm-backups/.queue/clean-vm/')
+
+const CLI_PATH = resolve(__dirname, 'cli.js')
+exports.run = async function runMergeWorker(remotePath) {
+  try {
+    // TODO: find a way to pass the acquire the lock and then pass it down the worker
+    if (await check(join(remotePath, CLEAN_VM_QUEUE))) {
+      // already locked, don't start another worker
+      return
+    }
+
+    spawn(CLI_PATH, {
+      cwd: remotePath,
+      detached: true,
+      stdio: 'inherit',
+    }).unref()
+  } catch (error) {
+    // we usually don't want to throw if the merge worker failed to start
+    return error
+  }
+}

@xen-orchestra/backups/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.12.2",
+  "version": "0.15.1",
   "engines": {
     "node": ">=14.6"
   },
@@ -20,25 +20,26 @@
     "@vates/disposable": "^0.1.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^0.17.0",
-    "@xen-orchestra/log": "^0.2.1",
+    "@xen-orchestra/fs": "^0.18.0",
+    "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/template": "^0.1.0",
-    "compare-versions": "^3.6.0",
+    "compare-versions": "^4.0.1",
     "d3-time-format": "^3.0.0",
     "end-of-stream": "^1.4.4",
-    "fs-extra": "^9.0.0",
+    "fs-extra": "^10.0.0",
     "golike-defer": "^0.5.1",
     "limit-concurrency-decorator": "^0.5.0",
     "lodash": "^4.17.20",
     "node-zone": "^0.4.0",
     "parse-pairs": "^1.1.0",
+    "promise-toolbox": "^0.20.0",
+    "proper-lockfile": "^4.1.2",
     "pump": "^3.0.0",
-    "promise-toolbox": "^0.19.2",
-    "vhd-lib": "^1.1.0",
+    "vhd-lib": "^1.3.0",
     "yazl": "^2.5.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^0.6.4"
+    "@xen-orchestra/xapi": "^0.8.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

@xen-orchestra/backups/writers/DeltaBackupWriter.js

@@ -3,7 +3,7 @@ const map = require('lodash/map.js')
 const mapValues = require('lodash/mapValues.js')
 const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
 const { asyncMap } = require('@xen-orchestra/async-map')
-const { chainVhd, checkVhdChain, default: Vhd } = require('vhd-lib')
+const { chainVhd, checkVhdChain, VhdFile } = require('vhd-lib')
 const { createLogger } = require('@xen-orchestra/log')
 const { dirname } = require('path')
 
@@ -38,7 +38,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           try {
             await checkVhdChain(handler, path)
 
-            const vhd = new Vhd(handler, path)
+            const vhd = new VhdFile(handler, path)
             await vhd.readHeaderAndFooter()
             found = found || vhd.footer.uuid.equals(packUuid(baseUuid))
           } catch (error) {
@@ -113,19 +113,13 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
   }
 
   async _deleteOldEntries() {
-    return Task.run({ name: 'merge' }, async () => {
-      const adapter = this._adapter
-      const oldEntries = this._oldEntries
+    const adapter = this._adapter
+    const oldEntries = this._oldEntries
 
-      let size = 0
-      // delete sequentially from newest to oldest to avoid unnecessary merges
-      for (let i = oldEntries.length; i-- > 0; ) {
-        size += await adapter.deleteDeltaVmBackups([oldEntries[i]])
-      }
-      return {
-        size,
-      }
-    })
+    // delete sequentially from newest to oldest to avoid unnecessary merges
+    for (let i = oldEntries.length; i-- > 0; ) {
+      await adapter.deleteDeltaVmBackups([oldEntries[i]])
+    }
   }
 
   async _transfer({ timestamp, deltaExport, sizeContainers }) {
@@ -206,7 +200,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           }
 
           // set the correct UUID in the VHD
-          const vhd = new Vhd(handler, path)
+          const vhd = new VhdFile(handler, path)
           await vhd.readHeaderAndFooter()
           vhd.footer.uuid = packUuid(vdi.uuid)
           await vhd.readBlockAllocationTable() // required by writeFooter()

@xen-orchestra/backups/writers/_MixinBackupWriter.js

@@ -1,34 +1,51 @@
 const { createLogger } = require('@xen-orchestra/log')
+const { join } = require('path')
 
-const { getVmBackupDir } = require('../_getVmBackupDir.js')
+const { BACKUP_DIR, getVmBackupDir } = require('../_getVmBackupDir.js')
+const MergeWorker = require('../merge-worker/index.js')
+const { formatFilenameDate } = require('../_filenameDate.js')
 
 const { warn } = createLogger('xo:backups:MixinBackupWriter')
 
 exports.MixinBackupWriter = (BaseClass = Object) =>
   class MixinBackupWriter extends BaseClass {
+    #lock
+    #vmBackupDir
+
     constructor({ remoteId, ...rest }) {
       super(rest)
 
       this._adapter = rest.backup.remoteAdapters[remoteId]
       this._remoteId = remoteId
-      this._lock = undefined
+
+      this.#vmBackupDir = getVmBackupDir(this._backup.vm.uuid)
     }
 
     _cleanVm(options) {
       return this._adapter
-        .cleanVm(getVmBackupDir(this._backup.vm.uuid), { ...options, fixMetadata: true, onLog: warn, lock: false })
+        .cleanVm(this.#vmBackupDir, { ...options, fixMetadata: true, onLog: warn, lock: false })
         .catch(warn)
     }
 
     async beforeBackup() {
       const { handler } = this._adapter
-      const vmBackupDir = getVmBackupDir(this._backup.vm.uuid)
+      const vmBackupDir = this.#vmBackupDir
       await handler.mktree(vmBackupDir)
-      this._lock = await handler.lock(vmBackupDir)
+      this.#lock = await handler.lock(vmBackupDir)
     }
 
     async afterBackup() {
-      await this._cleanVm({ remove: true, merge: true })
-      await this._lock.dispose()
+      const { disableMergeWorker } = this._backup.config
+
+      const { merge } = await this._cleanVm({ remove: true, merge: disableMergeWorker })
+      await this.#lock.dispose()
+
+      // merge worker only compatible with local remotes
+      const { handler } = this._adapter
+      if (merge && !disableMergeWorker && typeof handler._getRealPath === 'function') {
+        await handler.outputFile(join(MergeWorker.CLEAN_VM_QUEUE, formatFilenameDate(new Date())), this._backup.vm.uuid)
+        const remotePath = handler._getRealPath()
+        await MergeWorker.run(remotePath)
+      }
     }
   }

@xen-orchestra/backups/writers/_checkVhd.js

@@ -1,4 +1,4 @@
-const Vhd = require('vhd-lib').default
+const Vhd = require('vhd-lib').VhdFile
 
 exports.checkVhd = async function checkVhd(handler, path) {
   await new Vhd(handler, path).readHeaderAndFooter()

@xen-orchestra/cr-seed-cli/package.json

@@ -18,7 +18,7 @@
   "preferGlobal": true,
   "dependencies": {
     "golike-defer": "^0.5.1",
-    "xen-api": "^0.33.1"
+    "xen-api": "^0.35.1"
   },
   "scripts": {
     "postversion": "npm publish"

@xen-orchestra/cron/src/index.spec.js

@@ -2,6 +2,8 @@
 
 import { createSchedule } from './'
 
+jest.useFakeTimers()
+
 const wrap = value => () => value
 
 describe('issues', () => {

@xen-orchestra/fs/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "@xen-orchestra/fs",
-  "version": "0.17.0",
+  "version": "0.18.0",
   "license": "AGPL-3.0-or-later",
   "description": "The File System for Xen Orchestra backups.",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
@@ -17,19 +17,19 @@
     "node": ">=14"
   },
   "dependencies": {
-    "@marsaud/smb2": "^0.17.2",
+    "@marsaud/smb2": "^0.18.0",
     "@sindresorhus/df": "^3.1.1",
     "@sullux/aws-sdk": "^1.0.5",
     "@vates/coalesce-calls": "^0.1.0",
     "@xen-orchestra/async-map": "^0.1.2",
     "aws-sdk": "^2.686.0",
-    "decorator-synchronized": "^0.5.0",
+    "decorator-synchronized": "^0.6.0",
     "execa": "^5.0.0",
-    "fs-extra": "^9.0.0",
+    "fs-extra": "^10.0.0",
     "get-stream": "^6.0.0",
     "limit-concurrency-decorator": "^0.5.0",
     "lodash": "^4.17.4",
-    "promise-toolbox": "^0.19.2",
+    "promise-toolbox": "^0.20.0",
     "proper-lockfile": "^4.1.2",
     "readable-stream": "^3.0.6",
     "through2": "^4.0.2",
@@ -45,7 +45,7 @@
     "async-iterator-to-stream": "^1.1.0",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^7.0.2",
-    "dotenv": "^8.0.0",
+    "dotenv": "^10.0.0",
     "rimraf": "^3.0.0"
   },
   "scripts": {

@xen-orchestra/fs/src/abstract.js

@@ -1,13 +1,13 @@
 import asyncMapSettled from '@xen-orchestra/async-map/legacy'
 import getStream from 'get-stream'
 import path, { basename } from 'path'
-import synchronized from 'decorator-synchronized'
 import { coalesceCalls } from '@vates/coalesce-calls'
 import { fromCallback, fromEvent, ignoreErrors, timeout } from 'promise-toolbox'
 import { limitConcurrency } from 'limit-concurrency-decorator'
 import { parse } from 'xo-remote-parser'
 import { pipeline } from 'stream'
 import { randomBytes } from 'crypto'
+import { synchronized } from 'decorator-synchronized'
 
 import normalizePath from './_normalizePath'
 import { createChecksumStream, validChecksumOfReadStream } from './checksum'

@xen-orchestra/fs/src/index.js

@@ -27,3 +27,12 @@ export const getHandler = (remote, ...rest) => {
   }
   return new Handler(remote, ...rest)
 }
+
+export const getSyncedHandler = async (...opts) => {
+  const handler = getHandler(...opts)
+  await handler.sync()
+  return {
+    dispose: () => handler.forget(),
+    value: handler,
+  }
+}

@xen-orchestra/fs/src/s3.js

@@ -183,9 +183,21 @@ export default class S3Handler extends RemoteHandlerAbstract {
     }
     const params = this._createParams(file)
     params.Range = `bytes=${position}-${position + buffer.length - 1}`
-    const result = await this._s3.getObject(params)
-    result.Body.copy(buffer)
-    return { bytesRead: result.Body.length, buffer }
+    try {
+      const result = await this._s3.getObject(params)
+      result.Body.copy(buffer)
+      return { bytesRead: result.Body.length, buffer }
+    } catch (e) {
+      if (e.code === 'NoSuchKey') {
+        if (await this._isNotEmptyDir(file)) {
+          const error = new Error(`${file} is a directory`)
+          error.code = 'EISDIR'
+          error.path = file
+          throw error
+        }
+      }
+      throw e
+    }
   }
 
   async _rmdir(path) {
@@ -199,6 +211,23 @@ export default class S3Handler extends RemoteHandlerAbstract {
     // nothing to do, directories do not exist, they are part of the files' path
   }
 
+  // reimplement _rmTree to handle efficiantly path with more than 1000 entries in trees
+  // @todo : use parallel processing for unlink
+  async _rmTree(path) {
+    let NextContinuationToken
+    do {
+      const result = await this._s3.listObjectsV2({
+        Bucket: this._bucket,
+        Prefix: this._dir + path + '/',
+        ContinuationToken: NextContinuationToken,
+      })
+      NextContinuationToken = result.isTruncated ? null : result.NextContinuationToken
+      for (const path of result.Contents) {
+        await this._unlink(path)
+      }
+    } while (NextContinuationToken !== null)
+  }
+
   async _write(file, buffer, position) {
     if (typeof file !== 'string') {
       file = file.fd

@xen-orchestra/log/README.md

@@ -66,6 +66,10 @@ configure([
     // if filter is a string, then it is pattern
     // (https://github.com/visionmedia/debug#wildcards) which is
     // matched against the namespace of the logs
+    //
+    // If it's an array, it will be handled as an array of filters
+    // and the transport will be used if any one of them match the
+    // current log
     filter: process.env.DEBUG,
 
     transport: transportConsole(),

@xen-orchestra/log/USAGE.md

@@ -48,6 +48,10 @@ configure([
     // if filter is a string, then it is pattern
     // (https://github.com/visionmedia/debug#wildcards) which is
     // matched against the namespace of the logs
+    //
+    // If it's an array, it will be handled as an array of filters
+    // and the transport will be used if any one of them match the
+    // current log
     filter: process.env.DEBUG,
 
     transport: transportConsole(),

@xen-orchestra/log/configure.js

@@ -4,6 +4,42 @@ const { compileGlobPattern } = require('./utils')
 
 // ===================================================================
 
+const compileFilter = filter => {
+  if (filter === undefined) {
+    return
+  }
+
+  const type = typeof filter
+  if (type === 'function') {
+    return filter
+  }
+  if (type === 'string') {
+    const re = compileGlobPattern(filter)
+    return log => re.test(log.namespace)
+  }
+
+  if (Array.isArray(filter)) {
+    const filters = filter.map(compileFilter).filter(_ => _ !== undefined)
+    const { length } = filters
+    if (length === 0) {
+      return
+    }
+    if (length === 1) {
+      return filters[0]
+    }
+    return log => {
+      for (let i = 0; i < length; ++i) {
+        if (filters[i](log)) {
+          return true
+        }
+      }
+      return false
+    }
+  }
+
+  throw new TypeError('unsupported `filter`')
+}
+
 const createTransport = config => {
   if (typeof config === 'function') {
     return config
@@ -19,26 +55,15 @@ const createTransport = config => {
     }
   }
 
-  let { filter } = config
-  let transport = createTransport(config.transport)
   const level = resolve(config.level)
+  const filter = compileFilter([config.filter, level === undefined ? undefined : log => log.level >= level])
+
+  let transport = createTransport(config.transport)
 
   if (filter !== undefined) {
-    if (typeof filter === 'string') {
-      const re = compileGlobPattern(filter)
-      filter = log => re.test(log.namespace)
-    }
-
     const orig = transport
     transport = function (log) {
-      if ((level !== undefined && log.level >= level) || filter(log)) {
-        return orig.apply(this, arguments)
-      }
-    }
-  } else if (level !== undefined) {
-    const orig = transport
-    transport = function (log) {
-      if (log.level >= level) {
+      if (filter(log)) {
         return orig.apply(this, arguments)
       }
     }

@xen-orchestra/log/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "@xen-orchestra/log",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "license": "ISC",
   "description": "Logging system with decoupled producers/consumer",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/log",
@@ -24,7 +24,7 @@
   },
   "dependencies": {
     "lodash": "^4.17.4",
-    "promise-toolbox": "^0.19.2"
+    "promise-toolbox": "^0.20.0"
   },
   "scripts": {
     "postversion": "npm publish"

@xen-orchestra/mixins/Config.js

@@ -1,5 +1,6 @@
 const get = require('lodash/get')
 const identity = require('lodash/identity')
+const isEqual = require('lodash/isEqual')
 const { createLogger } = require('@xen-orchestra/log')
 const { parseDuration } = require('@vates/parse-duration')
 const { watch } = require('app-conf')
@@ -48,7 +49,7 @@ module.exports = class Config {
     const watcher = config => {
       try {
         const value = processor(get(config, path))
-        if (value !== prev) {
+        if (!isEqual(value, prev)) {
           prev = value
           cb(value)
         }

@xen-orchestra/mixins/package.json

@@ -14,14 +14,14 @@
     "url": "https://vates.fr"
   },
   "license": "AGPL-3.0-or-later",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "engines": {
     "node": ">=12"
   },
   "dependencies": {
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/emit-async": "^0.1.0",
-    "@xen-orchestra/log": "^0.2.1",
+    "@xen-orchestra/log": "^0.3.0",
     "app-conf": "^0.9.0",
     "lodash": "^4.17.21"
   },

@xen-orchestra/proxy-cli/package.json

@@ -23,7 +23,7 @@
     "xo-proxy-cli": "dist/index.js"
   },
   "engines": {
-    "node": ">=8.10"
+    "node": ">=12"
   },
   "dependencies": {
     "@iarna/toml": "^2.2.0",
@@ -33,12 +33,12 @@
     "content-type": "^1.0.4",
     "cson-parser": "^4.0.7",
     "getopts": "^2.2.3",
-    "http-request-plus": "^0.12",
+    "http-request-plus": "^0.13.0",
     "json-rpc-protocol": "^0.13.1",
-    "promise-toolbox": "^0.19.2",
+    "promise-toolbox": "^0.20.0",
     "pump": "^3.0.0",
     "pumpify": "^2.0.1",
-    "split2": "^3.1.1"
+    "split2": "^4.1.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

@xen-orchestra/proxy/config.toml

@@ -18,6 +18,7 @@ keepAliveInterval = 10e3
 #
 # https://en.wikipedia.org/wiki/File-system_permissions#Numeric_notation
 dirMode = 0o700
+disableMergeWorker = false
 snapshotNameLabelTpl = '[XO Backup {job.name}] {vm.name_label}'
 
 [backups.defaultSettings]
@@ -59,6 +60,13 @@ cert = '/var/lib/xo-proxy/certificate.pem'
 key = '/var/lib/xo-proxy/key.pem'
 port = 443
 
+[logs]
+# Display all logs matching this filter, regardless of their level
+#filter = 'xo:backups:*'
+
+# Display all logs with level >=, regardless of their namespace
+level = 'info'
+
 [remoteOptions]
 mountsDir = '/run/xo-proxy/mounts'
 

@xen-orchestra/proxy/package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "@xen-orchestra/proxy",
-  "version": "0.14.4",
+  "version": "0.15.2",
   "license": "AGPL-3.0-or-later",
   "description": "XO Proxy used to remotely execute backup jobs",
   "keywords": [
@@ -31,17 +31,17 @@
     "@vates/decorate-with": "^0.1.0",
     "@vates/disposable": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.12.2",
-    "@xen-orchestra/fs": "^0.17.0",
-    "@xen-orchestra/log": "^0.2.1",
+    "@xen-orchestra/backups": "^0.15.1",
+    "@xen-orchestra/fs": "^0.18.0",
+    "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/mixin": "^0.1.0",
-    "@xen-orchestra/mixins": "^0.1.0",
+    "@xen-orchestra/mixins": "^0.1.1",
     "@xen-orchestra/self-signed": "^0.1.0",
-    "@xen-orchestra/xapi": "^0.6.4",
+    "@xen-orchestra/xapi": "^0.8.0",
     "ajv": "^8.0.3",
     "app-conf": "^0.9.0",
     "async-iterator-to-stream": "^1.1.0",
-    "fs-extra": "^9.1.0",
+    "fs-extra": "^10.0.0",
     "get-stream": "^6.0.0",
     "getopts": "^2.2.3",
     "golike-defer": "^0.5.1",
@@ -54,11 +54,11 @@
     "lodash": "^4.17.10",
     "node-zone": "^0.4.0",
     "parse-pairs": "^1.0.0",
-    "promise-toolbox": "^0.19.2",
+    "promise-toolbox": "^0.20.0",
     "source-map-support": "^0.5.16",
     "stoppable": "^1.0.6",
     "xdg-basedir": "^4.0.0",
-    "xen-api": "^0.33.1",
+    "xen-api": "^0.35.1",
     "xo-common": "^0.7.0"
   },
   "devDependencies": {
@@ -72,7 +72,7 @@
     "@vates/toggle-scripts": "^1.0.0",
     "babel-plugin-transform-dev": "^2.0.1",
     "cross-env": "^7.0.2",
-    "index-modules": "^0.4.0"
+    "index-modules": "^0.4.3"
   },
   "scripts": {
     "_build": "index-modules --index-file index.mjs src/app/mixins && babel --delete-dir-on-start --keep-file-extension --source-maps --out-dir=dist/ src/",

@xen-orchestra/proxy/src/app/mixins/api.mjs

@@ -14,15 +14,27 @@ import { createLogger } from '@xen-orchestra/log'
 
 const { debug, warn } = createLogger('xo:proxy:api')
 
-const ndJsonStream = asyncIteratorToStream(async function* (responseId, iterable) {
-  yield format.response(responseId, { $responseType: 'ndjson' }) + '\n'
+const ndJsonStream = asyncIteratorToStream(async function*(responseId, iterable) {
   try {
-    for await (const data of iterable) {
+    let cursor, iterator
+    try {
+      const getIterator = iterable[Symbol.iterator] ?? iterable[Symbol.asyncIterator]
+      iterator = getIterator.call(iterable)
+
+      cursor = await iterator.next()
+      yield format.response(responseId, { $responseType: 'ndjson' }) + '\n'
+    } catch (error) {
+      yield format.error(responseId, error)
+      throw error
+    }
+
+    while (!cursor.done) {
       try {
-        yield JSON.stringify(data) + '\n'
+        yield JSON.stringify(cursor.value) + '\n'
       } catch (error) {
         warn('ndJsonStream, item error', { error })
       }
+      cursor = await iterator.next()
     }
   } catch (error) {
     warn('ndJsonStream, fatal error', { error })
@@ -40,7 +52,7 @@ export default class Api {
       ctx.req.setTimeout(0)
 
       const profile = await app.authentication.findProfile({
-        authenticationToken: ctx.cookies.get('authenticationToken'),
+        authenticationToken: ctx.cookies.get('authenticationToken')
       })
       if (profile === undefined) {
         ctx.status = 401
@@ -111,7 +123,7 @@ export default class Api {
     this.addMethods({
       system: {
         getMethodsInfo: [
-          function* () {
+          function*() {
             const methods = this._methods
             for (const name in methods) {
               const { description, params = {} } = methods[name]
@@ -119,25 +131,25 @@ export default class Api {
             }
           }.bind(this),
           {
-            description: 'returns the signatures of all available API methods',
-          },
+            description: 'returns the signatures of all available API methods'
+          }
         ],
         getServerVersion: [
           () => appVersion,
           {
-            description: 'returns the version of xo-server',
-          },
+            description: 'returns the version of xo-server'
+          }
         ],
         listMethods: [
-          function* () {
+          function*() {
             const methods = this._methods
             for (const name in methods) {
               yield name
             }
           }.bind(this),
           {
-            description: 'returns the name of all available API methods',
-          },
+            description: 'returns the name of all available API methods'
+          }
         ],
         methodSignature: [
           ({ method: name }) => {
@@ -152,14 +164,14 @@ export default class Api {
           {
             description: 'returns the signature of an API method',
             params: {
-              method: { type: 'string' },
-            },
-          },
-        ],
+              method: { type: 'string' }
+            }
+          }
+        ]
       },
       test: {
         range: [
-          function* ({ start = 0, stop, step }) {
+          function*({ start = 0, stop, step }) {
             if (step === undefined) {
               step = start > stop ? -1 : 1
             }
@@ -177,11 +189,11 @@ export default class Api {
             params: {
               start: { optional: true, type: 'number' },
               step: { optional: true, type: 'number' },
-              stop: { type: 'number' },
-            },
-          },
-        ],
-      },
+              stop: { type: 'number' }
+            }
+          }
+        ]
+      }
     })
   }
 
@@ -208,7 +220,7 @@ export default class Api {
         return required
       }),
 
-      type: 'object',
+      type: 'object'
     })
 
     const m = params => {

@xen-orchestra/proxy/src/app/mixins/backups.mjs

@@ -11,6 +11,7 @@ import { DurablePartition } from '@xen-orchestra/backups/DurablePartition.js'
 import { execFile } from 'child_process'
 import { formatVmBackups } from '@xen-orchestra/backups/formatVmBackups.js'
 import { ImportVmBackup } from '@xen-orchestra/backups/ImportVmBackup.js'
+import { JsonRpcError } from 'json-rpc-protocol'
 import { Readable } from 'stream'
 import { RemoteAdapter } from '@xen-orchestra/backups/RemoteAdapter.js'
 import { RestoreMetadataBackup } from '@xen-orchestra/backups/RestoreMetadataBackup.js'
@@ -108,7 +109,7 @@ export default class Backups {
         if (!__DEV__) {
           const license = await app.appliance.getSelfLicense()
           if (license === undefined) {
-            throw new Error('no valid proxy license')
+            throw new JsonRpcError('no valid proxy license')
           }
         }
         return run.apply(this, arguments)

@xen-orchestra/proxy/src/app/mixins/logs.mjs

@@ -0,0 +1,17 @@
+import transportConsole from '@xen-orchestra/log/transports/console.js'
+import { configure } from '@xen-orchestra/log/configure.js'
+
+export default class Logs {
+  constructor(app) {
+    const transport = transportConsole()
+    app.config.watch('logs', ({ filter, level }) => {
+      configure([
+        {
+          filter: [process.env.DEBUG, filter],
+          level,
+          transport,
+        },
+      ])
+    })
+  }
+}

@xen-orchestra/upload-ova/package.json

@@ -33,9 +33,9 @@
     "chalk": "^4.1.0",
     "exec-promise": "^0.7.0",
     "form-data": "^4.0.0",
-    "fs-extra": "^9.0.0",
+    "fs-extra": "^10.0.0",
     "get-stream": "^6.0.0",
-    "http-request-plus": "^0.12",
+    "http-request-plus": "^0.13.0",
     "human-format": "^0.11.0",
     "l33teral": "^3.0.3",
     "lodash": "^4.17.4",

@xen-orchestra/xapi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xen-orchestra/xapi",
-  "version": "0.6.4",
+  "version": "0.8.0",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/xapi",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
@@ -25,7 +25,7 @@
     "xo-common": "^0.7.0"
   },
   "peerDependencies": {
-    "xen-api": "^0.33.1"
+    "xen-api": "^0.35.1"
   },
   "scripts": {
     "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
@@ -40,11 +40,11 @@
   "dependencies": {
     "@vates/decorate-with": "^0.1.0",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/log": "^0.2.1",
+    "@xen-orchestra/log": "^0.3.0",
     "d3-time-format": "^3.0.0",
     "golike-defer": "^0.5.1",
     "lodash": "^4.17.15",
-    "promise-toolbox": "^0.19.2"
+    "promise-toolbox": "^0.20.0"
   },
   "private": false,
   "license": "AGPL-3.0-or-later",

@xen-orchestra/xapi/src/vdi.js

@@ -1,6 +1,7 @@
 const CancelToken = require('promise-toolbox/CancelToken.js')
 const pCatch = require('promise-toolbox/catch.js')
 const pRetry = require('promise-toolbox/retry.js')
+const { decorateWith } = require('@vates/decorate-with')
 
 const extractOpaqueRef = require('./_extractOpaqueRef.js')
 
@@ -11,10 +12,13 @@ module.exports = class Vdi {
     return extractOpaqueRef(await this.callAsync('VDI.clone', vdiRef))
   }
 
+  // work around a race condition in XCP-ng/XenServer where the disk is not fully unmounted yet
+  @decorateWith(pRetry.wrap, function () {
+    return this._vdiDestroyRetryWhenInUse
+  })
   async destroy(vdiRef) {
     await pCatch.call(
-      // work around a race condition in XCP-ng/XenServer where the disk is not fully unmounted yet
-      pRetry(() => this.callAsync('VDI.destroy', vdiRef), this._vdiDestroyRetryWhenInUse),
+      this.callAsync('VDI.destroy', vdiRef),
       // if this VDI is not found, consider it destroyed
       { code: 'HANDLE_INVALID' },
       noop

CHANGELOG.md

@@ -1,9 +1,124 @@
+## **5.64.0** (2021-10-29)
+
 # ChangeLog
 
-## **5.61.0** (2021-07-30)
-
 <img id="latest" src="https://badgen.net/badge/channel/latest/yellow" alt="Channel: latest" />
 
+## Highlights
+
+- [Netbox] Support older versions of Netbox and prevent "active is not a valid choice" error [#5898](https://github.com/vatesfr/xen-orchestra/issues/5898) (PR [#5946](https://github.com/vatesfr/xen-orchestra/pull/5946))
+- [Tasks] Filter out short tasks using a default filter (PR [#5921](https://github.com/vatesfr/xen-orchestra/pull/5921))
+- [Host] Handle evacuation failure during host shutdown (PR [#5966](https://github.com/vatesfr/xen-orchestra/pull/#5966))
+- [Menu] Notify user when proxies need to be upgraded (PR [#5930](https://github.com/vatesfr/xen-orchestra/pull/5930))
+- [Servers] Ability to use an HTTP proxy between XO and a server (PR [#5958](https://github.com/vatesfr/xen-orchestra/pull/5958))
+- [VM/export] Ability to copy the export URL (PR [#5948](https://github.com/vatesfr/xen-orchestra/pull/5948))
+- [Pool/advanced] Ability to define network for importing/exporting VMs/VDIs (PR [#5957](https://github.com/vatesfr/xen-orchestra/pull/5957))
+- [Host/advanced] Add button to enable/disable the host (PR [#5952](https://github.com/vatesfr/xen-orchestra/pull/5952))
+- [Backups] Enable merge worker by default
+
+### Enhancements
+
+- [Jobs] Ability to copy a job ID (PR [#5951](https://github.com/vatesfr/xen-orchestra/pull/5951))
+
+### Bug fixes
+
+- [Backups] Delete unused snapshots related to other schedules (even no longer existing) (PR [#5949](https://github.com/vatesfr/xen-orchestra/pull/5949))
+- [Jobs] Fix `job.runSequence` method (PR [#5944](https://github.com/vatesfr/xen-orchestra/pull/5944))
+- [Netbox] Fix error when testing plugin on versions older than 2.10 (PR [#5963](https://github.com/vatesfr/xen-orchestra/pull/5963))
+- [Snapshot] Fix "Create VM from snapshot" creating a template instead of a VM (PR [#5955](https://github.com/vatesfr/xen-orchestra/pull/5955))
+- [Host/Logs] Improve the display of log content (PR [#5943](https://github.com/vatesfr/xen-orchestra/pull/5943))
+- [XOA licenses] Fix expiration date displaying "Invalid date" in some rare cases (PR [#5967](https://github.com/vatesfr/xen-orchestra/pull/5967))
+- [API/pool.listPoolsMatchingCriteria] Fix `checkSrName`/`checkPoolName` `is not a function` error
+
+### Released packages
+
+- xo-server-netbox 0.3.3
+- vhd-lib 1.3.0
+- xen-api 0.35.1
+- @xen-orchestra/xapi 0.8.0
+- @xen-orchestra/backups 0.15.1
+- @xen-orchestra/proxy 0.15.2
+- vhd-cli 0.5.0
+- xapi-explore-sr 0.4.0
+- xo-server 5.83.0
+- xo-web 5.89.0
+
+## **5.63.0** (2021-09-30)
+
+<img id="stable" src="https://badgen.net/badge/channel/stable/green" alt="Channel: stable" />
+
+### Highlights
+
+- [Backup] Go back to previous page instead of going to the overview after editing a job: keeps current filters and page (PR [#5913](https://github.com/vatesfr/xen-orchestra/pull/5913))
+- [Health] Do not take into consideration duplicated MAC addresses from CR VMs (PR [#5916](https://github.com/vatesfr/xen-orchestra/pull/5916))
+- [Health] Ability to filter duplicated MAC addresses by running VMs (PR [#5917](https://github.com/vatesfr/xen-orchestra/pull/5917))
+- [Tables] Move the search bar and pagination to the top of the table (PR [#5914](https://github.com/vatesfr/xen-orchestra/pull/5914))
+- [Netbox] Handle nested prefixes by always assigning an IP to the smallest prefix it matches (PR [#5908](https://github.com/vatesfr/xen-orchestra/pull/5908))
+
+### Bug fixes
+
+- [SSH keys] Allow SSH key to be broken anywhere to avoid breaking page formatting (Thanks [@tstivers1990](https://github.com/tstivers1990)!) [#5891](https://github.com/vatesfr/xen-orchestra/issues/5891) (PR [#5892](https://github.com/vatesfr/xen-orchestra/pull/5892))
+- [Netbox] Better handling and error messages when encountering issues due to UUID custom field not being configured correctly [#5905](https://github.com/vatesfr/xen-orchestra/issues/5905) [#5806](https://github.com/vatesfr/xen-orchestra/issues/5806) [#5834](https://github.com/vatesfr/xen-orchestra/issues/5834) (PR [#5909](https://github.com/vatesfr/xen-orchestra/pull/5909))
+- [New VM] Don't send network config if untouched as all commented config can make Cloud-init fail [#5918](https://github.com/vatesfr/xen-orchestra/issues/5918) (PR [#5923](https://github.com/vatesfr/xen-orchestra/pull/5923))
+
+### Released packages
+
+- xen-api 0.34.3
+- vhd-lib 1.2.0
+- xo-server-netbox 0.3.1
+- @xen-orchestra/proxy 0.14.7
+- xo-server 5.82.3
+- xo-web 5.88.0
+
+## **5.62.1** (2021-09-17)
+
+### Bug fixes
+
+- [VM/Advanced] Fix conversion from UEFI to BIOS boot firmware (PR [#5895](https://github.com/vatesfr/xen-orchestra/pull/5895))
+- [VM/network] Support newline-delimited IP addresses reported by some guest tools
+- Fix VM/host stats, VM creation with Cloud-init, and VM backups, with NATted hosts [#5896](https://github.com/vatesfr/xen-orchestra/issues/5896)
+- [VM/import] Very small VMDK and OVA files were mangled upon import (PR [#5903](https://github.com/vatesfr/xen-orchestra/pull/5903))
+
+### Released packages
+
+- xen-api 0.34.2
+- @xen-orchestra/proxy 0.14.6
+- xo-server 5.82.2
+
+## **5.62.0** (2021-08-31)
+
+### Highlights
+
+- [Host] Add warning in case of unmaintained host version [#5840](https://github.com/vatesfr/xen-orchestra/issues/5840) (PR [#5847](https://github.com/vatesfr/xen-orchestra/pull/5847))
+- [Backup] Use default migration network if set when importing/exporting VMs/VDIs (PR [#5883](https://github.com/vatesfr/xen-orchestra/pull/5883))
+
+### Enhancements
+
+- [New network] Ability for pool's admin to create a new network within the pool (PR [#5873](https://github.com/vatesfr/xen-orchestra/pull/5873))
+- [Netbox] Synchronize primary IPv4 and IPv6 addresses [#5633](https://github.com/vatesfr/xen-orchestra/issues/5633) (PR [#5879](https://github.com/vatesfr/xen-orchestra/pull/5879))
+
+### Bug fixes
+
+- [VM/network] Fix an issue where multiple IPs would be displayed in the same tag when using old Xen tools. This also fixes Netbox's IP synchronization for the affected VMs. (PR [#5860](https://github.com/vatesfr/xen-orchestra/pull/5860))
+- [LDAP] Handle groups with no members (PR [#5862](https://github.com/vatesfr/xen-orchestra/pull/5862))
+- Fix empty button on small size screen (PR [#5874](https://github.com/vatesfr/xen-orchestra/pull/5874))
+- [Host] Fix `Cannot read property 'other_config' of undefined` error when enabling maintenance mode (PR [#5875](https://github.com/vatesfr/xen-orchestra/pull/5875))
+
+### Released packages
+
+- xen-api 0.34.1
+- @xen-orchestra/xapi 0.7.0
+- @xen-orchestra/backups 0.13.0
+- @xen-orchestra/fs 0.18.0
+- @xen-orchestra/log 0.3.0
+- @xen-orchestra/mixins 0.1.1
+- xo-server-auth-ldap 0.10.4
+- xo-server-netbox 0.3.0
+- xo-server 5.82.1
+- xo-web 5.87.0
+
+## **5.61.0** (2021-07-30)
+
 ### Highlights
 
 - [SR/disks] Display base copies' active VDIs (PR [#5826](https://github.com/vatesfr/xen-orchestra/pull/5826))
@@ -31,8 +146,6 @@
 
 ## **5.60.0** (2021-06-30)
 
-<img id="stable" src="https://badgen.net/badge/channel/stable/green" alt="Channel: stable" />
-
 ### Highlights
 
 - [VM/disks] Ability to rescan ISO SRs (PR [#5814](https://github.com/vatesfr/xen-orchestra/pull/5814))
@@ -78,7 +191,7 @@
 
 - [Smart backup] Report missing pools [#2844](https://github.com/vatesfr/xen-orchestra/issues/2844) (PR [#5768](https://github.com/vatesfr/xen-orchestra/pull/5768))
 - [Metadata Backup] Add a warning on restoring a metadata backup (PR [#5769](https://github.com/vatesfr/xen-orchestra/pull/5769))
-- [Netbox] [Plugin](https://xen-orchestra.com/docs/advanced.html#netbox) to synchronize pools, VMs and IPs with [Netbox](https://netbox.readthedocs.io/en/stable/) (PR [#5783](https://github.com/vatesfr/xen-orchestra/pull/5783))
+- [Netbox][plugin](https://xen-orchestra.com/docs/advanced.html#netbox) to synchronize pools, VMs and IPs with [Netbox](https://netbox.readthedocs.io/en/stable/) (PR [#5783](https://github.com/vatesfr/xen-orchestra/pull/5783))
 
 ### Enhancements
 

CHANGELOG.unreleased.md

@@ -7,12 +7,12 @@
 
 > Users must be able to say: “Nice enhancement, I'm eager to test it”
 
-- [Host/Load-balancer] Log events with `info` instead of `debug` (PR [#5870](https://github.com/vatesfr/xen-orchestra/pull/5870))
-
 ### Bug fixes
 
 > Users must be able to say: “I had this issue, happy to know it's fixed”
 
+[Import/VM] Fix the import of OVA files (PR [#5976](https://github.com/vatesfr/xen-orchestra/pull/5976))
+
 ### Packages to release
 
 > Packages will be released in the order they are here, therefore, they should
@@ -30,4 +30,7 @@
 >
 > In case of conflict, the highest (lowest in previous list) `$version` wins.
 
-- xo-server-load-balancer minor
+- @xen-orchestra/fs minor
+- vhd-lib minor
+- xo-server patch
+- vhd-cli minor

ISSUE_TEMPLATE.md

@@ -1,27 +0,0 @@
-<!--
-Welcome to the issue section of Xen Orchestra!
-
-Here you can:
-- report an issue
-- propose an enhancement
-- ask a question
-
-Please, respect this template as much as possible, it helps us sort
-the issues :)
--->
-
-### Context
-
-- **XO origin**: the sources / XO Appliance
-- **Versions**:
-  - Node: **FILL HERE**
-  - xo-web: **FILL HERE**
-  - xo-server: **FILL HERE**
-
-### Expected behavior
-
-<!-- What you expect to happen -->
-
-### Current behavior
-
-<!-- What is actually happening -->

docs/advanced.md

@@ -327,6 +327,8 @@ Synchronize your pools, VMs, network interfaces and IP addresses with your [Netb
 
 ![](./assets/netbox.png)
 
+### Netbox side
+
 - Go to your Netbox interface
 - Configure prefixes:
   - Go to IPAM > Prefixes > Add
@@ -339,13 +341,19 @@ XO will try to find the right prefix for each IP address. If it can't find a pre
 - Generate a token:
   - Go to Admin > Tokens > Add token
   - Create a token with "Write enabled"
-- Add a UUID custom field:
+- Add a UUID custom field (for **Netbox 2.x**):
   - Got to Admin > Custom fields > Add custom field
   - Create a custom field called "uuid" (lower case!)
   - Assign it to object types `virtualization > cluster` and `virtualization > virtual machine`
 
 ![](./assets/customfield.png)
 
+:::tip
+In Netbox 3.x, custom fields can be found directly in the site (no need to go in the admin section). It's available in "Other/Customization/Custom Fields". After creation of the `uuid` field, assign it to the object types `virtualization > cluster` and `virtualization > virtual machine`.
+:::
+
+### In Xen Orchestra
+
 - Go to Xen Orchestra > Settings > Plugins > Netbox and fill out the configuration:
   - Endpoint: the URL of your Netbox instance (e.g.: `https://netbox.company.net`)
   - Unauthorized certificate: only for HTTPS, enable this option if your Netbox instance uses a self-signed SSL certificate

docs/backups.md

@@ -286,8 +286,9 @@ When it's done exporting, we'll remove the snapshot. Note: this operation will t
 Concurrency is a parameter that let you define how many VMs your backup job will manage simultaneously.
 
 :::tip
+
 - Default concurrency value is 2 if left empty.
-:::
+  :::
 
 Let's say you want to backup 50 VMs (each with 1x disk) at 3:00 AM. There are **2 different strategies**:
 
@@ -297,7 +298,7 @@ Let's say you want to backup 50 VMs (each with 1x disk) at 3:00 AM. There are **
 The first purely sequential strategy will lead to the fact that: **you can't predict when a snapshot of your data will occur**. Because you can't predict the first VM export time (let's say 3 hours), then your second VM will have its snapshot taken 3 hours later, at 6 AM.
 
 :::tip
-If you need your backup to be done at a specific time  you should consider creating a specific backup task for this VM.
+If you need your backup to be done at a specific time you should consider creating a specific backup task for this VM.
 :::
 
 Strategy number 2 is to parallelise: all the snapshots will be taken at 3 AM. However **it's risky without limits**: it means potentially doing 50 snapshots or more at once on the same storage. **Since XenServer doesn't have a queue**, it will try to do all of them at once. This is also prone to race conditions and could cause crashes on your storage.
@@ -312,6 +313,7 @@ You should be aware of your hardware limitation when defining the best concurren
 The best way to define the best concurrency for you is by increasing it slowly and watching the result on backup time.
 
 So to summarize, if you set your concurrency at 6 and you have 20 Vms to backup the process will be the following:
+
 - We start the backup of the first 6 VMs.
 - When one VM backup as ended we will launch the next VM backup.
 - We're keep launching new VM backup until the 20 VMs are finished, keeping 6 backups running.

docs/full_backups.md

@@ -1,6 +1,6 @@
 # Full backups
 
-You can schedule full backups of your VMs, by exporting them to the local XOA file-system, or directly to an NFS or SMB share. The "rentention" parameter allows you to modify how many backups are retained (by removing the oldest one).
+You can schedule full backups of your VMs, by exporting them to the local XOA file-system, or directly to an NFS or SMB share. The "retention" parameter allows you to modify how many backups are retained (by removing the oldest one).
 
 [![](./assets/backupexample.png)](https://xen-orchestra.com/blog/backup-your-xenserver-vms-with-xen-orchestra/)
 

docs/updater.md

@@ -10,7 +10,7 @@ By design, the updater is only available in XOA. If you are using XO from the so
 
 ## Requirements
 
-In order to work, the updater needs access to `xen-orchestra.com` (port 443).
+In order to work, the updater needs access to `xen-orchestra.com` (port 443) and `nodejs.org` (port 443).
 
 ## Usage
 

docs/xoa.md

@@ -61,7 +61,7 @@ Please only use this if you have issues with [the default way to deploy XOA](ins
 Alternatively, you can deploy it by connecting to your XenServer host and executing the following:
 
 ```
-bash -c "$(curl -sS https://xoa.io/deploy)"
+bash -c "$(wget -qO- https://xoa.io/deploy)"
 ```
 
 :::tip
@@ -78,7 +78,7 @@ curl: (35) error:1407742E:SSL routines:SSL23_GET_SERVER_HELLO:tlsv1 alert protoc
 It means that the secure HTTPS protocol is not supported, you can bypass this using the unsecure command instead:
 
 ```
-bash -c "$(curl -sS http://xoa.io/deploy)"
+bash -c "$(wget -qO- http://xoa.io/deploy)"
 ```
 
 :::
@@ -97,59 +97,26 @@ After the VM is imported, you just need to start it with `xe vm-start vm="XOA"`
 
 ## First console connection
 
-If you connect via SSH or console, the default credentials are:
+### Deployed with the [web deploy form](https://xen-orchestra.com/#!/xoa)
 
-- user: xoa
-- password: xoa
+In that case, you already set the password for `xoa` user. If you forgot it, see below.
 
-During your first connection, the system will ask you to:
+### Manually deployed
 
-- enter the current password again (`xoa`)
-- enter your new password
-- retype your new password
+If you connect via SSH or console for the first time without using our [web deploy form](https://xen-orchestra.com/#!/xoa), be aware **there's NO default password set for security reasons**. To set it, you need to connect to your host to find the XOA VM UUID (eg via `xe vm-list`).
 
-When it's done, you'll be disconnected, so reconnect again with your new password.
-
-Here is an example when you connect via SSH for the first time:
+Then replace `<UUID>` with the previously find UUID, and `<password>` with your password:
 
 ```
-$ ssh xoa@192.168.100.146
-Warning: Permanently added '192.168.100.146' (ECDSA) to the list of known hosts.
-xoa@192.168.100.146's password:
-You are required to change your password immediately (root enforced)
- __   __             ____           _               _
- \ \ / /            / __ \         | |             | |
-  \ V / ___ _ __   | |  | |_ __ ___| |__   ___  ___| |_ _ __ __ _
-   > < / _ \ '_ \  | |  | | '__/ __| '_ \ / _ \/ __| __| '__/ _` |
-  / . \  __/ | | | | |__| | | | (__| | | |  __/\__ \ |_| | | (_| |
- /_/ \_\___|_| |_|  \____/|_|  \___|_| |_|\___||___/\__|_|  \__,_|
-
-Welcome to XOA Unified Edition, with Pro Support.
-
-* Restart XO: sudo systemctl restart xo-server.service
-* Display logs: sudo systemctl status xo-server.service
-* Register your XOA: sudo xoa-updater --register
-* Update your XOA: sudo xoa-updater --upgrade
-
-OFFICIAL XOA DOCUMENTATION HERE: https://xen-orchestra.com/docs/xoa.html
-
-Support available at https://xen-orchestra.com/#!/member/support
-
-Build number: 16.10.24
-
-Based on Debian GNU/Linux 8 (Stable) 64bits in PVHVM mode
-
-WARNING: Your password has expired.
-You must change your password now and login again!
-Changing password for xoa.
-(current) UNIX password:
-Enter new UNIX password:
-Retype new UNIX password:
-passwd: password updated successfully
-Connection to 192.168.100.146 closed.
-$
+xe vm-param-set uuid=<UUID> xenstore-data:vm-data/system-account-xoa-password=<password>
 ```
 
+:::tip
+Don't forget to use quotes for your password, eg: `xenstore-data:vm-data/system-account-xoa-password='MyPassW0rd!'`
+:::
+
+Then, you could connect with `xoa` username and the password you defined in the previous command, eg with `ssh xoa@<XOA IP ADDRESS>`.
+
 ### Using sudo
 
 To avoid typing `sudo` for any admin command, you can have a root shell with `sudo -s`:

docs/xosan.md

@@ -318,7 +318,7 @@ XOSAN is a 100% software defined solution for XenServer hyperconvergence. You ca
 
 You will need to be registered on our website in order to use Xen Orchestra. If you are not yet registered, [here is the way](https://xen-orchestra.com/#!/signup)
 
-SSH in your XenServer and use the command line `bash -c "$(curl -sS https://xoa.io/deploy)"` - it will deploy Xen Orchestra Appliance on your XenServer infrastructure which is required to use XOSAN.
+SSH in your XenServer and use the command line `bash -c "$(wget -qO- https://xoa.io/deploy)"` - it will deploy Xen Orchestra Appliance on your XenServer infrastructure which is required to use XOSAN.
 
 > Note: You can also download the XVA file and follow [these instructions](https://xen-orchestra.com/docs/xoa.html#the-alternative).
 

package.json

@@ -3,7 +3,7 @@
     "@babel/core": "^7.0.0",
     "@babel/eslint-parser": "^7.13.8",
     "@babel/register": "^7.0.0",
-    "babel-jest": "^26.0.1",
+    "babel-jest": "^27.3.1",
     "benchmark": "^2.1.4",
     "eslint": "^7.6.0",
     "eslint-config-prettier": "^8.1.0",
@@ -12,17 +12,17 @@
     "eslint-plugin-eslint-comments": "^3.2.0",
     "eslint-plugin-import": "^2.22.1",
     "eslint-plugin-node": "^11.1.0",
-    "eslint-plugin-promise": "^4.2.1",
+    "eslint-plugin-promise": "^5.1.0",
     "eslint-plugin-react": "^7.21.5",
     "exec-promise": "^0.7.0",
     "globby": "^11.0.1",
     "handlebars": "^4.7.6",
     "husky": "^4.2.5",
-    "jest": "^26.0.1",
-    "lint-staged": "^10.2.7",
+    "jest": "^27.3.1",
+    "lint-staged": "^11.1.2",
     "lodash": "^4.17.4",
     "prettier": "^2.0.5",
-    "promise-toolbox": "^0.19.2",
+    "promise-toolbox": "^0.20.0",
     "sorted-object": "^2.0.1",
     "vuepress": "^1.4.1"
   },
@@ -61,8 +61,7 @@
       "/xo-server-test/",
       "/xo-web/"
     ],
-    "testRegex": "\\.spec\\.js$",
-    "timers": "fake"
+    "testRegex": "\\.spec\\.js$"
   },
   "lint-staged": {
     "*.{md,ts,ts}": "prettier --write"

packages/vhd-cli/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "vhd-cli",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "license": "ISC",
   "description": "Tools to read/create and merge VHD files",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-cli",
@@ -24,11 +24,11 @@
     "node": ">=8.10"
   },
   "dependencies": {
-    "@xen-orchestra/fs": "^0.17.0",
+    "@xen-orchestra/fs": "^0.18.0",
     "cli-progress": "^3.1.0",
     "exec-promise": "^0.7.0",
     "getopts": "^2.2.3",
-    "vhd-lib": "^1.1.0"
+    "vhd-lib": "^1.3.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",
@@ -36,8 +36,8 @@
     "@babel/preset-env": "^7.0.0",
     "cross-env": "^7.0.2",
     "execa": "^5.0.0",
-    "index-modules": "^0.3.0",
-    "promise-toolbox": "^0.19.2",
+    "index-modules": "^0.4.3",
+    "promise-toolbox": "^0.20.0",
     "rimraf": "^3.0.0",
     "tmp": "^0.2.1"
   },

packages/vhd-cli/src/commands/check.js

@@ -1,9 +1,9 @@
-import Vhd, { checkVhdChain } from 'vhd-lib'
+import { VhdFile, checkVhdChain } from 'vhd-lib'
 import getopts from 'getopts'
 import { getHandler } from '@xen-orchestra/fs'
 import { resolve } from 'path'
 
-const checkVhd = (handler, path) => new Vhd(handler, path).readHeaderAndFooter()
+const checkVhd = (handler, path) => new VhdFile(handler, path).readHeaderAndFooter()
 
 export default async rawArgs => {
   const { chain, _: args } = getopts(rawArgs, {

packages/vhd-cli/src/commands/compare.js

@@ -0,0 +1,81 @@
+import { getSyncedHandler } from '@xen-orchestra/fs'
+import { openVhd, Constants } from 'vhd-lib'
+import Disposable from 'promise-toolbox/Disposable'
+import omit from 'lodash/omit'
+
+const deepCompareObjects = function (src, dest, path) {
+  for (const key of Object.keys(src)) {
+    const srcValue = src[key]
+    const destValue = dest[key]
+    if (srcValue !== destValue) {
+      const srcType = typeof srcValue
+      const destType = typeof destValue
+      if (srcType !== destType) {
+        throw new Error(`key ${path + '/' + key} is of type *${srcType}* in source and *${destType}* in dest`)
+      }
+
+      if (srcType !== 'object') {
+        throw new Error(`key ${path + '/' + key} is *${srcValue}* in source and *${destValue}* in dest`)
+      }
+
+      if (Buffer.isBuffer(srcValue)) {
+        if (!(Buffer.isBuffer(destValue) && srcValue.equals(destValue))) {
+          throw new Error(`key ${path + '/' + key} is buffer in source that does not equal dest`)
+        }
+      } else {
+        deepCompareObjects(src[key], dest[key], path + '/' + key)
+      }
+    }
+  }
+}
+
+export default async args => {
+  if (args.length < 4 || args.some(_ => _ === '-h' || _ === '--help')) {
+    return `Usage: compare <sourceRemoteUrl> <source VHD> <destionationRemoteUrl> <destination> `
+  }
+  const [sourceRemoteUrl, sourcePath, destRemoteUrl, destPath] = args
+
+  await Disposable.use(async function* () {
+    const sourceHandler = yield getSyncedHandler({ url: sourceRemoteUrl })
+    const src = yield openVhd(sourceHandler, sourcePath)
+    const destHandler = yield getSyncedHandler({ url: destRemoteUrl })
+    const dest = yield openVhd(destHandler, destPath)
+
+    // parent locator entries contains offset that can be different without impacting the vhd
+    // we'll compare them later
+    // table offset and checksum are also implementation specific
+    const ignoredEntries = ['checksum', 'parentLocatorEntry', 'tableOffset']
+    deepCompareObjects(omit(src.header, ignoredEntries), omit(dest.header, ignoredEntries), 'header')
+    deepCompareObjects(src.footer, dest.footer, 'footer')
+
+    await src.readBlockAllocationTable()
+    await dest.readBlockAllocationTable()
+
+    for (let i = 0; i < src.header.maxTableEntries; i++) {
+      if (src.containsBlock(i)) {
+        if (dest.containsBlock(i)) {
+          const srcBlock = await src.readBlock(i)
+          const destBlock = await dest.readBlock(i)
+
+          if (!srcBlock.buffer.equals(destBlock.buffer)) {
+            throw new Error(`Block  ${i} has different data in src and dest`)
+          }
+        } else {
+          throw new Error(`Block  ${i} is present in source but not in dest `)
+        }
+      } else if (dest.containsBlock(i)) {
+        throw new Error(`Block  ${i} is present in dest but not in source `)
+      }
+    }
+
+    for (let parentLocatorId = 0; parentLocatorId < Constants.PARENT_LOCATOR_ENTRIES; parentLocatorId++) {
+      const srcParentLocator = await src.readParentLocator(parentLocatorId)
+      const destParentLocator = await dest.readParentLocator(parentLocatorId)
+      if (!srcParentLocator.data || !srcParentLocator.data.equals(destParentLocator.data)) {
+        console.log(srcParentLocator, destParentLocator)
+        throw new Error(`Parent Locator  ${parentLocatorId} has different data in src and dest`)
+      }
+    }
+    console.log('there is no difference between theses vhd')
+  })
+}

packages/vhd-cli/src/commands/copy.js

@@ -0,0 +1,50 @@
+import { getSyncedHandler } from '@xen-orchestra/fs'
+import { openVhd, VhdFile, VhdDirectory } from 'vhd-lib'
+import Disposable from 'promise-toolbox/Disposable'
+import getopts from 'getopts'
+
+export default async rawArgs => {
+  const {
+    directory,
+    help,
+    _: args,
+  } = getopts(rawArgs, {
+    alias: {
+      directory: 'd',
+      help: 'h',
+    },
+    boolean: ['directory', 'force'],
+    default: {
+      directory: false,
+      help: false,
+    },
+  })
+  if (args.length < 4 || help) {
+    return `Usage: index.js copy <sourceRemoteUrl> <source VHD> <destionationRemoteUrl> <destination> --directory`
+  }
+  const [sourceRemoteUrl, sourcePath, destRemoteUrl, destPath] = args
+
+  await Disposable.use(async function* () {
+    const sourceHandler = yield getSyncedHandler({ url: sourceRemoteUrl })
+    const src = yield openVhd(sourceHandler, sourcePath)
+    await src.readBlockAllocationTable()
+    const destHandler = yield getSyncedHandler({ url: destRemoteUrl })
+    const dest = yield directory ? VhdDirectory.create(destHandler, destPath) : VhdFile.create(destHandler, destPath)
+    // copy data
+    dest.header = src.header
+    dest.footer = src.footer
+
+    for await (const block of src.blocks()) {
+      await dest.writeEntireBlock(block)
+    }
+
+    // copy parent locators
+    for (let parentLocatorId = 0; parentLocatorId < 8; parentLocatorId++) {
+      const parentLocator = await src.readParentLocator(parentLocatorId)
+      await dest.writeParentLocator(parentLocator)
+    }
+    await dest.writeFooter()
+    await dest.writeHeader()
+    await dest.writeBlockAllocationTable()
+  })
+}

packages/vhd-cli/src/commands/info.js

@@ -1,9 +1,9 @@
-import Vhd from 'vhd-lib'
+import { VhdFile } from 'vhd-lib'
 import { getHandler } from '@xen-orchestra/fs'
 import { resolve } from 'path'
 
 export default async args => {
-  const vhd = new Vhd(getHandler({ url: 'file:///' }), resolve(args[0]))
+  const vhd = new VhdFile(getHandler({ url: 'file:///' }), resolve(args[0]))
 
   try {
     await vhd.readHeaderAndFooter()

packages/vhd-cli/src/commands/repl.js

@@ -2,7 +2,7 @@ import { asCallback, fromCallback, fromEvent } from 'promise-toolbox'
 import { getHandler } from '@xen-orchestra/fs'
 import { relative } from 'path'
 import { start as createRepl } from 'repl'
-import Vhd, * as vhdLib from 'vhd-lib'
+import * as vhdLib from 'vhd-lib'
 
 export default async args => {
   const cwd = process.cwd()
@@ -14,7 +14,7 @@ export default async args => {
     })
     Object.assign(repl.context, vhdLib)
     repl.context.handler = handler
-    repl.context.open = path => new Vhd(handler, relative(cwd, path))
+    repl.context.open = path => new vhdLib.VhdFile(handler, relative(cwd, path))
 
     // Make the REPL waits for promise completion.
     repl.eval = (evaluate => (cmd, context, filename, cb) => {

packages/vhd-lib/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "vhd-lib",
-  "version": "1.1.0",
+  "version": "1.3.0",
   "license": "AGPL-3.0-or-later",
   "description": "Primitives for VHD file handling",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-lib",
@@ -17,12 +17,12 @@
   },
   "dependencies": {
     "@vates/read-chunk": "^0.1.2",
-    "@xen-orchestra/log": "^0.2.1",
+    "@xen-orchestra/log": "^0.3.0",
     "async-iterator-to-stream": "^1.0.2",
-    "fs-extra": "^9.0.0",
+    "fs-extra": "^10.0.0",
     "limit-concurrency-decorator": "^0.5.0",
     "lodash": "^4.17.4",
-    "promise-toolbox": "^0.19.2",
+    "promise-toolbox": "^0.20.0",
     "struct-fu": "^1.2.0",
     "uuid": "^8.3.1"
   },
@@ -30,7 +30,7 @@
     "@babel/cli": "^7.0.0",
     "@babel/core": "^7.0.0",
     "@babel/preset-env": "^7.0.0",
-    "@xen-orchestra/fs": "^0.17.0",
+    "@xen-orchestra/fs": "^0.18.0",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^7.0.2",
     "execa": "^5.0.0",

packages/vhd-lib/src/Vhd/VhdAbstract.integ.spec.js

@@ -0,0 +1,138 @@
+/* eslint-env jest */
+
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import fs from 'fs-extra'
+import { getSyncedHandler } from '@xen-orchestra/fs'
+import { Disposable, pFromCallback } from 'promise-toolbox'
+
+import { openVhd } from '../index'
+import { createRandomFile, convertFromRawToVhd, createRandomVhdDirectory } from '../tests/utils'
+import { VhdAbstract } from './VhdAbstract'
+
+let tempDir
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+test('It creates an alias', async () => {
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file://' + tempDir })
+    const aliasPath = `alias/alias.alias.vhd`
+    const aliasFsPath = `${tempDir}/${aliasPath}`
+    await fs.mkdirp(`${tempDir}/alias`)
+
+    const testOneCombination = async ({ targetPath, targetContent }) => {
+      await VhdAbstract.createAlias(handler, aliasPath, targetPath)
+      // alias file is created
+      expect(await fs.exists(aliasFsPath)).toEqual(true)
+      // content is the target path relative to the alias location
+      const content = await fs.readFile(aliasFsPath, 'utf-8')
+      expect(content).toEqual(targetContent)
+      // create alias fails if alias already exists, remove it before next loop step
+      await fs.unlink(aliasFsPath)
+    }
+
+    const combinations = [
+      { targetPath: `targets.vhd`, targetContent: `../targets.vhd` },
+      { targetPath: `alias/targets.vhd`, targetContent: `targets.vhd` },
+      { targetPath: `alias/sub/targets.vhd`, targetContent: `sub/targets.vhd` },
+      { targetPath: `sibling/targets.vhd`, targetContent: `../sibling/targets.vhd` },
+    ]
+
+    for (const { targetPath, targetContent } of combinations) {
+      await testOneCombination({ targetPath, targetContent })
+    }
+  })
+})
+
+test('alias must have *.alias.vhd extension', async () => {
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
+    const aliasPath = 'invalidalias.vhd'
+    const targetPath = 'targets.vhd'
+    expect(async () => await VhdAbstract.createAlias(handler, aliasPath, targetPath)).rejects.toThrow()
+
+    expect(await fs.exists(aliasPath)).toEqual(false)
+  })
+})
+
+test('alias must not be chained', async () => {
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
+    const aliasPath = 'valid.alias.vhd'
+    const targetPath = 'an.other.valid.alias.vhd'
+    expect(async () => await VhdAbstract.createAlias(handler, aliasPath, targetPath)).rejects.toThrow()
+    expect(await fs.exists(aliasPath)).toEqual(false)
+  })
+})
+
+test('It rename and unlink a VHDFile', async () => {
+  const initalSize = 4
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSize)
+  await convertFromRawToVhd(rawFileName, `${tempDir}/randomfile.vhd`)
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
+    const { size } = await fs.stat(`${tempDir}/randomfile.vhd`)
+
+    await VhdAbstract.rename(handler, 'randomfile.vhd', 'renamed.vhd')
+    expect(await fs.exists(`${tempDir}/randomfile.vhd`)).toEqual(false)
+    const { size: renamedSize } = await fs.stat(`${tempDir}/renamed.vhd`)
+    expect(size).toEqual(renamedSize)
+    await VhdAbstract.unlink(handler, 'renamed.vhd')
+    expect(await fs.exists(`${tempDir}/renamed.vhd`)).toEqual(false)
+  })
+})
+
+test('It rename and unlink a VhdDirectory', async () => {
+  const initalSize = 4
+  const vhdDirectory = `${tempDir}/randomfile.dir`
+  await createRandomVhdDirectory(vhdDirectory, initalSize)
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
+    const vhd = yield openVhd(handler, 'randomfile.dir')
+    expect(vhd.header.cookie).toEqual('cxsparse')
+    expect(vhd.footer.cookie).toEqual('conectix')
+
+    await VhdAbstract.rename(handler, 'randomfile.dir', 'renamed.vhd')
+    expect(await fs.exists(`${tempDir}/randomfile.dir`)).toEqual(false)
+    await VhdAbstract.unlink(handler, `renamed.vhd`)
+    expect(await fs.exists(`${tempDir}/renamed.vhd`)).toEqual(false)
+  })
+})
+
+test('It create , rename and unlink alias', async () => {
+  const initalSize = 4
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSize)
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  const aliasFileName = `${tempDir}/aliasFileName.alias.vhd`
+  const aliasFileNameRenamed = `${tempDir}/aliasFileNameRenamed.alias.vhd`
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
+    await VhdAbstract.createAlias(handler, 'aliasFileName.alias.vhd', 'randomfile.vhd')
+    expect(await fs.exists(aliasFileName)).toEqual(true)
+    expect(await fs.exists(vhdFileName)).toEqual(true)
+
+    await VhdAbstract.rename(handler, 'aliasFileName.alias.vhd', 'aliasFileNameRenamed.alias.vhd')
+    expect(await fs.exists(aliasFileName)).toEqual(false)
+    expect(await fs.exists(vhdFileName)).toEqual(true)
+    expect(await fs.exists(aliasFileNameRenamed)).toEqual(true)
+
+    await VhdAbstract.unlink(handler, 'aliasFileNameRenamed.alias.vhd')
+    expect(await fs.exists(aliasFileName)).toEqual(false)
+    expect(await fs.exists(vhdFileName)).toEqual(false)
+    expect(await fs.exists(aliasFileNameRenamed)).toEqual(false)
+  })
+})

packages/vhd-lib/src/Vhd/VhdAbstract.js

@@ -0,0 +1,207 @@
+import { computeBatSize, sectorsRoundUpNoZero, sectorsToBytes } from './_utils'
+import { PLATFORM_NONE, SECTOR_SIZE, PLATFORM_W2KU, PARENT_LOCATOR_ENTRIES } from '../_constants'
+import { resolveAlias, isVhdAlias } from '../_resolveAlias'
+
+import assert from 'assert'
+import path from 'path'
+
+export class VhdAbstract {
+  #header
+  bitmapSize
+  footer
+  fullBlockSize
+  sectorsOfBitmap
+  sectorsPerBlock
+
+  get header() {
+    assert.notStrictEqual(this.#header, undefined, `header must be read before it's used`)
+    return this.#header
+  }
+
+  set header(header) {
+    this.#header = header
+    this.sectorsPerBlock = header.blockSize / SECTOR_SIZE
+    this.sectorsOfBitmap = sectorsRoundUpNoZero(this.sectorsPerBlock >> 3)
+    this.fullBlockSize = sectorsToBytes(this.sectorsOfBitmap + this.sectorsPerBlock)
+    this.bitmapSize = sectorsToBytes(this.sectorsOfBitmap)
+  }
+
+  /**
+   * instantiate a Vhd
+   *
+   * @returns {AbstractVhd}
+   */
+  static async open() {
+    throw new Error('open not implemented')
+  }
+
+  /**
+   * Check if this vhd contains a block with id blockId
+   * Must be called after readBlockAllocationTable
+   *
+   * @param {number} blockId
+   * @returns {boolean}
+   *
+   */
+  containsBlock(blockId) {
+    throw new Error(`checking if this vhd contains the block ${blockId} is not implemented`)
+  }
+
+  /**
+   * Read the header and the footer
+   * check their integrity
+   * if checkSecondFooter also checks that the footer at the end is equal to the one at the beginning
+   *
+   * @param {boolean} checkSecondFooter
+   */
+  readHeaderAndFooter(checkSecondFooter = true) {
+    throw new Error(
+      `reading and checking footer, ${checkSecondFooter ? 'second footer,' : ''} and header is not implemented`
+    )
+  }
+
+  readBlockAllocationTable() {
+    throw new Error(`reading block allocation table is not implemented`)
+  }
+
+  /**
+   *
+   * @param {number} blockId
+   * @param {boolean} onlyBitmap
+   * @returns {Buffer}
+   */
+  readBlock(blockId, onlyBitmap = false) {
+    throw new Error(`reading  ${onlyBitmap ? 'bitmap of block' : 'block'} ${blockId} is not implemented`)
+  }
+
+  /**
+   * coalesce the block with id blockId from the child vhd into
+   * this vhd
+   *
+   * @param {AbstractVhd} child
+   * @param {number} blockId
+   *
+   * @returns {number} the merged data size
+   */
+  coalesceBlock(child, blockId) {
+    throw new Error(`coalescing the block ${blockId} from ${child} is not implemented`)
+  }
+
+  /**
+   * ensure the bat size can store at least entries block
+   * move blocks if needed
+   * @param {number} entries
+   */
+  ensureBatSize(entries) {
+    throw new Error(`ensuring batSize can store at least  ${entries} is not implemented`)
+  }
+
+  // Write a context footer. (At the end and beginning of a vhd file.)
+  writeFooter(onlyEndFooter = false) {
+    throw new Error(`writing footer   ${onlyEndFooter ? 'only at end' : 'on both side'} is not implemented`)
+  }
+
+  writeHeader() {
+    throw new Error(`writing header is not implemented`)
+  }
+
+  _writeParentLocatorData(parentLocatorId, platformDataOffset, data) {
+    throw new Error(`write Parent locator ${parentLocatorId} is not implemented`)
+  }
+
+  _readParentLocatorData(parentLocatorId, platformDataOffset, platformDataSpace) {
+    throw new Error(`read Parent locator ${parentLocatorId} is not implemented`)
+  }
+  // common
+  get batSize() {
+    return computeBatSize(this.header.maxTableEntries)
+  }
+
+  async writeParentLocator({ id, platformCode = PLATFORM_NONE, data = Buffer.alloc(0) }) {
+    assert(id >= 0, 'parent Locator id must be a positive number')
+    assert(id < PARENT_LOCATOR_ENTRIES, `parent Locator id  must be less than ${PARENT_LOCATOR_ENTRIES}`)
+
+    await this._writeParentLocatorData(id, data)
+
+    const entry = this.header.parentLocatorEntry[id]
+    const dataSpaceSectors = Math.ceil(data.length / SECTOR_SIZE)
+    entry.platformCode = platformCode
+    entry.platformDataSpace = dataSpaceSectors * SECTOR_SIZE
+    entry.platformDataLength = data.length
+  }
+
+  async readParentLocator(id) {
+    assert(id >= 0, 'parent Locator id must be a positive number')
+    assert(id < PARENT_LOCATOR_ENTRIES, `parent Locator id  must be less than ${PARENT_LOCATOR_ENTRIES}`)
+    const data = await this._readParentLocatorData(id)
+    // offset is storage specific, don't expose it
+    const { platformCode } = this.header.parentLocatorEntry[id]
+    return {
+      platformCode,
+      id,
+      data,
+    }
+  }
+
+  async setUniqueParentLocator(fileNameString) {
+    await this.writeParentLocator({
+      id: 0,
+      platformCode: PLATFORM_W2KU,
+      data: Buffer.from(fileNameString, 'utf16le'),
+    })
+
+    for (let i = 1; i < PARENT_LOCATOR_ENTRIES; i++) {
+      await this.writeParentLocator({
+        id: i,
+        platformCode: PLATFORM_NONE,
+        data: Buffer.alloc(0),
+      })
+    }
+  }
+
+  async *blocks() {
+    const nBlocks = this.header.maxTableEntries
+    for (let blockId = 0; blockId < nBlocks; ++blockId) {
+      if (await this.containsBlock(blockId)) {
+        yield await this.readBlock(blockId)
+      }
+    }
+  }
+
+  static async rename(handler, sourcePath, targetPath) {
+    await handler.rename(sourcePath, targetPath)
+  }
+
+  static async unlink(handler, path) {
+    const resolved = await resolveAlias(handler, path)
+    try {
+      await handler.unlink(resolved)
+    } catch (err) {
+      if (err.code === 'EISDIR') {
+        await handler.rmtree(resolved)
+      } else {
+        throw err
+      }
+    }
+
+    // also delete the alias file
+    if (path !== resolved) {
+      await handler.unlink(path)
+    }
+  }
+
+  static async createAlias(handler, aliasPath, targetPath) {
+    if (!isVhdAlias(aliasPath)) {
+      throw new Error(`Alias must be named *.alias.vhd,  ${aliasPath} given`)
+    }
+    if (isVhdAlias(targetPath)) {
+      throw new Error(`Chaining alias is forbidden ${aliasPath} to ${targetPath}`)
+    }
+    // aliasPath and targetPath are absolute path from the root of the handler
+    // normalize them so they can't  escape this dir
+    const aliasDir = path.dirname(path.resolve('/', aliasPath))
+    // only store the relative path from alias to target
+    const relativePathToTarget = path.relative(aliasDir, path.resolve('/', targetPath))
+    await handler.writeFile(aliasPath, relativePathToTarget)
+  }
+}

packages/vhd-lib/src/Vhd/VhdDirectory.js

@@ -0,0 +1,190 @@
+import { buildHeader, buildFooter } from './_utils'
+import { createLogger } from '@xen-orchestra/log'
+import { fuFooter, fuHeader, checksumStruct } from '../_structs'
+import { test, set as setBitmap } from '../_bitmap'
+import { VhdAbstract } from './VhdAbstract'
+import assert from 'assert'
+
+const { debug } = createLogger('vhd-lib:VhdDirectory')
+
+// ===================================================================
+// Directory format
+// <path>
+// ├─ header // raw content of the header
+// ├─ footer // raw content of the footer
+// ├─ bat // bit array. A zero bit indicates at a position that this block is not present
+// ├─ parentLocatorEntry{0-7} // data of a parent locator
+// ├─ blocks // blockId is the position in the BAT
+//    └─ <the first to  {blockId.length -3} numbers of blockId >
+//         └─ <the three last numbers  of blockID >  // block content.
+
+export class VhdDirectory extends VhdAbstract {
+  #uncheckedBlockTable
+
+  set header(header) {
+    super.header = header
+    this.#blockTable = Buffer.alloc(header.maxTableEntries)
+  }
+
+  get header() {
+    return super.header
+  }
+
+  get #blockTable() {
+    assert.notStrictEqual(this.#uncheckedBlockTable, undefined, 'Block table must be initialized before access')
+    return this.#uncheckedBlockTable
+  }
+
+  set #blockTable(blockTable) {
+    this.#uncheckedBlockTable = blockTable
+  }
+
+  static async open(handler, path) {
+    const vhd = new VhdDirectory(handler, path)
+
+    // openning a file for reading does not trigger EISDIR as long as we don't really read from it :
+    // https://man7.org/linux/man-pages/man2/open.2.html
+    // EISDIR pathname refers to a directory and the access requested
+    // involved writing (that is, O_WRONLY or O_RDWR is set).
+    // reading the header ensure we have a well formed directory immediatly
+    await vhd.readHeaderAndFooter()
+    return {
+      dispose: () => {},
+      value: vhd,
+    }
+  }
+
+  static async create(handler, path) {
+    await handler.mkdir(path)
+    const vhd = new VhdDirectory(handler, path)
+    return {
+      dispose: () => {},
+      value: vhd,
+    }
+  }
+
+  constructor(handler, path) {
+    super()
+    this._handler = handler
+    this._path = path
+  }
+
+  async readBlockAllocationTable() {
+    const { buffer } = await this._readChunk('bat')
+    this.#blockTable = buffer
+  }
+
+  containsBlock(blockId) {
+    return test(this.#blockTable, blockId)
+  }
+
+  getChunkPath(partName) {
+    return this._path + '/' + partName
+  }
+
+  async _readChunk(partName) {
+    // here we can implement compression and / or crypto
+    const buffer = await this._handler.readFile(this.getChunkPath(partName))
+
+    return {
+      buffer: Buffer.from(buffer),
+    }
+  }
+
+  async _writeChunk(partName, buffer) {
+    assert(Buffer.isBuffer(buffer))
+    // here we can implement compression and / or crypto
+
+    // chunks can be in sub directories :  create direcotries if necessary
+    const pathParts = partName.split('/')
+    let currentPath = this._path
+
+    // the last one is the file name
+    for (let i = 0; i < pathParts.length - 1; i++) {
+      currentPath += '/' + pathParts[i]
+      await this._handler.mkdir(currentPath)
+    }
+
+    return this._handler.writeFile(this.getChunkPath(partName), buffer)
+  }
+
+  // put block in subdirectories to limit impact when doing directory listing
+  _getBlockPath(blockId) {
+    const blockPrefix = Math.floor(blockId / 1e3)
+    const blockSuffix = blockId - blockPrefix * 1e3
+    return `blocks/${blockPrefix}/${blockSuffix}`
+  }
+
+  async readHeaderAndFooter() {
+    const { buffer: bufHeader } = await this._readChunk('header')
+    const { buffer: bufFooter } = await this._readChunk('footer')
+    const footer = buildFooter(bufFooter)
+    const header = buildHeader(bufHeader, footer)
+
+    this.footer = footer
+    this.header = header
+  }
+
+  async readBlock(blockId, onlyBitmap = false) {
+    if (onlyBitmap) {
+      throw new Error(`reading 'bitmap of block' ${blockId} in a VhdDirectory is not implemented`)
+    }
+    const { buffer } = await this._readChunk(this._getBlockPath(blockId))
+    return {
+      id: blockId,
+      bitmap: buffer.slice(0, this.bitmapSize),
+      data: buffer.slice(this.bitmapSize),
+      buffer,
+    }
+  }
+  ensureBatSize() {
+    // nothing to do in directory mode
+  }
+
+  async writeFooter() {
+    const { footer } = this
+
+    const rawFooter = fuFooter.pack(footer)
+
+    footer.checksum = checksumStruct(rawFooter, fuFooter)
+    debug(`Write footer  (checksum=${footer.checksum}). (data=${rawFooter.toString('hex')})`)
+
+    await this._writeChunk('footer', rawFooter)
+  }
+
+  writeHeader() {
+    const { header } = this
+    const rawHeader = fuHeader.pack(header)
+    header.checksum = checksumStruct(rawHeader, fuHeader)
+    debug(`Write header  (checksum=${header.checksum}). (data=${rawHeader.toString('hex')})`)
+    return this._writeChunk('header', rawHeader)
+  }
+
+  writeBlockAllocationTable() {
+    assert.notStrictEqual(this.#blockTable, undefined, 'Block allocation table has not been read')
+    assert.notStrictEqual(this.#blockTable.length, 0, 'Block allocation table is empty')
+
+    return this._writeChunk('bat', this.#blockTable)
+  }
+
+  // only works if data are in the same bucket
+  // and if the full block is modified in child ( which is the case whit xcp)
+
+  coalesceBlock(child, blockId) {
+    this._handler.copy(child.getChunkPath(blockId), this.getChunkPath(blockId))
+  }
+
+  async writeEntireBlock(block) {
+    await this._writeChunk(this._getBlockPath(block.id), block.buffer)
+    setBitmap(this.#blockTable, block.id)
+  }
+
+  async _readParentLocatorData(id) {
+    return (await this._readChunk('parentLocatorEntry' + id)).buffer
+  }
+
+  async _writeParentLocatorData(id, data) {
+    await this._writeChunk('parentLocatorEntry' + id, data)
+    this.header.parentLocatorEntry[id].platformDataOffset = 0
+  }
+}

packages/vhd-lib/src/Vhd/VhdFile.integ.spec.js

@@ -0,0 +1,164 @@
+/* eslint-env jest */
+
+import execa from 'execa'
+import fs from 'fs-extra'
+import getStream from 'get-stream'
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { getHandler } from '@xen-orchestra/fs'
+import { pFromCallback } from 'promise-toolbox'
+import { randomBytes } from 'crypto'
+
+import { VhdFile } from './VhdFile'
+
+import { SECTOR_SIZE } from '../_constants'
+import { checkFile, createRandomFile, convertFromRawToVhd, recoverRawContent } from '../tests/utils'
+
+let tempDir = null
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+test('blocks can be moved', async () => {
+  const initalSize = 4
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSize)
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  const handler = getHandler({ url: 'file://' })
+  const originalSize = await handler.getSize(rawFileName)
+  const newVhd = new VhdFile(handler, vhdFileName)
+  await newVhd.readHeaderAndFooter()
+  await newVhd.readBlockAllocationTable()
+  await newVhd._freeFirstBlockSpace(8000000)
+  const recoveredFileName = `${tempDir}/recovered`
+  await recoverRawContent(vhdFileName, recoveredFileName, originalSize)
+  expect(await fs.readFile(recoveredFileName)).toEqual(await fs.readFile(rawFileName))
+})
+
+test('the BAT MSB is not used for sign', async () => {
+  const randomBuffer = await pFromCallback(cb => randomBytes(SECTOR_SIZE, cb))
+  const emptyFileName = `${tempDir}/empty.vhd`
+  await execa('qemu-img', ['create', '-fvpc', emptyFileName, '1.8T'])
+  const handler = getHandler({ url: 'file://' })
+  const vhd = new VhdFile(handler, emptyFileName)
+  await vhd.readHeaderAndFooter()
+  await vhd.readBlockAllocationTable()
+  // we want the bit 31 to be on, to prove it's not been used for sign
+  const hugeWritePositionSectors = Math.pow(2, 31) + 200
+  await vhd.writeData(hugeWritePositionSectors, randomBuffer)
+  await checkFile(emptyFileName)
+  // here we are moving the first sector very far in the VHD to prove the BAT doesn't use signed int32
+  const hugePositionBytes = hugeWritePositionSectors * SECTOR_SIZE
+  await vhd._freeFirstBlockSpace(hugePositionBytes)
+
+  // we recover the data manually for speed reasons.
+  // fs.write() with offset is way faster than qemu-img when there is a 1.5To
+  // hole before the block of data
+  const recoveredFileName = `${tempDir}/recovered`
+  const recoveredFile = await fs.open(recoveredFileName, 'w')
+  try {
+    const vhd2 = new VhdFile(handler, emptyFileName)
+    await vhd2.readHeaderAndFooter()
+    await vhd2.readBlockAllocationTable()
+    for (let i = 0; i < vhd.header.maxTableEntries; i++) {
+      if (vhd.containsBlock(i)) {
+        const block = (await vhd2.readBlock(i)).data
+        await fs.write(recoveredFile, block, 0, block.length, vhd2.header.blockSize * i)
+      }
+    }
+  } finally {
+    fs.close(recoveredFile)
+  }
+  const recovered = await getStream.buffer(
+    await fs.createReadStream(recoveredFileName, {
+      start: hugePositionBytes,
+      end: hugePositionBytes + randomBuffer.length - 1,
+    })
+  )
+  expect(recovered).toEqual(randomBuffer)
+})
+
+test('writeData on empty file', async () => {
+  const mbOfRandom = 3
+  const rawFileName = `${tempDir}/randomfile`
+  const emptyFileName = `${tempDir}/empty.vhd`
+  await createRandomFile(rawFileName, mbOfRandom)
+  await execa('qemu-img', ['create', '-fvpc', emptyFileName, mbOfRandom + 'M'])
+  const randomData = await fs.readFile(rawFileName)
+  const handler = getHandler({ url: 'file://' })
+  const originalSize = await handler.getSize(rawFileName)
+  const newVhd = new VhdFile(handler, emptyFileName)
+  await newVhd.readHeaderAndFooter()
+  await newVhd.readBlockAllocationTable()
+  await newVhd.writeData(0, randomData)
+  const recoveredFileName = `${tempDir}/recovered`
+  await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
+  expect(await fs.readFile(recoveredFileName)).toEqual(randomData)
+})
+
+test('writeData in 2 non-overlaping operations', async () => {
+  const mbOfRandom = 3
+  const rawFileName = `${tempDir}/randomfile`
+  const emptyFileName = `${tempDir}/empty.vhd`
+  const recoveredFileName = `${tempDir}/recovered`
+  await createRandomFile(rawFileName, mbOfRandom)
+  await execa('qemu-img', ['create', '-fvpc', emptyFileName, mbOfRandom + 'M'])
+  const randomData = await fs.readFile(rawFileName)
+  const handler = getHandler({ url: 'file://' })
+  const originalSize = await handler.getSize(rawFileName)
+  const newVhd = new VhdFile(handler, emptyFileName)
+  await newVhd.readHeaderAndFooter()
+  await newVhd.readBlockAllocationTable()
+  const splitPointSectors = 2
+  await newVhd.writeData(0, randomData.slice(0, splitPointSectors * 512))
+  await newVhd.writeData(splitPointSectors, randomData.slice(splitPointSectors * 512))
+  await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
+  expect(await fs.readFile(recoveredFileName)).toEqual(randomData)
+})
+
+test('writeData in 2 overlaping operations', async () => {
+  const mbOfRandom = 3
+  const rawFileName = `${tempDir}/randomfile`
+  const emptyFileName = `${tempDir}/empty.vhd`
+  const recoveredFileName = `${tempDir}/recovered`
+  await createRandomFile(rawFileName, mbOfRandom)
+  await execa('qemu-img', ['create', '-fvpc', emptyFileName, mbOfRandom + 'M'])
+  const randomData = await fs.readFile(rawFileName)
+  const handler = getHandler({ url: 'file://' })
+  const originalSize = await handler.getSize(rawFileName)
+  const newVhd = new VhdFile(handler, emptyFileName)
+  await newVhd.readHeaderAndFooter()
+  await newVhd.readBlockAllocationTable()
+  const endFirstWrite = 3
+  const startSecondWrite = 2
+  await newVhd.writeData(0, randomData.slice(0, endFirstWrite * 512))
+  await newVhd.writeData(startSecondWrite, randomData.slice(startSecondWrite * 512))
+  await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
+  expect(await fs.readFile(recoveredFileName)).toEqual(randomData)
+})
+
+test('BAT can be extended and blocks moved', async () => {
+  const initalSize = 4
+  const rawFileName = `${tempDir}/randomfile`
+  const recoveredFileName = `${tempDir}/recovered`
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  await createRandomFile(rawFileName, initalSize)
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  const handler = getHandler({ url: 'file://' })
+  const originalSize = await handler.getSize(rawFileName)
+  const newVhd = new VhdFile(handler, vhdFileName)
+  await newVhd.readHeaderAndFooter()
+  await newVhd.readBlockAllocationTable()
+  await newVhd.ensureBatSize(2000)
+  await newVhd.writeBlockAllocationTable()
+  await recoverRawContent(vhdFileName, recoveredFileName, originalSize)
+  expect(await fs.readFile(recoveredFileName)).toEqual(await fs.readFile(rawFileName))
+})

packages/vhd-lib/src/Vhd/VhdFile.js

@@ -1,22 +1,20 @@
-import assert from 'assert'
-import { createLogger } from '@xen-orchestra/log'
-
-import checkFooter from './_checkFooter'
-import checkHeader from './_checkHeader'
-import getFirstAndLastBlocks from './_getFirstAndLastBlocks'
-import { fuFooter, fuHeader, checksumStruct, unpackField } from './_structs'
-import { set as mapSetBit, test as mapTestBit } from './_bitmap'
 import {
   BLOCK_UNUSED,
   FOOTER_SIZE,
   HEADER_SIZE,
-  PARENT_LOCATOR_ENTRIES,
   PLATFORM_NONE,
-  PLATFORM_W2KU,
   SECTOR_SIZE,
-} from './_constants'
+  PARENT_LOCATOR_ENTRIES,
+} from '../_constants'
+import { computeBatSize, sectorsToBytes, buildHeader, buildFooter, BUF_BLOCK_UNUSED } from './_utils'
+import { createLogger } from '@xen-orchestra/log'
+import { fuFooter, fuHeader, checksumStruct } from '../_structs'
+import { set as mapSetBit, test as mapTestBit } from '../_bitmap'
+import { VhdAbstract } from './VhdAbstract'
+import assert from 'assert'
+import getFirstAndLastBlocks from '../_getFirstAndLastBlocks'
 
-const { debug } = createLogger('vhd-lib:Vhd')
+const { debug } = createLogger('vhd-lib:VhdFile')
 
 // ===================================================================
 //
@@ -28,22 +26,6 @@ const { debug } = createLogger('vhd-lib:Vhd')
 //
 // ===================================================================
 
-const computeBatSize = entries => sectorsToBytes(sectorsRoundUpNoZero(entries * 4))
-
-// Sectors conversions.
-const sectorsRoundUpNoZero = bytes => Math.ceil(bytes / SECTOR_SIZE) || 1
-const sectorsToBytes = sectors => sectors * SECTOR_SIZE
-
-const assertChecksum = (name, buf, struct) => {
-  const actual = unpackField(struct.fields.checksum, buf)
-  const expected = checksumStruct(buf, struct)
-  assert.strictEqual(actual, expected, `invalid ${name} checksum ${actual}, expected ${expected}`)
-}
-
-// unused block as buffer containing a uint32BE
-const BUF_BLOCK_UNUSED = Buffer.allocUnsafe(4)
-BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0)
-
 // ===================================================================
 
 // Format:
@@ -68,12 +50,60 @@ BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0)
 // - parentLocatorSize(i) = header.parentLocatorEntry[i].platformDataSpace * sectorSize
 // - sectorSize = 512
 
-export default class Vhd {
+export class VhdFile extends VhdAbstract {
+  #uncheckedBlockTable
+
+  get #blockTable() {
+    assert.notStrictEqual(this.#uncheckedBlockTable, undefined, 'Block table must be initialized before access')
+    return this.#uncheckedBlockTable
+  }
+
+  set #blockTable(blockTable) {
+    this.#uncheckedBlockTable = blockTable
+  }
+
   get batSize() {
     return computeBatSize(this.header.maxTableEntries)
   }
 
+  set header(header) {
+    super.header = header
+    const size = this.batSize
+    this.#blockTable = Buffer.alloc(size)
+    for (let i = 0; i < this.header.maxTableEntries; i++) {
+      this.#blockTable.writeUInt32BE(BLOCK_UNUSED, i * 4)
+    }
+  }
+  get header() {
+    return super.header
+  }
+
+  static async open(handler, path) {
+    const fd = await handler.openFile(path, 'r+')
+    const vhd = new VhdFile(handler, fd)
+    // openning a file for reading does not trigger EISDIR as long as we don't really read from it :
+    // https://man7.org/linux/man-pages/man2/open.2.html
+    // EISDIR pathname refers to a directory and the access requested
+    // involved writing (that is, O_WRONLY or O_RDWR is set).
+    // reading the header ensure we have a well formed file immediatly
+    await vhd.readHeaderAndFooter()
+    return {
+      dispose: () => handler.closeFile(fd),
+      value: vhd,
+    }
+  }
+
+  static async create(handler, path) {
+    const fd = await handler.openFile(path, 'wx')
+    const vhd = new VhdFile(handler, fd)
+    return {
+      dispose: () => handler.closeFile(fd),
+      value: vhd,
+    }
+  }
+
   constructor(handler, path) {
+    super()
     this._handler = handler
     this._path = path
   }
@@ -87,11 +117,6 @@ export default class Vhd {
     assert.strictEqual(bytesRead, n)
     return buffer
   }
-
-  containsBlock(id) {
-    return this._getBatEntry(id) !== BLOCK_UNUSED
-  }
-
   // Returns the first address after metadata. (In bytes)
   _getEndOfHeaders() {
     const { header } = this
@@ -114,17 +139,24 @@ export default class Vhd {
     return end
   }
 
+  // return the first sector (bitmap) of a block
+  _getBatEntry(blockId) {
+    const i = blockId * 4
+    const blockTable = this.#blockTable
+    return i < blockTable.length ? blockTable.readUInt32BE(i) : BLOCK_UNUSED
+  }
+
   // Returns the first sector after data.
   _getEndOfData() {
     let end = Math.ceil(this._getEndOfHeaders() / SECTOR_SIZE)
 
-    const fullBlockSize = this.sectorsOfBitmap + this.sectorsPerBlock
+    const sectorsOfFullBlock = this.sectorsOfBitmap + this.sectorsPerBlock
     const { maxTableEntries } = this.header
     for (let i = 0; i < maxTableEntries; i++) {
       const blockAddr = this._getBatEntry(i)
 
       if (blockAddr !== BLOCK_UNUSED) {
-        end = Math.max(end, blockAddr + fullBlockSize)
+        end = Math.max(end, blockAddr + sectorsOfFullBlock)
       }
     }
 
@@ -133,7 +165,11 @@ export default class Vhd {
     return sectorsToBytes(end)
   }
 
-  // TODO: extract the checks into reusable functions:
+  containsBlock(id) {
+    return this._getBatEntry(id) !== BLOCK_UNUSED
+  }
+
+  // TODO:
   // - better human reporting
   // - auto repair if possible
   async readHeaderAndFooter(checkSecondFooter = true) {
@@ -141,50 +177,25 @@ export default class Vhd {
     const bufFooter = buf.slice(0, FOOTER_SIZE)
     const bufHeader = buf.slice(FOOTER_SIZE)
 
-    assertChecksum('footer', bufFooter, fuFooter)
-    assertChecksum('header', bufHeader, fuHeader)
+    const footer = buildFooter(bufFooter)
+    const header = buildHeader(bufHeader, footer)
 
     if (checkSecondFooter) {
       const size = await this._handler.getSize(this._path)
       assert(bufFooter.equals(await this._read(size - FOOTER_SIZE, FOOTER_SIZE)), 'footer1 !== footer2')
     }
 
-    const footer = (this.footer = fuFooter.unpack(bufFooter))
-    checkFooter(footer)
-
-    const header = (this.header = fuHeader.unpack(bufHeader))
-    checkHeader(header, footer)
-
-    // Compute the number of sectors in one block.
-    // Default: One block contains 4096 sectors of 512 bytes.
-    const sectorsPerBlock = (this.sectorsPerBlock = header.blockSize / SECTOR_SIZE)
-
-    // Compute bitmap size in sectors.
-    // Default: 1.
-    const sectorsOfBitmap = (this.sectorsOfBitmap = sectorsRoundUpNoZero(sectorsPerBlock >> 3))
-
-    // Full block size => data block size + bitmap size.
-    this.fullBlockSize = sectorsToBytes(sectorsPerBlock + sectorsOfBitmap)
-
-    // In bytes.
-    // Default: 512.
-    this.bitmapSize = sectorsToBytes(sectorsOfBitmap)
+    this.footer = footer
+    this.header = header
   }
 
   // Returns a buffer that contains the block allocation table of a vhd file.
   async readBlockAllocationTable() {
     const { header } = this
-    this.blockTable = await this._read(header.tableOffset, header.maxTableEntries * 4)
+    this.#blockTable = await this._read(header.tableOffset, header.maxTableEntries * 4)
   }
 
-  // return the first sector (bitmap) of a block
-  _getBatEntry(blockId) {
-    const i = blockId * 4
-    const { blockTable } = this
-    return i < blockTable.length ? blockTable.readUInt32BE(i) : BLOCK_UNUSED
-  }
-
-  _readBlock(blockId, onlyBitmap = false) {
+  readBlock(blockId, onlyBitmap = false) {
     const blockAddr = this._getBatEntry(blockId)
     if (blockAddr === BLOCK_UNUSED) {
       throw new Error(`no such block ${blockId}`)
@@ -214,7 +225,7 @@ export default class Vhd {
   }
 
   async _freeFirstBlockSpace(spaceNeededBytes) {
-    const firstAndLastBlocks = getFirstAndLastBlocks(this.blockTable)
+    const firstAndLastBlocks = getFirstAndLastBlocks(this.#blockTable)
     if (firstAndLastBlocks === undefined) {
       return
     }
@@ -249,8 +260,8 @@ export default class Vhd {
     const newBatSize = computeBatSize(entries)
     await this._freeFirstBlockSpace(newBatSize - this.batSize)
     const maxTableEntries = (header.maxTableEntries = entries)
-    const prevBat = this.blockTable
-    const bat = (this.blockTable = Buffer.allocUnsafe(newBatSize))
+    const prevBat = this.#blockTable
+    const bat = (this.#blockTable = Buffer.allocUnsafe(newBatSize))
     prevBat.copy(bat)
     bat.fill(BUF_BLOCK_UNUSED, prevMaxTableEntries * 4)
     debug(`ensureBatSize: extend BAT ${prevMaxTableEntries} -> ${maxTableEntries}`)
@@ -264,7 +275,7 @@ export default class Vhd {
   // set the first sector (bitmap) of a block
   _setBatEntry(block, blockSector) {
     const i = block * 4
-    const { blockTable } = this
+    const blockTable = this.#blockTable
 
     blockTable.writeUInt32BE(blockSector, i)
 
@@ -298,7 +309,7 @@ export default class Vhd {
     await this._write(bitmap, sectorsToBytes(blockAddr))
   }
 
-  async _writeEntireBlock(block) {
+  async writeEntireBlock(block) {
     let blockAddr = this._getBatEntry(block.id)
 
     if (blockAddr === BLOCK_UNUSED) {
@@ -314,7 +325,7 @@ export default class Vhd {
       blockAddr = await this._createBlock(block.id)
       parentBitmap = Buffer.alloc(this.bitmapSize, 0)
     } else if (parentBitmap === undefined) {
-      parentBitmap = (await this._readBlock(block.id, true)).bitmap
+      parentBitmap = (await this.readBlock(block.id, true)).bitmap
     }
 
     const offset = blockAddr + this.sectorsOfBitmap + beginSectorId
@@ -333,7 +344,7 @@ export default class Vhd {
   }
 
   async coalesceBlock(child, blockId) {
-    const block = await child._readBlock(blockId)
+    const block = await child.readBlock(blockId)
     const { bitmap, data } = block
 
     debug(`coalesceBlock block=${blockId}`)
@@ -358,10 +369,10 @@ export default class Vhd {
 
       const isFullBlock = i === 0 && endSector === sectorsPerBlock
       if (isFullBlock) {
-        await this._writeEntireBlock(block)
+        await this.writeEntireBlock(block)
       } else {
         if (parentBitmap === null) {
-          parentBitmap = (await this._readBlock(blockId, true)).bitmap
+          parentBitmap = (await this.readBlock(blockId, true)).bitmap
         }
         await this._writeBlockSectors(block, i, endSector, parentBitmap)
       }
@@ -399,6 +410,13 @@ export default class Vhd {
     return this._write(rawHeader, offset)
   }
 
+  writeBlockAllocationTable() {
+    const header = this.header
+    const blockTable = this.#blockTable
+    debug(`Write BlockAllocationTable at: ${header.tableOffset} ). (data=${blockTable.toString('hex')})`)
+    return this._write(blockTable, header.tableOffset)
+  }
+
   async writeData(offsetSectors, buffer) {
     const bufferSizeSectors = Math.ceil(buffer.length / SECTOR_SIZE)
     const startBlock = Math.floor(offsetSectors / this.sectorsPerBlock)
@@ -436,26 +454,47 @@ export default class Vhd {
       const deltaSectors = neededSectors - currentSpace
       await this._freeFirstBlockSpace(sectorsToBytes(deltaSectors))
       this.header.tableOffset += sectorsToBytes(deltaSectors)
-      await this._write(this.blockTable, this.header.tableOffset)
+      await this._write(this.#blockTable, this.header.tableOffset)
     }
     return firstLocatorOffset
   }
 
-  async setUniqueParentLocator(fileNameString) {
+  async _readParentLocatorData(parentLocatorId) {
+    const { platformDataOffset, platformDataLength } = this.header.parentLocatorEntry[parentLocatorId]
+    if (platformDataLength > 0) {
+      return (await this._read(platformDataOffset, platformDataLength)).buffer
+    }
+    return Buffer.alloc(0)
+  }
+
+  async _writeParentLocatorData(parentLocatorId, data) {
+    let position
     const { header } = this
-    header.parentLocatorEntry[0].platformCode = PLATFORM_W2KU
-    const encodedFilename = Buffer.from(fileNameString, 'utf16le')
-    const dataSpaceSectors = Math.ceil(encodedFilename.length / SECTOR_SIZE)
-    const position = await this._ensureSpaceForParentLocators(dataSpaceSectors)
-    await this._write(encodedFilename, position)
-    header.parentLocatorEntry[0].platformDataSpace = dataSpaceSectors * SECTOR_SIZE
-    header.parentLocatorEntry[0].platformDataLength = encodedFilename.length
-    header.parentLocatorEntry[0].platformDataOffset = position
-    for (let i = 1; i < 8; i++) {
-      header.parentLocatorEntry[i].platformCode = PLATFORM_NONE
-      header.parentLocatorEntry[i].platformDataSpace = 0
-      header.parentLocatorEntry[i].platformDataLength = 0
-      header.parentLocatorEntry[i].platformDataOffset = 0
+    if (data.length === 0) {
+      // reset offset if data is empty
+      header.parentLocatorEntry[parentLocatorId].platformDataOffset = 0
+    } else {
+      if (data.length <= header.parentLocatorEntry[parentLocatorId].platformDataSpace) {
+        // new parent locator length is smaller than available space : keep it in place
+        position = header.parentLocatorEntry[parentLocatorId].platformDataOffset
+      } else {
+        const firstAndLastBlocks = getFirstAndLastBlocks(this.#blockTable)
+        if (firstAndLastBlocks === undefined) {
+          // no block in data : put the parent locatorn entry at the end
+          position = this._getEndOfData()
+        } else {
+          // need more size
+
+          // since there can be multiple parent locator entry, we can't extend the entry in place
+          // move the first(s) block(s) at the end of the data
+          // move the parent locator to the  precedent position of the first block
+          const { firstSector } = firstAndLastBlocks
+          await this._freeFirstBlockSpace(header.parentLocatorEntry[parentLocatorId].platformDataSpace)
+          position = sectorsToBytes(firstSector)
+        }
+      }
+      await this._write(data, position)
+      header.parentLocatorEntry[parentLocatorId].platformDataOffset = position
     }
   }
 }

packages/vhd-lib/src/Vhd/_utils.js

@@ -0,0 +1,52 @@
+import assert from 'assert'
+import { BLOCK_UNUSED, SECTOR_SIZE } from '../_constants'
+import { fuFooter, fuHeader, checksumStruct, unpackField } from '../_structs'
+import checkFooter from '../checkFooter'
+import checkHeader from '../_checkHeader'
+
+export const computeBatSize = entries => sectorsToBytes(sectorsRoundUpNoZero(entries * 4))
+
+// Sectors conversions.
+export const sectorsRoundUpNoZero = bytes => Math.ceil(bytes / SECTOR_SIZE) || 1
+export const sectorsToBytes = sectors => sectors * SECTOR_SIZE
+
+export const assertChecksum = (name, buf, struct) => {
+  const actual = unpackField(struct.fields.checksum, buf)
+  const expected = checksumStruct(buf, struct)
+  assert.strictEqual(actual, expected, `invalid ${name} checksum ${actual}, expected ${expected}`)
+}
+
+// unused block as buffer containing a uint32BE
+export const BUF_BLOCK_UNUSED = Buffer.allocUnsafe(4)
+BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0)
+
+/**
+ * Check and parse the header buffer to build an header object
+ *
+ * @param {Buffer} bufHeader
+ * @param {Object} footer
+ * @returns {Object} the parsed header
+ */
+export const buildHeader = (bufHeader, footer) => {
+  assertChecksum('header', bufHeader, fuHeader)
+
+  const header = fuHeader.unpack(bufHeader)
+  checkHeader(header, footer)
+  return header
+}
+
+/**
+ * Check and parse the footer buffer to build a footer object
+ *
+ * @param {Buffer} bufHeader
+ * @param {Object} footer
+ * @returns {Object} the parsed footer
+ */
+
+export const buildFooter = bufFooter => {
+  assertChecksum('footer', bufFooter, fuFooter)
+
+  const footer = fuFooter.unpack(bufFooter)
+  checkFooter(footer)
+  return footer
+}

packages/vhd-lib/src/_createFooterHeader.integ.spec.js

@@ -0,0 +1,10 @@
+/* eslint-env jest */
+import { createFooter } from './_createFooterHeader'
+
+test('createFooter() does not crash', () => {
+  createFooter(104448, Math.floor(Date.now() / 1000), {
+    cylinders: 3,
+    heads: 4,
+    sectorsPerTrack: 17,
+  })
+})

packages/vhd-lib/src/_getFirstAndLastBlocks.js

@@ -6,7 +6,9 @@ import { BLOCK_UNUSED } from './_constants'
 // in the file
 export default bat => {
   const n = bat.length
-  assert.notStrictEqual(n, 0)
+  if (n === 0) {
+    return
+  }
   assert.strictEqual(n % 4, 0)
 
   let i = 0

packages/vhd-lib/src/_resolveAlias.integ.spec.js

@@ -0,0 +1,66 @@
+/* eslint-env jest */
+
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { getSyncedHandler } from '@xen-orchestra/fs'
+import { Disposable, pFromCallback } from 'promise-toolbox'
+
+import { isVhdAlias, resolveAlias } from './_resolveAlias'
+
+let tempDir
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+test('is vhd alias recognize only *.alias.vhd files', () => {
+  expect(isVhdAlias('filename.alias.vhd')).toEqual(true)
+  expect(isVhdAlias('alias.vhd')).toEqual(false)
+  expect(isVhdAlias('filename.vhd')).toEqual(false)
+  expect(isVhdAlias('filename.alias.vhd.other')).toEqual(false)
+})
+
+test('resolve return the path in argument for a non alias file ', async () => {
+  expect(await resolveAlias(null, 'filename.vhd')).toEqual('filename.vhd')
+})
+test('resolve get the path of the target file for an alias', async () => {
+  await Disposable.use(async function* () {
+    // same directory
+    const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
+    await handler.mkdir(`alias`)
+    const aliasPath = 'alias/alias.alias.vhd'
+    const testOneCombination = async ({ targetPath, targetContent }) => {
+      await handler.writeFile(aliasPath, targetPath, { flags: 'w' })
+      const resolved = await resolveAlias(handler, aliasPath)
+      expect(resolved).toEqual(targetContent)
+      await handler.unlink(aliasPath)
+    }
+    // the alias contain the relative path to the file. The resolved values is the full path from the root of the remote
+    const combinations = [
+      { targetPath: `../targets.vhd`, targetContent: `targets.vhd` },
+      { targetPath: `targets.vhd`, targetContent: `alias/targets.vhd` },
+      { targetPath: `sub/targets.vhd`, targetContent: `alias/sub/targets.vhd` },
+      { targetPath: `../sibling/targets.vhd`, targetContent: `sibling/targets.vhd` },
+    ]
+
+    for (const { targetPath, targetContent } of combinations) {
+      await testOneCombination({ targetPath, targetContent })
+    }
+  })
+})
+
+test('resolve throws an error an alias to an alias', async () => {
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
+    const alias = 'alias.alias.vhd'
+    const target = 'target.alias.vhd'
+    await handler.writeFile(alias, target)
+    expect(async () => await resolveAlias(handler, alias)).rejects.toThrow(Error)
+  })
+})

packages/vhd-lib/src/_resolveAlias.js

@@ -0,0 +1,18 @@
+import resolveRelativeFromFile from './_resolveRelativeFromFile'
+
+export function isVhdAlias(filename) {
+  return filename.endsWith('.alias.vhd')
+}
+
+export async function resolveAlias(handler, filename) {
+  if (!isVhdAlias(filename)) {
+    return filename
+  }
+  const aliasContent = (await handler.readFile(filename)).toString().trim()
+  // also handle circular references and unreasonnably long chains
+  if (isVhdAlias(aliasContent)) {
+    throw new Error(`Chaining alias is forbidden ${filename} to ${aliasContent}`)
+  }
+  // the target is relative to the alias location
+  return resolveRelativeFromFile(filename, aliasContent)
+}

packages/vhd-lib/src/chain.js

@@ -1,11 +1,11 @@
 import { dirname, relative } from 'path'
 
-import Vhd from './vhd'
+import { VhdFile } from './'
 import { DISK_TYPE_DIFFERENCING } from './_constants'
 
 export default async function chain(parentHandler, parentPath, childHandler, childPath, force = false) {
-  const parentVhd = new Vhd(parentHandler, parentPath)
-  const childVhd = new Vhd(childHandler, childPath)
+  const parentVhd = new VhdFile(parentHandler, parentPath)
+  const childVhd = new VhdFile(childHandler, childPath)
 
   await childVhd.readHeaderAndFooter()
   const { header, footer } = childVhd

packages/vhd-lib/src/checkChain.js

@@ -1,10 +1,10 @@
-import Vhd from './vhd'
+import { VhdFile } from '.'
 import resolveRelativeFromFile from './_resolveRelativeFromFile'
 import { DISK_TYPE_DYNAMIC } from './_constants'
 
 export default async function checkChain(handler, path) {
   while (true) {
-    const vhd = new Vhd(handler, path)
+    const vhd = new VhdFile(handler, path)
     await vhd.readHeaderAndFooter()
 
     if (vhd.footer.diskType === DISK_TYPE_DYNAMIC) {

packages/vhd-lib/src/createContentStream.js

@@ -1,11 +1,11 @@
 import asyncIteratorToStream from 'async-iterator-to-stream'
 
-import Vhd from './vhd'
+import { VhdFile } from '.'
 
 export default asyncIteratorToStream(async function* (handler, path) {
   const fd = await handler.openFile(path, 'r')
   try {
-    const vhd = new Vhd(handler, fd)
+    const vhd = new VhdFile(handler, fd)
     await vhd.readHeaderAndFooter()
     await vhd.readBlockAllocationTable()
     const {
@@ -17,10 +17,10 @@ export default asyncIteratorToStream(async function* (handler, path) {
 
     const emptyBlock = Buffer.alloc(blockSize)
     for (let i = 0; i < nFullBlocks; ++i) {
-      yield vhd.containsBlock(i) ? (await vhd._readBlock(i)).data : emptyBlock
+      yield vhd.containsBlock(i) ? (await vhd.readBlock(i)).data : emptyBlock
     }
     if (nLeftoverBytes !== 0) {
-      yield (vhd.containsBlock(nFullBlocks) ? (await vhd._readBlock(nFullBlocks)).data : emptyBlock).slice(
+      yield (vhd.containsBlock(nFullBlocks) ? (await vhd.readBlock(nFullBlocks)).data : emptyBlock).slice(
         0,
         nLeftoverBytes
       )

packages/vhd-lib/src/createReadableRawStream.integ.spec.js

@@ -8,7 +8,7 @@ import { pipeline } from 'readable-stream'
 
 import { createReadableRawStream, createReadableSparseStream } from './'
 
-import { createFooter } from './_createFooterHeader'
+import { checkFile, convertFromVhdToRaw } from './tests/utils'
 
 let tempDir = null
 
@@ -20,14 +20,6 @@ afterEach(async () => {
   await pFromCallback(cb => rimraf(tempDir, cb))
 })
 
-test('createFooter() does not crash', () => {
-  createFooter(104448, Math.floor(Date.now() / 1000), {
-    cylinders: 3,
-    heads: 4,
-    sectorsPerTrack: 17,
-  })
-})
-
 test('ReadableRawVHDStream does not crash', async () => {
   const data = [
     {
@@ -102,6 +94,7 @@ test('ReadableSparseVHDStream can handle a sparse file', async () => {
     },
   ]
   const fileSize = blockSize * 110
+
   const stream = await createReadableSparseStream(
     fileSize,
     blockSize,
@@ -110,9 +103,10 @@ test('ReadableSparseVHDStream can handle a sparse file', async () => {
   )
   expect(stream.length).toEqual(4197888)
   const pipe = stream.pipe(createWriteStream(`${tempDir}/output.vhd`))
+
   await fromEvent(pipe, 'finish')
-  await execa('vhd-util', ['check', '-t', '-i', '-n', `${tempDir}/output.vhd`])
-  await execa('qemu-img', ['convert', '-f', 'vpc', '-O', 'raw', `${tempDir}/output.vhd`, `${tempDir}/out1.raw`])
+  await checkFile(`${tempDir}/output.vhd`)
+  await convertFromVhdToRaw(`${tempDir}/output.vhd`, `${tempDir}/out1.raw`)
   const out1 = await readFile(`${tempDir}/out1.raw`)
   const expected = Buffer.alloc(fileSize)
   blocks.forEach(b => {

packages/vhd-lib/src/createSyntheticStream.integ.spec.js

@@ -0,0 +1,42 @@
+/* eslint-env jest */
+
+import execa from 'execa'
+import fs from 'fs-extra'
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { getHandler } from '@xen-orchestra/fs'
+import { pFromCallback } from 'promise-toolbox'
+import { pipeline } from 'readable-stream'
+
+import { checkFile, createRandomFile, convertFromRawToVhd } from './tests/utils'
+import { createSyntheticStream } from '.'
+
+let tempDir = null
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+test.only('createSyntheticStream passes vhd-util check', async () => {
+  const initalSize = 4
+  const rawFileName = `${tempDir}/randomfile`
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  const recoveredVhdFileName = `${tempDir}/recovered.vhd`
+  await createRandomFile(rawFileName, initalSize)
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  await checkFile(vhdFileName)
+  const handler = getHandler({ url: 'file://' })
+  const stream = await createSyntheticStream(handler, vhdFileName)
+  const expectedVhdSize = (await fs.stat(vhdFileName)).size
+  expect(stream.length).toEqual((await fs.stat(vhdFileName)).size)
+  await pFromCallback(cb => pipeline(stream, fs.createWriteStream(recoveredVhdFileName), cb))
+  await checkFile(recoveredVhdFileName)
+  const stats = await fs.stat(recoveredVhdFileName)
+  expect(stats.size).toEqual(expectedVhdSize)
+  await execa('qemu-img', ['compare', recoveredVhdFileName, rawFileName])
+})

packages/vhd-lib/src/createSyntheticStream.js

@@ -3,7 +3,7 @@ import { createLogger } from '@xen-orchestra/log'
 
 import resolveRelativeFromFile from './_resolveRelativeFromFile'
 
-import Vhd from './vhd'
+import { VhdFile } from '.'
 import { BLOCK_UNUSED, DISK_TYPE_DYNAMIC, FOOTER_SIZE, HEADER_SIZE, SECTOR_SIZE } from './_constants'
 import { fuFooter, fuHeader, checksumStruct } from './_structs'
 import { test as mapTestBit } from './_bitmap'
@@ -27,7 +27,7 @@ export default async function createSyntheticStream(handler, paths) {
     const open = async path => {
       const fd = await handler.openFile(path, 'r')
       fds.push(fd)
-      const vhd = new Vhd(handler, fd)
+      const vhd = new VhdFile(handler, fd)
       vhds.push(vhd)
       await vhd.readHeaderAndFooter()
       await vhd.readBlockAllocationTable()
@@ -126,7 +126,7 @@ export default async function createSyntheticStream(handler, paths) {
             }
             let block = blocksByVhd.get(vhd)
             if (block === undefined) {
-              block = yield vhd._readBlock(iBlock)
+              block = yield vhd.readBlock(iBlock)
               blocksByVhd.set(vhd, block)
             }
             const { bitmap, data } = block

packages/vhd-lib/src/createVhdStreamWithLength.integ.spec.js

@@ -1,6 +1,5 @@
 /* eslint-env jest */
 
-import asyncIteratorToStream from 'async-iterator-to-stream'
 import execa from 'execa'
 import fs from 'fs-extra'
 import rimraf from 'rimraf'
@@ -12,6 +11,7 @@ import { pipeline } from 'readable-stream'
 
 import { createVhdStreamWithLength } from '.'
 import { FOOTER_SIZE } from './_constants'
+import { createRandomFile, convertFromRawToVhd, convertFromVhdToRaw } from './tests/utils'
 
 let tempDir = null
 
@@ -23,27 +23,6 @@ afterEach(async () => {
   await pFromCallback(cb => rimraf(tempDir, cb))
 })
 
-const RAW = 'raw'
-const VHD = 'vpc'
-const convert = (inputFormat, inputFile, outputFormat, outputFile) =>
-  execa('qemu-img', ['convert', '-f', inputFormat, '-O', outputFormat, inputFile, outputFile])
-
-const createRandomStream = asyncIteratorToStream(function* (size) {
-  let requested = Math.min(size, yield)
-  while (size > 0) {
-    const buf = Buffer.allocUnsafe(requested)
-    for (let i = 0; i < requested; ++i) {
-      buf[i] = Math.floor(Math.random() * 256)
-    }
-    requested = Math.min((size -= requested), yield buf)
-  }
-})
-
-async function createRandomFile(name, size) {
-  const input = await createRandomStream(size)
-  await pFromCallback(cb => pipeline(input, fs.createWriteStream(name), cb))
-}
-
 const forOwn = (object, cb) => Object.keys(object).forEach(key => cb(object[key], key, object))
 
 describe('createVhdStreamWithLength', () => {
@@ -58,10 +37,10 @@ describe('createVhdStreamWithLength', () => {
     (size, title) =>
       it(title, async () => {
         const inputRaw = `${tempDir}/input.raw`
-        await createRandomFile(inputRaw, size)
+        await createRandomFile(inputRaw, size / 1024 / 1024)
 
         const inputVhd = `${tempDir}/input.vhd`
-        await convert(RAW, inputRaw, VHD, inputVhd)
+        await convertFromRawToVhd(inputRaw, inputVhd)
 
         const result = await createVhdStreamWithLength(await createReadStream(inputVhd))
         const { length } = result
@@ -75,18 +54,18 @@ describe('createVhdStreamWithLength', () => {
 
         // ensure the generated VHD is correct and contains the same data
         const outputRaw = `${tempDir}/output.raw`
-        await convert(VHD, outputVhd, RAW, outputRaw)
+        await convertFromVhdToRaw(outputVhd, outputRaw)
         await execa('cmp', [inputRaw, outputRaw])
       })
   )
 
   it('can skip blank after the last block and before the footer', async () => {
-    const initialSize = 4 * 1024
+    const initialSize = 4
     const rawFileName = `${tempDir}/randomfile`
     const vhdName = `${tempDir}/randomfile.vhd`
     const outputVhdName = `${tempDir}/output.vhd`
     await createRandomFile(rawFileName, initialSize)
-    await convert(RAW, rawFileName, VHD, vhdName)
+    await convertFromRawToVhd(rawFileName, vhdName)
     const { size: vhdSize } = await fs.stat(vhdName)
     // read file footer
     const footer = await getStream.buffer(createReadStream(vhdName, { start: vhdSize - FOOTER_SIZE }))

packages/vhd-lib/src/createVhdStreamWithLength.js

@@ -2,7 +2,7 @@ import assert from 'assert'
 import { pipeline, Transform } from 'readable-stream'
 import { readChunk } from '@vates/read-chunk'
 
-import checkFooter from './_checkFooter'
+import checkFooter from './checkFooter'
 import checkHeader from './_checkHeader'
 import noop from './_noop'
 import getFirstAndLastBlocks from './_getFirstAndLastBlocks'

packages/vhd-lib/src/index.js

@@ -1,10 +1,14 @@
-export { default } from './vhd'
 export { default as chainVhd } from './chain'
+export { default as checkFooter } from './checkFooter'
 export { default as checkVhdChain } from './checkChain'
 export { default as createContentStream } from './createContentStream'
 export { default as createReadableRawStream } from './createReadableRawStream'
 export { default as createReadableSparseStream } from './createReadableSparseStream'
 export { default as createSyntheticStream } from './createSyntheticStream'
-export { default as mergeVhd } from './merge'
 export { default as createVhdStreamWithLength } from './createVhdStreamWithLength'
+export { default as mergeVhd } from './merge'
 export { default as peekFooterFromVhdStream } from './peekFooterFromVhdStream'
+export { openVhd } from './openVhd'
+export { VhdDirectory } from './Vhd/VhdDirectory'
+export { VhdFile } from './Vhd/VhdFile'
+export * as Constants from './_constants'

packages/vhd-lib/src/merge.integ.spec.js

@@ -1,19 +1,16 @@
 /* eslint-env jest */
 
-import asyncIteratorToStream from 'async-iterator-to-stream'
 import execa from 'execa'
 import fs from 'fs-extra'
-import getStream from 'get-stream'
 import rimraf from 'rimraf'
 import tmp from 'tmp'
 import { getHandler } from '@xen-orchestra/fs'
 import { pFromCallback } from 'promise-toolbox'
-import { pipeline } from 'readable-stream'
-import { randomBytes } from 'crypto'
 
-import Vhd, { chainVhd, createSyntheticStream, mergeVhd as vhdMerge } from './index'
+import { VhdFile, chainVhd, mergeVhd as vhdMerge } from './index'
 
 import { SECTOR_SIZE } from './_constants'
+import { checkFile, createRandomFile, convertFromRawToVhd, recoverRawContent } from './tests/utils'
 
 let tempDir = null
 
@@ -27,256 +24,55 @@ afterEach(async () => {
   await pFromCallback(cb => rimraf(tempDir, cb))
 })
 
-async function createRandomFile(name, sizeMB) {
-  const createRandomStream = asyncIteratorToStream(function* (size) {
-    while (size-- > 0) {
-      yield Buffer.from([Math.floor(Math.random() * 256)])
-    }
-  })
-  const input = createRandomStream(sizeMB * 1024 * 1024)
-  await pFromCallback(cb => pipeline(input, fs.createWriteStream(name), cb))
-}
-
-async function checkFile(vhdName) {
-  await execa('vhd-util', ['check', '-p', '-b', '-t', '-n', vhdName])
-}
-
-async function recoverRawContent(vhdName, rawName, originalSize) {
-  await checkFile(vhdName)
-  await execa('qemu-img', ['convert', '-fvpc', '-Oraw', vhdName, rawName])
-  if (originalSize !== undefined) {
-    await execa('truncate', ['-s', originalSize, rawName])
-  }
-}
-
-async function convertFromRawToVhd(rawName, vhdName) {
-  await execa('qemu-img', ['convert', '-f', 'raw', '-Ovpc', rawName, vhdName])
-}
-
-test('blocks can be moved', async () => {
-  const initalSize = 4
-  const rawFileName = `${tempDir}/randomfile`
-  await createRandomFile(rawFileName, initalSize)
-  const vhdFileName = `${tempDir}/randomfile.vhd`
-  await convertFromRawToVhd(rawFileName, vhdFileName)
-  const handler = getHandler({ url: 'file://' })
-  const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new Vhd(handler, vhdFileName)
-  await newVhd.readHeaderAndFooter()
-  await newVhd.readBlockAllocationTable()
-  await newVhd._freeFirstBlockSpace(8000000)
-  const recoveredFileName = `${tempDir}/recovered`
-  await recoverRawContent(vhdFileName, recoveredFileName, originalSize)
-  expect(await fs.readFile(recoveredFileName)).toEqual(await fs.readFile(rawFileName))
-})
-
-test('the BAT MSB is not used for sign', async () => {
-  const randomBuffer = await pFromCallback(cb => randomBytes(SECTOR_SIZE, cb))
-  const emptyFileName = `${tempDir}/empty.vhd`
-  await execa('qemu-img', ['create', '-fvpc', emptyFileName, '1.8T'])
-  const handler = getHandler({ url: 'file://' })
-  const vhd = new Vhd(handler, emptyFileName)
-  await vhd.readHeaderAndFooter()
-  await vhd.readBlockAllocationTable()
-  // we want the bit 31 to be on, to prove it's not been used for sign
-  const hugeWritePositionSectors = Math.pow(2, 31) + 200
-  await vhd.writeData(hugeWritePositionSectors, randomBuffer)
-  await checkFile(emptyFileName)
-  // here we are moving the first sector very far in the VHD to prove the BAT doesn't use signed int32
-  const hugePositionBytes = hugeWritePositionSectors * SECTOR_SIZE
-  await vhd._freeFirstBlockSpace(hugePositionBytes)
-
-  // we recover the data manually for speed reasons.
-  // fs.write() with offset is way faster than qemu-img when there is a 1.5To
-  // hole before the block of data
-  const recoveredFileName = `${tempDir}/recovered`
-  const recoveredFile = await fs.open(recoveredFileName, 'w')
-  try {
-    const vhd2 = new Vhd(handler, emptyFileName)
-    await vhd2.readHeaderAndFooter()
-    await vhd2.readBlockAllocationTable()
-    for (let i = 0; i < vhd.header.maxTableEntries; i++) {
-      const entry = vhd._getBatEntry(i)
-      if (entry !== 0xffffffff) {
-        const block = (await vhd2._readBlock(i)).data
-        await fs.write(recoveredFile, block, 0, block.length, vhd2.header.blockSize * i)
-      }
-    }
-  } finally {
-    fs.close(recoveredFile)
-  }
-  const recovered = await getStream.buffer(
-    await fs.createReadStream(recoveredFileName, {
-      start: hugePositionBytes,
-      end: hugePositionBytes + randomBuffer.length - 1,
-    })
-  )
-  expect(recovered).toEqual(randomBuffer)
-})
-
-test('writeData on empty file', async () => {
-  const mbOfRandom = 3
-  const rawFileName = `${tempDir}/randomfile`
-  const emptyFileName = `${tempDir}/empty.vhd`
-  await createRandomFile(rawFileName, mbOfRandom)
-  await execa('qemu-img', ['create', '-fvpc', emptyFileName, mbOfRandom + 'M'])
-  const randomData = await fs.readFile(rawFileName)
-  const handler = getHandler({ url: 'file://' })
-  const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new Vhd(handler, emptyFileName)
-  await newVhd.readHeaderAndFooter()
-  await newVhd.readBlockAllocationTable()
-  await newVhd.writeData(0, randomData)
-  const recoveredFileName = `${tempDir}/recovered`
-  await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
-  expect(await fs.readFile(recoveredFileName)).toEqual(randomData)
-})
-
-test('writeData in 2 non-overlaping operations', async () => {
-  const mbOfRandom = 3
-  const rawFileName = `${tempDir}/randomfile`
-  const emptyFileName = `${tempDir}/empty.vhd`
-  const recoveredFileName = `${tempDir}/recovered`
-  await createRandomFile(rawFileName, mbOfRandom)
-  await execa('qemu-img', ['create', '-fvpc', emptyFileName, mbOfRandom + 'M'])
-  const randomData = await fs.readFile(rawFileName)
-  const handler = getHandler({ url: 'file://' })
-  const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new Vhd(handler, emptyFileName)
-  await newVhd.readHeaderAndFooter()
-  await newVhd.readBlockAllocationTable()
-  const splitPointSectors = 2
-  await newVhd.writeData(0, randomData.slice(0, splitPointSectors * 512))
-  await newVhd.writeData(splitPointSectors, randomData.slice(splitPointSectors * 512))
-  await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
-  expect(await fs.readFile(recoveredFileName)).toEqual(randomData)
-})
-
-test('writeData in 2 overlaping operations', async () => {
-  const mbOfRandom = 3
-  const rawFileName = `${tempDir}/randomfile`
-  const emptyFileName = `${tempDir}/empty.vhd`
-  const recoveredFileName = `${tempDir}/recovered`
-  await createRandomFile(rawFileName, mbOfRandom)
-  await execa('qemu-img', ['create', '-fvpc', emptyFileName, mbOfRandom + 'M'])
-  const randomData = await fs.readFile(rawFileName)
-  const handler = getHandler({ url: 'file://' })
-  const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new Vhd(handler, emptyFileName)
-  await newVhd.readHeaderAndFooter()
-  await newVhd.readBlockAllocationTable()
-  const endFirstWrite = 3
-  const startSecondWrite = 2
-  await newVhd.writeData(0, randomData.slice(0, endFirstWrite * 512))
-  await newVhd.writeData(startSecondWrite, randomData.slice(startSecondWrite * 512))
-  await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
-  expect(await fs.readFile(recoveredFileName)).toEqual(randomData)
-})
-
-test('BAT can be extended and blocks moved', async () => {
-  const initalSize = 4
-  const rawFileName = `${tempDir}/randomfile`
-  const recoveredFileName = `${tempDir}/recovered`
-  const vhdFileName = `${tempDir}/randomfile.vhd`
-  await createRandomFile(rawFileName, initalSize)
-  await convertFromRawToVhd(rawFileName, vhdFileName)
-  const handler = getHandler({ url: 'file://' })
-  const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new Vhd(handler, vhdFileName)
-  await newVhd.readHeaderAndFooter()
-  await newVhd.readBlockAllocationTable()
-  await newVhd.ensureBatSize(2000)
-  await recoverRawContent(vhdFileName, recoveredFileName, originalSize)
-  expect(await fs.readFile(recoveredFileName)).toEqual(await fs.readFile(rawFileName))
-})
-
-test('coalesce works with empty parent files', async () => {
-  const mbOfRandom = 2
-  const rawFileName = `${tempDir}/randomfile`
-  const emptyFileName = `${tempDir}/empty.vhd`
-  const vhdFileName = `${tempDir}/randomfile.vhd`
-  const recoveredFileName = `${tempDir}/recovered`
-  await createRandomFile(rawFileName, mbOfRandom)
-  await convertFromRawToVhd(rawFileName, vhdFileName)
-  await execa('qemu-img', ['create', '-fvpc', emptyFileName, mbOfRandom + 1 + 'M'])
-  await checkFile(vhdFileName)
-  await checkFile(emptyFileName)
-  const handler = getHandler({ url: 'file://' })
-  const originalSize = await handler._getSize(rawFileName)
-  await chainVhd(handler, emptyFileName, handler, vhdFileName, true)
-  await checkFile(vhdFileName)
-  await checkFile(emptyFileName)
-  await vhdMerge(handler, emptyFileName, handler, vhdFileName)
-  await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
-  expect(await fs.readFile(recoveredFileName)).toEqual(await fs.readFile(rawFileName))
-})
-
 test('coalesce works in normal cases', async () => {
   const mbOfRandom = 5
-  const randomFileName = `${tempDir}/randomfile`
-  const random2FileName = `${tempDir}/randomfile2`
-  const smallRandomFileName = `${tempDir}/small_randomfile`
-  const parentFileName = `${tempDir}/parent.vhd`
-  const child1FileName = `${tempDir}/child1.vhd`
-  const child2FileName = `${tempDir}/child2.vhd`
-  const recoveredFileName = `${tempDir}/recovered`
-  await createRandomFile(randomFileName, mbOfRandom)
-  await createRandomFile(smallRandomFileName, Math.ceil(mbOfRandom / 2))
-  await execa('qemu-img', ['create', '-fvpc', parentFileName, mbOfRandom + 1 + 'M'])
-  await convertFromRawToVhd(randomFileName, child1FileName)
-  const handler = getHandler({ url: 'file://' })
-  await execa('vhd-util', ['snapshot', '-n', child2FileName, '-p', child1FileName])
-  const vhd = new Vhd(handler, child2FileName)
+  const randomFilePath = `${tempDir}/randomfile`
+  const random2FilePath = `${tempDir}/randomfile2`
+  const smallRandomFilePath = `${tempDir}/small_randomfile`
+  const parentFilePath = `${tempDir}/parent.vhd`
+  const child1FilePath = `${tempDir}/child1.vhd`
+  const child2FilePath = `${tempDir}/child2.vhd`
+  const recoveredFilePath = `${tempDir}/recovered`
+  await createRandomFile(randomFilePath, mbOfRandom)
+  await createRandomFile(smallRandomFilePath, Math.ceil(mbOfRandom / 2))
+  await execa('qemu-img', ['create', '-fvpc', parentFilePath, mbOfRandom + 1 + 'M'])
+  await checkFile(parentFilePath)
+  await convertFromRawToVhd(randomFilePath, child1FilePath)
+  const handler = getHandler({ url: `file://${tempDir}/` })
+  await execa('vhd-util', ['snapshot', '-n', child2FilePath, '-p', child1FilePath])
+  const vhd = new VhdFile(handler, 'child2.vhd')
   await vhd.readHeaderAndFooter()
   await vhd.readBlockAllocationTable()
   vhd.footer.creatorApplication = 'xoa'
   await vhd.writeFooter()
 
-  const originalSize = await handler._getSize(randomFileName)
-  await chainVhd(handler, parentFileName, handler, child1FileName, true)
-  await execa('vhd-util', ['check', '-t', '-n', child1FileName])
-  await chainVhd(handler, child1FileName, handler, child2FileName, true)
-  await execa('vhd-util', ['check', '-t', '-n', child2FileName])
-  const smallRandom = await fs.readFile(smallRandomFileName)
-  const newVhd = new Vhd(handler, child2FileName)
+  const originalSize = await handler._getSize('randomfile')
+  await checkFile(child1FilePath)
+  await chainVhd(handler, 'parent.vhd', handler, 'child1.vhd', true)
+  await checkFile(child1FilePath)
+  await chainVhd(handler, 'child1.vhd', handler, 'child2.vhd', true)
+  await checkFile(child2FilePath)
+  const smallRandom = await fs.readFile(smallRandomFilePath)
+  const newVhd = new VhdFile(handler, 'child2.vhd')
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   await newVhd.writeData(5, smallRandom)
-  await checkFile(child2FileName)
-  await checkFile(child1FileName)
-  await checkFile(parentFileName)
-  await vhdMerge(handler, parentFileName, handler, child1FileName)
-  await checkFile(parentFileName)
-  await chainVhd(handler, parentFileName, handler, child2FileName, true)
-  await checkFile(child2FileName)
-  await vhdMerge(handler, parentFileName, handler, child2FileName)
-  await checkFile(parentFileName)
-  await recoverRawContent(parentFileName, recoveredFileName, originalSize)
-  await execa('cp', [randomFileName, random2FileName])
-  const fd = await fs.open(random2FileName, 'r+')
+  await checkFile(child2FilePath)
+  await checkFile(child1FilePath)
+  await checkFile(parentFilePath)
+  await vhdMerge(handler, 'parent.vhd', handler, 'child1.vhd')
+  await checkFile(parentFilePath)
+  await chainVhd(handler, 'parent.vhd', handler, 'child2.vhd', true)
+  await checkFile(child2FilePath)
+  await vhdMerge(handler, 'parent.vhd', handler, 'child2.vhd')
+  await checkFile(parentFilePath)
+  await recoverRawContent(parentFilePath, recoveredFilePath, originalSize)
+  await execa('cp', [randomFilePath, random2FilePath])
+  const fd = await fs.open(random2FilePath, 'r+')
   try {
     await fs.write(fd, smallRandom, 0, smallRandom.length, 5 * SECTOR_SIZE)
   } finally {
     await fs.close(fd)
   }
-  expect(await fs.readFile(recoveredFileName)).toEqual(await fs.readFile(random2FileName))
-})
-
-test.only('createSyntheticStream passes vhd-util check', async () => {
-  const initalSize = 4
-  const rawFileName = `${tempDir}/randomfile`
-  const vhdFileName = `${tempDir}/randomfile.vhd`
-  const recoveredVhdFileName = `${tempDir}/recovered.vhd`
-  await createRandomFile(rawFileName, initalSize)
-  await convertFromRawToVhd(rawFileName, vhdFileName)
-  await checkFile(vhdFileName)
-  const handler = getHandler({ url: 'file://' })
-  const stream = await createSyntheticStream(handler, vhdFileName)
-  const expectedVhdSize = (await fs.stat(vhdFileName)).size
-  expect(stream.length).toEqual((await fs.stat(vhdFileName)).size)
-  await pFromCallback(cb => pipeline(stream, fs.createWriteStream(recoveredVhdFileName), cb))
-  await checkFile(recoveredVhdFileName)
-  const stats = await fs.stat(recoveredVhdFileName)
-  expect(stats.size).toEqual(expectedVhdSize)
-  await execa('qemu-img', ['compare', recoveredVhdFileName, rawFileName])
+  expect(await fs.readFile(recoveredFilePath)).toEqual(await fs.readFile(random2FilePath))
 })

packages/vhd-lib/src/merge.js

@@ -5,7 +5,7 @@ import noop from './_noop'
 import { createLogger } from '@xen-orchestra/log'
 import { limitConcurrency } from 'limit-concurrency-decorator'
 
-import Vhd from './vhd'
+import { VhdFile } from '.'
 import { basename, dirname } from 'path'
 import { DISK_TYPE_DIFFERENCING, DISK_TYPE_DYNAMIC } from './_constants'
 
@@ -25,10 +25,10 @@ export default limitConcurrency(2)(async function merge(
 
   const parentFd = await parentHandler.openFile(parentPath, 'r+')
   try {
-    const parentVhd = new Vhd(parentHandler, parentFd)
+    const parentVhd = new VhdFile(parentHandler, parentFd)
     const childFd = await childHandler.openFile(childPath, 'r')
     try {
-      const childVhd = new Vhd(childHandler, childFd)
+      const childVhd = new VhdFile(childHandler, childFd)
 
       let mergeState = await parentHandler.readFile(mergeStatePath).catch(error => {
         if (error.code !== 'ENOENT') {

packages/vhd-lib/src/openVhd.integ.spec.js

@@ -0,0 +1,62 @@
+/* eslint-env jest */
+
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { getSyncedHandler } from '@xen-orchestra/fs'
+import { Disposable, pFromCallback } from 'promise-toolbox'
+
+import { openVhd } from './index'
+import { createRandomFile, convertFromRawToVhd, createRandomVhdDirectory } from './tests/utils'
+
+import { VhdAbstract } from './Vhd/VhdAbstract'
+
+let tempDir
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+test('It opens a vhd file ( alias or not)', async () => {
+  const initalSize = 4
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSize)
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: `file://${tempDir}/` })
+    const vhd = yield openVhd(handler, 'randomfile.vhd')
+    expect(vhd.header.cookie).toEqual('cxsparse')
+    expect(vhd.footer.cookie).toEqual('conectix')
+
+    await VhdAbstract.createAlias(handler, 'out.alias.vhd', 'randomfile.vhd')
+    const alias = yield openVhd(handler, 'out.alias.vhd')
+    expect(alias.header.cookie).toEqual('cxsparse')
+    expect(alias.footer.cookie).toEqual('conectix')
+    expect(alias._path?.path).toEqual('/randomfile.vhd')
+  })
+})
+
+test('It opens a vhd directory', async () => {
+  const initalSize = 4
+  const vhdDirectory = `${tempDir}/randomfile.dir`
+  await createRandomVhdDirectory(vhdDirectory, initalSize)
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: `file://${tempDir}/` })
+    const vhd = yield openVhd(handler, 'randomfile.dir')
+    expect(vhd.header.cookie).toEqual('cxsparse')
+    expect(vhd.footer.cookie).toEqual('conectix')
+
+    await VhdAbstract.createAlias(handler, 'out.alias.vhd', 'randomfile.dir')
+    const alias = yield openVhd(handler, 'out.alias.vhd')
+    expect(alias.header.cookie).toEqual('cxsparse')
+    expect(alias.footer.cookie).toEqual('conectix')
+    expect(alias._path).toEqual('randomfile.dir')
+  })
+})

packages/vhd-lib/src/openVhd.js

@@ -0,0 +1,14 @@
+import { resolveAlias } from './_resolveAlias'
+import { VhdFile, VhdDirectory } from './'
+
+export async function openVhd(handler, path) {
+  const resolved = await resolveAlias(handler, path)
+  try {
+    return await VhdFile.open(handler, resolved)
+  } catch (e) {
+    if (e.code !== 'EISDIR') {
+      throw e
+    }
+    return await VhdDirectory.open(handler, resolved)
+  }
+}

packages/vhd-lib/src/tests/utils.js

@@ -0,0 +1,87 @@
+import { pFromCallback } from 'promise-toolbox'
+import { pipeline } from 'readable-stream'
+import asyncIteratorToStream from 'async-iterator-to-stream'
+import execa from 'execa'
+import fs from 'fs-extra'
+import { randomBytes } from 'crypto'
+
+const createRandomStream = asyncIteratorToStream(function* (size) {
+  while (size > 0) {
+    yield randomBytes(Math.min(size, 1024))
+    size -= 1024
+  }
+})
+
+export async function createRandomFile(name, sizeMB) {
+  const input = createRandomStream(sizeMB * 1024 * 1024)
+  await pFromCallback(cb => pipeline(input, fs.createWriteStream(name), cb))
+}
+
+export async function checkFile(vhdName) {
+  await execa('vhd-util', ['check', '-p', '-b', '-t', '-n', vhdName])
+}
+
+const RAW = 'raw'
+const VHD = 'vpc'
+const VMDK = 'vmdk'
+
+async function convert(inputFormat, inputFile, outputFormat, outputFile) {
+  await execa('qemu-img', ['convert', `-f${inputFormat}`, '-O', outputFormat, inputFile, outputFile])
+}
+
+export async function convertFromRawToVhd(rawName, vhdName) {
+  await convert(RAW, rawName, VHD, vhdName)
+}
+
+export async function convertFromVhdToRaw(vhdName, rawName) {
+  await convert(VHD, vhdName, RAW, rawName)
+}
+
+export async function convertFromVmdkToRaw(vmdkName, rawName) {
+  await convert(VMDK, vmdkName, RAW, rawName)
+}
+
+export async function recoverRawContent(vhdName, rawName, originalSize) {
+  await checkFile(vhdName)
+  await convertFromVhdToRaw(vhdName, rawName)
+  if (originalSize !== undefined) {
+    await execa('truncate', ['-s', originalSize, rawName])
+  }
+}
+
+export async function createRandomVhdDirectory(path, sizeMB) {
+  fs.mkdir(path)
+  const rawFileName = `${path}/temp.raw`
+  await createRandomFile(rawFileName, sizeMB)
+  const vhdFileName = `${path}/vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+
+  const srcVhd = await fs.open(vhdFileName, 'r')
+
+  const footer = Buffer.alloc(512)
+  await fs.read(srcVhd, footer, 0, footer.length, 0)
+  await fs.writeFile(path + '/footer', footer)
+
+  const header = Buffer.alloc(1024)
+  await fs.read(srcVhd, header, 0, header.length, 512)
+  await fs.writeFile(path + '/header', header)
+
+  await fs.close(srcVhd)
+
+  // a BAT , with at most 512 blocks of 2MB
+  const bat = Buffer.alloc(512, 1)
+  await fs.writeFile(path + '/bat', bat)
+
+  // copy blocks
+  const srcRaw = await fs.open(rawFileName, 'r')
+  const blockDataSize = 512 * 4096
+  const bitmap = Buffer.alloc(4096)
+  await fs.mkdir(path + '/blocks/')
+  await fs.mkdir(path + '/blocks/1/')
+  for (let i = 0, offset = 0; i < sizeMB; i++, offset += blockDataSize) {
+    const blockData = Buffer.alloc(blockDataSize)
+    await fs.read(srcRaw, blockData, offset)
+    await fs.writeFile(path + '/blocks/1/' + i, Buffer.concat([bitmap, blockData]))
+  }
+  await fs.close(srcRaw)
+}

packages/xapi-explore-sr/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "xapi-explore-sr",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "license": "ISC",
   "description": "Display the list of VDIs (unmanaged and snapshots included) of a SR",
   "keywords": [
@@ -39,7 +39,7 @@
     "human-format": "^0.11.0",
     "lodash": "^4.17.4",
     "pw": "^0.0.4",
-    "xen-api": "^0.33.1"
+    "xen-api": "^0.35.1"
   },
   "devDependencies": {
     "@babel/cli": "^7.1.5",

packages/xapi-explore-sr/src/index.js

@@ -3,6 +3,7 @@
 import archy from 'archy'
 import chalk from 'chalk'
 import execPromise from 'exec-promise'
+import firstDefined from '@xen-orchestra/defined'
 import humanFormat from 'human-format'
 import pw from 'pw'
 import { createClient } from 'xen-api'
@@ -69,11 +70,13 @@ execPromise(async args => {
     url = required('Host URL'),
     user = required('Host user'),
     password = await askPassword('Host password'),
+    httpProxy = firstDefined(process.env.http_proxy, process.env.HTTP_PROXY),
   ] = args
 
   const xapi = createClient({
     allowUnauthorized: true,
     auth: { user, password },
+    httpProxy,
     readOnly: true,
     url,
     watchEvents: false,

packages/xen-api/README.md

@@ -52,6 +52,7 @@ Options:
 - `auth`: credentials used to sign in (can also be specified in the URL)
 - `readOnly = false`: if true, no methods with side-effects can be called
 - `callTimeout`: number of milliseconds after which a call is considered failed (can also be a map of timeouts by methods)
+- `httpProxy`: URL of the HTTP/HTTPS proxy used to reach the host, can include credentials
 
 ```js
 // Force connection.

packages/xen-api/USAGE.md

@@ -34,6 +34,7 @@ Options:
 - `auth`: credentials used to sign in (can also be specified in the URL)
 - `readOnly = false`: if true, no methods with side-effects can be called
 - `callTimeout`: number of milliseconds after which a call is considered failed (can also be a map of timeouts by methods)
+- `httpProxy`: URL of the HTTP/HTTPS proxy used to reach the host, can include credentials
 
 ```js
 // Force connection.

packages/xen-api/examples/package.json

@@ -8,6 +8,6 @@
     "promise-toolbox": "^0.19.2",
     "readable-stream": "^3.1.1",
     "throttle": "^1.0.3",
-    "vhd-lib": "^1.1.0"
+    "vhd-lib": "^1.3.0"
   }
 }

packages/xen-api/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "xen-api",
-  "version": "0.33.1",
+  "version": "0.35.1",
   "license": "ISC",
   "description": "Connector to the Xen API",
   "keywords": [
@@ -34,8 +34,8 @@
     "bind-property-descriptor": "^1.0.0",
     "blocked": "^1.2.1",
     "debug": "^4.0.1",
-    "http-request-plus": "^0.12",
-    "jest-diff": "^26.4.2",
+    "http-request-plus": "^0.13.0",
+    "jest-diff": "^27.3.1",
     "json-rpc-protocol": "^0.13.1",
     "kindof": "^2.0.0",
     "limit-concurrency-decorator": "^0.5.0",
@@ -43,7 +43,8 @@
     "make-error": "^1.3.0",
     "minimist": "^1.2.0",
     "ms": "^2.1.1",
-    "promise-toolbox": "^0.19.2",
+    "promise-toolbox": "^0.20.0",
+    "proxy-agent": "^5.0.0",
     "pw": "0.0.4",
     "xmlrpc": "^1.3.2",
     "xo-collection": "^0.5.0"

packages/xen-api/src/cli.js

@@ -2,10 +2,10 @@
 
 import blocked from 'blocked'
 import createDebug from 'debug'
-import diff from 'jest-diff'
 import minimist from 'minimist'
 import pw from 'pw'
 import { asCallback, fromCallback, fromEvent } from 'promise-toolbox'
+import { diff } from 'jest-diff'
 import { filter, find } from 'lodash'
 import { getBoundPropertyDescriptor } from 'bind-property-descriptor'
 import { start as createRepl } from 'repl'

packages/xen-api/src/index.js

@@ -115,6 +115,7 @@ export class Xapi extends EventEmitter {
     }
 
     this._allowUnauthorized = opts.allowUnauthorized
+    this._httpProxy = opts.httpProxy
     this._setUrl(url)
 
     this._connected = new Promise(resolve => {
@@ -359,22 +360,35 @@ export class Xapi extends EventEmitter {
       }
     }
 
-    const response = await httpRequest(
-      $cancelToken,
-      this._url,
-      host !== undefined && {
-        hostname: await this._getHostAddress(this.getObject(host)),
-      },
+    let url = new URL('http://localhost')
+    url.protocol = this._url.protocol
+    url.pathname = pathname
+    url.search = new URLSearchParams(query)
+    await this._setHostAddressInUrl(url, host)
+
+    const response = await pRetry(
+      async () =>
+        httpRequest($cancelToken, url.href, {
+          rejectUnauthorized: !this._allowUnauthorized,
+
+          // this is an inactivity timeout (unclear in Node doc)
+          timeout: this._httpInactivityTimeout,
+
+          maxRedirects: 0,
+
+          // Support XS <= 6.5 with Node => 12
+          minVersion: 'TLSv1',
+        }),
       {
-        pathname,
-        query,
-        rejectUnauthorized: !this._allowUnauthorized,
-
-        // this is an inactivity timeout (unclear in Node doc)
-        timeout: this._httpInactivityTimeout,
-
-        // Support XS <= 6.5 with Node => 12
-        minVersion: 'TLSv1',
+        when: { code: 302 },
+        onRetry: async error => {
+          const response = error.response
+          if (response === undefined) {
+            throw error
+          }
+          response.cancel()
+          url = await this._replaceHostAddressInUrl(new URL(response.headers.location, url))
+        },
       }
     )
 
@@ -421,32 +435,28 @@ export class Xapi extends EventEmitter {
       headers['content-length'] = '1125899906842624'
     }
 
-    const doRequest = httpRequest.put.bind(
-      undefined,
-      $cancelToken,
-      this._url,
-      host !== undefined && {
-        hostname: await this._getHostAddress(this.getObject(host)),
-      },
-      {
-        body,
-        headers,
-        pathname,
-        query,
-        rejectUnauthorized: !this._allowUnauthorized,
+    const url = new URL('http://localhost')
+    url.protocol = this._url.protocol
+    url.pathname = pathname
+    url.search = new URLSearchParams(query)
+    await this._setHostAddressInUrl(url, host)
 
-        // this is an inactivity timeout (unclear in Node doc)
-        timeout: this._httpInactivityTimeout,
+    const doRequest = httpRequest.put.bind(undefined, $cancelToken, {
+      body,
+      headers,
+      rejectUnauthorized: !this._allowUnauthorized,
 
-        // Support XS <= 6.5 with Node => 12
-        minVersion: 'TLSv1',
-      }
-    )
+      // this is an inactivity timeout (unclear in Node doc)
+      timeout: this._httpInactivityTimeout,
+
+      // Support XS <= 6.5 with Node => 12
+      minVersion: 'TLSv1',
+    })
 
     // if body is a stream, sends a dummy request to probe for a redirection
     // before consuming body
     const response = await (isStream
-      ? doRequest({
+      ? doRequest(url.href, {
           body: '',
 
           // omit task_id because this request will fail on purpose
@@ -456,9 +466,9 @@ export class Xapi extends EventEmitter {
         }).then(
           response => {
             response.cancel()
-            return doRequest()
+            return doRequest(url.href)
           },
-          error => {
+          async error => {
             let response
             if (error != null && (response = error.response) != null) {
               response.cancel()
@@ -469,14 +479,16 @@ export class Xapi extends EventEmitter {
               } = response
               if (statusCode === 302 && location !== undefined) {
                 // ensure the original query is sent
-                return doRequest(location, { query })
+                const newUrl = new URL(location, url)
+                newUrl.searchParams.set('task_id', query.task_id)
+                return doRequest((await this._replaceHostAddressInUrl(newUrl)).href)
               }
             }
 
             throw error
           }
         )
-      : doRequest())
+      : doRequest(url.href))
 
     if (pTaskResult !== undefined) {
       pTaskResult = pTaskResult.catch(error => {
@@ -792,7 +804,35 @@ export class Xapi extends EventEmitter {
     }
   }
 
-  async _getHostAddress({ address }) {
+  async _setHostAddressInUrl(url, host) {
+    const pool = this._pool
+
+    const poolBackupNetwork = pool.other_config['xo:backupNetwork']
+    if (host === undefined) {
+      if (poolBackupNetwork === undefined) {
+        const xapiUrl = this._url
+        url.hostname = xapiUrl.hostname
+        url.port = xapiUrl.port
+        return
+      }
+
+      host = await this.getRecord('host', pool.master)
+    }
+
+    let { address } = host
+    if (poolBackupNetwork !== undefined) {
+      const hostPifs = new Set(host.PIFs)
+      try {
+        const networkRef = await this._roCall('network.get_by_uuid', [poolBackupNetwork])
+        const networkPifs = await this.getField('network', networkRef, 'PIFs')
+
+        const backupNetworkPifRef = networkPifs.find(hostPifs.has, hostPifs)
+        address = await this.getField('PIF', backupNetworkPifRef, 'IP')
+      } catch (error) {
+        console.warn('unable to get the host address linked to the pool backup network', poolBackupNetwork, error)
+      }
+    }
+
     if (this._reverseHostIpAddresses) {
       try {
         ;[address] = await fromCallback(dns.reverse, address)
@@ -800,7 +840,8 @@ export class Xapi extends EventEmitter {
         console.warn('reversing host address', address, error)
       }
     }
-    return address
+
+    url.hostname = address
   }
 
   _setUrl(url) {
@@ -811,6 +852,7 @@ export class Xapi extends EventEmitter {
         rejectUnauthorized: !this._allowUnauthorized,
       },
       url,
+      httpProxy: this._httpProxy,
     })
     this._url = url
   }
@@ -862,6 +904,19 @@ export class Xapi extends EventEmitter {
     }
   }
 
+  async _replaceHostAddressInUrl(url) {
+    try {
+      // TODO: look for hostname in all addresses of this host (including all its PIFs)
+      const host = (await this.getAllRecords('host')).find(host => host.address === url.hostname)
+      if (host !== undefined) {
+        await this._setHostAddressInUrl(url, host)
+      }
+    } catch (error) {
+      console.warn('_replaceHostAddressInUrl', url, error)
+    }
+    return url
+  }
+
   _processEvents(events) {
     const flush = this._objects.bufferEvents()
     events.forEach(event => {

packages/xen-api/src/transports/json-rpc.js

@@ -1,4 +1,5 @@
 import httpRequestPlus from 'http-request-plus'
+import ProxyAgent from 'proxy-agent'
 import { format, parse } from 'json-rpc-protocol'
 
 import XapiError from '../_XapiError'
@@ -6,7 +7,11 @@ import XapiError from '../_XapiError'
 import UnsupportedTransport from './_UnsupportedTransport'
 
 // https://github.com/xenserver/xenadmin/blob/0df39a9d83cd82713f32d24704852a0fd57b8a64/XenModel/XenAPI/Session.cs#L403-L433
-export default ({ secureOptions, url }) => {
+export default ({ secureOptions, url, httpProxy }) => {
+  let agent
+  if (httpProxy !== undefined) {
+    agent = new ProxyAgent(httpProxy)
+  }
   return (method, args) =>
     httpRequestPlus
       .post(url, {
@@ -17,6 +22,7 @@ export default ({ secureOptions, url }) => {
           'Content-Type': 'application/json',
         },
         path: '/jsonrpc',
+        agent,
       })
       .readAll('utf8')
       .then(

packages/xen-api/src/transports/xml-rpc-json.js

@@ -1,5 +1,6 @@
 import { createClient, createSecureClient } from 'xmlrpc'
 import { promisify } from 'promise-toolbox'
+import ProxyAgent from 'proxy-agent'
 
 import XapiError from '../_XapiError'
 
@@ -70,10 +71,15 @@ const parseResult = result => {
   throw new UnsupportedTransport()
 }
 
-export default ({ secureOptions, url: { hostname, port, protocol } }) => {
+export default ({ secureOptions, url: { hostname, port, protocol }, httpProxy }) => {
   const secure = protocol === 'https:'
+  let agent
+  if (httpProxy !== undefined) {
+    agent = new ProxyAgent(httpProxy)
+  }
   const client = (secure ? createSecureClient : createClient)({
     ...(secure ? secureOptions : undefined),
+    agent,
     host: hostname,
     path: '/json',
     port,

packages/xen-api/src/transports/xml-rpc.js

@@ -1,5 +1,6 @@
 import { createClient, createSecureClient } from 'xmlrpc'
 import { promisify } from 'promise-toolbox'
+import ProxyAgent from 'proxy-agent'
 
 import XapiError from '../_XapiError'
 
@@ -30,10 +31,15 @@ const parseResult = result => {
   return result.Value
 }
 
-export default ({ secureOptions, url: { hostname, port, protocol } }) => {
+export default ({ secureOptions, url: { hostname, port, protocol, httpProxy } }) => {
   const secure = protocol === 'https:'
+  let agent
+  if (httpProxy !== undefined) {
+    agent = new ProxyAgent(httpProxy)
+  }
   const client = (secure ? createSecureClient : createClient)({
     ...(secure ? secureOptions : undefined),
+    agent,
     host: hostname,
     port,
   })

packages/xo-cli/package.json

@@ -33,8 +33,8 @@
     "bluebird": "^3.5.1",
     "chalk": "^4.1.0",
     "exec-promise": "^0.7.0",
-    "fs-extra": "^9.0.0",
-    "http-request-plus": "^0.12",
+    "fs-extra": "^10.0.0",
+    "http-request-plus": "^0.13.0",
     "human-format": "^0.11.0",
     "l33teral": "^3.0.3",
     "lodash": "^4.17.4",
@@ -42,7 +42,7 @@
     "mkdirp": "^1.0.4",
     "pretty-ms": "^7.0.0",
     "progress-stream": "^2.0.0",
-    "promise-toolbox": "^0.19.2",
+    "promise-toolbox": "^0.20.0",
     "pump": "^3.0.0",
     "pw": "^0.0.4",
     "strip-indent": "^3.0.0",

packages/xo-cli/src/index.js

@@ -14,7 +14,7 @@ const chalk = require('chalk')
 const forEach = require('lodash/forEach')
 const fromCallback = require('promise-toolbox/fromCallback')
 const getKeys = require('lodash/keys')
-const hrp = require('http-request-plus').default
+const hrp = require('http-request-plus')
 const humanFormat = require('human-format')
 const identity = require('lodash/identity')
 const isObject = require('lodash/isObject')

packages/xo-collection/package.json

@@ -30,7 +30,7 @@
     "@babel/core": "^7.0.0",
     "@babel/preset-env": "^7.0.0",
     "cross-env": "^7.0.2",
-    "promise-toolbox": "^0.19.2",
+    "promise-toolbox": "^0.20.0",
     "rimraf": "^3.0.0"
   },
   "scripts": {

packages/xo-import-servers-csv/package.json

@@ -40,7 +40,7 @@
     "xo-lib": "^0.10.1"
   },
   "devDependencies": {
-    "@types/node": "^14.0.9",
+    "@types/node": "^16.11.6",
     "@types/through2": "^2.0.31",
     "typescript": "^3.1.6"
   },

packages/xo-server-audit/package.json

@@ -47,9 +47,9 @@
   "dependencies": {
     "@xen-orchestra/audit-core": "^0.2.0",
     "@xen-orchestra/cron": "^1.0.6",
-    "@xen-orchestra/log": "^0.2.1",
+    "@xen-orchestra/log": "^0.3.0",
     "async-iterator-to-stream": "^1.1.0",
-    "promise-toolbox": "^0.19.2",
+    "promise-toolbox": "^0.20.0",
     "readable-stream": "^3.5.0",
     "xo-common": "^0.7.0"
   },

packages/xo-server-auth-ldap/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-server-auth-ldap",
-  "version": "0.10.2",
+  "version": "0.10.4",
   "license": "AGPL-3.0-or-later",
   "description": "LDAP authentication plugin for XO-Server",
   "keywords": [
@@ -28,13 +28,15 @@
     "xo-server-auth-ldap": "dist/test-cli.js"
   },
   "engines": {
-    "node": ">=10"
+    "node": ">=12"
   },
   "dependencies": {
+    "@xen-orchestra/log": "^0.3.0",
+    "ensure-array": "^1.0.0",
     "exec-promise": "^0.7.0",
     "inquirer": "^8.0.0",
-    "ldapts": "^2.2.1",
-    "promise-toolbox": "^0.19.2"
+    "ldapts": "^3.1.1",
+    "promise-toolbox": "^0.20.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/xo-server-auth-ldap/src/index.js

@@ -1,10 +1,14 @@
 /* eslint no-throw-literal: 0 */
 
+import ensureArray from 'ensure-array'
 import fromCallback from 'promise-toolbox/fromCallback'
 import { Client } from 'ldapts'
+import { createLogger } from '@xen-orchestra/log'
 import { Filter } from 'ldapts/filters/Filter'
 import { readFile } from 'fs'
 
+const logger = createLogger('xo:xo-server-auth-ldap')
+
 // ===================================================================
 
 const DEFAULTS = {
@@ -26,8 +30,6 @@ const evalFilter = (filter, vars) =>
     return escape(value)
   })
 
-const noop = Function.prototype
-
 export const configurationSchema = {
   type: 'object',
   properties: {
@@ -183,8 +185,7 @@ export const testSchema = {
 // ===================================================================
 
 class AuthLdap {
-  constructor({ logger = noop, xo }) {
-    this._logger = logger
+  constructor({ xo } = {}) {
     this._xo = xo
 
     this._authenticate = this._authenticate.bind(this)
@@ -256,10 +257,8 @@ class AuthLdap {
   }
 
   async _authenticate({ username, password }) {
-    const logger = this._logger
-
     if (username === undefined || password === undefined) {
-      logger('require `username` and `password` to authenticate!')
+      logger.debug('require `username` and `password` to authenticate!')
 
       return null
     }
@@ -275,29 +274,34 @@ class AuthLdap {
       {
         const { _credentials: credentials } = this
         if (credentials) {
-          logger(`attempting to bind with as ${credentials.dn}...`)
+          logger.debug(`attempting to bind with as ${credentials.dn}...`)
           await client.bind(credentials.dn, credentials.password)
-          logger(`successfully bound as ${credentials.dn}`)
+          logger.debug(`successfully bound as ${credentials.dn}`)
         }
       }
 
       // Search for the user.
-      logger('searching for entries...')
+      logger.debug('searching for entries...')
       const { searchEntries: entries } = await client.search(this._searchBase, {
         scope: 'sub',
         filter: evalFilter(this._searchFilter, {
           name: username,
         }),
       })
-      logger(`${entries.length} entries found`)
+      logger.debug(`${entries.length} entries found`)
 
       // Try to find an entry which can be bind with the given password.
       for (const entry of entries) {
         try {
-          logger(`attempting to bind as ${entry.dn}`)
+          logger.debug(`attempting to bind as ${entry.dn}`)
           await client.bind(entry.dn, password)
-          logger(`successfully bound as ${entry.dn} => ${username} authenticated`)
-          logger(JSON.stringify(entry, null, 2))
+          logger.info(`successfully bound as ${entry.dn} => ${username} authenticated`)
+          logger.debug(JSON.stringify(entry, null, 2))
+
+          // CLI test: don't register user/sync groups
+          if (this._xo === undefined) {
+            return
+          }
 
           let user
           if (this._userIdAttribute === undefined) {
@@ -314,18 +318,18 @@ class AuthLdap {
               try {
                 await this._synchronizeGroups(user, entry[groupsConfig.membersMapping.userAttribute])
               } catch (error) {
-                logger(`failed to synchronize groups: ${error.message}`)
+                logger.error(`failed to synchronize groups: ${error.message}`)
               }
             }
           }
 
           return { userId: user.id }
         } catch (error) {
-          logger(`failed to bind as ${entry.dn}: ${error.message}`)
+          logger.debug(`failed to bind as ${entry.dn}: ${error.message}`)
         }
       }
 
-      logger(`could not authenticate ${username}`)
+      logger.debug(`could not authenticate ${username}`)
       return null
     } finally {
       await client.unbind()
@@ -334,7 +338,6 @@ class AuthLdap {
 
   // Synchronize user's groups OR all groups if no user is passed
   async _synchronizeGroups(user, memberId) {
-    const logger = this._logger
     const client = new Client(this._clientOpts)
 
     try {
@@ -346,12 +349,12 @@ class AuthLdap {
       {
         const { _credentials: credentials } = this
         if (credentials) {
-          logger(`attempting to bind with as ${credentials.dn}...`)
+          logger.debug(`attempting to bind with as ${credentials.dn}...`)
           await client.bind(credentials.dn, credentials.password)
-          logger(`successfully bound as ${credentials.dn}`)
+          logger.debug(`successfully bound as ${credentials.dn}`)
         }
       }
-      logger('syncing groups...')
+      logger.info('syncing groups...')
       const { base, displayNameAttribute, filter, idAttribute, membersMapping } = this._groupsConfig
       const { searchEntries: ldapGroups } = await client.search(base, {
         scope: 'sub',
@@ -373,12 +376,11 @@ class AuthLdap {
 
         // Empty or undefined names/IDs are invalid
         if (!groupLdapId || !groupLdapName) {
-          logger(`Invalid group ID (${groupLdapId}) or name (${groupLdapName})`)
+          logger.error(`Invalid group ID (${groupLdapId}) or name (${groupLdapName})`)
           continue
         }
 
-        let ldapGroupMembers = ldapGroup[membersMapping.groupAttribute]
-        ldapGroupMembers = Array.isArray(ldapGroupMembers) ? ldapGroupMembers : [ldapGroupMembers]
+        const ldapGroupMembers = ensureArray(ldapGroup[membersMapping.groupAttribute])
 
         // If a user was passed, only update the user's groups
         if (user !== undefined && !ldapGroupMembers.includes(memberId)) {
@@ -393,7 +395,7 @@ class AuthLdap {
         if (xoGroupIndex === -1) {
           if (xoGroups.find(group => group.name === groupLdapName) !== undefined) {
             // TODO: check against LDAP groups that are being created as well
-            logger(`A group called ${groupLdapName} already exists`)
+            logger.error(`A group called ${groupLdapName} already exists`)
             continue
           }
           xoGroup = await this._xo.createGroup({
@@ -459,6 +461,8 @@ class AuthLdap {
           xoGroups.filter(group => group.provider === 'ldap').map(group => this._xo.deleteGroup(group.id))
         )
       }
+
+      logger.info('done syncing groups')
     } finally {
       await client.unbind()
     }

packages/xo-server-auth-ldap/src/test-cli.js

@@ -1,6 +1,8 @@
 #!/usr/bin/env node
 
 import execPromise from 'exec-promise'
+import transportConsole from '@xen-orchestra/log/transports/console'
+import { configure } from '@xen-orchestra/log/configure.js'
 import { fromCallback } from 'promise-toolbox'
 import { readFile, writeFile } from 'fs'
 
@@ -28,9 +30,14 @@ execPromise(async args => {
     }
   )
 
-  const plugin = createPlugin({
-    logger: console.log.bind(console),
-  })
+  configure([
+    {
+      filter: process.env.DEBUG ?? 'xo:xo-server-auth-ldap',
+      transport: transportConsole(),
+    },
+  ])
+
+  const plugin = createPlugin()
   await plugin.configure(config)
 
   await plugin._authenticate({

packages/xo-server-backup-reports/package.json

@@ -33,7 +33,7 @@
   },
   "dependencies": {
     "@xen-orchestra/defined": "^0.0.1",
-    "@xen-orchestra/log": "^0.2.1",
+    "@xen-orchestra/log": "^0.3.0",
     "human-format": "^0.11.0",
     "lodash": "^4.13.1",
     "moment-timezone": "^0.5.13"

packages/xo-server-load-balancer/package.json

@@ -28,7 +28,7 @@
   },
   "dependencies": {
     "@xen-orchestra/cron": "^1.0.6",
-    "@xen-orchestra/log": "^0.2.1",
+    "@xen-orchestra/log": "^0.3.0",
     "lodash": "^4.16.2"
   },
   "devDependencies": {

packages/xo-server-load-balancer/src/density-plan.js

@@ -1,9 +1,9 @@
 import { clone, filter, map as mapToArray } from 'lodash'
 
 import Plan from './plan'
-import { log as logP } from './utils'
+import { debug as debugP } from './utils'
 
-export const log = str => logP(`density: ${str}`)
+export const debug = str => debugP(`density: ${str}`)
 
 // ===================================================================
 
@@ -48,7 +48,7 @@ export default class DensityPlan extends Plan {
 
       // A host to optimize needs the ability to be restarted.
       if (hostToOptimize.powerOnMode === '') {
-        log(`Host (${hostId}) does not have a power on mode.`)
+        debug(`Host (${hostId}) does not have a power on mode.`)
         continue
       }
 
@@ -94,20 +94,20 @@ export default class DensityPlan extends Plan {
       }
     }
 
-    log(`Density mode: ${optimizationsCount} optimizations.`)
+    debug(`Density mode: ${optimizationsCount} optimizations.`)
   }
 
   async _simulate({ host, destinations, hostsAverages }) {
     const { id: hostId } = host
 
-    log(`Try to optimize Host (${hostId}).`)
+    debug(`Try to optimize Host (${hostId}).`)
 
     const vms = filter(this._getAllRunningVms(), vm => vm.$container === hostId)
     const vmsAverages = await this._getVmsAverages(vms, { [host.id]: host })
 
     for (const vm of vms) {
       if (!vm.xenTools) {
-        log(`VM (${vm.id}) of Host (${hostId}) does not support pool migration.`)
+        debug(`VM (${vm.id}) of Host (${hostId}) does not support pool migration.`)
         return
       }
 
@@ -118,7 +118,7 @@ export default class DensityPlan extends Plan {
         // - It's necessary to maintain a dictionary of tags for each host.
         // - ...
         if (this._antiAffinityTags.includes(tag)) {
-          log(`VM (${vm.id}) of Host (${hostId}) cannot be migrated. It contains anti-affinity tag '${tag}'.`)
+          debug(`VM (${vm.id}) of Host (${hostId}) cannot be migrated. It contains anti-affinity tag '${tag}'.`)
           return
         }
       }
@@ -205,19 +205,19 @@ export default class DensityPlan extends Plan {
       mapToArray(moves, move => {
         const { vm, destination } = move
         const xapiDest = this.xo.getXapi(destination)
-        log(
+        debug(
           `Migrate VM (${vm.id} "${vm.name_label}") to Host (${destination.id} "${destination.name_label}") from Host (${fmtSrcHost}).`
         )
         return xapiDest.migrateVm(vm._xapiId, this.xo.getXapi(destination), destination._xapiId)
       })
     )
 
-    log(`Shutdown Host (${fmtSrcHost}).`)
+    debug(`Shutdown Host (${fmtSrcHost}).`)
 
     try {
       await xapiSrc.shutdownHost(srcHost.id)
     } catch (error) {
-      log(`Unable to shutdown Host (${fmtSrcHost}).`, { error })
+      debug(`Unable to shutdown Host (${fmtSrcHost}).`, { error })
     }
   }
 }