feat(backup/VmBackup): regularly report speed in task info

Fixes #6040

.github/ISSUE_TEMPLATE/bug_report.md

@@ -4,7 +4,6 @@ about: Create a report to help us improve
 title: ''
 labels: 'status: triaging :triangular_flag_on_post:, type: bug :bug:'
 assignees: ''
-
 ---
 
 **Describe the bug**
@@ -12,6 +11,7 @@ A clear and concise description of what the bug is.
 
 **To Reproduce**
 Steps to reproduce the behavior:
+
 1. Go to '...'
 2. Click on '....'
 3. Scroll down to '....'
@@ -24,10 +24,11 @@ A clear and concise description of what you expected to happen.
 If applicable, add screenshots to help explain your problem.
 
 **Desktop (please complete the following information):**
- - Node: [e.g. 16.12.1]
- - xo-server: [e.g. 5.82.3]
- - xo-web: [e.g. 5.87.0]
- - hypervisor: [e.g. XCP-ng 8.2.0]
+
+- Node: [e.g. 16.12.1]
+- xo-server: [e.g. 5.82.3]
+- xo-web: [e.g. 5.87.0]
+- hypervisor: [e.g. XCP-ng 8.2.0]
 
 **Additional context**
 Add any other context about the problem here.

@vates/async-each/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/async-each/README.md

@@ -0,0 +1,68 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/async-each
+
+[![Package Version](https://badgen.net/npm/v/@vates/async-each)](https://npmjs.org/package/@vates/async-each) ![License](https://badgen.net/npm/license/@vates/async-each) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/async-each)](https://bundlephobia.com/result?p=@vates/async-each) [![Node compatibility](https://badgen.net/npm/node/@vates/async-each)](https://npmjs.org/package/@vates/async-each)
+
+> Run async fn for each item in (async) iterable
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/async-each):
+
+```
+> npm install --save @vates/async-each
+```
+
+## Usage
+
+### `asyncEach(iterable, iteratee, [opts])`
+
+Executes `iteratee` in order for each value yielded by `iterable`.
+
+Returns a promise wich rejects as soon as a call to `iteratee` throws or a promise returned by it rejects, and which resolves when all promises returned by `iteratee` have resolved.
+
+`iterable` must be an iterable or async iterable.
+
+`iteratee` is called with the same `this` value as `asyncEach`, and with the following arguments:
+
+- `value`: the value yielded by `iterable`
+- `index`: the 0-based index for this value
+- `iterable`: the iterable itself
+
+`opts` is an object that can contains the following options:
+
+- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
+- `signal`: an abort signal to stop the iteration
+- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
+
+```js
+import { asyncEach } from '@vates/async-each'
+
+const contents = []
+await asyncEach(
+  ['foo.txt', 'bar.txt', 'baz.txt'],
+  async function (filename, i) {
+    contents[i] = await readFile(filename)
+  },
+  {
+    // reads two files at a time
+    concurrency: 2,
+  }
+)
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/async-each/USAGE.md

@@ -0,0 +1,35 @@
+### `asyncEach(iterable, iteratee, [opts])`
+
+Executes `iteratee` in order for each value yielded by `iterable`.
+
+Returns a promise wich rejects as soon as a call to `iteratee` throws or a promise returned by it rejects, and which resolves when all promises returned by `iteratee` have resolved.
+
+`iterable` must be an iterable or async iterable.
+
+`iteratee` is called with the same `this` value as `asyncEach`, and with the following arguments:
+
+- `value`: the value yielded by `iterable`
+- `index`: the 0-based index for this value
+- `iterable`: the iterable itself
+
+`opts` is an object that can contains the following options:
+
+- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
+- `signal`: an abort signal to stop the iteration
+- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
+
+```js
+import { asyncEach } from '@vates/async-each'
+
+const contents = []
+await asyncEach(
+  ['foo.txt', 'bar.txt', 'baz.txt'],
+  async function (filename, i) {
+    contents[i] = await readFile(filename)
+  },
+  {
+    // reads two files at a time
+    concurrency: 2,
+  }
+)
+```

@vates/async-each/index.js

@@ -0,0 +1,99 @@
+'use strict'
+
+const noop = Function.prototype
+
+class AggregateError extends Error {
+  constructor(errors, message) {
+    super(message)
+    this.errors = errors
+  }
+}
+
+exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 1, signal, stopOnError = true } = {}) {
+  return new Promise((resolve, reject) => {
+    const it = (iterable[Symbol.iterator] || iterable[Symbol.asyncIterator]).call(iterable)
+    const errors = []
+    let running = 0
+    let index = 0
+
+    let onAbort
+    if (signal !== undefined) {
+      onAbort = () => {
+        onRejectedWrapper(new Error('asyncEach aborted'))
+      }
+      signal.addEventListener('abort', onAbort)
+    }
+
+    const clean = () => {
+      onFulfilled = onRejected = noop
+      if (onAbort !== undefined) {
+        signal.removeEventListener('abort', onAbort)
+      }
+    }
+
+    resolve = (resolve =>
+      function resolveAndClean(value) {
+        resolve(value)
+        clean()
+      })(resolve)
+    reject = (reject =>
+      function rejectAndClean(reason) {
+        reject(reason)
+        clean()
+      })(reject)
+
+    let onFulfilled = value => {
+      --running
+      next()
+    }
+    const onFulfilledWrapper = value => onFulfilled(value)
+
+    let onRejected = stopOnError
+      ? reject
+      : error => {
+          --running
+          errors.push(error)
+          next()
+        }
+    const onRejectedWrapper = reason => onRejected(reason)
+
+    let nextIsRunning = false
+    let next = async () => {
+      if (nextIsRunning) {
+        return
+      }
+      nextIsRunning = true
+      if (running < concurrency) {
+        const cursor = await it.next()
+        if (cursor.done) {
+          next = () => {
+            if (running === 0) {
+              if (errors.length === 0) {
+                resolve()
+              } else {
+                reject(new AggregateError(errors))
+              }
+            }
+          }
+        } else {
+          ++running
+          try {
+            const result = iteratee.call(this, cursor.value, index++, iterable)
+            let then
+            if (result != null && typeof result === 'object' && typeof (then = result.then) === 'function') {
+              then.call(result, onFulfilledWrapper, onRejectedWrapper)
+            } else {
+              onFulfilled(result)
+            }
+          } catch (error) {
+            onRejected(error)
+          }
+        }
+        nextIsRunning = false
+        return next()
+      }
+      nextIsRunning = false
+    }
+    next()
+  })
+}

@vates/async-each/index.spec.js

@@ -0,0 +1,99 @@
+'use strict'
+
+/* eslint-env jest */
+
+const { asyncEach } = require('./')
+
+const randomDelay = (max = 10) =>
+  new Promise(resolve => {
+    setTimeout(resolve, Math.floor(Math.random() * max + 1))
+  })
+
+const rejectionOf = p =>
+  new Promise((resolve, reject) => {
+    p.then(reject, resolve)
+  })
+
+describe('asyncEach', () => {
+  const thisArg = 'qux'
+  const values = ['foo', 'bar', 'baz']
+
+  Object.entries({
+    'sync iterable': () => values,
+    'async iterable': async function* () {
+      for (const value of values) {
+        await randomDelay()
+        yield value
+      }
+    },
+  }).forEach(([what, getIterable]) =>
+    describe('with ' + what, () => {
+      let iterable
+      beforeEach(() => {
+        iterable = getIterable()
+      })
+
+      it('works', async () => {
+        const iteratee = jest.fn(async () => {})
+
+        await asyncEach.call(thisArg, iterable, iteratee)
+
+        expect(iteratee.mock.instances).toEqual(Array.from(values, () => thisArg))
+        expect(iteratee.mock.calls).toEqual(Array.from(values, (value, index) => [value, index, iterable]))
+      })
+      ;[1, 2, 4].forEach(concurrency => {
+        it('respects a concurrency of ' + concurrency, async () => {
+          let running = 0
+
+          await asyncEach(
+            values,
+            async () => {
+              ++running
+              expect(running).toBeLessThanOrEqual(concurrency)
+              await randomDelay()
+              --running
+            },
+            { concurrency }
+          )
+        })
+      })
+
+      it('stops on first error when stopOnError is true', async () => {
+        const error = new Error()
+        const iteratee = jest.fn((_, i) => {
+          if (i === 1) {
+            throw error
+          }
+        })
+
+        expect(await rejectionOf(asyncEach(iterable, iteratee, { stopOnError: true }))).toBe(error)
+        expect(iteratee).toHaveBeenCalledTimes(2)
+      })
+
+      it('rejects AggregateError when stopOnError is false', async () => {
+        const errors = []
+        const iteratee = jest.fn(() => {
+          const error = new Error()
+          errors.push(error)
+          throw error
+        })
+
+        const error = await rejectionOf(asyncEach(iterable, iteratee, { stopOnError: false }))
+        expect(error.errors).toEqual(errors)
+        expect(iteratee.mock.calls).toEqual(Array.from(values, (value, index) => [value, index, iterable]))
+      })
+
+      it('can be interrupted with an AbortSignal', async () => {
+        const ac = new AbortController()
+        const iteratee = jest.fn((_, i) => {
+          if (i === 1) {
+            ac.abort()
+          }
+        })
+
+        await expect(asyncEach(iterable, iteratee, { signal: ac.signal })).rejects.toThrow('asyncEach aborted')
+        expect(iteratee).toHaveBeenCalledTimes(2)
+      })
+    })
+  )
+})

@vates/async-each/package.json

@@ -0,0 +1,34 @@
+{
+  "private": false,
+  "name": "@vates/async-each",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/async-each",
+  "description": "Run async fn for each item in (async) iterable",
+  "keywords": [
+    "array",
+    "async",
+    "collection",
+    "each",
+    "for",
+    "foreach",
+    "iterable",
+    "iterator"
+  ],
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/async-each",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "0.1.0",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@vates/compose/README.md

@@ -65,6 +65,23 @@ const f = compose(
 )
 ```
 
+Functions can receive extra parameters:
+
+```js
+const isIn = (value, min, max) => min <= value && value <= max
+
+// Only compatible when `fns` is passed as an array!
+const f = compose([
+  [add, 2],
+  [isIn, 3, 10],
+])
+
+console.log(f(1))
+// → true
+```
+
+> Note: if the first function is defined with extra parameters, it will only receive the first value passed to the composed function, instead of all the parameters.
+
 ## Contributions
 
 Contributions are _very_ welcomed, either on the documentation or on

@vates/compose/USAGE.md

@@ -46,3 +46,20 @@ const f = compose(
   [add2, mul3]
 )
 ```
+
+Functions can receive extra parameters:
+
+```js
+const isIn = (value, min, max) => min <= value && value <= max
+
+// Only compatible when `fns` is passed as an array!
+const f = compose([
+  [add, 2],
+  [isIn, 3, 10],
+])
+
+console.log(f(1))
+// → true
+```
+
+> Note: if the first function is defined with extra parameters, it will only receive the first value passed to the composed function, instead of all the parameters.

@vates/compose/index.js

@@ -4,11 +4,13 @@ const defaultOpts = { async: false, right: false }
 
 exports.compose = function compose(opts, fns) {
   if (Array.isArray(opts)) {
-    fns = opts
+    fns = opts.slice() // don't mutate passed array
     opts = defaultOpts
   } else if (typeof opts === 'object') {
     opts = Object.assign({}, defaultOpts, opts)
-    if (!Array.isArray(fns)) {
+    if (Array.isArray(fns)) {
+      fns = fns.slice() // don't mutate passed array
+    } else {
       fns = Array.prototype.slice.call(arguments, 1)
     }
   } else {
@@ -20,6 +22,24 @@ exports.compose = function compose(opts, fns) {
   if (n === 0) {
     throw new TypeError('at least one function must be passed')
   }
+
+  for (let i = 0; i < n; ++i) {
+    const entry = fns[i]
+    if (Array.isArray(entry)) {
+      const fn = entry[0]
+      const args = entry.slice()
+      args[0] = undefined
+      fns[i] = function composeWithArgs(value) {
+        args[0] = value
+        try {
+          return fn.apply(this, args)
+        } finally {
+          args[0] = undefined
+        }
+      }
+    }
+  }
+
   if (n === 1) {
     return fns[0]
   }

@vates/compose/package.json

@@ -14,7 +14,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "engines": {
     "node": ">=7.6"
   },

@vates/decorate-with/README.md

@@ -59,6 +59,36 @@ decorateMethodsWith(Foo, {
 
 The decorated class is returned, so you can export it directly.
 
+To apply multiple transforms to a method, you can either call `decorateMethodsWith` multiple times or use [`@vates/compose`](https://www.npmjs.com/package/@vates/compose):
+
+```js
+decorateMethodsWith(Foo, {
+  bar: compose([
+    [lodash.debounce, 150]
+    lodash.curry,
+  ])
+})
+```
+
+### `perInstance(fn, ...args)`
+
+Helper to decorate the method by instance instead of for the whole class.
+
+This is often necessary for caching or deduplicating calls.
+
+```js
+import { perInstance } from '@vates/decorateWith'
+
+class Foo {
+  @decorateWith(perInstance, lodash.memoize)
+  bar() {
+    // body
+  }
+}
+```
+
+Because it's a normal function, it can also be used with `decorateMethodsWith`, with `compose` or even by itself.
+
 ## Contributions
 
 Contributions are _very_ welcomed, either on the documentation or on

@vates/decorate-with/USAGE.md

@@ -40,3 +40,33 @@ decorateMethodsWith(Foo, {
 ```
 
 The decorated class is returned, so you can export it directly.
+
+To apply multiple transforms to a method, you can either call `decorateMethodsWith` multiple times or use [`@vates/compose`](https://www.npmjs.com/package/@vates/compose):
+
+```js
+decorateMethodsWith(Foo, {
+  bar: compose([
+    [lodash.debounce, 150]
+    lodash.curry,
+  ])
+})
+```
+
+### `perInstance(fn, ...args)`
+
+Helper to decorate the method by instance instead of for the whole class.
+
+This is often necessary for caching or deduplicating calls.
+
+```js
+import { perInstance } from '@vates/decorateWith'
+
+class Foo {
+  @decorateWith(perInstance, lodash.memoize)
+  bar() {
+    // body
+  }
+}
+```
+
+Because it's a normal function, it can also be used with `decorateMethodsWith`, with `compose` or even by itself.

@vates/decorate-with/index.js

@@ -19,3 +19,15 @@ exports.decorateMethodsWith = function decorateMethodsWith(klass, map) {
   }
   return klass
 }
+
+exports.perInstance = function perInstance(fn, decorator, ...args) {
+  const map = new WeakMap()
+  return function () {
+    let decorated = map.get(this)
+    if (decorated === undefined) {
+      decorated = decorator(fn, ...args)
+      map.set(this, decorated)
+    }
+    return decorated.apply(this, arguments)
+  }
+}

@vates/decorate-with/package.json

@@ -20,7 +20,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "0.1.0",
+  "version": "1.0.0",
   "engines": {
     "node": ">=8.10"
   },

@xen-orchestra/audit-core/package.json

@@ -30,7 +30,7 @@
     "rimraf": "^3.0.0"
   },
   "dependencies": {
-    "@vates/decorate-with": "^0.1.0",
+    "@vates/decorate-with": "^1.0.0",
     "@xen-orchestra/log": "^0.3.0",
     "golike-defer": "^0.5.1",
     "object-hash": "^2.0.1"

@xen-orchestra/backups-cli/package.json

@@ -7,8 +7,8 @@
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.15.1",
-    "@xen-orchestra/fs": "^0.18.0",
+    "@xen-orchestra/backups": "^0.18.3",
+    "@xen-orchestra/fs": "^0.19.3",
     "filenamify": "^4.1.0",
     "getopts": "^2.2.5",
     "lodash": "^4.17.15",
@@ -27,7 +27,7 @@
   "scripts": {
     "postversion": "npm publish --access public"
   },
-  "version": "0.6.0",
+  "version": "0.6.1",
   "license": "AGPL-3.0-or-later",
   "author": {
     "name": "Vates SAS",

@xen-orchestra/backups/RemoteAdapter.js

@@ -3,19 +3,20 @@ const Disposable = require('promise-toolbox/Disposable.js')
 const fromCallback = require('promise-toolbox/fromCallback.js')
 const fromEvent = require('promise-toolbox/fromEvent.js')
 const pDefer = require('promise-toolbox/defer.js')
-const pump = require('pump')
-const { basename, dirname, join, normalize, resolve } = require('path')
+const groupBy = require('lodash/groupBy.js')
+const { dirname, join, normalize, resolve } = require('path')
 const { createLogger } = require('@xen-orchestra/log')
-const { createSyntheticStream, mergeVhd, VhdFile } = require('vhd-lib')
+const { Constants, createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } = require('vhd-lib')
 const { deduped } = require('@vates/disposable/deduped.js')
 const { execFile } = require('child_process')
 const { readdir, stat } = require('fs-extra')
+const { v4: uuidv4 } = require('uuid')
 const { ZipFile } = require('yazl')
 
 const { BACKUP_DIR } = require('./_getVmBackupDir.js')
 const { cleanVm } = require('./_cleanVm.js')
 const { getTmpDir } = require('./_getTmpDir.js')
-const { isMetadataFile, isVhdFile } = require('./_backupType.js')
+const { isMetadataFile } = require('./_backupType.js')
 const { isValidXva } = require('./_isValidXva.js')
 const { listPartitions, LVM_PARTITION_TYPE } = require('./_listPartitions.js')
 const { lvs, pvs } = require('./_lvm.js')
@@ -67,58 +68,17 @@ const debounceResourceFactory = factory =>
   }
 
 class RemoteAdapter {
-  constructor(handler, { debounceResource = res => res, dirMode } = {}) {
+  constructor(handler, { debounceResource = res => res, dirMode, vhdDirectoryCompression } = {}) {
     this._debounceResource = debounceResource
     this._dirMode = dirMode
     this._handler = handler
+    this._vhdDirectoryCompression = vhdDirectoryCompression
   }
 
   get handler() {
     return this._handler
   }
 
-  async _deleteVhd(path) {
-    const handler = this._handler
-    const vhds = await asyncMapSettled(
-      await handler.list(dirname(path), {
-        filter: isVhdFile,
-        prependDir: true,
-      }),
-      async path => {
-        try {
-          const vhd = new VhdFile(handler, path)
-          await vhd.readHeaderAndFooter()
-          return {
-            footer: vhd.footer,
-            header: vhd.header,
-            path,
-          }
-        } catch (error) {
-          // Do not fail on corrupted VHDs (usually uncleaned temporary files),
-          // they are probably inconsequent to the backup process and should not
-          // fail it.
-          warn(`BackupNg#_deleteVhd ${path}`, { error })
-        }
-      }
-    )
-    const base = basename(path)
-    const child = vhds.find(_ => _ !== undefined && _.header.parentUnicodeName === base)
-    if (child === undefined) {
-      await handler.unlink(path)
-      return 0
-    }
-
-    try {
-      const childPath = child.path
-      const mergedDataSize = await mergeVhd(handler, path, handler, childPath)
-      await handler.rename(path, childPath)
-      return mergedDataSize
-    } catch (error) {
-      handler.unlink(path).catch(warn)
-      throw error
-    }
-  }
-
   async _findPartition(devicePath, partitionId) {
     const partitions = await listPartitions(devicePath)
     const partition = partitions.find(_ => _.id === partitionId)
@@ -232,6 +192,22 @@ class RemoteAdapter {
     return files
   }
 
+  // check if we will be allowed to merge a a vhd created in this adapter
+  // with the vhd at path `path`
+  async isMergeableParent(packedParentUid, path) {
+    return await Disposable.use(openVhd(this.handler, path), vhd => {
+      // this baseUuid is not linked with this vhd
+      if (!vhd.footer.uuid.equals(packedParentUid)) {
+        return false
+      }
+
+      const isVhdDirectory = vhd instanceof VhdDirectory
+      return isVhdDirectory
+        ? this.#useVhdDirectory() && this.#getCompressionType() === vhd.compressionType
+        : !this.#useVhdDirectory()
+    })
+  }
+
   fetchPartitionFiles(diskId, partitionId, paths) {
     const { promise, reject, resolve } = pDefer()
     Disposable.use(
@@ -254,7 +230,7 @@ class RemoteAdapter {
   async deleteDeltaVmBackups(backups) {
     const handler = this._handler
 
-    // unused VHDs will be detected by `cleanVm`
+    // this will delete the json, unused VHDs will be detected by `cleanVm`
     await asyncMapSettled(backups, ({ _filename }) => handler.unlink(_filename))
   }
 
@@ -285,17 +261,40 @@ class RemoteAdapter {
     )
   }
 
-  async deleteVmBackup(filename) {
-    const metadata = JSON.parse(String(await this._handler.readFile(filename)))
-    metadata._filename = filename
+  deleteVmBackup(file) {
+    return this.deleteVmBackups([file])
+  }
 
-    if (metadata.mode === 'delta') {
-      await this.deleteDeltaVmBackups([metadata])
-    } else if (metadata.mode === 'full') {
-      await this.deleteFullVmBackups([metadata])
-    } else {
-      throw new Error(`no deleter for backup mode ${metadata.mode}`)
+  async deleteVmBackups(files) {
+    const { delta, full, ...others } = groupBy(await asyncMap(files, file => this.readVmBackupMetadata(file)), 'mode')
+
+    const unsupportedModes = Object.keys(others)
+    if (unsupportedModes.length !== 0) {
+      throw new Error('no deleter for backup modes: ' + unsupportedModes.join(', '))
     }
+
+    await Promise.all([
+      delta !== undefined && this.deleteDeltaVmBackups(delta),
+      full !== undefined && this.deleteFullVmBackups(full),
+    ])
+
+    const dirs = new Set(files.map(file => dirname(file)))
+    for (const dir of dirs) {
+      // don't merge in main process, unused VHDs will be merged in the next backup run
+      await this.cleanVm(dir, { remove: true, onLog: warn })
+    }
+  }
+
+  #getCompressionType() {
+    return this._vhdDirectoryCompression
+  }
+
+  #useVhdDirectory() {
+    return this.handler.type === 's3'
+  }
+
+  #useAlias() {
+    return this.#useVhdDirectory()
   }
 
   getDisk = Disposable.factory(this.getDisk)
@@ -354,13 +353,26 @@ class RemoteAdapter {
     return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId))
   }
 
+  // if we use alias on this remote, we have to name the file alias.vhd
+  getVhdFileName(baseName) {
+    if (this.#useAlias()) {
+      return `${baseName}.alias.vhd`
+    }
+    return `${baseName}.vhd`
+  }
+
   async listAllVmBackups() {
     const handler = this._handler
 
     const backups = { __proto__: null }
-    await asyncMap(await handler.list(BACKUP_DIR), async vmUuid => {
-      const vmBackups = await this.listVmBackups(vmUuid)
-      backups[vmUuid] = vmBackups
+    await asyncMap(await handler.list(BACKUP_DIR), async entry => {
+      // ignore hidden and lock files
+      if (entry[0] !== '.' && !entry.endsWith('.lock')) {
+        const vmBackups = await this.listVmBackups(entry)
+        if (vmBackups.length !== 0) {
+          backups[entry] = vmBackups
+        }
+      }
     })
 
     return backups
@@ -498,6 +510,25 @@ class RemoteAdapter {
     return backups.sort(compareTimestamp)
   }
 
+  async writeVhd(path, input, { checksum = true, validator = noop } = {}) {
+    const handler = this._handler
+
+    if (this.#useVhdDirectory()) {
+      const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
+      await createVhdDirectoryFromStream(handler, dataPath, input, {
+        concurrency: 16,
+        compression: this.#getCompressionType(),
+        async validator() {
+          await input.task
+          return validator.apply(this, arguments)
+        },
+      })
+      await VhdAbstract.createAlias(handler, path, dataPath)
+    } else {
+      await this.outputStream(path, input, { checksum, validator })
+    }
+  }
+
   async outputStream(path, input, { checksum = true, validator = noop } = {}) {
     await this._handler.outputStream(path, input, {
       checksum,
@@ -509,6 +540,52 @@ class RemoteAdapter {
     })
   }
 
+  async _createSyntheticStream(handler, paths) {
+    let disposableVhds = []
+
+    // if it's a path : open all hierarchy of parent
+    if (typeof paths === 'string') {
+      let vhd,
+        vhdPath = paths
+      do {
+        const disposable = await openVhd(handler, vhdPath)
+        vhd = disposable.value
+        disposableVhds.push(disposable)
+        vhdPath = resolveRelativeFromFile(vhdPath, vhd.header.parentUnicodeName)
+      } while (vhd.footer.diskType !== Constants.DISK_TYPES.DYNAMIC)
+    } else {
+      // only open the list of path given
+      disposableVhds = paths.map(path => openVhd(handler, path))
+    }
+
+    // I don't want the vhds to be disposed on return
+    // but only when the stream is done ( or failed )
+    const disposables = await Disposable.all(disposableVhds)
+    const vhds = disposables.value
+
+    let disposed = false
+    const disposeOnce = async () => {
+      if (!disposed) {
+        disposed = true
+
+        try {
+          await disposables.dispose()
+        } catch (error) {
+          warn('_createSyntheticStream: failed to dispose VHDs', { error })
+        }
+      }
+    }
+
+    const synthetic = new VhdSynthetic(vhds)
+    await synthetic.readHeaderAndFooter()
+    await synthetic.readBlockAllocationTable()
+    const stream = await synthetic.stream()
+    stream.on('end', disposeOnce)
+    stream.on('close', disposeOnce)
+    stream.on('error', disposeOnce)
+    return stream
+  }
+
   async readDeltaVmBackup(metadata) {
     const handler = this._handler
     const { vbds, vdis, vhds, vifs, vm } = metadata
@@ -516,7 +593,7 @@ class RemoteAdapter {
 
     const streams = {}
     await asyncMapSettled(Object.keys(vdis), async id => {
-      streams[`${id}.vhd`] = await createSyntheticStream(handler, join(dir, vhds[id]))
+      streams[`${id}.vhd`] = await this._createSyntheticStream(handler, join(dir, vhds[id]))
     })
 
     return {

@xen-orchestra/backups/_VmBackup.js

@@ -36,6 +36,11 @@ const forkDeltaExport = deltaExport =>
 
 exports.VmBackup = class VmBackup {
   constructor({ config, getSnapshotNameLabel, job, remoteAdapters, remotes, schedule, settings, srs, vm }) {
+    if (vm.other_config['xo:backup:job'] === job.id) {
+      // otherwise replicated VMs would be matched and replicated again and again
+      throw new Error('cannot backup a VM created by this very job')
+    }
+
     this.config = config
     this.job = job
     this.remoteAdapters = remoteAdapters
@@ -246,6 +251,20 @@ exports.VmBackup = class VmBackup {
 
     const timestamp = Date.now()
 
+    const progress = {
+      handle: setInterval(() => {
+        const { size } = sizeContainer
+        const timestamp = Date.now()
+        Task.info('transfer', {
+          speed: (size - progress.size) / (timestamp - progress.timestamp),
+        })
+        progress.size = size
+        progress.timestamp = timestamp
+      }, 5e3 * 60),
+      size: sizeContainer.size,
+      timestamp,
+    }
+
     await this._callWriters(
       writer =>
         writer.run({
@@ -256,6 +275,8 @@ exports.VmBackup = class VmBackup {
       'writer.run()'
     )
 
+    clearInterval(progress.handle)
+
     const { size } = sizeContainer
     const end = Date.now()
     const duration = end - timestamp
@@ -333,13 +354,16 @@ exports.VmBackup = class VmBackup {
 
     const baseUuidToSrcVdi = new Map()
     await asyncMap(await baseVm.$getDisks(), async baseRef => {
-      const snapshotOf = await xapi.getField('VDI', baseRef, 'snapshot_of')
+      const [baseUuid, snapshotOf] = await Promise.all([
+        xapi.getField('VDI', baseRef, 'uuid'),
+        xapi.getField('VDI', baseRef, 'snapshot_of'),
+      ])
       const srcVdi = srcVdis[snapshotOf]
       if (srcVdi !== undefined) {
-        baseUuidToSrcVdi.set(await xapi.getField('VDI', baseRef, 'uuid'), srcVdi)
+        baseUuidToSrcVdi.set(baseUuid, srcVdi)
       } else {
-        debug('no base VDI found', {
-          vdi: srcVdi.uuid,
+        debug('ignore snapshot VDI because no longer present on VM', {
+          vdi: baseUuid,
         })
       }
     })
@@ -351,6 +375,11 @@ exports.VmBackup = class VmBackup {
       false
     )
 
+    if (presentBaseVdis.size === 0) {
+      debug('no base VM found')
+      return
+    }
+
     const fullVdisRequired = new Set()
     baseUuidToSrcVdi.forEach((srcVdi, baseUuid) => {
       if (presentBaseVdis.has(baseUuid)) {

@xen-orchestra/backups/_backupWorker.js

@@ -70,6 +70,7 @@ class BackupWorker {
       yield new RemoteAdapter(handler, {
         debounceResource: this.debounceResource,
         dirMode: this.#config.dirMode,
+        vhdDirectoryCompression: this.#config.vhdDirectoryCompression,
       })
     } finally {
       await handler.forget()

@xen-orchestra/backups/_cleanVm.integ.spec.js

@@ -0,0 +1,437 @@
+/* eslint-env jest */
+
+const rimraf = require('rimraf')
+const tmp = require('tmp')
+const fs = require('fs-extra')
+const { getHandler } = require('@xen-orchestra/fs')
+const { pFromCallback } = require('promise-toolbox')
+const crypto = require('crypto')
+const { RemoteAdapter } = require('./RemoteAdapter')
+const { VHDFOOTER, VHDHEADER } = require('./tests.fixtures.js')
+const { VhdFile, Constants, VhdDirectory, VhdAbstract } = require('vhd-lib')
+const { checkAliases } = require('./_cleanVm')
+const { dirname, basename } = require('path')
+
+let tempDir, adapter, handler, jobId, vdiId, basePath
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+  handler = getHandler({ url: `file://${tempDir}` })
+  await handler.sync()
+  adapter = new RemoteAdapter(handler)
+  jobId = uniqueId()
+  vdiId = uniqueId()
+  basePath = `vdis/${jobId}/${vdiId}`
+  await fs.mkdirp(`${tempDir}/${basePath}`)
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+  await handler.forget()
+})
+
+const uniqueId = () => crypto.randomBytes(16).toString('hex')
+
+async function generateVhd(path, opts = {}) {
+  let vhd
+
+  let dataPath = path
+  if (opts.useAlias) {
+    await handler.mkdir(dirname(path) + '/data/')
+    dataPath = dirname(path) + '/data/' + basename(path)
+  }
+  if (opts.mode === 'directory') {
+    await handler.mkdir(dataPath)
+    vhd = new VhdDirectory(handler, dataPath)
+  } else {
+    const fd = await handler.openFile(dataPath, 'wx')
+    vhd = new VhdFile(handler, fd)
+  }
+
+  vhd.header = { ...VHDHEADER, ...opts.header }
+  vhd.footer = { ...VHDFOOTER, ...opts.footer }
+  vhd.footer.uuid = Buffer.from(crypto.randomBytes(16))
+
+  if (vhd.header.parentUnicodeName) {
+    vhd.footer.diskType = Constants.DISK_TYPES.DIFFERENCING
+  } else {
+    vhd.footer.diskType = Constants.DISK_TYPES.DYNAMIC
+  }
+
+  if (opts.useAlias === true) {
+    await VhdAbstract.createAlias(handler, path + '.alias.vhd', dataPath)
+  }
+
+  await vhd.writeBlockAllocationTable()
+  await vhd.writeHeader()
+  await vhd.writeFooter()
+  return vhd
+}
+
+test('It remove broken vhd', async () => {
+  // todo also tests a directory and an alias
+
+  await handler.writeFile(`${basePath}/notReallyAVhd.vhd`, 'I AM NOT A VHD')
+  expect((await handler.list(basePath)).length).toEqual(1)
+  let loggued = ''
+  const onLog = message => {
+    loggued += message
+  }
+  await adapter.cleanVm('/', { remove: false, onLog })
+  expect(loggued).toEqual(`error while checking the VHD with path /${basePath}/notReallyAVhd.vhd`)
+  // not removed
+  expect((await handler.list(basePath)).length).toEqual(1)
+  // really remove it
+  await adapter.cleanVm('/', { remove: true, onLog })
+  expect((await handler.list(basePath)).length).toEqual(0)
+})
+
+test('it remove vhd with missing or multiple ancestors', async () => {
+  // one with a broken parent
+  await generateVhd(`${basePath}/abandonned.vhd`, {
+    header: {
+      parentUnicodeName: 'gone.vhd',
+      parentUid: Buffer.from(crypto.randomBytes(16)),
+    },
+  })
+
+  // one orphan, which is a full vhd, no parent
+  const orphan = await generateVhd(`${basePath}/orphan.vhd`)
+  // a child to the orphan
+  await generateVhd(`${basePath}/child.vhd`, {
+    header: {
+      parentUnicodeName: 'orphan.vhd',
+      parentUid: orphan.footer.uuid,
+    },
+  })
+
+  // clean
+  let loggued = ''
+  const onLog = message => {
+    loggued += message + '\n'
+  }
+  await adapter.cleanVm('/', { remove: true, onLog })
+
+  const deletedOrphanVhd = loggued.match(/deleting orphan VHD/g) || []
+  expect(deletedOrphanVhd.length).toEqual(1) // only one vhd should have been deleted
+  const deletedAbandonnedVhd = loggued.match(/abandonned.vhd is missing/g) || []
+  expect(deletedAbandonnedVhd.length).toEqual(1) // and it must be abandonned.vhd
+
+  // we don't test the filew on disk, since they will all be marker as unused and deleted without a metadata.json file
+})
+
+test('it remove backup meta data referencing a missing vhd in delta backup', async () => {
+  // create a metadata file marking child and orphan as ok
+  await handler.writeFile(
+    `metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      vhds: [
+        `${basePath}/orphan.vhd`,
+        `${basePath}/child.vhd`,
+        // abandonned.json is not here
+      ],
+    })
+  )
+
+  await generateVhd(`${basePath}/abandonned.vhd`)
+
+  // one orphan, which is a full vhd, no parent
+  const orphan = await generateVhd(`${basePath}/orphan.vhd`)
+
+  // a child to the orphan
+  await generateVhd(`${basePath}/child.vhd`, {
+    header: {
+      parentUnicodeName: 'orphan.vhd',
+      parentUid: orphan.footer.uuid,
+    },
+  })
+
+  let loggued = ''
+  const onLog = message => {
+    loggued += message + '\n'
+  }
+  await adapter.cleanVm('/', { remove: true, onLog })
+  let matched = loggued.match(/deleting unused VHD /g) || []
+  expect(matched.length).toEqual(1) // only one vhd should have been deleted
+  matched = loggued.match(/abandonned.vhd is unused/g) || []
+  expect(matched.length).toEqual(1) // and it must be abandonned.vhd
+
+  // a missing vhd cause clean to remove all vhds
+  await handler.writeFile(
+    `metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      vhds: [
+        `${basePath}/deleted.vhd`, // in metadata but not in vhds
+        `${basePath}/orphan.vhd`,
+        `${basePath}/child.vhd`,
+        // abandonned.vhd is not here anymore
+      ],
+    }),
+    { flags: 'w' }
+  )
+  loggued = ''
+  await adapter.cleanVm('/', { remove: true, onLog })
+  matched = loggued.match(/deleting unused VHD /g) || []
+  expect(matched.length).toEqual(2) // all vhds (orphan and  child  ) should have been deleted
+})
+
+test('it merges delta of non destroyed chain', async () => {
+  await handler.writeFile(
+    `metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      size: 12000, // a size too small
+      vhds: [
+        `${basePath}/grandchild.vhd`, // grand child should not be merged
+        `${basePath}/child.vhd`,
+        // orphan is not here, he should be merged in child
+      ],
+    })
+  )
+
+  // one orphan, which is a full vhd, no parent
+  const orphan = await generateVhd(`${basePath}/orphan.vhd`)
+  // a child to the orphan
+  const child = await generateVhd(`${basePath}/child.vhd`, {
+    header: {
+      parentUnicodeName: 'orphan.vhd',
+      parentUid: orphan.footer.uuid,
+    },
+  })
+  // a grand child
+  await generateVhd(`${basePath}/grandchild.vhd`, {
+    header: {
+      parentUnicodeName: 'child.vhd',
+      parentUid: child.footer.uuid,
+    },
+  })
+
+  let loggued = []
+  const onLog = message => {
+    loggued.push(message)
+  }
+  await adapter.cleanVm('/', { remove: true, onLog })
+  expect(loggued[0]).toEqual(`the parent /${basePath}/orphan.vhd of the child /${basePath}/child.vhd is unused`)
+  expect(loggued[1]).toEqual(`incorrect size in metadata: 12000 instead of 209920`)
+
+  loggued = []
+  await adapter.cleanVm('/', { remove: true, merge: true, onLog })
+  const [unused, merging] = loggued
+  expect(unused).toEqual(`the parent /${basePath}/orphan.vhd of the child /${basePath}/child.vhd is unused`)
+  expect(merging).toEqual(`merging /${basePath}/child.vhd into /${basePath}/orphan.vhd`)
+
+  const metadata = JSON.parse(await handler.readFile(`metadata.json`))
+  // size should be the size of children + grand children after the merge
+  expect(metadata.size).toEqual(209920)
+
+  // merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
+  // only check deletion
+  const remainingVhds = await handler.list(basePath)
+  expect(remainingVhds.length).toEqual(2)
+  expect(remainingVhds.includes('child.vhd')).toEqual(true)
+  expect(remainingVhds.includes('grandchild.vhd')).toEqual(true)
+})
+
+test('it finish unterminated merge ', async () => {
+  await handler.writeFile(
+    `metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      size: 209920,
+      vhds: [`${basePath}/orphan.vhd`, `${basePath}/child.vhd`],
+    })
+  )
+
+  // one orphan, which is a full vhd, no parent
+  const orphan = await generateVhd(`${basePath}/orphan.vhd`)
+  // a child to the orphan
+  const child = await generateVhd(`${basePath}/child.vhd`, {
+    header: {
+      parentUnicodeName: 'orphan.vhd',
+      parentUid: orphan.footer.uuid,
+    },
+  })
+  // a merge in progress file
+  await handler.writeFile(
+    `${basePath}/.orphan.vhd.merge.json`,
+    JSON.stringify({
+      parent: {
+        header: orphan.header.checksum,
+      },
+      child: {
+        header: child.header.checksum,
+      },
+    })
+  )
+
+  await adapter.cleanVm('/', { remove: true, merge: true })
+  // merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
+
+  // only check deletion
+  const remainingVhds = await handler.list(basePath)
+  expect(remainingVhds.length).toEqual(1)
+  expect(remainingVhds.includes('child.vhd')).toEqual(true)
+})
+
+// each of the vhd can be a file, a directory, an alias to a file or an alias to a directory
+// the message an resulting files should be identical to the output with vhd files which is tested independantly
+
+describe('tests multiple combination ', () => {
+  for (const useAlias of [true, false]) {
+    for (const vhdMode of ['file', 'directory']) {
+      test(`alias : ${useAlias}, mode: ${vhdMode}`, async () => {
+        // a broken VHD
+        if (useAlias) {
+          await handler.mkdir(basePath + '/data')
+        }
+
+        const brokenVhdDataPath = basePath + (useAlias ? '/data/broken.vhd' : '/broken.vhd')
+
+        if (vhdMode === 'directory') {
+          await handler.mkdir(brokenVhdDataPath)
+        } else {
+          await handler.writeFile(brokenVhdDataPath, 'notreallyavhd')
+        }
+        if (useAlias) {
+          await VhdAbstract.createAlias(handler, 'broken.alias.vhd', brokenVhdDataPath)
+        }
+
+        // a vhd non referenced in metada
+        await generateVhd(`${basePath}/nonreference.vhd`, { useAlias, mode: vhdMode })
+        // an abandonded delta vhd without its parent
+        await generateVhd(`${basePath}/abandonned.vhd`, {
+          useAlias,
+          mode: vhdMode,
+          header: {
+            parentUnicodeName: 'gone.vhd',
+            parentUid: crypto.randomBytes(16),
+          },
+        })
+
+        // an ancestor of a vhd present in metadata
+        const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, {
+          useAlias,
+          mode: vhdMode,
+        })
+        const child = await generateVhd(`${basePath}/child.vhd`, {
+          useAlias,
+          mode: vhdMode,
+          header: {
+            parentUnicodeName: 'ancestor.vhd' + (useAlias ? '.alias.vhd' : ''),
+            parentUid: ancestor.footer.uuid,
+          },
+        })
+        // a grand child  vhd in metadata
+        await generateVhd(`${basePath}/grandchild.vhd`, {
+          useAlias,
+          mode: vhdMode,
+          header: {
+            parentUnicodeName: 'child.vhd' + (useAlias ? '.alias.vhd' : ''),
+            parentUid: child.footer.uuid,
+          },
+        })
+
+        // an older parent that was merging in clean
+        const cleanAncestor = await generateVhd(`${basePath}/cleanAncestor.vhd`, {
+          useAlias,
+          mode: vhdMode,
+        })
+        // a clean  vhd in metadata
+        const clean = await generateVhd(`${basePath}/clean.vhd`, {
+          useAlias,
+          mode: vhdMode,
+          header: {
+            parentUnicodeName: 'cleanAncestor.vhd' + (useAlias ? '.alias.vhd' : ''),
+            parentUid: cleanAncestor.footer.uuid,
+          },
+        })
+
+        await handler.writeFile(
+          `${basePath}/.cleanAncestor.vhd${useAlias ? '.alias.vhd' : ''}.merge.json`,
+          JSON.stringify({
+            parent: {
+              header: cleanAncestor.header.checksum,
+            },
+            child: {
+              header: clean.header.checksum,
+            },
+          })
+        )
+
+        // the metadata file
+        await handler.writeFile(
+          `metadata.json`,
+          JSON.stringify({
+            mode: 'delta',
+            vhds: [
+              `${basePath}/grandchild.vhd` + (useAlias ? '.alias.vhd' : ''), // grand child should not be merged
+              `${basePath}/child.vhd` + (useAlias ? '.alias.vhd' : ''),
+              `${basePath}/clean.vhd` + (useAlias ? '.alias.vhd' : ''),
+            ],
+          })
+        )
+
+        await adapter.cleanVm('/', { remove: true, merge: true })
+
+        const metadata = JSON.parse(await handler.readFile(`metadata.json`))
+        // size should be the size of children + grand children + clean after the merge
+        expect(metadata.size).toEqual(vhdMode === 'file' ? 314880 : undefined)
+
+        // broken vhd, non referenced, abandonned should be deleted ( alias and data)
+        // ancestor and child should be merged
+        // grand child and clean vhd should not have changed
+        const survivors = await handler.list(basePath)
+        // console.log(survivors)
+        if (useAlias) {
+          const dataSurvivors = await handler.list(basePath + '/data')
+          // the goal of the alias : do not move a full folder
+          expect(dataSurvivors).toContain('ancestor.vhd')
+          expect(dataSurvivors).toContain('grandchild.vhd')
+          expect(dataSurvivors).toContain('cleanAncestor.vhd')
+          expect(survivors).toContain('clean.vhd.alias.vhd')
+          expect(survivors).toContain('child.vhd.alias.vhd')
+          expect(survivors).toContain('grandchild.vhd.alias.vhd')
+          expect(survivors.length).toEqual(4) // the 3 ok + data
+          expect(dataSurvivors.length).toEqual(3) // the 3 ok + data
+        } else {
+          expect(survivors).toContain('clean.vhd')
+          expect(survivors).toContain('child.vhd')
+          expect(survivors).toContain('grandchild.vhd')
+          expect(survivors.length).toEqual(3)
+        }
+      })
+    }
+  }
+})
+
+test('it cleans orphan merge states ', async () => {
+  await handler.writeFile(`${basePath}/.orphan.vhd.merge.json`, '')
+
+  await adapter.cleanVm('/', { remove: true })
+
+  expect(await handler.list(basePath)).toEqual([])
+})
+
+test('check Aliases should work alone', async () => {
+  await handler.mkdir('vhds')
+  await handler.mkdir('vhds/data')
+  await generateVhd(`vhds/data/ok.vhd`)
+  await VhdAbstract.createAlias(handler, 'vhds/ok.alias.vhd', 'vhds/data/ok.vhd')
+
+  await VhdAbstract.createAlias(handler, 'vhds/missingData.alias.vhd', 'vhds/data/nonexistent.vhd')
+
+  await generateVhd(`vhds/data/missingalias.vhd`)
+
+  await checkAliases(['vhds/missingData.alias.vhd', 'vhds/ok.alias.vhd'], 'vhds/data', { remove: true, handler })
+
+  // only ok have suvived
+  const alias = (await handler.list('vhds')).filter(f => f.endsWith('.vhd'))
+  expect(alias.length).toEqual(1)
+
+  const data = await handler.list('vhds/data')
+  expect(data.length).toEqual(1)
+})

@xen-orchestra/backups/_cleanVm.js

@@ -1,13 +1,33 @@
 const assert = require('assert')
 const sum = require('lodash/sum')
 const { asyncMap } = require('@xen-orchestra/async-map')
-const { VhdFile, mergeVhd } = require('vhd-lib')
+const { Constants, mergeVhd, openVhd, VhdAbstract, VhdFile } = require('vhd-lib')
+const { isVhdAlias, resolveVhdAlias } = require('vhd-lib/aliases')
 const { dirname, resolve } = require('path')
-const { DISK_TYPE_DIFFERENCING } = require('vhd-lib/dist/_constants.js')
+const { DISK_TYPES } = Constants
 const { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } = require('./_backupType.js')
 const { limitConcurrency } = require('limit-concurrency-decorator')
 
 const { Task } = require('./Task.js')
+const { Disposable } = require('promise-toolbox')
+
+// checking the size of a vhd directory is costly
+// 1 Http Query per 1000 blocks
+// we only check size of all the vhd are VhdFiles
+function shouldComputeVhdsSize(vhds) {
+  return vhds.every(vhd => vhd instanceof VhdFile)
+}
+
+const computeVhdsSize = (handler, vhdPaths) =>
+  Disposable.use(
+    vhdPaths.map(vhdPath => openVhd(handler, vhdPath)),
+    async vhds => {
+      if (shouldComputeVhdsSize(vhds)) {
+        const sizes = await asyncMap(vhds, vhd => vhd.getSize())
+        return sum(sizes)
+      }
+    }
+  )
 
 // chain is an array of VHDs from child to parent
 //
@@ -63,14 +83,13 @@ async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
     )
 
     clearInterval(handle)
-
     await Promise.all([
-      handler.rename(parent, child),
+      VhdAbstract.rename(handler, parent, child),
       asyncMap(children.slice(0, -1), child => {
         onLog(`the VHD ${child} is unused`)
         if (remove) {
           onLog(`deleting unused VHD ${child}`)
-          return handler.unlink(child)
+          return VhdAbstract.unlink(handler, child)
         }
       }),
     ])
@@ -81,10 +100,11 @@ async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
 
 const noop = Function.prototype
 
-const INTERRUPTED_VHDS_REG = /^(?:(.+)\/)?\.(.+)\.merge.json$/
+const INTERRUPTED_VHDS_REG = /^\.(.+)\.merge.json$/
 const listVhds = async (handler, vmDir) => {
-  const vhds = []
-  const interruptedVhds = new Set()
+  const vhds = new Set()
+  const aliases = {}
+  const interruptedVhds = new Map()
 
   await asyncMap(
     await handler.list(`${vmDir}/vdis`, {
@@ -99,25 +119,77 @@ const listVhds = async (handler, vmDir) => {
         async vdiDir => {
           const list = await handler.list(vdiDir, {
             filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file),
-            prependDir: true,
           })
-
+          aliases[vdiDir] = list.filter(vhd => isVhdAlias(vhd)).map(file => `${vdiDir}/${file}`)
           list.forEach(file => {
             const res = INTERRUPTED_VHDS_REG.exec(file)
             if (res === null) {
-              vhds.push(file)
+              vhds.add(`${vdiDir}/${file}`)
             } else {
-              const [, dir, file] = res
-              interruptedVhds.add(`${dir}/${file}`)
+              interruptedVhds.set(`${vdiDir}/${res[1]}`, `${vdiDir}/${file}`)
             }
           })
         }
       )
   )
 
-  return { vhds, interruptedVhds }
+  return { vhds, interruptedVhds, aliases }
 }
 
+async function checkAliases(aliasPaths, targetDataRepository, { handler, onLog = noop, remove = false }) {
+  const aliasFound = []
+  for (const path of aliasPaths) {
+    const target = await resolveVhdAlias(handler, path)
+
+    if (!isVhdFile(target)) {
+      onLog(`Alias ${path} references a non vhd target:  ${target}`)
+      if (remove) {
+        await handler.unlink(target)
+        await handler.unlink(path)
+      }
+      continue
+    }
+
+    try {
+      const { dispose } = await openVhd(handler, target)
+      try {
+        await dispose()
+      } catch (e) {
+        // error during dispose should not trigger a deletion
+      }
+    } catch (error) {
+      onLog(`target ${target} of alias ${path} is missing or broken`, { error })
+      if (remove) {
+        try {
+          await VhdAbstract.unlink(handler, path)
+        } catch (e) {
+          if (e.code !== 'ENOENT') {
+            onLog(`Error while deleting target ${target} of alias ${path}`, { error: e })
+          }
+        }
+      }
+      continue
+    }
+
+    aliasFound.push(resolve('/', target))
+  }
+
+  const entries = await handler.list(targetDataRepository, {
+    ignoreMissing: true,
+    prependDir: true,
+  })
+
+  entries.forEach(async entry => {
+    if (!aliasFound.includes(entry)) {
+      onLog(`the Vhd  ${entry} is not referenced by a an alias`)
+      if (remove) {
+        await VhdAbstract.unlink(handler, entry)
+      }
+    }
+  })
+}
+exports.checkAliases = checkAliases
+
 const defaultMergeLimiter = limitConcurrency(1)
 
 exports.cleanVm = async function cleanVm(
@@ -128,62 +200,85 @@ exports.cleanVm = async function cleanVm(
 
   const handler = this._handler
 
-  const vhds = new Set()
+  const vhdsToJSons = new Set()
   const vhdParents = { __proto__: null }
   const vhdChildren = { __proto__: null }
 
-  const vhdsList = await listVhds(handler, vmDir)
+  const { vhds, interruptedVhds, aliases } = await listVhds(handler, vmDir)
 
   // remove broken VHDs
-  await asyncMap(vhdsList.vhds, async path => {
+  await asyncMap(vhds, async path => {
     try {
-      const vhd = new VhdFile(handler, path)
-      await vhd.readHeaderAndFooter(!vhdsList.interruptedVhds.has(path))
-      vhds.add(path)
-      if (vhd.footer.diskType === DISK_TYPE_DIFFERENCING) {
-        const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
-        vhdParents[path] = parent
-        if (parent in vhdChildren) {
-          const error = new Error('this script does not support multiple VHD children')
-          error.parent = parent
-          error.child1 = vhdChildren[parent]
-          error.child2 = path
-          throw error // should we throw?
+      await Disposable.use(openVhd(handler, path, { checkSecondFooter: !interruptedVhds.has(path) }), vhd => {
+        if (vhd.footer.diskType === DISK_TYPES.DIFFERENCING) {
+          const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
+          vhdParents[path] = parent
+          if (parent in vhdChildren) {
+            const error = new Error('this script does not support multiple VHD children')
+            error.parent = parent
+            error.child1 = vhdChildren[parent]
+            error.child2 = path
+            throw error // should we throw?
+          }
+          vhdChildren[parent] = path
         }
-        vhdChildren[parent] = path
-      }
+      })
     } catch (error) {
+      vhds.delete(path)
       onLog(`error while checking the VHD with path ${path}`, { error })
       if (error?.code === 'ERR_ASSERTION' && remove) {
         onLog(`deleting broken ${path}`)
-        await handler.unlink(path)
+        return VhdAbstract.unlink(handler, path)
       }
     }
   })
 
+  // remove interrupted merge states for missing VHDs
+  for (const interruptedVhd of interruptedVhds.keys()) {
+    if (!vhds.has(interruptedVhd)) {
+      const statePath = interruptedVhds.get(interruptedVhd)
+      interruptedVhds.delete(interruptedVhd)
+
+      onLog('orphan merge state', {
+        mergeStatePath: statePath,
+        missingVhdPath: interruptedVhd,
+      })
+      if (remove) {
+        onLog(`deleting orphan merge state ${statePath}`)
+        await handler.unlink(statePath)
+      }
+    }
+  }
+
+  // check if alias are correct
+  // check if all vhd in data subfolder have a corresponding alias
+  await asyncMap(Object.keys(aliases), async dir => {
+    await checkAliases(aliases[dir], `${dir}/data`, { handler, onLog, remove })
+  })
+
   // remove VHDs with missing ancestors
   {
     const deletions = []
 
     // return true if the VHD has been deleted or is missing
-    const deleteIfOrphan = vhd => {
-      const parent = vhdParents[vhd]
+    const deleteIfOrphan = vhdPath => {
+      const parent = vhdParents[vhdPath]
       if (parent === undefined) {
         return
       }
 
       // no longer needs to be checked
-      delete vhdParents[vhd]
+      delete vhdParents[vhdPath]
 
       deleteIfOrphan(parent)
 
       if (!vhds.has(parent)) {
-        vhds.delete(vhd)
+        vhds.delete(vhdPath)
 
-        onLog(`the parent ${parent} of the VHD ${vhd} is missing`)
+        onLog(`the parent ${parent} of the VHD ${vhdPath} is missing`)
         if (remove) {
-          onLog(`deleting orphan VHD ${vhd}`)
-          deletions.push(handler.unlink(vhd))
+          onLog(`deleting orphan VHD ${vhdPath}`)
+          deletions.push(VhdAbstract.unlink(handler, vhdPath))
         }
       }
     }
@@ -199,7 +294,7 @@ exports.cleanVm = async function cleanVm(
     await Promise.all(deletions)
   }
 
-  const jsons = []
+  const jsons = new Set()
   const xvas = new Set()
   const xvaSums = []
   const entries = await handler.list(vmDir, {
@@ -207,7 +302,7 @@ exports.cleanVm = async function cleanVm(
   })
   entries.forEach(path => {
     if (isMetadataFile(path)) {
-      jsons.push(path)
+      jsons.add(path)
     } else if (isXvaFile(path)) {
       xvas.add(path)
     } else if (isXvaSumFile(path)) {
@@ -229,22 +324,25 @@ exports.cleanVm = async function cleanVm(
   // compile the list of unused XVAs and VHDs, and remove backup metadata which
   // reference a missing XVA/VHD
   await asyncMap(jsons, async json => {
-    const metadata = JSON.parse(await handler.readFile(json))
+    let metadata
+    try {
+      metadata = JSON.parse(await handler.readFile(json))
+    } catch (error) {
+      onLog(`failed to read metadata file ${json}`, { error })
+      jsons.delete(json)
+      return
+    }
+
     const { mode } = metadata
-    let size
     if (mode === 'full') {
       const linkedXva = resolve('/', vmDir, metadata.xva)
-
       if (xvas.has(linkedXva)) {
         unusedXvas.delete(linkedXva)
-
-        size = await handler.getSize(linkedXva).catch(error => {
-          onLog(`failed to get size of ${json}`, { error })
-        })
       } else {
         onLog(`the XVA linked to the metadata ${json} is missing`)
         if (remove) {
           onLog(`deleting incomplete backup ${json}`)
+          jsons.delete(json)
           await handler.unlink(json)
         }
       }
@@ -254,38 +352,24 @@ exports.cleanVm = async function cleanVm(
         return Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
       })()
 
+      const missingVhds = linkedVhds.filter(_ => !vhds.has(_))
+
       // FIXME: find better approach by keeping as much of the backup as
       // possible (existing disks) even if one disk is missing
-      if (linkedVhds.every(_ => vhds.has(_))) {
+      if (missingVhds.length === 0) {
         linkedVhds.forEach(_ => unusedVhds.delete(_))
-
-        size = await asyncMap(linkedVhds, vhd => handler.getSize(vhd)).then(sum, error => {
-          onLog(`failed to get size of ${json}`, { error })
+        linkedVhds.forEach(path => {
+          vhdsToJSons[path] = json
         })
       } else {
-        onLog(`Some VHDs linked to the metadata ${json} are missing`)
+        onLog(`Some VHDs linked to the metadata ${json} are missing`, { missingVhds })
         if (remove) {
           onLog(`deleting incomplete backup ${json}`)
+          jsons.delete(json)
           await handler.unlink(json)
         }
       }
     }
-
-    const metadataSize = metadata.size
-    if (size !== undefined && metadataSize !== size) {
-      onLog(`incorrect size in metadata: ${metadataSize ?? 'none'} instead of ${size}`)
-
-      // don't update if the the stored size is greater than found files,
-      // it can indicates a problem
-      if (fixMetadata && (metadataSize === undefined || metadataSize < size)) {
-        try {
-          metadata.size = size
-          await handler.writeFile(json, JSON.stringify(metadata), { flags: 'w' })
-        } catch (error) {
-          onLog(`failed to update size in backup metadata ${json}`, { error })
-        }
-      }
-    }
   })
 
   // TODO: parallelize by vm/job/vdi
@@ -324,7 +408,7 @@ exports.cleanVm = async function cleanVm(
       onLog(`the VHD ${vhd} is unused`)
       if (remove) {
         onLog(`deleting unused VHD ${vhd}`)
-        unusedVhdsDeletion.push(handler.unlink(vhd))
+        unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))
       }
     }
 
@@ -333,9 +417,9 @@ exports.cleanVm = async function cleanVm(
     })
 
     // merge interrupted VHDs
-    vhdsList.interruptedVhds.forEach(parent => {
+    for (const parent of interruptedVhds.keys()) {
       vhdChainsToMerge[parent] = [vhdChildren[parent], parent]
-    })
+    }
 
     Object.values(vhdChainsToMerge).forEach(chain => {
       if (chain !== undefined) {
@@ -344,9 +428,15 @@ exports.cleanVm = async function cleanVm(
     })
   }
 
-  const doMerge = () => {
-    const promise = asyncMap(toMerge, async chain => limitedMergeVhdChain(chain, { handler, onLog, remove, merge }))
-    return merge ? promise.then(sizes => ({ size: sum(sizes) })) : promise
+  const metadataWithMergedVhd = {}
+  const doMerge = async () => {
+    await asyncMap(toMerge, async chain => {
+      const merged = await limitedMergeVhdChain(chain, { handler, onLog, remove, merge })
+      if (merged !== undefined) {
+        const metadataPath = vhdsToJSons[chain[0]] // all the chain should have the same metada file
+        metadataWithMergedVhd[metadataPath] = true
+      }
+    })
   }
 
   await Promise.all([
@@ -371,6 +461,52 @@ exports.cleanVm = async function cleanVm(
     }),
   ])
 
+  // update size for delta metadata with merged VHD
+  // check for the other that the size is the same as the real file size
+
+  await asyncMap(jsons, async metadataPath => {
+    const metadata = JSON.parse(await handler.readFile(metadataPath))
+
+    let fileSystemSize
+    const merged = metadataWithMergedVhd[metadataPath] !== undefined
+
+    const { mode, size, vhds, xva } = metadata
+
+    try {
+      if (mode === 'full') {
+        // a full backup : check size
+        const linkedXva = resolve('/', vmDir, xva)
+        fileSystemSize = await handler.getSize(linkedXva)
+      } else if (mode === 'delta') {
+        const linkedVhds = Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
+        fileSystemSize = await computeVhdsSize(handler, linkedVhds)
+
+        // the size is not computed in some cases (e.g. VhdDirectory)
+        if (fileSystemSize === undefined) {
+          return
+        }
+
+        // don't warn if the size has changed after a merge
+        if (!merged && fileSystemSize !== size) {
+          onLog(`incorrect size in metadata: ${size ?? 'none'} instead of ${fileSystemSize}`)
+        }
+      }
+    } catch (error) {
+      onLog(`failed to get size of ${metadataPath}`, { error })
+      return
+    }
+
+    // systematically update size after a merge
+    if ((merged || fixMetadata) && size !== fileSystemSize) {
+      metadata.size = fileSystemSize
+      try {
+        await handler.writeFile(metadataPath, JSON.stringify(metadata), { flags: 'w' })
+      } catch (error) {
+        onLog(`failed to update size in backup metadata ${metadataPath} after merge`, { error })
+      }
+    }
+  })
+
   return {
     // boolean whether some VHDs were merged (or should be merged)
     merge: toMerge.length !== 0,

@xen-orchestra/backups/_isValidXva.js

@@ -1,11 +1,24 @@
 const assert = require('assert')
 
-const isGzipFile = async (handler, fd) => {
+const COMPRESSED_MAGIC_NUMBERS = [
   // https://tools.ietf.org/html/rfc1952.html#page-5
-  const magicNumber = Buffer.allocUnsafe(2)
+  Buffer.from('1F8B', 'hex'),
 
-  assert.strictEqual((await handler.read(fd, magicNumber, 0)).bytesRead, magicNumber.length)
-  return magicNumber[0] === 31 && magicNumber[1] === 139
+  // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#zstandard-frames
+  Buffer.from('28B52FFD', 'hex'),
+]
+const MAGIC_NUMBER_MAX_LENGTH = Math.max(...COMPRESSED_MAGIC_NUMBERS.map(_ => _.length))
+
+const isCompressedFile = async (handler, fd) => {
+  const header = Buffer.allocUnsafe(MAGIC_NUMBER_MAX_LENGTH)
+  assert.strictEqual((await handler.read(fd, header, 0)).bytesRead, header.length)
+
+  for (const magicNumber of COMPRESSED_MAGIC_NUMBERS) {
+    if (magicNumber.compare(header, 0, magicNumber.length) === 0) {
+      return true
+    }
+  }
+  return false
 }
 
 // TODO: better check?
@@ -43,8 +56,8 @@ async function isValidXva(path) {
         return false
       }
 
-      return (await isGzipFile(handler, fd))
-        ? true // gzip files cannot be validated at this time
+      return (await isCompressedFile(handler, fd))
+        ? true // compressed files cannot be validated at this time
         : await isValidTar(handler, size, fd)
     } finally {
       handler.closeFile(fd).catch(noop)

@xen-orchestra/backups/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.15.1",
+  "version": "0.18.3",
   "engines": {
     "node": ">=14.6"
   },
@@ -16,11 +16,11 @@
     "postversion": "npm publish --access public"
   },
   "dependencies": {
-    "@vates/compose": "^2.0.0",
+    "@vates/compose": "^2.1.0",
     "@vates/disposable": "^0.1.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^0.18.0",
+    "@xen-orchestra/fs": "^0.19.3",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/template": "^0.1.0",
     "compare-versions": "^4.0.1",
@@ -35,11 +35,12 @@
     "promise-toolbox": "^0.20.0",
     "proper-lockfile": "^4.1.2",
     "pump": "^3.0.0",
-    "vhd-lib": "^1.3.0",
+    "uuid": "^8.3.2",
+    "vhd-lib": "^3.0.0",
     "yazl": "^2.5.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^0.8.0"
+    "@xen-orchestra/xapi": "^0.8.5"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

@xen-orchestra/backups/tests.fixtures.js

@@ -0,0 +1,92 @@
+// a valid footer of a 2
+exports.VHDFOOTER = {
+  cookie: 'conectix',
+  features: 2,
+  fileFormatVersion: 65536,
+  dataOffset: 512,
+  timestamp: 0,
+  creatorApplication: 'caml',
+  creatorVersion: 1,
+  creatorHostOs: 0,
+  originalSize: 53687091200,
+  currentSize: 53687091200,
+  diskGeometry: { cylinders: 25700, heads: 16, sectorsPerTrackCylinder: 255 },
+  diskType: 3,
+  checksum: 4294962945,
+  uuid: Buffer.from('d8dbcad85265421e8b298d99c2eec551', 'utf-8'),
+  saved: '',
+  hidden: '',
+  reserved: '',
+}
+exports.VHDHEADER = {
+  cookie: 'cxsparse',
+  dataOffset: undefined,
+  tableOffset: 2048,
+  headerVersion: 65536,
+  maxTableEntries: 25600,
+  blockSize: 2097152,
+  checksum: 4294964241,
+  parentUuid: null,
+  parentTimestamp: 0,
+  reserved1: 0,
+  parentUnicodeName: '',
+  parentLocatorEntry: [
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+  ],
+  reserved2: '',
+}

@xen-orchestra/backups/writers/DeltaBackupWriter.js

@@ -3,7 +3,7 @@ const map = require('lodash/map.js')
 const mapValues = require('lodash/mapValues.js')
 const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
 const { asyncMap } = require('@xen-orchestra/async-map')
-const { chainVhd, checkVhdChain, VhdFile } = require('vhd-lib')
+const { chainVhd, checkVhdChain, openVhd, VhdAbstract } = require('vhd-lib')
 const { createLogger } = require('@xen-orchestra/log')
 const { dirname } = require('path')
 
@@ -16,6 +16,7 @@ const { MixinBackupWriter } = require('./_MixinBackupWriter.js')
 const { AbstractDeltaWriter } = require('./_AbstractDeltaWriter.js')
 const { checkVhd } = require('./_checkVhd.js')
 const { packUuid } = require('./_packUuid.js')
+const { Disposable } = require('promise-toolbox')
 
 const { warn } = createLogger('xo:backups:DeltaBackupWriter')
 
@@ -23,6 +24,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
   async checkBaseVdis(baseUuidToSrcVdi) {
     const { handler } = this._adapter
     const backup = this._backup
+    const adapter = this._adapter
 
     const backupDir = getVmBackupDir(backup.vm.uuid)
     const vdisDir = `${backupDir}/vdis/${backup.job.id}`
@@ -34,16 +36,21 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           filter: _ => _[0] !== '.' && _.endsWith('.vhd'),
           prependDir: true,
         })
+        const packedBaseUuid = packUuid(baseUuid)
         await asyncMap(vhds, async path => {
           try {
             await checkVhdChain(handler, path)
+            // Warning, this should not be written as found = found || await adapter.isMergeableParent(packedBaseUuid, path)
+            //
+            // since all the checks of a path are done in parallel, found would be containing
+            // only the last answer of isMergeableParent which is probably not the right one
+            // this led to the support tickets  https://help.vates.fr/#ticket/zoom/4751 , 4729, 4665 and 4300
 
-            const vhd = new VhdFile(handler, path)
-            await vhd.readHeaderAndFooter()
-            found = found || vhd.footer.uuid.equals(packUuid(baseUuid))
+            const isMergeable = await adapter.isMergeableParent(packedBaseUuid, path)
+            found = found || isMergeable
           } catch (error) {
             warn('checkBaseVdis', { error })
-            await ignoreErrors.call(handler.unlink(path))
+            await ignoreErrors.call(VhdAbstract.unlink(handler, path))
           }
         })
       } catch (error) {
@@ -144,7 +151,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
               // don't do delta for it
               vdi.uuid
             : vdi.$snapshot_of$uuid
-        }/${basename}.vhd`
+        }/${adapter.getVhdFileName(basename)}`
     )
 
     const metadataFilename = `${backupDir}/${basename}.json`
@@ -188,7 +195,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
             await checkVhd(handler, parentPath)
           }
 
-          await adapter.outputStream(path, deltaExport.streams[`${id}.vhd`], {
+          await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
             // no checksum for VHDs, because they will be invalidated by
             // merges and chainings
             checksum: false,
@@ -200,11 +207,11 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           }
 
           // set the correct UUID in the VHD
-          const vhd = new VhdFile(handler, path)
-          await vhd.readHeaderAndFooter()
-          vhd.footer.uuid = packUuid(vdi.uuid)
-          await vhd.readBlockAllocationTable() // required by writeFooter()
-          await vhd.writeFooter()
+          await Disposable.use(openVhd(handler, path), async vhd => {
+            vhd.footer.uuid = packUuid(vdi.uuid)
+            await vhd.readBlockAllocationTable() // required by writeFooter()
+            await vhd.writeFooter()
+          })
         })
       )
       return {

@xen-orchestra/backups/writers/_MixinBackupWriter.js

@@ -21,10 +21,18 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
       this.#vmBackupDir = getVmBackupDir(this._backup.vm.uuid)
     }
 
-    _cleanVm(options) {
-      return this._adapter
-        .cleanVm(this.#vmBackupDir, { ...options, fixMetadata: true, onLog: warn, lock: false })
-        .catch(warn)
+    async _cleanVm(options) {
+      try {
+        return await this._adapter.cleanVm(this.#vmBackupDir, {
+          ...options,
+          fixMetadata: true,
+          onLog: warn,
+          lock: false,
+        })
+      } catch (error) {
+        warn(error)
+        return {}
+      }
     }
 
     async beforeBackup() {
@@ -36,14 +44,21 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
 
     async afterBackup() {
       const { disableMergeWorker } = this._backup.config
-
-      const { merge } = await this._cleanVm({ remove: true, merge: disableMergeWorker })
-      await this.#lock.dispose()
-
       // merge worker only compatible with local remotes
       const { handler } = this._adapter
-      if (merge && !disableMergeWorker && typeof handler._getRealPath === 'function') {
-        await handler.outputFile(join(MergeWorker.CLEAN_VM_QUEUE, formatFilenameDate(new Date())), this._backup.vm.uuid)
+      const willMergeInWorker = !disableMergeWorker && typeof handler._getRealPath === 'function'
+
+      const { merge } = await this._cleanVm({ remove: true, merge: !willMergeInWorker })
+      await this.#lock.dispose()
+
+      if (merge && willMergeInWorker) {
+        const taskFile =
+          join(MergeWorker.CLEAN_VM_QUEUE, formatFilenameDate(new Date())) +
+          '-' +
+          // add a random suffix to avoid collision in case multiple tasks are created at the same second
+          Math.random().toString(36).slice(2)
+
+        await handler.outputFile(taskFile, this._backup.vm.uuid)
         const remotePath = handler._getRealPath()
         await MergeWorker.run(remotePath)
       }

@xen-orchestra/backups/writers/_checkVhd.js

@@ -1,5 +1,6 @@
-const Vhd = require('vhd-lib').VhdFile
+const openVhd = require('vhd-lib').openVhd
+const Disposable = require('promise-toolbox/Disposable')
 
 exports.checkVhd = async function checkVhd(handler, path) {
-  await new Vhd(handler, path).readHeaderAndFooter()
+  await Disposable.use(openVhd(handler, path), () => {})
 }

@xen-orchestra/cron/src/index.spec.js

@@ -2,6 +2,8 @@
 
 import { createSchedule } from './'
 
+jest.useFakeTimers()
+
 const wrap = value => () => value
 
 describe('issues', () => {

@xen-orchestra/fs/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "@xen-orchestra/fs",
-  "version": "0.18.0",
+  "version": "0.19.3",
   "license": "AGPL-3.0-or-later",
   "description": "The File System for Xen Orchestra backups.",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
@@ -21,7 +21,9 @@
     "@sindresorhus/df": "^3.1.1",
     "@sullux/aws-sdk": "^1.0.5",
     "@vates/coalesce-calls": "^0.1.0",
+    "@vates/decorate-with": "^1.0.0",
     "@xen-orchestra/async-map": "^0.1.2",
+    "@xen-orchestra/log": "^0.3.0",
     "aws-sdk": "^2.686.0",
     "decorator-synchronized": "^0.6.0",
     "execa": "^5.0.0",
@@ -33,7 +35,7 @@
     "proper-lockfile": "^4.1.2",
     "readable-stream": "^3.0.6",
     "through2": "^4.0.2",
-    "xo-remote-parser": "^0.7.0"
+    "xo-remote-parser": "^0.8.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

@xen-orchestra/fs/src/abstract.js

@@ -76,6 +76,7 @@ export default class RemoteHandlerAbstract {
 
     const sharedLimit = limitConcurrency(options.maxParallelOperations ?? DEFAULT_MAX_PARALLEL_OPERATIONS)
     this.closeFile = sharedLimit(this.closeFile)
+    this.copy = sharedLimit(this.copy)
     this.getInfo = sharedLimit(this.getInfo)
     this.getSize = sharedLimit(this.getSize)
     this.list = sharedLimit(this.list)
@@ -307,6 +308,17 @@ export default class RemoteHandlerAbstract {
     return p
   }
 
+  async copy(oldPath, newPath, { checksum = false } = {}) {
+    oldPath = normalizePath(oldPath)
+    newPath = normalizePath(newPath)
+
+    let p = timeout.call(this._copy(oldPath, newPath), this._timeout)
+    if (checksum) {
+      p = Promise.all([p, this._copy(checksumFile(oldPath), checksumFile(newPath))])
+    }
+    return p
+  }
+
   async rmdir(dir) {
     await timeout.call(this._rmdir(normalizePath(dir)).catch(ignoreEnoent), this._timeout)
   }
@@ -519,6 +531,9 @@ export default class RemoteHandlerAbstract {
   async _rename(oldPath, newPath) {
     throw new Error('Not implemented')
   }
+  async _copy(oldPath, newPath) {
+    throw new Error('Not implemented')
+  }
 
   async _rmdir(dir) {
     throw new Error('Not implemented')

@xen-orchestra/fs/src/local.js

@@ -33,6 +33,10 @@ export default class LocalHandler extends RemoteHandlerAbstract {
     return fs.close(fd)
   }
 
+  async _copy(oldPath, newPath) {
+    return fs.copy(this._getFilePath(oldPath), this._getFilePath(newPath))
+  }
+
   async _createReadStream(file, options) {
     if (typeof file === 'string') {
       const stream = fs.createReadStream(this._getFilePath(file), options)

@xen-orchestra/fs/src/s3.js

@@ -1,9 +1,14 @@
 import aws from '@sullux/aws-sdk'
 import assert from 'assert'
 import http from 'http'
+import https from 'https'
+import pRetry from 'promise-toolbox/retry'
+import { createLogger } from '@xen-orchestra/log'
+import { decorateWith } from '@vates/decorate-with'
 import { parse } from 'xo-remote-parser'
 
 import RemoteHandlerAbstract from './abstract'
+import { asyncEach } from '@vates/async-each'
 
 // endpoints https://docs.aws.amazon.com/general/latest/gr/s3.html
 
@@ -13,10 +18,13 @@ const MAX_PART_SIZE = 1024 * 1024 * 1024 * 5 // 5GB
 const MAX_PARTS_COUNT = 10000
 const MAX_OBJECT_SIZE = 1024 * 1024 * 1024 * 1024 * 5 // 5TB
 const IDEAL_FRAGMENT_SIZE = Math.ceil(MAX_OBJECT_SIZE / MAX_PARTS_COUNT) // the smallest fragment size that still allows a 5TB upload in 10000 fragments, about 524MB
+
+const { warn } = createLogger('xo:fs:s3')
+
 export default class S3Handler extends RemoteHandlerAbstract {
   constructor(remote, _opts) {
     super(remote)
-    const { host, path, username, password, protocol, region } = parse(remote.url)
+    const { allowUnauthorized, host, path, username, password, protocol, region } = parse(remote.url)
     const params = {
       accessKeyId: username,
       apiVersion: '2006-03-01',
@@ -29,8 +37,13 @@ export default class S3Handler extends RemoteHandlerAbstract {
       },
     }
     if (protocol === 'http') {
-      params.httpOptions.agent = new http.Agent()
+      params.httpOptions.agent = new http.Agent({ keepAlive: true })
       params.sslEnabled = false
+    } else if (protocol === 'https') {
+      params.httpOptions.agent = new https.Agent({
+        rejectUnauthorized: !allowUnauthorized,
+        keepAlive: true,
+      })
     }
     if (region !== undefined) {
       params.region = region
@@ -51,6 +64,44 @@ export default class S3Handler extends RemoteHandlerAbstract {
     return { Bucket: this._bucket, Key: this._dir + file }
   }
 
+  async _multipartCopy(oldPath, newPath) {
+    const size = await this._getSize(oldPath)
+    const CopySource = `/${this._bucket}/${this._dir}${oldPath}`
+    const multipartParams = await this._s3.createMultipartUpload({ ...this._createParams(newPath) })
+    const param2 = { ...multipartParams, CopySource }
+    try {
+      const parts = []
+      let start = 0
+      while (start < size) {
+        const range = `bytes=${start}-${Math.min(start + MAX_PART_SIZE, size) - 1}`
+        const partParams = { ...param2, PartNumber: parts.length + 1, CopySourceRange: range }
+        const upload = await this._s3.uploadPartCopy(partParams)
+        parts.push({ ETag: upload.CopyPartResult.ETag, PartNumber: partParams.PartNumber })
+        start += MAX_PART_SIZE
+      }
+      await this._s3.completeMultipartUpload({ ...multipartParams, MultipartUpload: { Parts: parts } })
+    } catch (e) {
+      await this._s3.abortMultipartUpload(multipartParams)
+      throw e
+    }
+  }
+
+  async _copy(oldPath, newPath) {
+    const CopySource = `/${this._bucket}/${this._dir}${oldPath}`
+    try {
+      await this._s3.copyObject({
+        ...this._createParams(newPath),
+        CopySource,
+      })
+    } catch (e) {
+      // object > 5GB must be copied part by part
+      if (e.code === 'EntityTooLarge') {
+        return this._multipartCopy(oldPath, newPath)
+      }
+      throw e
+    }
+  }
+
   async _isNotEmptyDir(path) {
     const result = await this._s3.listObjectsV2({
       Bucket: this._bucket,
@@ -90,6 +141,21 @@ export default class S3Handler extends RemoteHandlerAbstract {
     }
   }
 
+  // some objectstorage provider like backblaze, can answer a 500/503 routinely
+  // in this case we should retry,  and let their load balancing do its magic
+  // https://www.backblaze.com/b2/docs/calling.html#error_handling
+  @decorateWith(pRetry.wrap, {
+    delays: [100, 200, 500, 1000, 2000],
+    when: e => e.code === 'InternalError',
+    onRetry(error) {
+      warn('retrying writing file', {
+        attemptNumber: this.attemptNumber,
+        delay: this.delay,
+        error,
+        file: this.arguments[0],
+      })
+    },
+  })
   async _writeFile(file, data, options) {
     return this._s3.putObject({ ...this._createParams(file), Body: data })
   }
@@ -125,16 +191,30 @@ export default class S3Handler extends RemoteHandlerAbstract {
     const splitPrefix = splitPath(prefix)
     const result = await this._s3.listObjectsV2({
       Bucket: this._bucket,
-      Prefix: splitPrefix.join('/'),
+      Prefix: splitPrefix.join('/') + '/', // need slash at the end with the use of delimiters
+      Delimiter: '/', // will only return path until delimiters
     })
-    const uniq = new Set()
+
+    if (result.IsTruncated) {
+      const error = new Error('more than 1000 objects, unsupported in this implementation')
+      error.dir = dir
+      throw error
+    }
+
+    const uniq = []
+
+    // sub directories
+    for (const entry of result.CommonPrefixes) {
+      const line = splitPath(entry.Prefix)
+      uniq.push(line[line.length - 1])
+    }
+    // files
     for (const entry of result.Contents) {
       const line = splitPath(entry.Key)
-      if (line.length > splitPrefix.length) {
-        uniq.add(line[splitPrefix.length])
-      }
+      uniq.push(line[line.length - 1])
     }
-    return [...uniq]
+
+    return uniq
   }
 
   async _mkdir(path) {
@@ -147,25 +227,9 @@ export default class S3Handler extends RemoteHandlerAbstract {
     // nothing to do, directories do not exist, they are part of the files' path
   }
 
+  // s3 doesn't have a rename operation, so copy + delete source
   async _rename(oldPath, newPath) {
-    const size = await this._getSize(oldPath)
-    const multipartParams = await this._s3.createMultipartUpload({ ...this._createParams(newPath) })
-    const param2 = { ...multipartParams, CopySource: `/${this._bucket}/${this._dir}${oldPath}` }
-    try {
-      const parts = []
-      let start = 0
-      while (start < size) {
-        const range = `bytes=${start}-${Math.min(start + MAX_PART_SIZE, size) - 1}`
-        const partParams = { ...param2, PartNumber: parts.length + 1, CopySourceRange: range }
-        const upload = await this._s3.uploadPartCopy(partParams)
-        parts.push({ ETag: upload.CopyPartResult.ETag, PartNumber: partParams.PartNumber })
-        start += MAX_PART_SIZE
-      }
-      await this._s3.completeMultipartUpload({ ...multipartParams, MultipartUpload: { Parts: parts } })
-    } catch (e) {
-      await this._s3.abortMultipartUpload(multipartParams)
-      throw e
-    }
+    await this.copy(oldPath, newPath)
     await this._s3.deleteObject(this._createParams(oldPath))
   }
 
@@ -211,6 +275,34 @@ export default class S3Handler extends RemoteHandlerAbstract {
     // nothing to do, directories do not exist, they are part of the files' path
   }
 
+  // reimplement _rmtree to handle efficiantly path with more than 1000 entries in trees
+  // @todo : use parallel processing for unlink
+  async _rmtree(path) {
+    let NextContinuationToken
+    do {
+      const result = await this._s3.listObjectsV2({
+        Bucket: this._bucket,
+        Prefix: this._dir + path + '/',
+        ContinuationToken: NextContinuationToken,
+      })
+      NextContinuationToken = result.IsTruncated ? result.NextContinuationToken : undefined
+      await asyncEach(
+        result.Contents,
+        async ({ Key }) => {
+          // _unlink will add the prefix, but Key contains everything
+          // also we don't need to check if we delete a directory, since the list only return files
+          await this._s3.deleteObject({
+            Bucket: this._bucket,
+            Key,
+          })
+        },
+        {
+          concurrency: 16,
+        }
+      )
+    } while (NextContinuationToken !== undefined)
+  }
+
   async _write(file, buffer, position) {
     if (typeof file !== 'string') {
       file = file.fd

@xen-orchestra/log/USAGE.md

@@ -48,6 +48,10 @@ configure([
     // if filter is a string, then it is pattern
     // (https://github.com/visionmedia/debug#wildcards) which is
     // matched against the namespace of the logs
+    //
+    // If it's an array, it will be handled as an array of filters
+    // and the transport will be used if any one of them match the
+    // current log
     filter: process.env.DEBUG,
 
     transport: transportConsole(),

@xen-orchestra/mixin/package.json

@@ -19,7 +19,7 @@
     "node": ">=6"
   },
   "dependencies": {
-    "bind-property-descriptor": "^1.0.0",
+    "bind-property-descriptor": "^2.0.0",
     "lodash": "^4.17.21"
   },
   "scripts": {

@xen-orchestra/mixins/Config.js

@@ -13,7 +13,7 @@ module.exports = class Config {
     const watchers = (this._watchers = new Set())
 
     app.hooks.on('start', async () => {
-      app.hooks.on(
+      app.hooks.once(
         'stop',
         await watch({ appDir, appName, ignoreUnknownFormats: true }, (error, config) => {
           if (error != null) {
@@ -32,7 +32,7 @@ module.exports = class Config {
   get(path) {
     const value = get(this._config, path)
     if (value === undefined) {
-      throw new TypeError('missing config entry: ' + value)
+      throw new TypeError('missing config entry: ' + path)
     }
     return value
   }

@xen-orchestra/mixins/package.json

@@ -14,7 +14,7 @@
     "url": "https://vates.fr"
   },
   "license": "AGPL-3.0-or-later",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "engines": {
     "node": ">=12"
   },
@@ -22,7 +22,7 @@
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/emit-async": "^0.1.0",
     "@xen-orchestra/log": "^0.3.0",
-    "app-conf": "^0.9.0",
+    "app-conf": "^1.0.0",
     "lodash": "^4.17.21"
   },
   "scripts": {

@xen-orchestra/proxy-cli/package.json

@@ -29,7 +29,7 @@
     "@iarna/toml": "^2.2.0",
     "@vates/read-chunk": "^0.1.2",
     "ansi-colors": "^4.1.1",
-    "app-conf": "^0.9.0",
+    "app-conf": "^1.0.0",
     "content-type": "^1.0.4",
     "cson-parser": "^4.0.7",
     "getopts": "^2.2.3",

@xen-orchestra/proxy/config.toml

@@ -20,6 +20,7 @@ keepAliveInterval = 10e3
 dirMode = 0o700
 disableMergeWorker = false
 snapshotNameLabelTpl = '[XO Backup {job.name}] {vm.name_label}'
+vhdDirectoryCompression = 'brotli'
 
 [backups.defaultSettings]
 reportWhen = 'failure'
@@ -87,3 +88,20 @@ ignoreNobakVdis = false
 
 maxUncoalescedVdis = 1
 watchEvents = ['network', 'PIF', 'pool', 'SR', 'task', 'VBD', 'VDI', 'VIF', 'VM']
+
+
+
+#compact mode
+[reverseProxies]
+# '/http/' = 'http://localhost:8081/'
+#The target can have a path ( like `http://target/sub/directory/`),
+# parameters (`?param=one`) and hash (`#jwt:32154`) that are automatically added to all queries transfered by the proxy.
+# If a parameter is present in the configuration and in the query, only the config parameter is transferred.
+# '/another' = http://hiddenServer:8765/path/
+
+# And use the extended mode when required
+# The additionnal options of a proxy's configuraiton's section are used to instantiate the `https` Agent(respectively the `http`).
+# A notable option is `rejectUnauthorized` which allow to connect to a HTTPS backend  with an invalid/ self signed certificate
+#[reverseProxies.'/https/']
+# target = 'https://localhost:8080/'
+# rejectUnauthorized = false

@xen-orchestra/proxy/package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "@xen-orchestra/proxy",
-  "version": "0.15.2",
+  "version": "0.17.3",
   "license": "AGPL-3.0-or-later",
   "description": "XO Proxy used to remotely execute backup jobs",
   "keywords": [
@@ -22,32 +22,33 @@
     "xo-proxy": "dist/index.mjs"
   },
   "engines": {
-    "node": ">=14.13"
+    "node": ">=14.18"
   },
   "dependencies": {
     "@iarna/toml": "^2.2.0",
     "@koa/router": "^10.0.0",
-    "@vates/compose": "^2.0.0",
-    "@vates/decorate-with": "^0.1.0",
+    "@vates/compose": "^2.1.0",
+    "@vates/decorate-with": "^1.0.0",
     "@vates/disposable": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.15.1",
-    "@xen-orchestra/fs": "^0.18.0",
+    "@xen-orchestra/backups": "^0.18.3",
+    "@xen-orchestra/fs": "^0.19.3",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/mixin": "^0.1.0",
-    "@xen-orchestra/mixins": "^0.1.1",
+    "@xen-orchestra/mixins": "^0.1.2",
     "@xen-orchestra/self-signed": "^0.1.0",
-    "@xen-orchestra/xapi": "^0.8.0",
+    "@xen-orchestra/xapi": "^0.8.5",
     "ajv": "^8.0.3",
-    "app-conf": "^0.9.0",
+    "app-conf": "^1.0.0",
     "async-iterator-to-stream": "^1.1.0",
     "fs-extra": "^10.0.0",
     "get-stream": "^6.0.0",
     "getopts": "^2.2.3",
     "golike-defer": "^0.5.1",
     "http-server-plus": "^0.11.0",
+    "http2-proxy": "^5.0.53",
     "json-rpc-protocol": "^0.13.1",
-    "jsonrpc-websocket-client": "^0.6.0",
+    "jsonrpc-websocket-client": "^0.7.2",
     "koa": "^2.5.1",
     "koa-compress": "^5.0.1",
     "koa-helmet": "^5.1.0",
@@ -57,7 +58,7 @@
     "promise-toolbox": "^0.20.0",
     "source-map-support": "^0.5.16",
     "stoppable": "^1.0.6",
-    "xdg-basedir": "^4.0.0",
+    "xdg-basedir": "^5.1.0",
     "xen-api": "^0.35.1",
     "xo-common": "^0.7.0"
   },

@xen-orchestra/proxy/src/app/mixins/api.mjs

@@ -14,7 +14,7 @@ import { createLogger } from '@xen-orchestra/log'
 
 const { debug, warn } = createLogger('xo:proxy:api')
 
-const ndJsonStream = asyncIteratorToStream(async function*(responseId, iterable) {
+const ndJsonStream = asyncIteratorToStream(async function* (responseId, iterable) {
   try {
     let cursor, iterator
     try {
@@ -45,14 +45,14 @@ export default class Api {
   constructor(app, { appVersion, httpServer }) {
     this._ajv = new Ajv({ allErrors: true })
     this._methods = { __proto__: null }
-
-    const router = new Router({ prefix: '/api/v1' }).post('/', async ctx => {
+    const PREFIX = '/api/v1'
+    const router = new Router({ prefix: PREFIX }).post('/', async ctx => {
       // Before Node 13.0 there was an inactivity timeout of 2 mins, which may
       // not be enough for the API.
       ctx.req.setTimeout(0)
 
       const profile = await app.authentication.findProfile({
-        authenticationToken: ctx.cookies.get('authenticationToken')
+        authenticationToken: ctx.cookies.get('authenticationToken'),
       })
       if (profile === undefined) {
         ctx.status = 401
@@ -102,6 +102,7 @@ export default class Api {
           // breaks, send some data every 10s to keep it opened.
           const stopTimer = clearInterval.bind(
             undefined,
+            // @to check : can this add space inside binary data ?
             setInterval(() => stream.push(' '), keepAliveInterval)
           )
           stream.on('end', stopTimer).on('error', stopTimer)
@@ -118,12 +119,19 @@ export default class Api {
       .use(router.routes())
       .use(router.allowedMethods())
 
-    httpServer.on('request', koa.callback())
+    const callback = koa.callback()
+    httpServer.on('request', (req, res) => {
+      // only answers to query to the root url of this mixin
+      // do it before giving the request to Koa to ensure it's not modified
+      if (req.url.startsWith(PREFIX)) {
+        callback(req, res)
+      }
+    })
 
     this.addMethods({
       system: {
         getMethodsInfo: [
-          function*() {
+          function* () {
             const methods = this._methods
             for (const name in methods) {
               const { description, params = {} } = methods[name]
@@ -131,25 +139,25 @@ export default class Api {
             }
           }.bind(this),
           {
-            description: 'returns the signatures of all available API methods'
-          }
+            description: 'returns the signatures of all available API methods',
+          },
         ],
         getServerVersion: [
           () => appVersion,
           {
-            description: 'returns the version of xo-server'
-          }
+            description: 'returns the version of xo-server',
+          },
         ],
         listMethods: [
-          function*() {
+          function* () {
             const methods = this._methods
             for (const name in methods) {
               yield name
             }
           }.bind(this),
           {
-            description: 'returns the name of all available API methods'
-          }
+            description: 'returns the name of all available API methods',
+          },
         ],
         methodSignature: [
           ({ method: name }) => {
@@ -164,14 +172,14 @@ export default class Api {
           {
             description: 'returns the signature of an API method',
             params: {
-              method: { type: 'string' }
-            }
-          }
-        ]
+              method: { type: 'string' },
+            },
+          },
+        ],
       },
       test: {
         range: [
-          function*({ start = 0, stop, step }) {
+          function* ({ start = 0, stop, step }) {
             if (step === undefined) {
               step = start > stop ? -1 : 1
             }
@@ -189,11 +197,11 @@ export default class Api {
             params: {
               start: { optional: true, type: 'number' },
               step: { optional: true, type: 'number' },
-              stop: { type: 'number' }
-            }
-          }
-        ]
-      }
+              stop: { type: 'number' },
+            },
+          },
+        ],
+      },
     })
   }
 
@@ -220,7 +228,7 @@ export default class Api {
         return required
       }),
 
-      type: 'object'
+      type: 'object',
     })
 
     const m = params => {

@xen-orchestra/proxy/src/app/mixins/authentication.mjs

@@ -1,5 +1,6 @@
+import assert from 'assert'
 import fse from 'fs-extra'
-import xdg from 'xdg-basedir'
+import { xdgConfig } from 'xdg-basedir'
 import { createLogger } from '@xen-orchestra/log'
 import { execFileSync } from 'child_process'
 
@@ -10,33 +11,48 @@ const { warn } = createLogger('xo:proxy:authentication')
 const isValidToken = t => typeof t === 'string' && t.length !== 0
 
 export default class Authentication {
-  constructor(_, { appName, config: { authenticationToken: token } }) {
-    if (!isValidToken(token)) {
-      token = JSON.parse(execFileSync('xenstore-read', ['vm-data/xo-proxy-authenticationToken']))
+  #token
 
-      if (!isValidToken(token)) {
-        throw new Error('missing authenticationToken in configuration')
-      }
+  constructor(app, { appName, config: { authenticationToken: token } }) {
+    const setToken = ({ token }) => {
+      assert(isValidToken(token), 'invalid authentication token: ' + token)
+
+      // save this token in the automatically handled conf file
+      fse.outputFileSync(
+        // this file must take precedence over normal user config
+        `${xdgConfig}/${appName}/config.z-auto.json`,
+        JSON.stringify({ authenticationToken: token }),
+        { mode: 0o600 }
+      )
+
+      this.#token = token
+    }
+
+    if (isValidToken(token)) {
+      this.#token = token
+    } else {
+      setToken({ token: JSON.parse(execFileSync('xenstore-read', ['vm-data/xo-proxy-authenticationToken'])) })
 
       try {
-        // save this token in the automatically handled conf file
-        fse.outputFileSync(
-          // this file must take precedence over normal user config
-          `${xdg.config}/${appName}/config.z-auto.json`,
-          JSON.stringify({ authenticationToken: token }),
-          { mode: 0o600 }
-        )
         execFileSync('xenstore-rm', ['vm-data/xo-proxy-authenticationToken'])
       } catch (error) {
         warn('failed to remove token from XenStore', { error })
       }
     }
 
-    this._token = token
+    app.api.addMethod('authentication.setToken', setToken, {
+      description: 'change the authentication token used by this XO Proxy',
+      params: {
+        token: {
+          type: 'string',
+          minLength: 1,
+        },
+      },
+    })
   }
 
   async findProfile(credentials) {
-    if (credentials?.authenticationToken === this._token) {
+    if (credentials?.authenticationToken === this.#token) {
       return new Profile()
     }
   }

@xen-orchestra/proxy/src/app/mixins/backups.mjs

@@ -164,6 +164,17 @@ export default class Backups {
             },
           },
         ],
+        deleteVmBackups: [
+          ({ filenames, remote }) =>
+            Disposable.use(this.getAdapter(remote), adapter => adapter.deleteVmBackups(filenames)),
+          {
+            description: 'delete VM backups',
+            params: {
+              filenames: { type: 'array', items: { type: 'string' } },
+              remote: { type: 'object' },
+            },
+          },
+        ],
         fetchPartitionFiles: [
           ({ disk: diskId, remote, partition: partitionId, paths }) =>
             Disposable.use(this.getAdapter(remote), adapter => adapter.fetchPartitionFiles(diskId, partitionId, paths)),
@@ -403,6 +414,7 @@ export default class Backups {
     return new RemoteAdapter(yield app.remotes.getHandler(remote), {
       debounceResource: app.debounceResource.bind(app),
       dirMode: app.config.get('backups.dirMode'),
+      vhdDirectoryCompression: app.config.get('backups.vhdDirectoryCompression'),
     })
   }
 

@xen-orchestra/proxy/src/app/mixins/reverseProxy.mjs

@@ -0,0 +1,120 @@
+import { urlToHttpOptions } from 'url'
+import proxy from 'http2-proxy'
+
+function removeSlash(str) {
+  return str.replace(/^\/|\/$/g, '')
+}
+
+function mergeUrl(relative, base) {
+  const res = new URL(base)
+  const relativeUrl = new URL(relative, base)
+  res.pathname = relativeUrl.pathname
+  relativeUrl.searchParams.forEach((value, name) => {
+    // we do not allow to modify params already specified by config
+    if (!res.searchParams.has(name)) {
+      res.searchParams.append(name, value)
+    }
+  })
+  res.hash = relativeUrl.hash.length > 0 ? relativeUrl.hash : res.hash
+  return res
+}
+
+export function backendToLocalPath(basePath, target, backendUrl) {
+  // keep redirect url relative to local server
+  const localPath = `${basePath}/${backendUrl.pathname.substring(target.pathname.length)}${backendUrl.search}${
+    backendUrl.hash
+  }`
+  return localPath
+}
+
+export function localToBackendUrl(basePath, target, localPath) {
+  let localPathWithoutBase = removeSlash(localPath).substring(basePath.length)
+  localPathWithoutBase = './' + removeSlash(localPathWithoutBase)
+  const url = mergeUrl(localPathWithoutBase, target)
+  return url
+}
+
+export default class ReverseProxy {
+  constructor(app, { httpServer }) {
+    app.config.watch('reverseProxies', proxies => {
+      this._proxies = Object.keys(proxies)
+        .sort((a, b) => b.length - a.length)
+        .map(path => {
+          let config = proxies[path]
+          if (typeof config === 'string') {
+            config = { target: config }
+          }
+          config.path = '/proxy/v1/' + removeSlash(path) + '/'
+
+          return config
+        })
+    })
+
+    httpServer.on('request', (req, res) => this._proxy(req, res))
+    httpServer.on('upgrade', (req, socket, head) => this._upgrade(req, socket, head))
+  }
+
+  _getConfigFromRequest(req) {
+    return this._proxies.find(({ path }) => req.url.startsWith(path))
+  }
+
+  _proxy(req, res) {
+    const config = this._getConfigFromRequest(req)
+
+    if (config === undefined) {
+      res.writeHead(404)
+      res.end('404')
+      return
+    }
+
+    const url = new URL(config.target)
+    const targetUrl = localToBackendUrl(config.path, url, req.originalUrl || req.url)
+    proxy.web(req, res, {
+      ...urlToHttpOptions(targetUrl),
+      ...config.options,
+      onReq: (req, { headers }) => {
+        headers['x-forwarded-for'] = req.socket.remoteAddress
+        headers['x-forwarded-proto'] = req.socket.encrypted ? 'https' : 'http'
+        if (req.headers.host !== undefined) {
+          headers['x-forwarded-host'] = req.headers.host
+        }
+      },
+      onRes: (req, res, proxyRes) => {
+        // rewrite redirect to pass through this proxy
+        if (proxyRes.statusCode === 301 || proxyRes.statusCode === 302) {
+          // handle relative/ absolute location
+          const redirectTargetLocation = new URL(proxyRes.headers.location, url)
+
+          // this proxy should only allow communication between known hosts. Don't open it too much
+          if (redirectTargetLocation.hostname !== url.hostname || redirectTargetLocation.protocol !== url.protocol) {
+            throw new Error(`Can't redirect from ${url.hostname} to ${redirectTargetLocation.hostname} `)
+          }
+          res.writeHead(proxyRes.statusCode, {
+            ...proxyRes.headers,
+            location: backendToLocalPath(config.path, url, redirectTargetLocation),
+          })
+          res.end()
+          return
+        }
+        // pass through the answer of the remote server
+        res.writeHead(proxyRes.statusCode, proxyRes.headers)
+        // pass through content
+        proxyRes.pipe(res)
+      },
+    })
+  }
+
+  _upgrade(req, socket, head) {
+    const config = this._getConfigFromRequest(req)
+    if (config === undefined) {
+      return
+    }
+
+    const { path, target, options } = config
+    const targetUrl = localToBackendUrl(path, target, req.originalUrl || req.url)
+    proxy.ws(req, socket, head, {
+      ...urlToHttpOptions(targetUrl),
+      ...options,
+    })
+  }
+}

@xen-orchestra/proxy/tests/cert.pem

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDETCCAfkCFHXO1U7YJHI61bPNhYDvyBNJYH4LMA0GCSqGSIb3DQEBCwUAMEUx
+CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
+cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjIwMTEwMTI0MTU4WhcNNDkwNTI3MTI0
+MTU4WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE
+CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEA1jMLdHuZu2R1fETyB2iRect1alwv76clp/7A8tx4zNaVA9qB
+BcHbI83mkozuyrXpsEUblTvvcWkheBPAvWD4gj0eWSDSiuf0edcIS6aky+Lr/n0T
+W/vL5kVNrgPTlsO8OyQcXjDeuUOR1xDWIa8G71Ynd6wtATB7oXe7kaV/Z6b2fENr
+4wlW0YEDnMHik59c9jXDshhQYDlErwZsSyHuLwkC7xuYO26SUW9fPcHJA3uOfxeG
+BrCxMuSMOJtdmslRWhLCjbk0PT12OYCCRlvuTvPHa8N57GEQbi4xAu+XATgO1DUm
+Dq/oCSj0TcWUXXOykN/PAC2cjIyqkU2e7orGaQIDAQABMA0GCSqGSIb3DQEBCwUA
+A4IBAQCTshhF3V5WVhnpFGHd+tPfeHmUVrUnbC+xW7fSeWpamNmTjHb7XB6uDR0O
+DGswhEitbbSOsCiwz4/zpfE3/3+X07O8NPbdHTVHCei6D0uyegEeWQ2HoocfZs3X
+8CORe8TItuvQAevV17D0WkGRoJGVAOiKo+izpjI55QXQ+FjkJ0bfl1iksnUJk0+I
+ZNmRRNjNyOxo7NAzomSBHfJ5rDE+E440F2uvXIE9OIwHRiq6FGvQmvGijPeeP5J0
+LzcSK98jfINFSsA/Wn5vWE+gfH9ySD2G3r2cDTS904T77PNiYH+cNSP6ujtmNzvK
+Bgoa3jXZPRBi82TUOb2jj5DB33bg
+-----END CERTIFICATE-----

@xen-orchestra/proxy/tests/key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA1jMLdHuZu2R1fETyB2iRect1alwv76clp/7A8tx4zNaVA9qB
+BcHbI83mkozuyrXpsEUblTvvcWkheBPAvWD4gj0eWSDSiuf0edcIS6aky+Lr/n0T
+W/vL5kVNrgPTlsO8OyQcXjDeuUOR1xDWIa8G71Ynd6wtATB7oXe7kaV/Z6b2fENr
+4wlW0YEDnMHik59c9jXDshhQYDlErwZsSyHuLwkC7xuYO26SUW9fPcHJA3uOfxeG
+BrCxMuSMOJtdmslRWhLCjbk0PT12OYCCRlvuTvPHa8N57GEQbi4xAu+XATgO1DUm
+Dq/oCSj0TcWUXXOykN/PAC2cjIyqkU2e7orGaQIDAQABAoIBAQC65uVq6WLWGa1O
+FtbdUggGL1svyGrngYChGvB/uZMKoX57U1DbljDCCCrV23WNmbfkYBjWWervmZ1j
+qlC2roOJGQ1/Fd3A6O7w1YnegPUxFrt3XunijE55iiVi3uHknryDGlpKcfgVzfjW
+oVFHKPMzKYjcqnbGn+hwlwoq5y7JYFTOa57/dZbyommbodRyy9Dpn0OES0grQqwR
+VD1amrQ7XJhukcxQgYPuDc/jM3CuowoBsv9f+Q2zsPgr6CpHxxLLIs+kt8NQJT9v
+neg/pm8ojcwOa9qoILdtu6ue7ee3VE9cFnB1vutxS1+MPeI5wgTJjaYrgPCMxXBM
+2LdJJEmBAoGBAPA6LpuU1vv5R3x66hzenSk4LS1fj24K0WuBdTwFvzQmCr70oKdo
+Yywxt+ZkBw5aEtzQlB8GewolHobDJrzxMorU+qEXX3jP2BIPDVQl2orfjr03Yyus
+s5mYS/Qa6Zf1yObrjulTNm8oTn1WaG3TIvi8c5DyG2OK28N/9oMI1XGRAoGBAORD
+YKyII/S66gZsJSf45qmrhq1hHuVt1xae5LUPP6lVD+MCCAmuoJnReV8fc9h7Dvgd
+YPMINkWUTePFr3o4p1mh2ZC7ldczgDn6X4TldY2J3Zg47xJa5hL0L6JL4NiCGRIE
+FV5rLJxkGh/DDBfmC9hQQ6Yg6cHvyewso5xVnBtZAoGAI+OdWPMIl0ZrrqYyWbPM
+aP8SiMfRBtCo7tW9bQUyxpi0XEjxw3Dt+AlJfysMftFoJgMnTedK9H4NLHb1T579
+PQ6KjwyN39+1WSVUiXDKUJsLmSswLrMzdcvx9PscUO6QYCdrB2K+LCcqasFBAr9b
+ZyvIXCw/eUSihneUnYjxUnECgYAoPgCzKiU8ph9QFozOaUExNH4/3tl1lVHQOR8V
+FKUik06DtP35xwGlXJrLPF5OEhPnhjZrYk0/IxBAUb/ICmjmknQq4gdes0Ot9QgW
+A+Yfl+irR45ObBwXx1kGgd4YDYeh93pU9QweXj+Ezfw50mLQNgZXKYJMoJu2uX/2
+tdkZsQKBgQCTfDcW8qBntI6V+3Gh+sIThz+fjdv5+qT54heO4EHadc98ykEZX0M1
+sCWJiAQWM/zWXcsTndQDgDsvo23jpoulVPDitSEISp5gSe9FEN2njsVVID9h1OIM
+f30s5kwcJoiV9kUCya/BFtuS7kbuQfAyPU0v3I+lUey6VCW6A83OTg==
+-----END RSA PRIVATE KEY-----

@xen-orchestra/proxy/tests/localServer.mjs

@@ -0,0 +1,60 @@
+import { createServer as creatServerHttps } from 'https'
+import { createServer as creatServerHttp } from 'http'
+
+import { WebSocketServer } from 'ws'
+import fs from 'fs'
+
+const httpsServer = creatServerHttps({
+  key: fs.readFileSync('key.pem'),
+  cert: fs.readFileSync('cert.pem'),
+})
+const httpServer = creatServerHttp()
+
+const wss = new WebSocketServer({ noServer: true, perMessageDeflate: false })
+
+function upgrade(request, socket, head) {
+  const { pathname } = new URL(request.url)
+  // web socket server only on /foo url
+  if (pathname === '/foo') {
+    wss.handleUpgrade(request, socket, head, function done(ws) {
+      wss.emit('connection', ws, request)
+      ws.on('message', function message(data) {
+        ws.send(data)
+      })
+    })
+  } else {
+    socket.destroy()
+  }
+}
+
+function httpHandler(req, res) {
+  switch (req.url) {
+    case '/index.html':
+      res.end('hi')
+      return
+    case '/redirect':
+      res.writeHead(301, {
+        Location: 'index.html',
+      })
+      res.end()
+      return
+    case '/chainRedirect':
+      res.writeHead(301, {
+        Location: '/redirect',
+      })
+      res.end()
+      return
+    default:
+      res.writeHead(404)
+      res.end()
+  }
+}
+
+httpsServer.on('upgrade', upgrade)
+httpServer.on('upgrade', upgrade)
+
+httpsServer.on('request', httpHandler)
+httpServer.on('request', httpHandler)
+
+httpsServer.listen(8080)
+httpServer.listen(8081)

@xen-orchestra/proxy/tests/reverseProxy.unit.spec.mjs

@@ -0,0 +1,123 @@
+import ReverseProxy, { backendToLocalPath, localToBackendUrl } from '../dist/app/mixins/reverseProxy.mjs'
+import { deepEqual, strictEqual } from 'assert'
+
+function makeApp(reverseProxies) {
+  return {
+    config: {
+      get: () => reverseProxies,
+    },
+  }
+}
+
+const app = makeApp({
+  https: {
+    target: 'https://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
+    oneOption: true,
+  },
+  http: 'http://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
+})
+
+// test localToBackendUrl
+const expectedLocalToRemote = {
+  https: [
+    {
+      local: '/proxy/v1/https/',
+      remote: 'https://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
+    },
+    {
+      local: '/proxy/v1/https/sub',
+      remote: 'https://localhost:8080/remotePath/sub?baseParm=1#one=2&another=3',
+    },
+    {
+      local: '/proxy/v1/https/sub/index.html',
+      remote: 'https://localhost:8080/remotePath/sub/index.html?baseParm=1#one=2&another=3',
+    },
+    {
+      local: '/proxy/v1/https/sub?param=1',
+      remote: 'https://localhost:8080/remotePath/sub?baseParm=1&param=1#one=2&another=3',
+    },
+    {
+      local: '/proxy/v1/https/sub?baseParm=willbeoverwritten&param=willstay',
+      remote: 'https://localhost:8080/remotePath/sub?baseParm=1&param=willstay#one=2&another=3',
+    },
+    {
+      local: '/proxy/v1/https/sub?param=1#another=willoverwrite',
+      remote: 'https://localhost:8080/remotePath/sub?baseParm=1&param=1#another=willoverwrite',
+    },
+  ],
+}
+const proxy = new ReverseProxy(app, { httpServer: { on: () => {} } })
+for (const proxyId in expectedLocalToRemote) {
+  for (const { local, remote } of expectedLocalToRemote[proxyId]) {
+    const config = proxy._getConfigFromRequest({ url: local })
+    const url = new URL(config.target)
+    strictEqual(localToBackendUrl(config.path, url, local).href, remote, 'error converting to backend')
+  }
+}
+
+// test backendToLocalPath
+const expectedRemoteToLocal = {
+  https: [
+    {
+      local: '/proxy/v1/https/',
+      remote: 'https://localhost:8080/remotePath/',
+    },
+    {
+      local: '/proxy/v1/https/sub/index.html',
+      remote: '/remotePath/sub/index.html',
+    },
+    {
+      local: '/proxy/v1/https/?baseParm=1#one=2&another=3',
+      remote: '?baseParm=1#one=2&another=3',
+    },
+    {
+      local: '/proxy/v1/https/sub?baseParm=1#one=2&another=3',
+      remote: 'https://localhost:8080/remotePath/sub?baseParm=1#one=2&another=3',
+    },
+  ],
+}
+
+for (const proxyId in expectedRemoteToLocal) {
+  for (const { local, remote } of expectedRemoteToLocal[proxyId]) {
+    const config = proxy._getConfigFromRequest({ url: local })
+    const targetUrl = new URL('https://localhost:8080/remotePath/?baseParm=1#one=2&another=3')
+    const remoteUrl = new URL(remote, targetUrl)
+    strictEqual(backendToLocalPath(config.path, targetUrl, remoteUrl), local, 'error converting to local')
+  }
+}
+
+// test _getConfigFromRequest
+
+const expectedConfig = [
+  {
+    local: '/proxy/v1/http/other',
+    config: {
+      target: 'http://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
+      options: {},
+      path: '/proxy/v1/http',
+    },
+  },
+  {
+    local: '/proxy/v1/http',
+    config: undefined,
+  },
+
+  {
+    local: '/proxy/v1/other',
+    config: undefined,
+  },
+  {
+    local: '/proxy/v1/https/',
+    config: {
+      target: 'https://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
+      options: {
+        oneOption: true,
+      },
+      path: '/proxy/v1/https',
+    },
+  },
+]
+
+for (const { local, config } of expectedConfig) {
+  deepEqual(proxy._getConfigFromRequest({ url: local }), config)
+}

@xen-orchestra/upload-ova/package.json

@@ -44,8 +44,8 @@
     "pw": "^0.0.4",
     "strip-indent": "^3.0.0",
     "xdg-basedir": "^4.0.0",
-    "xo-lib": "^0.10.1",
-    "xo-vmdk-to-vhd": "^2.0.0"
+    "xo-lib": "^0.11.1",
+    "xo-vmdk-to-vhd": "^2.0.3"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

@xen-orchestra/xapi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xen-orchestra/xapi",
-  "version": "0.8.0",
+  "version": "0.8.5",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/xapi",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
@@ -38,7 +38,7 @@
     "prepublishOnly": "yarn run build"
   },
   "dependencies": {
-    "@vates/decorate-with": "^0.1.0",
+    "@vates/decorate-with": "^1.0.0",
     "@xen-orchestra/async-map": "^0.1.2",
     "@xen-orchestra/log": "^0.3.0",
     "d3-time-format": "^3.0.0",

@xen-orchestra/xapi/src/index.js

@@ -98,18 +98,16 @@ function removeWatcher(predicate, cb) {
 
 class Xapi extends Base {
   constructor({
-    callRetryWhenTooManyPendingTasks,
+    callRetryWhenTooManyPendingTasks = { delay: 5e3, tries: 10 },
     ignoreNobakVdis,
     maxUncoalescedVdis,
-    vdiDestroyRetryWhenInUse,
+    vdiDestroyRetryWhenInUse = { delay: 5e3, tries: 10 },
     ...opts
   }) {
     assert.notStrictEqual(ignoreNobakVdis, undefined)
 
     super(opts)
     this._callRetryWhenTooManyPendingTasks = {
-      delay: 5e3,
-      tries: 10,
       ...callRetryWhenTooManyPendingTasks,
       onRetry,
       when: { code: 'TOO_MANY_PENDING_TASKS' },
@@ -117,8 +115,6 @@ class Xapi extends Base {
     this._ignoreNobakVdis = ignoreNobakVdis
     this._maxUncoalescedVdis = maxUncoalescedVdis
     this._vdiDestroyRetryWhenInUse = {
-      delay: 5e3,
-      retries: 10,
       ...vdiDestroyRetryWhenInUse,
       onRetry,
       when: { code: 'VDI_IN_USE' },

@xen-orchestra/xapi/src/vm.js

@@ -60,7 +60,7 @@ module.exports = class Vm {
       try {
         vdi = await this[vdiRefOrUuid.startsWith('OpaqueRef:') ? 'getRecord' : 'getRecordByUuid']('VDI', vdiRefOrUuid)
       } catch (error) {
-        warn(error)
+        warn('_assertHealthyVdiChain, could not fetch VDI', { error })
         return
       }
       cache[vdi.$ref] = vdi
@@ -81,7 +81,7 @@ module.exports = class Vm {
                   try {
                     vdi = await this.getRecord('VDI', vdiRef)
                   } catch (error) {
-                    warn(error)
+                    warn('_assertHealthyVdiChain, could not fetch VDI', { error })
                     return
                   }
                   cache[vdiRef] = vdi
@@ -99,6 +99,7 @@ module.exports = class Vm {
       // should coalesce
       const children = childrenMap[vdi.uuid]
       if (
+        children !== undefined && // unused unmanaged VDI, will be GC-ed
         children.length === 1 &&
         !children[0].managed && // some SRs do not coalesce the leaf
         tolerance-- <= 0
@@ -166,7 +167,7 @@ module.exports = class Vm {
       memory_static_min,
       name_description,
       name_label,
-      // NVRAM, // experimental
+      NVRAM,
       order,
       other_config = {},
       PCI_bus = '',
@@ -255,6 +256,7 @@ module.exports = class Vm {
       is_vmss_snapshot,
       name_description,
       name_label,
+      NVRAM,
       order,
       reference_label,
       shutdown_delay,

CHANGELOG.md

@@ -1,9 +1,152 @@
-## **5.64.0** (2021-10-29)
-
 # ChangeLog
 
+## **5.66.2** (2022-01-05)
+
 <img id="latest" src="https://badgen.net/badge/channel/latest/yellow" alt="Channel: latest" />
 
+### Bug fixes
+
+- [Import/Disk] Fix `JSON.parse` and `createReadableSparseStream is not a function` errors [#6068](https://github.com/vatesfr/xen-orchestra/issues/6068)
+- [Backup] Fix delta backup are almost always full backup instead of differentials [Forum#5256](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it/69) [Forum#5371](https://xcp-ng.org/forum/topic/5371/delta-backup-changes-in-5-66) (PR [#6075](https://github.com/vatesfr/xen-orchestra/pull/6075))
+
+### Released packages
+
+- vhd-lib 3.0.0
+- xo-vmdk-to-vhd 2.0.3
+- @xen-orchestra/backups 0.18.3
+- @xen-orchestra/proxy 0.17.3
+- xo-server 5.86.3
+- xo-web 5.91.2
+
+## **5.66.1** (2021-12-23)
+
+### Bug fixes
+
+- [Dashboard/Health] Fix `error has occured` when a pool has no default SR
+- [Delta Backup] Fix unnecessary full backup when not using S3 [Forum#5371](https://xcp-ng.org/forum/topic/5371/delta-backup-changes-in-5-66) (PR [#6070](https://github.com/vatesfr/xen-orchestra/pull/6070))
+- [Backup] Fix incorrect warnings `incorrect size [...] instead of undefined`
+
+### Released packages
+
+- @xen-orchestra/backups 0.18.2
+- @xen-orchestra/proxy 0.17.2
+- xo-server 5.86.2
+- xo-web 5.91.1
+
+## **5.66.0** (2021-12-21)
+
+### Enhancements
+
+- [About] Show commit instead of version numbers for source users (PR [#6045](https://github.com/vatesfr/xen-orchestra/pull/6045))
+- [Health] Display default SRs that aren't shared [#5871](https://github.com/vatesfr/xen-orchestra/issues/5871) (PR [#6033](https://github.com/vatesfr/xen-orchestra/pull/6033))
+- [Pool,VM/advanced] Ability to change the suspend SR [#4163](https://github.com/vatesfr/xen-orchestra/issues/4163) (PR [#6044](https://github.com/vatesfr/xen-orchestra/pull/6044))
+- [Home/VMs/Backup filter] Filter out VMs in disabled backup jobs (PR [#6037](https://github.com/vatesfr/xen-orchestra/pull/6037))
+- [Rolling Pool Update] Automatically disable High Availability during the update [#5711](https://github.com/vatesfr/xen-orchestra/issues/5711) (PR [#6057](https://github.com/vatesfr/xen-orchestra/pull/6057))
+- [Delta Backup on S3] Compress blocks by default ([Brotli](https://en.wikipedia.org/wiki/Brotli)) which reduces remote usage and increase backup speed (PR [#5932](https://github.com/vatesfr/xen-orchestra/pull/5932))
+
+### Bug fixes
+
+- [Tables/actions] Fix collapsed actions being clickable despite being disabled (PR [#6023](https://github.com/vatesfr/xen-orchestra/pull/6023))
+- [Backup] Remove incorrect size warning following a merge [Forum#5727](https://xcp-ng.org/forum/topic/4769/warnings-showing-in-system-logs-following-each-backup-job/4) (PR [#6010](https://github.com/vatesfr/xen-orchestra/pull/6010))
+- [Delta Backup] Preserve UEFI boot parameters [#6054](https://github.com/vatesfr/xen-orchestra/issues/6054) [Forum#5319](https://xcp-ng.org/forum/topic/5319/bug-uefi-boot-parameters-not-preserved-with-delta-backups)
+
+### Released packages
+
+- @xen-orchestra/mixins 0.1.2
+- @xen-orchestra/xapi 0.8.5
+- vhd-lib 2.1.0
+- xo-vmdk-to-vhd 2.0.2
+- @xen-orchestra/backups 0.18.1
+- @xen-orchestra/proxy 0.17.1
+- xo-server 5.86.1
+- xo-web 5.91.0
+
+## **5.65.3** (2021-12-20)
+
+<img id="stable" src="https://badgen.net/badge/channel/stable/green" alt="Channel: stable" />
+
+### Bug fixes
+
+- [Continuous Replication] Fix `could not find the base VM`
+- [Backup/Smart mode] Always ignore replicated VMs created by the current job
+- [Backup] Fix `Unexpected end of JSON input` during merge step
+- [Backup] Fix stuck jobs when using S3 remotes (PR [#6067](https://github.com/vatesfr/xen-orchestra/pull/6067))
+
+### Released packages
+
+- @xen-orchestra/fs 0.19.3
+- vhd-lib 2.0.4
+- @xen-orchestra/backups 0.17.1
+- xo-server 5.85.1
+
+## **5.65.2** (2021-12-10)
+
+### Bug fixes
+
+- [Backup] Fix `handler.rmTree` is not a function [Forum#5256](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it/29) (PR [#6041](https://github.com/vatesfr/xen-orchestra/pull/6041))
+- [Backup] Fix `EEXIST` in logs when multiple merge tasks are created at the same time [Forum#5301](https://xcp-ng.org/forum/topic/5301/warnings-errors-in-journalctl)
+- [Backup] Fix missing backup on restore [Forum#5256](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it/29) (PR [#6048](https://github.com/vatesfr/xen-orchestra/pull/6048))
+
+### Released packages
+
+- @xen-orchestra/fs 0.19.2
+- vhd-lib 2.0.3
+- @xen-orchestra/backups 0.16.2
+- xo-server 5.84.3
+- @xen-orchestra/proxy 0.15.5
+
+## **5.65.1** (2021-12-03)
+
+### Bug fixes
+
+- [Delta Backup Restoration] Fix assertion error [Forum#5257](https://xcp-ng.org/forum/topic/5257/problems-building-from-source/16)
+- [Delta Backup Restoration] `TypeError: this disposable has already been disposed` [Forum#5257](https://xcp-ng.org/forum/topic/5257/problems-building-from-source/20)
+- [Backups] Fix: `Error: Chaining alias is forbidden xo-vm-backups/..alias.vhd to xo-vm-backups/....alias.vhd` when backuping a file to s3 [Forum#5226](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it)
+- [Delta Backup Restoration] `VDI_IO_ERROR(Device I/O errors)` [Forum#5727](https://xcp-ng.org/forum/topic/5257/problems-building-from-source/4) (PR [#6031](https://github.com/vatesfr/xen-orchestra/pull/6031))
+- [Delta Backup] Fix `Cannot read property 'uuid' of undefined` when a VDI has been removed from a backed up VM (PR [#6034](https://github.com/vatesfr/xen-orchestra/pull/6034))
+
+### Released packages
+
+- @vates/compose 2.1.0
+- vhd-lib 2.0.2
+- xo-vmdk-to-vhd 2.0.1
+- @xen-orchestra/backups 0.16.1
+- @xen-orchestra/proxy 0.15.4
+- xo-server 5.84.2
+
+## **5.65.0** (2021-11-30)
+
+### Highlights
+
+- [VM] Ability to export a snapshot's memory (PR [#6015](https://github.com/vatesfr/xen-orchestra/pull/6015))
+- [Cloud config] Ability to create a network cloud config template and reuse it in the VM creation [#5931](https://github.com/vatesfr/xen-orchestra/issues/5931) (PR [#5979](https://github.com/vatesfr/xen-orchestra/pull/5979))
+- [Backup/logs] identify XAPI errors (PR [#6001](https://github.com/vatesfr/xen-orchestra/pull/6001))
+- [lite] Highlight selected VM (PR [#5939](https://github.com/vatesfr/xen-orchestra/pull/5939))
+
+### Enhancements
+
+- [S3] Ability to authorize self signed certificates for S3 remote (PR [#5961](https://github.com/vatesfr/xen-orchestra/pull/5961))
+
+### Bug fixes
+
+- [Import/VM] Fix the import of OVA files (PR [#5976](https://github.com/vatesfr/xen-orchestra/pull/5976))
+
+### Released packages
+
+- @vates/async-each 0.1.0
+- xo-remote-parser 0.8.4
+- @xen-orchestra/fs 0.19.0
+- @xen-orchestra/xapi patch
+- vhd-lib 2.0.1
+- @xen-orchestra/backups 0.16.0
+- xo-lib 0.11.1
+- @xen-orchestra/proxy 0.15.3
+- xo-server 5.84.1
+- vhd-cli 0.6.0
+- xo-web 5.90.0
+
+## **5.64.0** (2021-10-29)
+
 ## Highlights
 
 - [Netbox] Support older versions of Netbox and prevent "active is not a valid choice" error [#5898](https://github.com/vatesfr/xen-orchestra/issues/5898) (PR [#5946](https://github.com/vatesfr/xen-orchestra/pull/5946))
@@ -11,8 +154,8 @@
 - [Host] Handle evacuation failure during host shutdown (PR [#5966](https://github.com/vatesfr/xen-orchestra/pull/#5966))
 - [Menu] Notify user when proxies need to be upgraded (PR [#5930](https://github.com/vatesfr/xen-orchestra/pull/5930))
 - [Servers] Ability to use an HTTP proxy between XO and a server (PR [#5958](https://github.com/vatesfr/xen-orchestra/pull/5958))
-- [VM/export] Ability to copy the export URL (PR [#5948](https://github.com/vatesfr/xen-orchestra/pull/5948))
-- [Pool/advanced] Ability to define network for importing/exporting VMs/VDIs (PR [#5957](https://github.com/vatesfr/xen-orchestra/pull/5957))
+- [VM/export] Ability to copy the export URL (PR [#5948](https://github.com/vatesfr/xen-orchestra/pull/5948))
+- [Pool/advanced] Ability to define network for importing/exporting VMs/VDIs (PR [#5957](https://github.com/vatesfr/xen-orchestra/pull/5957))
 - [Host/advanced] Add button to enable/disable the host (PR [#5952](https://github.com/vatesfr/xen-orchestra/pull/5952))
 - [Backups] Enable merge worker by default
 
@@ -45,13 +188,11 @@
 
 ## **5.63.0** (2021-09-30)
 
-<img id="stable" src="https://badgen.net/badge/channel/stable/green" alt="Channel: stable" />
-
 ### Highlights
 
 - [Backup] Go back to previous page instead of going to the overview after editing a job: keeps current filters and page (PR [#5913](https://github.com/vatesfr/xen-orchestra/pull/5913))
-- [Health] Do not take into consideration duplicated MAC addresses from CR VMs (PR [#5916](https://github.com/vatesfr/xen-orchestra/pull/5916))
-- [Health] Ability to filter duplicated MAC addresses by running VMs (PR [#5917](https://github.com/vatesfr/xen-orchestra/pull/5917))
+- [Health] Do not take into consideration duplicated MAC addresses from CR VMs (PR [#5916](https://github.com/vatesfr/xen-orchestra/pull/5916))
+- [Health] Ability to filter duplicated MAC addresses by running VMs (PR [#5917](https://github.com/vatesfr/xen-orchestra/pull/5917))
 - [Tables] Move the search bar and pagination to the top of the table (PR [#5914](https://github.com/vatesfr/xen-orchestra/pull/5914))
 - [Netbox] Handle nested prefixes by always assigning an IP to the smallest prefix it matches (PR [#5908](https://github.com/vatesfr/xen-orchestra/pull/5908))
 
@@ -199,7 +340,7 @@
 
 ### Bug fixes
 
-- [SDN Controller] Private network creation failure when the tunnels were created on different devices [Forum #4620](https://xcp-ng.org/forum/topic/4620/no-pif-found-in-center) (PR [#5793](https://github.com/vatesfr/xen-orchestra/pull/5793))
+- [SDN Controller] Private network creation failure when the tunnels were created on different devices [Forum#4620](https://xcp-ng.org/forum/topic/4620/no-pif-found-in-center) (PR [#5793](https://github.com/vatesfr/xen-orchestra/pull/5793))
 
 ### Released packages
 
@@ -318,7 +459,7 @@
 - [Proxy] _Redeploy_ now works when the bound VM is missing
 - [VM template] Fix confirmation modal doesn't appear on deleting a default template (PR [#5644](https://github.com/vatesfr/xen-orchestra/pull/5644))
 - [OVA VM Import] Fix imported VMs all having the same MAC addresses
-- [Disk import] Fix `an error has occurred` when importing wrong format or corrupted files [#5663](https://github.com/vatesfr/xen-orchestra/issues/5663) (PR [#5683](https://github.com/vatesfr/xen-orchestra/pull/5683))
+- [Disk import] Fix `an error has occurred` when importing wrong format or corrupted files [#5663](https://github.com/vatesfr/xen-orchestra/issues/5663) (PR [#5683](https://github.com/vatesfr/xen-orchestra/pull/5683))
 
 ### Released packages
 

CHANGELOG.unreleased.md

@@ -7,11 +7,24 @@
 
 > Users must be able to say: “Nice enhancement, I'm eager to test it”
 
+- Limit number of concurrent VM migrations per pool to `3`. Can be changed in `xo-server`'s configuration file: `xapiOptions.vmMigrationConcurrency` [#6065](https://github.com/vatesfr/xen-orchestra/issues/6065) (PR [#6076](https://github.com/vatesfr/xen-orchestra/pull/6076))
+- [Proxy] Now ships a reverse proxy (PR [#6072](https://github.com/vatesfr/xen-orchestra/pull/6072))
+- [Delta Backup] When using S3 remote, retry uploading VHD parts on Internal Error to support [Blackblaze](https://www.backblaze.com/b2/docs/calling.html#error_handling) [Forum#5397](https://xcp-ng.org/forum/topic/5397/delta-backups-failing-aws-s3-uploadpartcopy-cpu-too-busy/5) (PR [#6086](https://github.com/vatesfr/xen-orchestra/issues/6086))
+- [Backup] Add sanity check of aliases on S3 remotes (PR [#6043](https://github.com/vatesfr/xen-orchestra/pull/6043))
+- [Export/Disks] Allow the export of disks in VMDK format (PR [#5982](https://github.com/vatesfr/xen-orchestra/pull/5982))
+- [Rolling Pool Update] Automatically pause load balancer plugin during the update [#5711](https://github.com/vatesfr/xen-orchestra/issues/5711)
+- [Backup] Speedup merge and cleanup speed for S3 backup by a factor 10 (PR [#6100](https://github.com/vatesfr/xen-orchestra/pull/6100))
+
 ### Bug fixes
 
 > Users must be able to say: “I had this issue, happy to know it's fixed”
 
-[Import/VM] Fix the import of OVA files (PR [#5976](https://github.com/vatesfr/xen-orchestra/pull/5976))
+- [Backup] Detect and clear orphan merge states, fix `ENOENT` errors (PR [#6087](https://github.com/vatesfr/xen-orchestra/pull/6087))
+- [Backup] Ensure merges are also executed after backup on S3, maintaining the size of the VHD chain under control [Forum#45743](https://xcp-ng.org/forum/post/45743) (PR [#6095](https://github.com/vatesfr/xen-orchestra/pull/6095))
+- [Backup] Delete backups immediately instead of waiting for the next backup (PR [#6081](https://github.com/vatesfr/xen-orchestra/pull/6081))
+- [Backup] Delete S3 backups completely, even if there are more than 1000 files (PR [#6103](https://github.com/vatesfr/xen-orchestra/pull/6103))
+- [Backup] Fix merge resuming (PR [#6099](https://github.com/vatesfr/xen-orchestra/pull/6099))
+- [Plugin/Audit] Fix `key cannot be 'null' or 'undefined'` error when no audit log in the database [#6040](https://github.com/vatesfr/xen-orchestra/issues/6040) (PR [#6071](https://github.com/vatesfr/xen-orchestra/pull/6071))
 
 ### Packages to release
 
@@ -31,5 +44,11 @@
 > In case of conflict, the highest (lowest in previous list) `$version` wins.
 
 - @xen-orchestra/fs minor
-- xo-server patch
-- vhd-cli minor
+- vhd-lib minor
+- xo-vmdk-to-vhd minor
+- @xen-orchestra/backups minor
+- @xen-orchestra/backups-cli minor
+- @xen-orchestra/proxy minor
+- xo-server-audit patch
+- xo-server minor
+- xo-web minor

docs/advanced.md

@@ -327,6 +327,8 @@ Synchronize your pools, VMs, network interfaces and IP addresses with your [Netb
 
 ![](./assets/netbox.png)
 
+### Netbox side
+
 - Go to your Netbox interface
 - Configure prefixes:
   - Go to IPAM > Prefixes > Add
@@ -339,13 +341,29 @@ XO will try to find the right prefix for each IP address. If it can't find a pre
 - Generate a token:
   - Go to Admin > Tokens > Add token
   - Create a token with "Write enabled"
-- Add a UUID custom field:
+  - The owner of the token must have at least the following permissions:
+    - View permissions on:
+      - extras > custom-fields
+      - ipam > prefixes
+    - All permissions on:
+      - ipam > ip-addresses
+      - virtualization > cluster-types
+      - virtualization > clusters
+      - virtualization > interfaces
+      - virtualization > virtual-machines
+- Add a UUID custom field (for **Netbox 2.x**):
   - Got to Admin > Custom fields > Add custom field
   - Create a custom field called "uuid" (lower case!)
   - Assign it to object types `virtualization > cluster` and `virtualization > virtual machine`
 
 ![](./assets/customfield.png)
 
+:::tip
+In Netbox 3.x, custom fields can be found directly in the site (no need to go in the admin section). It's available in "Other/Customization/Custom Fields". After creation of the `uuid` field, assign it to the object types `virtualization > cluster` and `virtualization > virtual machine`.
+:::
+
+### In Xen Orchestra
+
 - Go to Xen Orchestra > Settings > Plugins > Netbox and fill out the configuration:
   - Endpoint: the URL of your Netbox instance (e.g.: `https://netbox.company.net`)
   - Unauthorized certificate: only for HTTPS, enable this option if your Netbox instance uses a self-signed SSL certificate

docs/backups.md

@@ -26,6 +26,12 @@ Each backups' job execution is identified by a `runId`. You can find this `runId
 
 ![](./assets/log-runId.png)
 
+## Exclude disks
+
+During a backup job, you can avoid saving all disks of the VM. To do that is trivial: just edit the VM disk name and add `[NOBAK]` before the current name, eg: `data-disk` will become `[NOBAK] data-disk` (with a space or not, doesn't matter).
+
+The disks marked with `[NOBAK]` will be now ignored in all following backups.
+
 ## Schedule
 
 :::tip

docs/delta_backups.md

@@ -43,12 +43,6 @@ Just go into your "Backup" view, and select Delta Backup. Then, it's the same as
 
 Unlike other types of backup jobs which delete the associated snapshot when the job is done and it has been exported, delta backups always keep a snapshot of every VM in the backup job, and uses it for the delta. Do not delete these snapshots!
 
-## Exclude disks
-
-During a delta backup job, you can avoid saving all disks of the VM. To do that is trivial: just edit the VM disk name and add `[NOBAK]` before the current name, eg: `data-disk` will become `[NOBAK] data-disk` (with a space or not, doesn't matter).
-
-The disks marked with `[NOBAK]` will be now ignored in all following backups.
-
 ## Delta backup initial seed
 
 If you don't want to do an initial full directly toward the destination, you can create a local delta backup first, then transfer the files to your destination.

docs/xoa.md

@@ -61,7 +61,7 @@ Please only use this if you have issues with [the default way to deploy XOA](ins
 Alternatively, you can deploy it by connecting to your XenServer host and executing the following:
 
 ```
-bash -c "$(curl -sS https://xoa.io/deploy)"
+bash -c "$(wget -qO- https://xoa.io/deploy)"
 ```
 
 :::tip
@@ -78,7 +78,7 @@ curl: (35) error:1407742E:SSL routines:SSL23_GET_SERVER_HELLO:tlsv1 alert protoc
 It means that the secure HTTPS protocol is not supported, you can bypass this using the unsecure command instead:
 
 ```
-bash -c "$(curl -sS http://xoa.io/deploy)"
+bash -c "$(wget -qO- http://xoa.io/deploy)"
 ```
 
 :::
@@ -103,9 +103,9 @@ In that case, you already set the password for `xoa` user. If you forgot it, see
 
 ### Manually deployed
 
-If you connect via SSH or console for the first time without using our [web deploy form](https://xen-orchestra.com/#!/xoa), be aware **there's NO default password set for security reasons**. To set it, you need to connect to your host to find the XOA VM UUID (eg via `xe vm-list`).
+If you connect via SSH or console for the first time without using our [web deploy form](https://xen-orchestra.com/#!/xoa), be aware **there is NO default password set for security reasons**. To set it, you need to connect to your host to find the XOA VM UUID (eg via `xe vm-list`).
 
-Then replace `<UUID>` with the previously find UUID, and `<password>` with your password:
+Next, you can replace `<UUID>` with the UUID you found previously, and `<password>` with your password:
 
 ```
 xe vm-param-set uuid=<UUID> xenstore-data:vm-data/system-account-xoa-password=<password>
@@ -115,7 +115,9 @@ xe vm-param-set uuid=<UUID> xenstore-data:vm-data/system-account-xoa-password=<p
 Don't forget to use quotes for your password, eg: `xenstore-data:vm-data/system-account-xoa-password='MyPassW0rd!'`
 :::
 
-Then, you could connect with `xoa` username and the password you defined in the previous command, eg with `ssh xoa@<XOA IP ADDRESS>`.
+Finally, you must reboot the VM to implement the changes.
+
+You can now connect with the `xoa` username and password you defined in the previous command, eg with `ssh xoa@<XOA IP ADDRESS>`.
 
 ### Using sudo
 

docs/xosan.md

@@ -318,7 +318,7 @@ XOSAN is a 100% software defined solution for XenServer hyperconvergence. You ca
 
 You will need to be registered on our website in order to use Xen Orchestra. If you are not yet registered, [here is the way](https://xen-orchestra.com/#!/signup)
 
-SSH in your XenServer and use the command line `bash -c "$(curl -sS https://xoa.io/deploy)"` - it will deploy Xen Orchestra Appliance on your XenServer infrastructure which is required to use XOSAN.
+SSH in your XenServer and use the command line `bash -c "$(wget -qO- https://xoa.io/deploy)"` - it will deploy Xen Orchestra Appliance on your XenServer infrastructure which is required to use XOSAN.
 
 > Note: You can also download the XVA file and follow [these instructions](https://xen-orchestra.com/docs/xoa.html#the-alternative).
 

package.json

@@ -19,7 +19,7 @@
     "handlebars": "^4.7.6",
     "husky": "^4.2.5",
     "jest": "^27.3.1",
-    "lint-staged": "^11.1.2",
+    "lint-staged": "^12.0.3",
     "lodash": "^4.17.4",
     "prettier": "^2.0.5",
     "promise-toolbox": "^0.20.0",
@@ -46,7 +46,6 @@
       ],
       "^(value-matcher)$": "$1/src",
       "^(vhd-cli)$": "$1/src",
-      "^(vhd-lib)$": "$1/src",
       "^(xo-[^/]+)$": [
         "$1/src",
         "$1"
@@ -61,8 +60,7 @@
       "/xo-server-test/",
       "/xo-web/"
     ],
-    "testRegex": "\\.spec\\.js$",
-    "timers": "fake"
+    "testRegex": "\\.spec\\.js$"
   },
   "lint-staged": {
     "*.{md,ts,ts}": "prettier --write"

packages/value-matcher/README.md

@@ -68,7 +68,7 @@ predicate([false, { foo: 'bar', baz: 42 }, null, 42]) // true
 predicate('foo') // false
 ```
 
-### `{ __all: Pattern[] }`
+### `{ __and: Pattern[] }`
 
 All patterns must match.
 

packages/value-matcher/USAGE.md

@@ -50,7 +50,7 @@ predicate([false, { foo: 'bar', baz: 42 }, null, 42]) // true
 predicate('foo') // false
 ```
 
-### `{ __all: Pattern[] }`
+### `{ __and: Pattern[] }`
 
 All patterns must match.
 

packages/vhd-cli/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "vhd-cli",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "license": "ISC",
   "description": "Tools to read/create and merge VHD files",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-cli",
@@ -24,11 +24,12 @@
     "node": ">=8.10"
   },
   "dependencies": {
-    "@xen-orchestra/fs": "^0.18.0",
+    "@xen-orchestra/fs": "^0.19.3",
     "cli-progress": "^3.1.0",
     "exec-promise": "^0.7.0",
     "getopts": "^2.2.3",
-    "vhd-lib": "^1.3.0"
+    "human-format": "^0.11.0",
+    "vhd-lib": "^3.0.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/vhd-cli/src/commands/info.js

@@ -1,6 +1,34 @@
-import { VhdFile } from 'vhd-lib'
+import { Constants, VhdFile } from 'vhd-lib'
 import { getHandler } from '@xen-orchestra/fs'
 import { resolve } from 'path'
+import humanFormat from 'human-format'
+import invert from 'lodash/invert.js'
+
+const { PLATFORMS } = Constants
+
+const DISK_TYPES_MAP = invert(Constants.DISK_TYPES)
+const PLATFORMS_MAP = invert(PLATFORMS)
+
+const MAPPERS = {
+  bytes: humanFormat.bytes,
+  date: _ => (_ !== 0 ? new Date(_) : 0),
+  diskType: _ => DISK_TYPES_MAP[_],
+  platform: _ => PLATFORMS_MAP[_],
+}
+function mapProperties(object, mapping) {
+  const result = { ...object }
+  for (const prop of Object.keys(mapping)) {
+    const value = object[prop]
+    if (value !== undefined) {
+      let mapper = mapping[prop]
+      if (typeof mapper === 'string') {
+        mapper = MAPPERS[mapper]
+      }
+      result[prop] = mapper(value)
+    }
+  }
+  return result
+}
 
 export default async args => {
   const vhd = new VhdFile(getHandler({ url: 'file:///' }), resolve(args[0]))
@@ -12,6 +40,26 @@ export default async args => {
     await vhd.readHeaderAndFooter(false)
   }
 
-  console.log(vhd.header)
-  console.log(vhd.footer)
+  console.log(
+    mapProperties(vhd.footer, {
+      currentSize: 'bytes',
+      diskType: 'diskType',
+      originalSize: 'bytes',
+      timestamp: 'date',
+    })
+  )
+
+  console.log(
+    mapProperties(vhd.header, {
+      blockSize: 'bytes',
+      parentTimestamp: 'date',
+      parentLocatorEntry: _ =>
+        _.filter(_ => _.platformCode !== PLATFORMS.NONE) // hide empty
+          .map(_ =>
+            mapProperties(_, {
+              platformCode: 'platform',
+            })
+          ),
+    })
+  )
 }

packages/vhd-cli/src/commands/raw.js

@@ -1,13 +1,18 @@
-import { createContentStream } from 'vhd-lib'
-import { getHandler } from '@xen-orchestra/fs'
+import { openVhd } from 'vhd-lib'
+import { getSyncedHandler } from '@xen-orchestra/fs'
 import { resolve } from 'path'
 
 import { writeStream } from '../_utils'
+import { Disposable } from 'promise-toolbox'
 
 export default async args => {
   if (args.length < 2 || args.some(_ => _ === '-h' || _ === '--help')) {
     return `Usage: ${this.command} <input VHD> [<output raw>]`
   }
 
-  await writeStream(createContentStream(getHandler({ url: 'file:///' }), resolve(args[0])), args[1])
+  await Disposable.use(async function* () {
+    const handler = getSyncedHandler({ url: 'file:///' })
+    const vhd = openVhd(handler, resolve(args[0]))
+    await writeStream(vhd.rawContent())
+  })
 }

packages/vhd-lib/.babelrc.js

@@ -1 +0,0 @@
-module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))

packages/vhd-lib/Vhd/VhdAbstract.integ.spec.js

@@ -0,0 +1,275 @@
+/* eslint-env jest */
+
+const rimraf = require('rimraf')
+const tmp = require('tmp')
+const fs = require('fs-extra')
+const { getSyncedHandler } = require('@xen-orchestra/fs')
+const { Disposable, pFromCallback } = require('promise-toolbox')
+
+const { openVhd } = require('../index')
+const { checkFile, createRandomFile, convertFromRawToVhd, createRandomVhdDirectory } = require('../tests/utils')
+const { VhdAbstract } = require('./VhdAbstract')
+const { BLOCK_UNUSED, FOOTER_SIZE, HEADER_SIZE, PLATFORMS, SECTOR_SIZE } = require('../_constants')
+const { unpackHeader, unpackFooter } = require('./_utils')
+
+let tempDir
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+const streamToBuffer = stream => {
+  let buffer = Buffer.alloc(0)
+
+  return new Promise((resolve, reject) => {
+    stream.on('data', data => (buffer = Buffer.concat([buffer, data])))
+    stream.on('end', () => resolve(buffer))
+  })
+}
+
+test('It creates an alias', async () => {
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file://' + tempDir })
+    const aliasPath = `alias/alias.alias.vhd`
+    const aliasFsPath = `${tempDir}/${aliasPath}`
+    await fs.mkdirp(`${tempDir}/alias`)
+
+    const testOneCombination = async ({ targetPath, targetContent }) => {
+      await VhdAbstract.createAlias(handler, aliasPath, targetPath)
+      // alias file is created
+      expect(await fs.exists(aliasFsPath)).toEqual(true)
+      // content is the target path relative to the alias location
+      const content = await fs.readFile(aliasFsPath, 'utf-8')
+      expect(content).toEqual(targetContent)
+      // create alias fails if alias already exists, remove it before next loop step
+      await fs.unlink(aliasFsPath)
+    }
+
+    const combinations = [
+      { targetPath: `targets.vhd`, targetContent: `../targets.vhd` },
+      { targetPath: `alias/targets.vhd`, targetContent: `targets.vhd` },
+      { targetPath: `alias/sub/targets.vhd`, targetContent: `sub/targets.vhd` },
+      { targetPath: `sibling/targets.vhd`, targetContent: `../sibling/targets.vhd` },
+    ]
+
+    for (const { targetPath, targetContent } of combinations) {
+      await testOneCombination({ targetPath, targetContent })
+    }
+  })
+})
+
+test('alias must have *.alias.vhd extension', async () => {
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file:///' })
+    const aliasPath = `${tempDir}/invalidalias.vhd`
+    const targetPath = `${tempDir}/targets.vhd`
+    expect(async () => await VhdAbstract.createAlias(handler, aliasPath, targetPath)).rejects.toThrow()
+
+    expect(await fs.exists(aliasPath)).toEqual(false)
+  })
+})
+
+test('alias must not be chained', async () => {
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file:///' })
+    const aliasPath = `${tempDir}/valid.alias.vhd`
+    const targetPath = `${tempDir}/an.other.valid.alias.vhd`
+    expect(async () => await VhdAbstract.createAlias(handler, aliasPath, targetPath)).rejects.toThrow()
+    expect(await fs.exists(aliasPath)).toEqual(false)
+  })
+})
+
+test('It rename and unlink a VHDFile', async () => {
+  const initalSize = 4
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSize)
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file:///' })
+    const { size } = await fs.stat(vhdFileName)
+    const targetFileName = `${tempDir}/renamed.vhd`
+
+    await VhdAbstract.rename(handler, vhdFileName, targetFileName)
+    expect(await fs.exists(vhdFileName)).toEqual(false)
+    const { size: renamedSize } = await fs.stat(targetFileName)
+    expect(size).toEqual(renamedSize)
+    await VhdAbstract.unlink(handler, targetFileName)
+    expect(await fs.exists(targetFileName)).toEqual(false)
+  })
+})
+
+test('It rename and unlink a VhdDirectory', async () => {
+  const initalSize = 4
+  const vhdDirectory = `${tempDir}/randomfile.dir`
+  await createRandomVhdDirectory(vhdDirectory, initalSize)
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file:///' })
+    const vhd = yield openVhd(handler, vhdDirectory)
+    expect(vhd.header.cookie).toEqual('cxsparse')
+    expect(vhd.footer.cookie).toEqual('conectix')
+
+    const targetFileName = `${tempDir}/renamed.vhd`
+    // it should clean an existing directory
+    await fs.mkdir(targetFileName)
+    await fs.writeFile(`${targetFileName}/dummy`, 'I exists')
+    await VhdAbstract.rename(handler, vhdDirectory, targetFileName)
+    expect(await fs.exists(vhdDirectory)).toEqual(false)
+    expect(await fs.exists(targetFileName)).toEqual(true)
+    expect(await fs.exists(`${targetFileName}/dummy`)).toEqual(false)
+    await VhdAbstract.unlink(handler, targetFileName)
+    expect(await fs.exists(targetFileName)).toEqual(false)
+  })
+})
+
+test('It create , rename and unlink alias', async () => {
+  const initalSize = 4
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSize)
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  const aliasFileName = `${tempDir}/aliasFileName.alias.vhd`
+  const aliasFileNameRenamed = `${tempDir}/aliasFileNameRenamed.alias.vhd`
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file:///' })
+    await VhdAbstract.createAlias(handler, aliasFileName, vhdFileName)
+    expect(await fs.exists(aliasFileName)).toEqual(true)
+    expect(await fs.exists(vhdFileName)).toEqual(true)
+
+    await VhdAbstract.rename(handler, aliasFileName, aliasFileNameRenamed)
+    expect(await fs.exists(aliasFileName)).toEqual(false)
+    expect(await fs.exists(vhdFileName)).toEqual(true)
+    expect(await fs.exists(aliasFileNameRenamed)).toEqual(true)
+
+    await VhdAbstract.unlink(handler, aliasFileNameRenamed)
+    expect(await fs.exists(aliasFileName)).toEqual(false)
+    expect(await fs.exists(vhdFileName)).toEqual(false)
+    expect(await fs.exists(aliasFileNameRenamed)).toEqual(false)
+  })
+})
+
+test('it can create a vhd stream', async () => {
+  const initialNbBlocks = 3
+  const initalSize = initialNbBlocks * 2
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSize)
+  const vhdFileName = `${tempDir}/vhd.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file://' + tempDir })
+
+    const vhd = yield openVhd(handler, 'vhd.vhd')
+    await vhd.readBlockAllocationTable()
+
+    const parentLocatorBase = Buffer.from('a file path, not aligned', 'utf16le')
+    const aligned = Buffer.alloc(SECTOR_SIZE, 0)
+    parentLocatorBase.copy(aligned)
+    await vhd.writeParentLocator({
+      id: 0,
+      platformCode: PLATFORMS.W2KU,
+      data: parentLocatorBase,
+    })
+    await vhd.writeFooter()
+    const stream = vhd.stream()
+
+    // read all the stream into a buffer
+
+    const buffer = await streamToBuffer(stream)
+    const length = buffer.length
+    const bufFooter = buffer.slice(0, FOOTER_SIZE)
+
+    // footer is still valid
+    expect(() => unpackFooter(bufFooter)).not.toThrow()
+    const footer = unpackFooter(bufFooter)
+
+    // header is still valid
+    const bufHeader = buffer.slice(FOOTER_SIZE, HEADER_SIZE + FOOTER_SIZE)
+    expect(() => unpackHeader(bufHeader, footer)).not.toThrow()
+
+    // 1 deleted block should be in ouput
+    let start = FOOTER_SIZE + HEADER_SIZE + vhd.batSize
+
+    const parentLocatorData = buffer.slice(start, start + SECTOR_SIZE)
+    expect(parentLocatorData.equals(aligned)).toEqual(true)
+    start += SECTOR_SIZE // parent locator
+    expect(length).toEqual(start + initialNbBlocks * vhd.fullBlockSize + FOOTER_SIZE)
+    expect(stream.length).toEqual(buffer.length)
+    // blocks
+    const blockBuf = Buffer.alloc(vhd.sectorsPerBlock * SECTOR_SIZE, 0)
+    for (let i = 0; i < initialNbBlocks; i++) {
+      const blockDataStart = start + i * vhd.fullBlockSize + 512 /* block bitmap */
+      const blockDataEnd = blockDataStart + vhd.sectorsPerBlock * SECTOR_SIZE
+      const content = buffer.slice(blockDataStart, blockDataEnd)
+      await handler.read('randomfile', blockBuf, i * vhd.sectorsPerBlock * SECTOR_SIZE)
+      expect(content.equals(blockBuf)).toEqual(true)
+    }
+    // footer
+    const endFooter = buffer.slice(length - FOOTER_SIZE)
+    expect(bufFooter).toEqual(endFooter)
+
+    await handler.writeFile('out.vhd', buffer)
+    // check that the vhd is still valid
+    await checkFile(`${tempDir}/out.vhd`)
+  })
+})
+
+it('can stream content', async () => {
+  const initalSizeMb = 5 // 2 block and an half
+  const initialNbBlocks = Math.ceil(initalSizeMb / 2)
+  const initialByteSize = initalSizeMb * 1024 * 1024
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSizeMb)
+  const vhdFileName = `${tempDir}/vhd.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  const bat = Buffer.alloc(512)
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file://' + tempDir })
+
+    const vhd = yield openVhd(handler, 'vhd.vhd')
+    // mark first block as unused
+    await handler.read('vhd.vhd', bat, vhd.header.tableOffset)
+    bat.writeUInt32BE(BLOCK_UNUSED, 0)
+    await handler.write('vhd.vhd', bat, vhd.header.tableOffset)
+
+    // read our modified block allocation table
+    await vhd.readBlockAllocationTable()
+    const stream = vhd.rawContent()
+    const buffer = await streamToBuffer(stream)
+
+    // qemu can modify size, to align it to geometry
+
+    // check that data didn't change
+    const blockDataLength = vhd.sectorsPerBlock * SECTOR_SIZE
+
+    // first block should be empty
+    const EMPTY = Buffer.alloc(blockDataLength, 0)
+    const firstBlock = buffer.slice(0, blockDataLength)
+    // using buffer1 toEquals buffer2 make jest crash trying to stringify it on failure
+    expect(firstBlock.equals(EMPTY)).toEqual(true)
+
+    let remainingLength = initialByteSize - blockDataLength // already checked the first block
+    for (let i = 1; i < initialNbBlocks; i++) {
+      // last block will be truncated
+      const blockSize = Math.min(blockDataLength, remainingLength - blockDataLength)
+      const blockDataStart = i * blockDataLength // first block have been deleted
+      const blockDataEnd = blockDataStart + blockSize
+      const content = buffer.slice(blockDataStart, blockDataEnd)
+
+      const blockBuf = Buffer.alloc(blockSize, 0)
+
+      await handler.read('randomfile', blockBuf, i * blockDataLength)
+      expect(content.equals(blockBuf)).toEqual(true)
+      remainingLength -= blockSize
+    }
+  })
+})

packages/vhd-lib/Vhd/VhdAbstract.js

@@ -0,0 +1,335 @@
+const {
+  computeBatSize,
+  computeFullBlockSize,
+  computeSectorOfBitmap,
+  computeSectorsPerBlock,
+  sectorsToBytes,
+} = require('./_utils')
+const {
+  ALIAS_MAX_PATH_LENGTH,
+  PLATFORMS,
+  SECTOR_SIZE,
+  PARENT_LOCATOR_ENTRIES,
+  FOOTER_SIZE,
+  HEADER_SIZE,
+  BLOCK_UNUSED,
+} = require('../_constants')
+const assert = require('assert')
+const path = require('path')
+const asyncIteratorToStream = require('async-iterator-to-stream')
+const { checksumStruct, fuFooter, fuHeader } = require('../_structs')
+const { isVhdAlias, resolveVhdAlias } = require('../aliases')
+
+exports.VhdAbstract = class VhdAbstract {
+  get bitmapSize() {
+    return sectorsToBytes(this.sectorsOfBitmap)
+  }
+
+  get fullBlockSize() {
+    return computeFullBlockSize(this.header.blockSize)
+  }
+
+  get sectorsOfBitmap() {
+    return computeSectorOfBitmap(this.header.blockSize)
+  }
+
+  get sectorsPerBlock() {
+    return computeSectorsPerBlock(this.header.blockSize)
+  }
+
+  get header() {
+    throw new Error('get header is not implemented')
+  }
+
+  get footer() {
+    throw new Error('get footer not implemented')
+  }
+
+  /**
+   * instantiate a Vhd
+   *
+   * @returns {AbstractVhd}
+   */
+  static async open() {
+    throw new Error('open not implemented')
+  }
+
+  /**
+   * Check if this vhd contains a block with id blockId
+   * Must be called after readBlockAllocationTable
+   *
+   * @param {number} blockId
+   * @returns {boolean}
+   *
+   */
+  containsBlock(blockId) {
+    throw new Error(`checking if this vhd contains the block ${blockId} is not implemented`)
+  }
+
+  /**
+   * Read the header and the footer
+   * check their integrity
+   * if checkSecondFooter also checks that the footer at the end is equal to the one at the beginning
+   *
+   * @param {boolean} checkSecondFooter
+   */
+  readHeaderAndFooter(checkSecondFooter = true) {
+    throw new Error(
+      `reading and checking footer, ${checkSecondFooter ? 'second footer,' : ''} and header is not implemented`
+    )
+  }
+
+  readBlockAllocationTable() {
+    throw new Error(`reading block allocation table is not implemented`)
+  }
+
+  /**
+   *
+   * @param {number} blockId
+   * @param {boolean} onlyBitmap
+   * @returns {Buffer}
+   */
+  readBlock(blockId, onlyBitmap = false) {
+    throw new Error(`reading  ${onlyBitmap ? 'bitmap of block' : 'block'} ${blockId} is not implemented`)
+  }
+
+  /**
+   * coalesce the block with id blockId from the child vhd into
+   * this vhd
+   *
+   * @param {AbstractVhd} child
+   * @param {number} blockId
+   *
+   * @returns {number} the merged data size
+   */
+  async coalesceBlock(child, blockId) {
+    const block = await child.readBlock(blockId)
+    await this.writeEntireBlock(block)
+    return block.data.length
+  }
+
+  /**
+   * ensure the bat size can store at least entries block
+   * move blocks if needed
+   * @param {number} entries
+   */
+  ensureBatSize(entries) {
+    throw new Error(`ensuring batSize can store at least  ${entries} is not implemented`)
+  }
+
+  // Write a context footer. (At the end and beginning of a vhd file.)
+  writeFooter(onlyEndFooter = false) {
+    throw new Error(`writing footer   ${onlyEndFooter ? 'only at end' : 'on both side'} is not implemented`)
+  }
+
+  writeHeader() {
+    throw new Error(`writing header is not implemented`)
+  }
+
+  _writeParentLocatorData(parentLocatorId, platformDataOffset, data) {
+    throw new Error(`write Parent locator ${parentLocatorId} is not implemented`)
+  }
+
+  _readParentLocatorData(parentLocatorId, platformDataOffset, platformDataSpace) {
+    throw new Error(`read Parent locator ${parentLocatorId} is not implemented`)
+  }
+  // common
+  get batSize() {
+    return computeBatSize(this.header.maxTableEntries)
+  }
+
+  async writeParentLocator({ id, platformCode = PLATFORMS.NONE, data = Buffer.alloc(0) }) {
+    assert(id >= 0, 'parent Locator id must be a positive number')
+    assert(id < PARENT_LOCATOR_ENTRIES, `parent Locator id  must be less than ${PARENT_LOCATOR_ENTRIES}`)
+
+    await this._writeParentLocatorData(id, data)
+
+    const entry = this.header.parentLocatorEntry[id]
+    const dataSpaceSectors = Math.ceil(data.length / SECTOR_SIZE)
+    entry.platformCode = platformCode
+    entry.platformDataSpace = dataSpaceSectors
+    entry.platformDataLength = data.length
+  }
+
+  async readParentLocator(id) {
+    assert(id >= 0, 'parent Locator id must be a positive number')
+    assert(id < PARENT_LOCATOR_ENTRIES, `parent Locator id  must be less than ${PARENT_LOCATOR_ENTRIES}`)
+    const data = await this._readParentLocatorData(id)
+    // offset is storage specific, don't expose it
+    const { platformCode } = this.header.parentLocatorEntry[id]
+    return {
+      platformCode,
+      id,
+      data,
+    }
+  }
+
+  async setUniqueParentLocator(fileNameString) {
+    await this.writeParentLocator({
+      id: 0,
+      platformCode: PLATFORMS.W2KU,
+      data: Buffer.from(fileNameString, 'utf16le'),
+    })
+
+    for (let i = 1; i < PARENT_LOCATOR_ENTRIES; i++) {
+      await this.writeParentLocator({
+        id: i,
+        platformCode: PLATFORMS.NONE,
+        data: Buffer.alloc(0),
+      })
+    }
+  }
+
+  async *blocks() {
+    const nBlocks = this.header.maxTableEntries
+    for (let blockId = 0; blockId < nBlocks; ++blockId) {
+      if (await this.containsBlock(blockId)) {
+        yield await this.readBlock(blockId)
+      }
+    }
+  }
+
+  static async rename(handler, sourcePath, targetPath) {
+    try {
+      // delete target if it already exists
+      await VhdAbstract.unlink(handler, targetPath)
+    } catch (e) {}
+    await handler.rename(sourcePath, targetPath)
+  }
+
+  static async unlink(handler, path) {
+    const resolved = await resolveVhdAlias(handler, path)
+    try {
+      await handler.unlink(resolved)
+    } catch (err) {
+      if (err.code === 'EISDIR') {
+        await handler.rmtree(resolved)
+      } else {
+        throw err
+      }
+    }
+
+    // also delete the alias file
+    if (path !== resolved) {
+      await handler.unlink(path)
+    }
+  }
+
+  static async createAlias(handler, aliasPath, targetPath) {
+    if (!isVhdAlias(aliasPath)) {
+      throw new Error(`Alias must be named *.alias.vhd,  ${aliasPath} given`)
+    }
+    if (isVhdAlias(targetPath)) {
+      throw new Error(`Chaining alias is forbidden ${aliasPath} to ${targetPath}`)
+    }
+    // aliasPath and targetPath are absolute path from the root of the handler
+    // normalize them so they can't  escape this dir
+    const aliasDir = path.dirname(path.resolve('/', aliasPath))
+    // only store the relative path from alias to target
+    const relativePathToTarget = path.relative(aliasDir, path.resolve('/', targetPath))
+
+    if (relativePathToTarget.length > ALIAS_MAX_PATH_LENGTH) {
+      throw new Error(
+        `Alias relative path ${relativePathToTarget} is too long : ${relativePathToTarget.length} chars, max is ${ALIAS_MAX_PATH_LENGTH}`
+      )
+    }
+    await handler.writeFile(aliasPath, relativePathToTarget)
+  }
+
+  stream() {
+    const { footer, batSize } = this
+    const { ...header } = this.header // copy since we don't ant to modifiy the current header
+    const rawFooter = fuFooter.pack(footer)
+    checksumStruct(rawFooter, fuFooter)
+
+    // update them in header
+    // update checksum in header
+
+    let offset = FOOTER_SIZE + HEADER_SIZE + batSize
+
+    const rawHeader = fuHeader.pack(header)
+    checksumStruct(rawHeader, fuHeader)
+
+    // add parentlocator size
+    for (let i = 0; i < PARENT_LOCATOR_ENTRIES; i++) {
+      header.parentLocatorEntry[i] = {
+        ...header.parentLocatorEntry[i],
+        platformDataOffset: offset,
+      }
+      offset += header.parentLocatorEntry[i].platformDataSpace * SECTOR_SIZE
+    }
+
+    assert.strictEqual(offset % SECTOR_SIZE, 0)
+
+    const bat = Buffer.allocUnsafe(batSize)
+    let offsetSector = offset / SECTOR_SIZE
+    const blockSizeInSectors = this.fullBlockSize / SECTOR_SIZE
+    let fileSize = offsetSector * SECTOR_SIZE + FOOTER_SIZE /* the footer at the end */
+    // compute BAT , blocks starts after parent locator entries
+    for (let i = 0; i < header.maxTableEntries; i++) {
+      if (this.containsBlock(i)) {
+        bat.writeUInt32BE(offsetSector, i * 4)
+        offsetSector += blockSizeInSectors
+        fileSize += this.fullBlockSize
+      } else {
+        bat.writeUInt32BE(BLOCK_UNUSED, i * 4)
+      }
+    }
+
+    const self = this
+    async function* iterator() {
+      yield rawFooter
+      yield rawHeader
+      yield bat
+
+      // yield parent locator
+
+      for (let i = 0; i < PARENT_LOCATOR_ENTRIES; i++) {
+        const space = header.parentLocatorEntry[i].platformDataSpace * SECTOR_SIZE
+        if (space > 0) {
+          const data = (await self.readParentLocator(i)).data
+          // align data to a sector
+          const buffer = Buffer.alloc(space, 0)
+          data.copy(buffer)
+          yield buffer
+        }
+      }
+
+      // yield all blocks
+      // since contains() can be costly for synthetic vhd, use the computed bat
+      for (let i = 0; i < header.maxTableEntries; i++) {
+        if (bat.readUInt32BE(i * 4) !== BLOCK_UNUSED) {
+          const block = await self.readBlock(i)
+          yield block.buffer
+        }
+      }
+      // yield footer again
+      yield rawFooter
+    }
+
+    const stream = asyncIteratorToStream(iterator())
+    stream.length = fileSize
+    return stream
+  }
+
+  rawContent() {
+    const { header, footer } = this
+    const { blockSize } = header
+    const self = this
+    async function* iterator() {
+      const nBlocks = header.maxTableEntries
+      let remainingSize = footer.currentSize
+      const EMPTY = Buffer.alloc(blockSize, 0)
+      for (let blockId = 0; blockId < nBlocks; ++blockId) {
+        let buffer = self.containsBlock(blockId) ? (await self.readBlock(blockId)).data : EMPTY
+        // the last block can be truncated since raw size is not a multiple of blockSize
+        buffer = remainingSize < blockSize ? buffer.slice(0, remainingSize) : buffer
+        remainingSize -= blockSize
+        yield buffer
+      }
+    }
+    const stream = asyncIteratorToStream(iterator())
+    stream.length = footer.currentSize
+    return stream
+  }
+}

packages/vhd-lib/Vhd/VhdDirectory.integ.spec.js

@@ -0,0 +1,114 @@
+/* eslint-env jest */
+
+const rimraf = require('rimraf')
+const tmp = require('tmp')
+const fs = require('fs-extra')
+const { getHandler, getSyncedHandler } = require('@xen-orchestra/fs')
+const { Disposable, pFromCallback } = require('promise-toolbox')
+
+const { openVhd, VhdDirectory } = require('../')
+const { createRandomFile, convertFromRawToVhd, convertToVhdDirectory } = require('../tests/utils')
+
+let tempDir = null
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+test('Can coalesce block', async () => {
+  const initalSize = 4
+  const parentrawFileName = `${tempDir}/randomfile`
+  const parentFileName = `${tempDir}/parent.vhd`
+  const parentDirectoryName = `${tempDir}/parent.dir.vhd`
+
+  await createRandomFile(parentrawFileName, initalSize)
+  await convertFromRawToVhd(parentrawFileName, parentFileName)
+  await convertToVhdDirectory(parentrawFileName, parentFileName, parentDirectoryName)
+
+  const childrawFileName = `${tempDir}/randomfile`
+  const childFileName = `${tempDir}/childFile.vhd`
+  await createRandomFile(childrawFileName, initalSize)
+  await convertFromRawToVhd(childrawFileName, childFileName)
+  const childRawDirectoryName = `${tempDir}/randomFile2.vhd`
+  const childDirectoryFileName = `${tempDir}/childDirFile.vhd`
+  const childDirectoryName = `${tempDir}/childDir.vhd`
+  await createRandomFile(childRawDirectoryName, initalSize)
+  await convertFromRawToVhd(childRawDirectoryName, childDirectoryFileName)
+  await convertToVhdDirectory(childRawDirectoryName, childDirectoryFileName, childDirectoryName)
+
+  await Disposable.use(async function* () {
+    const handler = getHandler({ url: 'file://' })
+    const parentVhd = yield openVhd(handler, parentDirectoryName, { flags: 'w' })
+    await parentVhd.readBlockAllocationTable()
+    const childFileVhd = yield openVhd(handler, childFileName)
+    await childFileVhd.readBlockAllocationTable()
+    const childDirectoryVhd = yield openVhd(handler, childDirectoryName)
+    await childDirectoryVhd.readBlockAllocationTable()
+
+    await parentVhd.coalesceBlock(childFileVhd, 0)
+    await parentVhd.writeFooter()
+    await parentVhd.writeBlockAllocationTable()
+    let parentBlockData = (await parentVhd.readBlock(0)).data
+    let childBlockData = (await childFileVhd.readBlock(0)).data
+    expect(parentBlockData.equals(childBlockData)).toEqual(true)
+
+    await parentVhd.coalesceBlock(childDirectoryVhd, 0)
+    await parentVhd.writeFooter()
+    await parentVhd.writeBlockAllocationTable()
+    parentBlockData = (await parentVhd.readBlock(0)).data
+    childBlockData = (await childDirectoryVhd.readBlock(0)).data
+    expect(parentBlockData).toEqual(childBlockData)
+  })
+})
+
+test('compressed blocks and metadata works', async () => {
+  const initalSize = 4
+  const rawFileName = `${tempDir}/randomfile`
+  const vhdName = `${tempDir}/parent.vhd`
+
+  await createRandomFile(rawFileName, initalSize)
+  await convertFromRawToVhd(rawFileName, vhdName)
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
+    const vhd = yield openVhd(handler, 'parent.vhd')
+    await vhd.readBlockAllocationTable()
+    const compressedVhd = yield VhdDirectory.create(handler, 'compressed.vhd', { compression: 'gzip' })
+    compressedVhd.header = vhd.header
+    compressedVhd.footer = vhd.footer
+    for await (const block of vhd.blocks()) {
+      await compressedVhd.writeEntireBlock(block)
+    }
+    await Promise
+      .all[(await compressedVhd.writeHeader(), await compressedVhd.writeFooter(), await compressedVhd.writeBlockAllocationTable())]
+
+    // compressed vhd have a metadata file
+    expect(await fs.exists(`${tempDir}/compressed.vhd/metadata.json`)).toEqual(true)
+    const metada = JSON.parse(await handler.readFile('compressed.vhd/metadata.json'))
+    expect(metada.compression.type).toEqual('gzip')
+    expect(metada.compression.options.level).toEqual(1)
+
+    // compressed vhd should not be broken
+    await compressedVhd.readHeaderAndFooter()
+    await compressedVhd.readBlockAllocationTable()
+
+    // check that footer and header are not modified
+    expect(compressedVhd.footer).toEqual(vhd.footer)
+    expect(compressedVhd.header).toEqual(vhd.header)
+
+    // their block content should not have changed
+    let counter = 0
+    for await (const block of compressedVhd.blocks()) {
+      const source = await vhd.readBlock(block.id)
+      expect(source.data.equals(block.data)).toEqual(true)
+      counter++
+    }
+    // neither the number of blocks
+    expect(counter).toEqual(2)
+  })
+})

packages/vhd-lib/Vhd/VhdDirectory.js

@@ -0,0 +1,291 @@
+const { unpackHeader, unpackFooter, sectorsToBytes } = require('./_utils')
+const { createLogger } = require('@xen-orchestra/log')
+const { fuFooter, fuHeader, checksumStruct } = require('../_structs')
+const { test, set: setBitmap } = require('../_bitmap')
+const { VhdAbstract } = require('./VhdAbstract')
+const assert = require('assert')
+const promisify = require('promise-toolbox/promisify')
+const zlib = require('zlib')
+
+const { debug } = createLogger('vhd-lib:VhdDirectory')
+
+const NULL_COMPRESSOR = {
+  compress: buffer => buffer,
+  decompress: buffer => buffer,
+  baseOptions: {},
+}
+
+const COMPRESSORS = {
+  gzip: {
+    compress: (
+      gzip => buffer =>
+        gzip(buffer, { level: zlib.constants.Z_BEST_SPEED })
+    )(promisify(zlib.gzip)),
+    decompress: promisify(zlib.gunzip),
+  },
+  brotli: {
+    compress: (
+      brotliCompress => buffer =>
+        brotliCompress(buffer, {
+          params: {
+            [zlib.constants.BROTLI_PARAM_QUALITY]: zlib.constants.BROTLI_MIN_QUALITY,
+          },
+        })
+    )(promisify(zlib.brotliCompress)),
+    decompress: promisify(zlib.brotliDecompress),
+  },
+}
+
+// inject identifiers
+for (const id of Object.keys(COMPRESSORS)) {
+  COMPRESSORS[id].id = id
+}
+
+function getCompressor(compressorType) {
+  if (compressorType === undefined) {
+    return NULL_COMPRESSOR
+  }
+
+  const compressor = COMPRESSORS[compressorType]
+
+  if (compressor === undefined) {
+    throw new Error(`Compression type ${compressorType} is not supported`)
+  }
+
+  return compressor
+}
+
+// ===================================================================
+// Directory format
+// <path>
+// ├─ chunk-filters.json
+// │    Ordered array of filters that have been applied before writing chunks.
+// │    These filters needs to be applied in reverse order to read them.
+// │
+// ├─ header // raw content of the header
+// ├─ footer // raw content of the footer
+// ├─ bat // bit array. A zero bit indicates at a position that this block is not present
+// ├─ parentLocatorEntry{0-7} // data of a parent locator
+// ├─ blocks // blockId is the position in the BAT
+//    └─ <the first to  {blockId.length -3} numbers of blockId >
+//         └─ <the three last numbers  of blockID >  // block content.
+
+exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
+  #uncheckedBlockTable
+  #header
+  footer
+  #compressor
+
+  get compressionType() {
+    return this.#compressor.id
+  }
+
+  set header(header) {
+    this.#header = header
+    this.#blockTable = Buffer.alloc(header.maxTableEntries)
+  }
+
+  get header() {
+    assert.notStrictEqual(this.#header, undefined, `header must be read before it's used`)
+    return this.#header
+  }
+
+  get #blockTable() {
+    assert.notStrictEqual(this.#uncheckedBlockTable, undefined, 'Block table must be initialized before access')
+    return this.#uncheckedBlockTable
+  }
+
+  set #blockTable(blockTable) {
+    this.#uncheckedBlockTable = blockTable
+  }
+
+  static async open(handler, path, { flags = 'r+' } = {}) {
+    const vhd = new VhdDirectory(handler, path, { flags })
+
+    // openning a file for reading does not trigger EISDIR as long as we don't really read from it :
+    // https://man7.org/linux/man-pages/man2/open.2.html
+    // EISDIR pathname refers to a directory and the access requested
+    // involved writing (that is, O_WRONLY or O_RDWR is set).
+    // reading the header ensure we have a well formed directory immediatly
+    await vhd.readHeaderAndFooter()
+    return {
+      dispose: () => {},
+      value: vhd,
+    }
+  }
+
+  static async create(handler, path, { flags = 'wx+', compression } = {}) {
+    await handler.mkdir(path)
+    const vhd = new VhdDirectory(handler, path, { flags, compression })
+    return {
+      dispose: () => {},
+      value: vhd,
+    }
+  }
+
+  constructor(handler, path, opts) {
+    super()
+    this._handler = handler
+    this._path = path
+    this._opts = opts
+    this.#compressor = getCompressor(opts?.compression)
+  }
+
+  async readBlockAllocationTable() {
+    const { buffer } = await this._readChunk('bat')
+    this.#blockTable = buffer
+  }
+
+  containsBlock(blockId) {
+    return test(this.#blockTable, blockId)
+  }
+
+  _getChunkPath(partName) {
+    return this._path + '/' + partName
+  }
+
+  async _readChunk(partName) {
+    // here we can implement compression and / or crypto
+    const buffer = await this._handler.readFile(this._getChunkPath(partName))
+
+    const uncompressed = await this.#compressor.decompress(buffer)
+    return {
+      buffer: uncompressed,
+    }
+  }
+
+  async _writeChunk(partName, buffer) {
+    assert.notStrictEqual(
+      this._opts?.flags,
+      'r',
+      `Can't write a chunk ${partName} in ${this._path} with read permission`
+    )
+
+    const compressed = await this.#compressor.compress(buffer)
+    return this._handler.outputFile(this._getChunkPath(partName), compressed, this._opts)
+  }
+
+  // put block in subdirectories to limit impact when doing directory listing
+  _getBlockPath(blockId) {
+    const blockPrefix = Math.floor(blockId / 1e3)
+    const blockSuffix = blockId - blockPrefix * 1e3
+    return `blocks/${blockPrefix}/${blockSuffix}`
+  }
+
+  async readHeaderAndFooter() {
+    await this.#readChunkFilters()
+
+    let bufHeader, bufFooter
+    try {
+      bufHeader = (await this._readChunk('header')).buffer
+      bufFooter = (await this._readChunk('footer')).buffer
+    } catch (error) {
+      // emit an AssertionError if the VHD is broken to stay as close as possible to the VhdFile API
+      if (error.code === 'ENOENT') {
+        assert(false, 'Header And Footer should exists')
+      } else {
+        throw error
+      }
+    }
+    const footer = unpackFooter(bufFooter)
+    const header = unpackHeader(bufHeader, footer)
+
+    this.footer = footer
+    this.header = header
+  }
+
+  async readBlock(blockId, onlyBitmap = false) {
+    if (onlyBitmap) {
+      throw new Error(`reading 'bitmap of block' ${blockId} in a VhdDirectory is not implemented`)
+    }
+    const { buffer } = await this._readChunk(this._getBlockPath(blockId))
+    return {
+      id: blockId,
+      bitmap: buffer.slice(0, this.bitmapSize),
+      data: buffer.slice(this.bitmapSize),
+      buffer,
+    }
+  }
+  ensureBatSize() {
+    // nothing to do in directory mode
+  }
+
+  async writeFooter() {
+    const { footer } = this
+
+    const rawFooter = fuFooter.pack(footer)
+
+    footer.checksum = checksumStruct(rawFooter, fuFooter)
+    debug(`Write footer  (checksum=${footer.checksum}). (data=${rawFooter.toString('hex')})`)
+
+    await this._writeChunk('footer', rawFooter)
+  }
+
+  async writeHeader() {
+    const { header } = this
+    const rawHeader = fuHeader.pack(header)
+    header.checksum = checksumStruct(rawHeader, fuHeader)
+    debug(`Write header  (checksum=${header.checksum}). (data=${rawHeader.toString('hex')})`)
+    await this._writeChunk('header', rawHeader)
+    await this.#writeChunkFilters()
+  }
+
+  writeBlockAllocationTable() {
+    assert.notStrictEqual(this.#blockTable, undefined, 'Block allocation table has not been read')
+    assert.notStrictEqual(this.#blockTable.length, 0, 'Block allocation table is empty')
+
+    return this._writeChunk('bat', this.#blockTable)
+  }
+
+  // only works if data are in the same handler
+  // and if the full block is modified in child ( which is the case whit xcp)
+  // and if the compression type is same on both sides
+  async coalesceBlock(child, blockId) {
+    if (
+      !(child instanceof VhdDirectory) ||
+      this._handler !== child._handler ||
+      child.compressionType !== this.compressionType
+    ) {
+      return super.coalesceBlock(child, blockId)
+    }
+    await this._handler.copy(
+      child._getChunkPath(child._getBlockPath(blockId)),
+      this._getChunkPath(this._getBlockPath(blockId))
+    )
+    return sectorsToBytes(this.sectorsPerBlock)
+  }
+
+  async writeEntireBlock(block) {
+    await this._writeChunk(this._getBlockPath(block.id), block.buffer)
+    setBitmap(this.#blockTable, block.id)
+  }
+
+  async _readParentLocatorData(id) {
+    return (await this._readChunk('parentLocatorEntry' + id)).buffer
+  }
+
+  async _writeParentLocatorData(id, data) {
+    await this._writeChunk('parentLocatorEntry' + id, data)
+    this.header.parentLocatorEntry[id].platformDataOffset = 0
+  }
+
+  async #writeChunkFilters() {
+    const compressionType = this.compressionType
+    const path = this._path + '/chunk-filters.json'
+    if (compressionType === undefined) {
+      await this._handler.unlink(path)
+    } else {
+      await this._handler.writeFile(path, JSON.stringify([compressionType]))
+    }
+  }
+
+  async #readChunkFilters() {
+    const chunkFilters = await this._handler.readFile(this._path + '/chunk-filters.json').then(JSON.parse, error => {
+      if (error.code === 'ENOENT') {
+        return []
+      }
+      throw error
+    })
+    this.#compressor = getCompressor(chunkFilters[0])
+  }
+}

packages/vhd-lib/Vhd/VhdFile.integ.spec.js

@@ -1,19 +1,25 @@
 /* eslint-env jest */
 
-import execa from 'execa'
-import fs from 'fs-extra'
-import getStream from 'get-stream'
-import rimraf from 'rimraf'
-import tmp from 'tmp'
-import { getHandler } from '@xen-orchestra/fs'
-import { pFromCallback } from 'promise-toolbox'
-import { pipeline } from 'readable-stream'
-import { randomBytes } from 'crypto'
+const execa = require('execa')
+const fs = require('fs-extra')
+const getStream = require('get-stream')
+const rimraf = require('rimraf')
+const tmp = require('tmp')
+const { getHandler } = require('@xen-orchestra/fs')
+const { Disposable, pFromCallback } = require('promise-toolbox')
+const { randomBytes } = require('crypto')
 
-import { VhdFile, chainVhd, createSyntheticStream, mergeVhd as vhdMerge } from './index'
+const { VhdFile } = require('./VhdFile')
+const { openVhd } = require('../openVhd')
 
-import { SECTOR_SIZE } from './_constants'
-import { checkFile, createRandomFile, convertFromRawToVhd, recoverRawContent } from './tests/utils'
+const { SECTOR_SIZE } = require('../_constants')
+const {
+  checkFile,
+  createRandomFile,
+  convertFromRawToVhd,
+  convertToVhdDirectory,
+  recoverRawContent,
+} = require('../tests/utils')
 
 let tempDir = null
 
@@ -27,6 +33,29 @@ afterEach(async () => {
   await pFromCallback(cb => rimraf(tempDir, cb))
 })
 
+test('respect the checkSecondFooter flag', async () => {
+  const initalSize = 0
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSize)
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+
+  const handler = getHandler({ url: `file://${tempDir}` })
+
+  const size = await handler.getSize('randomfile.vhd')
+  const fd = await handler.openFile('randomfile.vhd', 'r+')
+  const buffer = Buffer.alloc(512, 0)
+  // add a fake footer at the end
+  handler.write(fd, buffer, size)
+  await handler.closeFile(fd)
+  // not using openVhd to be able to call readHeaderAndFooter separatly
+  const vhd = new VhdFile(handler, 'randomfile.vhd')
+
+  await expect(async () => await vhd.readHeaderAndFooter()).rejects.toThrow()
+  await expect(async () => await vhd.readHeaderAndFooter(true)).rejects.toThrow()
+  await expect(await vhd.readHeaderAndFooter(false)).toEqual(undefined)
+})
+
 test('blocks can be moved', async () => {
   const initalSize = 4
   const rawFileName = `${tempDir}/randomfile`
@@ -59,6 +88,7 @@ test('the BAT MSB is not used for sign', async () => {
   // here we are moving the first sector very far in the VHD to prove the BAT doesn't use signed int32
   const hugePositionBytes = hugeWritePositionSectors * SECTOR_SIZE
   await vhd._freeFirstBlockSpace(hugePositionBytes)
+  await vhd.writeFooter()
 
   // we recover the data manually for speed reasons.
   // fs.write() with offset is way faster than qemu-img when there is a 1.5To
@@ -159,97 +189,49 @@ test('BAT can be extended and blocks moved', async () => {
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   await newVhd.ensureBatSize(2000)
+  await newVhd.writeBlockAllocationTable()
   await recoverRawContent(vhdFileName, recoveredFileName, originalSize)
   expect(await fs.readFile(recoveredFileName)).toEqual(await fs.readFile(rawFileName))
 })
 
-test('coalesce works with empty parent files', async () => {
-  const mbOfRandom = 2
-  const rawFileName = `${tempDir}/randomfile`
-  const emptyFileName = `${tempDir}/empty.vhd`
-  const vhdFileName = `${tempDir}/randomfile.vhd`
-  const recoveredFileName = `${tempDir}/recovered`
-  await createRandomFile(rawFileName, mbOfRandom)
-  await convertFromRawToVhd(rawFileName, vhdFileName)
-  await execa('qemu-img', ['create', '-fvpc', emptyFileName, mbOfRandom + 1 + 'M'])
-  await checkFile(vhdFileName)
-  await checkFile(emptyFileName)
-  const handler = getHandler({ url: 'file://' })
-  const originalSize = await handler._getSize(rawFileName)
-  await chainVhd(handler, emptyFileName, handler, vhdFileName, true)
-  await checkFile(vhdFileName)
-  await checkFile(emptyFileName)
-  await vhdMerge(handler, emptyFileName, handler, vhdFileName)
-  await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
-  expect(await fs.readFile(recoveredFileName)).toEqual(await fs.readFile(rawFileName))
-})
-
-test('coalesce works in normal cases', async () => {
-  const mbOfRandom = 5
-  const randomFileName = `${tempDir}/randomfile`
-  const random2FileName = `${tempDir}/randomfile2`
-  const smallRandomFileName = `${tempDir}/small_randomfile`
-  const parentFileName = `${tempDir}/parent.vhd`
-  const child1FileName = `${tempDir}/child1.vhd`
-  const child2FileName = `${tempDir}/child2.vhd`
-  const recoveredFileName = `${tempDir}/recovered`
-  await createRandomFile(randomFileName, mbOfRandom)
-  await createRandomFile(smallRandomFileName, Math.ceil(mbOfRandom / 2))
-  await execa('qemu-img', ['create', '-fvpc', parentFileName, mbOfRandom + 1 + 'M'])
-  await convertFromRawToVhd(randomFileName, child1FileName)
-  const handler = getHandler({ url: 'file://' })
-  await execa('vhd-util', ['snapshot', '-n', child2FileName, '-p', child1FileName])
-  const vhd = new VhdFile(handler, child2FileName)
-  await vhd.readHeaderAndFooter()
-  await vhd.readBlockAllocationTable()
-  vhd.footer.creatorApplication = 'xoa'
-  await vhd.writeFooter()
-
-  const originalSize = await handler._getSize(randomFileName)
-  await chainVhd(handler, parentFileName, handler, child1FileName, true)
-  await execa('vhd-util', ['check', '-t', '-n', child1FileName])
-  await chainVhd(handler, child1FileName, handler, child2FileName, true)
-  await execa('vhd-util', ['check', '-t', '-n', child2FileName])
-  const smallRandom = await fs.readFile(smallRandomFileName)
-  const newVhd = new VhdFile(handler, child2FileName)
-  await newVhd.readHeaderAndFooter()
-  await newVhd.readBlockAllocationTable()
-  await newVhd.writeData(5, smallRandom)
-  await checkFile(child2FileName)
-  await checkFile(child1FileName)
-  await checkFile(parentFileName)
-  await vhdMerge(handler, parentFileName, handler, child1FileName)
-  await checkFile(parentFileName)
-  await chainVhd(handler, parentFileName, handler, child2FileName, true)
-  await checkFile(child2FileName)
-  await vhdMerge(handler, parentFileName, handler, child2FileName)
-  await checkFile(parentFileName)
-  await recoverRawContent(parentFileName, recoveredFileName, originalSize)
-  await execa('cp', [randomFileName, random2FileName])
-  const fd = await fs.open(random2FileName, 'r+')
-  try {
-    await fs.write(fd, smallRandom, 0, smallRandom.length, 5 * SECTOR_SIZE)
-  } finally {
-    await fs.close(fd)
-  }
-  expect(await fs.readFile(recoveredFileName)).toEqual(await fs.readFile(random2FileName))
-})
-
-test.only('createSyntheticStream passes vhd-util check', async () => {
+test('Can coalesce block', async () => {
   const initalSize = 4
-  const rawFileName = `${tempDir}/randomfile`
-  const vhdFileName = `${tempDir}/randomfile.vhd`
-  const recoveredVhdFileName = `${tempDir}/recovered.vhd`
-  await createRandomFile(rawFileName, initalSize)
-  await convertFromRawToVhd(rawFileName, vhdFileName)
-  await checkFile(vhdFileName)
-  const handler = getHandler({ url: 'file://' })
-  const stream = await createSyntheticStream(handler, vhdFileName)
-  const expectedVhdSize = (await fs.stat(vhdFileName)).size
-  expect(stream.length).toEqual((await fs.stat(vhdFileName)).size)
-  await pFromCallback(cb => pipeline(stream, fs.createWriteStream(recoveredVhdFileName), cb))
-  await checkFile(recoveredVhdFileName)
-  const stats = await fs.stat(recoveredVhdFileName)
-  expect(stats.size).toEqual(expectedVhdSize)
-  await execa('qemu-img', ['compare', recoveredVhdFileName, rawFileName])
+  const parentrawFileName = `${tempDir}/randomfile`
+  const parentFileName = `${tempDir}/parent.vhd`
+  await createRandomFile(parentrawFileName, initalSize)
+  await convertFromRawToVhd(parentrawFileName, parentFileName)
+  const childrawFileName = `${tempDir}/randomfile`
+  const childFileName = `${tempDir}/childFile.vhd`
+  await createRandomFile(childrawFileName, initalSize)
+  await convertFromRawToVhd(childrawFileName, childFileName)
+  const childRawDirectoryName = `${tempDir}/randomFile2.vhd`
+  const childDirectoryFileName = `${tempDir}/childDirFile.vhd`
+  const childDirectoryName = `${tempDir}/childDir.vhd`
+  await createRandomFile(childRawDirectoryName, initalSize)
+  await convertFromRawToVhd(childRawDirectoryName, childDirectoryFileName)
+  await convertToVhdDirectory(childRawDirectoryName, childDirectoryFileName, childDirectoryName)
+
+  await Disposable.use(async function* () {
+    const handler = getHandler({ url: 'file://' })
+    const parentVhd = yield openVhd(handler, parentFileName, { flags: 'r+' })
+    await parentVhd.readBlockAllocationTable()
+    const childFileVhd = yield openVhd(handler, childFileName)
+    await childFileVhd.readBlockAllocationTable()
+    const childDirectoryVhd = yield openVhd(handler, childDirectoryName)
+    await childDirectoryVhd.readBlockAllocationTable()
+
+    await parentVhd.coalesceBlock(childFileVhd, 0)
+    await parentVhd.writeFooter()
+    await parentVhd.writeBlockAllocationTable()
+    let parentBlockData = (await parentVhd.readBlock(0)).data
+    let childBlockData = (await childFileVhd.readBlock(0)).data
+    expect(parentBlockData).toEqual(childBlockData)
+
+    await parentVhd.coalesceBlock(childDirectoryVhd, 0)
+    await parentVhd.writeFooter()
+    await parentVhd.writeBlockAllocationTable()
+    parentBlockData = (await parentVhd.readBlock(0)).data
+    childBlockData = (await childDirectoryVhd.readBlock(0)).data
+    expect(parentBlockData).toEqual(childBlockData)
+  })
 })

packages/vhd-lib/Vhd/VhdFile.js

@@ -1,18 +1,18 @@
-import {
+const {
   BLOCK_UNUSED,
   FOOTER_SIZE,
   HEADER_SIZE,
-  PLATFORM_NONE,
+  PLATFORMS,
   SECTOR_SIZE,
   PARENT_LOCATOR_ENTRIES,
-} from '../_constants'
-import { computeBatSize, sectorsToBytes, buildHeader, buildFooter, BUF_BLOCK_UNUSED } from './_utils'
-import { createLogger } from '@xen-orchestra/log'
-import { fuFooter, fuHeader, checksumStruct } from '../_structs'
-import { set as mapSetBit, test as mapTestBit } from '../_bitmap'
-import { VhdAbstract } from './VhdAbstract'
-import assert from 'assert'
-import getFirstAndLastBlocks from '../_getFirstAndLastBlocks'
+} = require('../_constants')
+const { computeBatSize, sectorsToBytes, unpackHeader, unpackFooter, BUF_BLOCK_UNUSED } = require('./_utils')
+const { createLogger } = require('@xen-orchestra/log')
+const { fuFooter, fuHeader, checksumStruct } = require('../_structs')
+const { set: mapSetBit } = require('../_bitmap')
+const { VhdAbstract } = require('./VhdAbstract')
+const assert = require('assert')
+const getFirstAndLastBlocks = require('../_getFirstAndLastBlocks')
 
 const { debug } = createLogger('vhd-lib:VhdFile')
 
@@ -50,8 +50,10 @@ const { debug } = createLogger('vhd-lib:VhdFile')
 // - parentLocatorSize(i) = header.parentLocatorEntry[i].platformDataSpace * sectorSize
 // - sectorSize = 512
 
-export class VhdFile extends VhdAbstract {
+exports.VhdFile = class VhdFile extends VhdAbstract {
   #uncheckedBlockTable
+  #header
+  footer
 
   get #blockTable() {
     assert.notStrictEqual(this.#uncheckedBlockTable, undefined, 'Block table must be initialized before access')
@@ -67,7 +69,7 @@ export class VhdFile extends VhdAbstract {
   }
 
   set header(header) {
-    super.header = header
+    this.#header = header
     const size = this.batSize
     this.#blockTable = Buffer.alloc(size)
     for (let i = 0; i < this.header.maxTableEntries; i++) {
@@ -75,26 +77,26 @@ export class VhdFile extends VhdAbstract {
     }
   }
   get header() {
-    return super.header
+    return this.#header
   }
 
-  static async open(handler, path) {
-    const fd = await handler.openFile(path, 'r+')
+  static async open(handler, path, { flags, checkSecondFooter = true } = {}) {
+    const fd = await handler.openFile(path, flags ?? 'r+')
     const vhd = new VhdFile(handler, fd)
     // openning a file for reading does not trigger EISDIR as long as we don't really read from it :
     // https://man7.org/linux/man-pages/man2/open.2.html
     // EISDIR pathname refers to a directory and the access requested
     // involved writing (that is, O_WRONLY or O_RDWR is set).
     // reading the header ensure we have a well formed file immediatly
-    await vhd.readHeaderAndFooter()
+    await vhd.readHeaderAndFooter(checkSecondFooter)
     return {
       dispose: () => handler.closeFile(fd),
       value: vhd,
     }
   }
 
-  static async create(handler, path) {
-    const fd = await handler.openFile(path, 'wx')
+  static async create(handler, path, { flags } = {}) {
+    const fd = await handler.openFile(path, flags ?? 'wx')
     const vhd = new VhdFile(handler, fd)
     return {
       dispose: () => handler.closeFile(fd),
@@ -129,7 +131,7 @@ export class VhdFile extends VhdAbstract {
     for (let i = 0; i < PARENT_LOCATOR_ENTRIES; i++) {
       const entry = header.parentLocatorEntry[i]
 
-      if (entry.platformCode !== PLATFORM_NONE) {
+      if (entry.platformCode !== PLATFORMS.NONE) {
         end = Math.max(end, entry.platformDataOffset + sectorsToBytes(entry.platformDataSpace))
       }
     }
@@ -177,8 +179,8 @@ export class VhdFile extends VhdAbstract {
     const bufFooter = buf.slice(0, FOOTER_SIZE)
     const bufHeader = buf.slice(FOOTER_SIZE)
 
-    const footer = buildFooter(bufFooter)
-    const header = buildHeader(bufHeader, footer)
+    const footer = unpackFooter(bufFooter)
+    const header = unpackHeader(bufHeader, footer)
 
     if (checkSecondFooter) {
       const size = await this._handler.getSize(this._path)
@@ -343,47 +345,6 @@ export class VhdFile extends VhdAbstract {
     )
   }
 
-  async coalesceBlock(child, blockId) {
-    const block = await child.readBlock(blockId)
-    const { bitmap, data } = block
-
-    debug(`coalesceBlock block=${blockId}`)
-
-    // For each sector of block data...
-    const { sectorsPerBlock } = child
-    let parentBitmap = null
-    for (let i = 0; i < sectorsPerBlock; i++) {
-      // If no changes on one sector, skip.
-      if (!mapTestBit(bitmap, i)) {
-        continue
-      }
-      let endSector = i + 1
-
-      // Count changed sectors.
-      while (endSector < sectorsPerBlock && mapTestBit(bitmap, endSector)) {
-        ++endSector
-      }
-
-      // Write n sectors into parent.
-      debug(`coalesceBlock: write sectors=${i}...${endSector}`)
-
-      const isFullBlock = i === 0 && endSector === sectorsPerBlock
-      if (isFullBlock) {
-        await this.writeEntireBlock(block)
-      } else {
-        if (parentBitmap === null) {
-          parentBitmap = (await this.readBlock(blockId, true)).bitmap
-        }
-        await this._writeBlockSectors(block, i, endSector, parentBitmap)
-      }
-
-      i = endSector
-    }
-
-    // Return the merged data size
-    return data.length
-  }
-
   // Write a context footer. (At the end and beginning of a vhd file.)
   async writeFooter(onlyEndFooter = false) {
     const { footer } = this
@@ -462,7 +423,7 @@ export class VhdFile extends VhdAbstract {
   async _readParentLocatorData(parentLocatorId) {
     const { platformDataOffset, platformDataLength } = this.header.parentLocatorEntry[parentLocatorId]
     if (platformDataLength > 0) {
-      return (await this._read(platformDataOffset, platformDataLength)).buffer
+      return await this._read(platformDataOffset, platformDataLength)
     }
     return Buffer.alloc(0)
   }
@@ -474,15 +435,32 @@ export class VhdFile extends VhdAbstract {
       // reset offset if data is empty
       header.parentLocatorEntry[parentLocatorId].platformDataOffset = 0
     } else {
-      if (data.length <= header.parentLocatorEntry[parentLocatorId].platformDataSpace) {
+      const space = header.parentLocatorEntry[parentLocatorId].platformDataSpace * SECTOR_SIZE
+      if (data.length <= space) {
         // new parent locator length is smaller than available space : keep it in place
         position = header.parentLocatorEntry[parentLocatorId].platformDataOffset
       } else {
-        // new parent locator length is bigger than available space : move it to the end
-        position = this._getEndOfData()
+        const firstAndLastBlocks = getFirstAndLastBlocks(this.#blockTable)
+        if (firstAndLastBlocks === undefined) {
+          // no block in data : put the parent locatorn entry at the end
+          position = this._getEndOfData()
+        } else {
+          // need more size
+
+          // since there can be multiple parent locator entry, we can't extend the entry in place
+          // move the first(s) block(s) at the end of the data
+          // move the parent locator to the  precedent position of the first block
+          const { firstSector } = firstAndLastBlocks
+          await this._freeFirstBlockSpace(space)
+          position = sectorsToBytes(firstSector)
+        }
       }
       await this._write(data, position)
       header.parentLocatorEntry[parentLocatorId].platformDataOffset = position
     }
   }
+
+  async getSize() {
+    return await this._handler.getSize(this._path)
+  }
 }

packages/vhd-lib/Vhd/VhdSynthetic.integ.spec.js

@@ -0,0 +1,83 @@
+/* eslint-env jest */
+
+const rimraf = require('rimraf')
+const tmp = require('tmp')
+const { Disposable, pFromCallback } = require('promise-toolbox')
+const { getSyncedHandler } = require('@xen-orchestra/fs')
+
+const { SECTOR_SIZE, PLATFORMS } = require('../_constants')
+const { createRandomFile, convertFromRawToVhd } = require('../tests/utils')
+const { openVhd, chainVhd } = require('..')
+const { VhdSynthetic } = require('./VhdSynthetic')
+
+let tempDir = null
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+test('It can read block and parent locator from a synthetic vhd', async () => {
+  const bigRawFileName = `/bigrandomfile`
+  await createRandomFile(`${tempDir}/${bigRawFileName}`, 8)
+  const bigVhdFileName = `/bigrandomfile.vhd`
+  await convertFromRawToVhd(`${tempDir}/${bigRawFileName}`, `${tempDir}/${bigVhdFileName}`)
+
+  const smallRawFileName = `/smallrandomfile`
+  await createRandomFile(`${tempDir}/${smallRawFileName}`, 4)
+  const smallVhdFileName = `/smallrandomfile.vhd`
+  await convertFromRawToVhd(`${tempDir}/${smallRawFileName}`, `${tempDir}/${smallVhdFileName}`)
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
+    // ensure the two VHD are linked, with the child of type DISK_TYPES.DIFFERENCING
+    await chainVhd(handler, bigVhdFileName, handler, smallVhdFileName, true)
+
+    const [smallVhd, bigVhd] = yield Disposable.all([
+      openVhd(handler, smallVhdFileName),
+      openVhd(handler, bigVhdFileName),
+    ])
+    // add parent locato
+    // this will also scramble the block inside the vhd files
+    await bigVhd.writeParentLocator({
+      id: 0,
+      platformCode: PLATFORMS.W2KU,
+      data: Buffer.from('I am in the big one'),
+    })
+    const syntheticVhd = new VhdSynthetic([smallVhd, bigVhd])
+    await syntheticVhd.readBlockAllocationTable()
+
+    expect(syntheticVhd.header.diskType).toEqual(bigVhd.header.diskType)
+    expect(syntheticVhd.header.parentTimestamp).toEqual(bigVhd.header.parentTimestamp)
+
+    // first two block should be from small
+    const buf = Buffer.alloc(syntheticVhd.sectorsPerBlock * SECTOR_SIZE, 0)
+    let content = (await syntheticVhd.readBlock(0)).data
+    await handler.read(smallRawFileName, buf, 0)
+    expect(content).toEqual(buf)
+
+    content = (await syntheticVhd.readBlock(1)).data
+    await handler.read(smallRawFileName, buf, buf.length)
+    expect(content).toEqual(buf)
+
+    // the next one from big
+
+    content = (await syntheticVhd.readBlock(2)).data
+    await handler.read(bigRawFileName, buf, buf.length * 2)
+    expect(content).toEqual(buf)
+
+    content = (await syntheticVhd.readBlock(3)).data
+    await handler.read(bigRawFileName, buf, buf.length * 3)
+    expect(content).toEqual(buf)
+
+    // the parent locator should the one of the root vhd
+    const parentLocator = await syntheticVhd.readParentLocator(0)
+    expect(parentLocator.platformCode).toEqual(PLATFORMS.W2KU)
+    expect(Buffer.from(parentLocator.data, 'utf-8').toString()).toEqual('I am in the big one')
+  })
+})

packages/vhd-lib/Vhd/VhdSynthetic.js

@@ -0,0 +1,88 @@
+const UUID = require('uuid')
+const cloneDeep = require('lodash/cloneDeep.js')
+const { asyncMap } = require('@xen-orchestra/async-map')
+const { VhdAbstract } = require('./VhdAbstract')
+const { DISK_TYPES, FOOTER_SIZE, HEADER_SIZE } = require('../_constants')
+
+const assert = require('assert')
+
+exports.VhdSynthetic = class VhdSynthetic extends VhdAbstract {
+  #vhds = []
+
+  get header() {
+    // this the VHD we want to synthetize
+    const vhd = this.#vhds[0]
+
+    // this is the root VHD
+    const rootVhd = this.#vhds[this.#vhds.length - 1]
+
+    // data of our synthetic VHD
+    // TODO: set parentLocatorEntry-s in header
+    return {
+      ...vhd.header,
+      parentLocatorEntry: cloneDeep(rootVhd.header.parentLocatorEntry),
+      tableOffset: FOOTER_SIZE + HEADER_SIZE,
+      parentTimestamp: rootVhd.header.parentTimestamp,
+      parentUnicodeName: rootVhd.header.parentUnicodeName,
+      parentUuid: rootVhd.header.parentUuid,
+    }
+  }
+
+  get footer() {
+    // this is the root VHD
+    const rootVhd = this.#vhds[this.#vhds.length - 1]
+    return {
+      ...this.#vhds[0].footer,
+      dataOffset: FOOTER_SIZE,
+      diskType: rootVhd.footer.diskType,
+    }
+  }
+
+  static async open(vhds) {
+    const vhd = new VhdSynthetic(vhds)
+    return {
+      dispose: () => {},
+      value: vhd,
+    }
+  }
+  /**
+   * @param {Array<VhdAbstract>} vhds the chain of Vhds used to compute this Vhd, from the deepest child (in position 0), to the root (in the last position)
+   * only the last one can have any type. Other must have type DISK_TYPES.DIFFERENCING (delta)
+   */
+  constructor(vhds) {
+    assert(vhds.length > 0)
+    super()
+    this.#vhds = vhds
+  }
+
+  async readBlockAllocationTable() {
+    await asyncMap(this.#vhds, vhd => vhd.readBlockAllocationTable())
+  }
+
+  containsBlock(blockId) {
+    return this.#vhds.some(vhd => vhd.containsBlock(blockId))
+  }
+
+  async readHeaderAndFooter() {
+    const vhds = this.#vhds
+
+    await asyncMap(vhds, vhd => vhd.readHeaderAndFooter())
+
+    for (let i = 0, n = vhds.length - 1; i < n; ++i) {
+      const child = vhds[i]
+      const parent = vhds[i + 1]
+      assert.strictEqual(child.footer.diskType, DISK_TYPES.DIFFERENCING)
+      assert.strictEqual(UUID.stringify(child.header.parentUuid), UUID.stringify(parent.footer.uuid))
+    }
+  }
+
+  async readBlock(blockId, onlyBitmap = false) {
+    const index = this.#vhds.findIndex(vhd => vhd.containsBlock(blockId))
+    // only read the content of the first vhd containing this block
+    return await this.#vhds[index].readBlock(blockId, onlyBitmap)
+  }
+
+  _readParentLocatorData(id) {
+    return this.#vhds[this.#vhds.length - 1]._readParentLocatorData(id)
+  }
+}

packages/vhd-lib/Vhd/_utils.js

@@ -0,0 +1,67 @@
+const assert = require('assert')
+const { BLOCK_UNUSED, SECTOR_SIZE } = require('../_constants')
+const { fuFooter, fuHeader, checksumStruct, unpackField } = require('../_structs')
+const checkFooter = require('../checkFooter')
+const checkHeader = require('../_checkHeader')
+
+const computeBatSize = entries => sectorsToBytes(sectorsRoundUpNoZero(entries * 4))
+exports.computeBatSize = computeBatSize
+
+const computeSectorsPerBlock = blockSize => blockSize / SECTOR_SIZE
+exports.computeSectorsPerBlock = computeSectorsPerBlock
+// one bit per sector
+const computeBlockBitmapSize = blockSize => computeSectorsPerBlock(blockSize) >>> 3
+exports.computeBlockBitmapSize = computeBlockBitmapSize
+const computeFullBlockSize = blockSize => blockSize + SECTOR_SIZE * computeSectorOfBitmap(blockSize)
+exports.computeFullBlockSize = computeFullBlockSize
+const computeSectorOfBitmap = blockSize => sectorsRoundUpNoZero(computeBlockBitmapSize(blockSize))
+exports.computeSectorOfBitmap = computeSectorOfBitmap
+
+// Sectors conversions.
+const sectorsRoundUpNoZero = bytes => Math.ceil(bytes / SECTOR_SIZE) || 1
+exports.sectorsRoundUpNoZero = sectorsRoundUpNoZero
+const sectorsToBytes = sectors => sectors * SECTOR_SIZE
+exports.sectorsToBytes = sectorsToBytes
+
+const assertChecksum = (name, buf, struct) => {
+  const actual = unpackField(struct.fields.checksum, buf)
+  const expected = checksumStruct(buf, struct)
+  assert.strictEqual(actual, expected, `invalid ${name} checksum ${actual}, expected ${expected}`)
+}
+exports.assertChecksum = assertChecksum
+
+// unused block as buffer containing a uint32BE
+const BUF_BLOCK_UNUSED = Buffer.allocUnsafe(4)
+BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0)
+exports.BUF_BLOCK_UNUSED = BUF_BLOCK_UNUSED
+
+/**
+ * Check and parse the header buffer to build an header object
+ *
+ * @param {Buffer} bufHeader
+ * @param {Object} footer
+ * @returns {Object} the parsed header
+ */
+exports.unpackHeader = (bufHeader, footer) => {
+  assertChecksum('header', bufHeader, fuHeader)
+
+  const header = fuHeader.unpack(bufHeader)
+  checkHeader(header, footer)
+  return header
+}
+
+/**
+ * Check and parse the footer buffer to build a footer object
+ *
+ * @param {Buffer} bufHeader
+ * @param {Object} footer
+ * @returns {Object} the parsed footer
+ */
+
+exports.unpackFooter = bufFooter => {
+  assertChecksum('footer', bufFooter, fuFooter)
+
+  const footer = fuFooter.unpack(bufFooter)
+  checkFooter(footer)
+  return footer
+}

packages/vhd-lib/_bitmap.js

@@ -0,0 +1,7 @@
+const MASK = 0x80
+
+exports.set = (map, bit) => {
+  map[bit >> 3] |= MASK >> (bit & 7)
+}
+
+exports.test = (map, bit) => ((map[bit >> 3] << (bit & 7)) & MASK) !== 0

packages/vhd-lib/_checkHeader.js

@@ -1,8 +1,8 @@
-import assert from 'assert'
+const assert = require('assert')
 
-import { HEADER_COOKIE, HEADER_VERSION, SECTOR_SIZE } from './_constants'
+const { HEADER_COOKIE, HEADER_VERSION, SECTOR_SIZE } = require('./_constants')
 
-export default (header, footer) => {
+module.exports = (header, footer) => {
   assert.strictEqual(header.cookie, HEADER_COOKIE)
   assert.strictEqual(header.dataOffset, undefined)
   assert.strictEqual(header.headerVersion, HEADER_VERSION)

packages/vhd-lib/_computeGeometryForSize.js

@@ -1,6 +1,6 @@
-import { SECTOR_SIZE } from './_constants'
+const { SECTOR_SIZE } = require('./_constants')
 
-export default function computeGeometryForSize(size) {
+module.exports = function computeGeometryForSize(size) {
   const totalSectors = Math.min(Math.ceil(size / 512), 65535 * 16 * 255)
   let sectorsPerTrackCylinder
   let heads

packages/vhd-lib/_constants.js

@@ -0,0 +1,40 @@
+exports.BLOCK_UNUSED = 0xffffffff
+
+// This lib has been extracted from the Xen Orchestra project.
+exports.CREATOR_APPLICATION = 'xo  '
+
+// Sizes in bytes.
+exports.FOOTER_SIZE = 512
+exports.HEADER_SIZE = 1024
+exports.SECTOR_SIZE = 512
+exports.DEFAULT_BLOCK_SIZE = 0x00200000 // from the spec
+
+exports.FOOTER_COOKIE = 'conectix'
+exports.HEADER_COOKIE = 'cxsparse'
+
+exports.DISK_TYPES = {
+  __proto__: null,
+
+  FIXED: 2,
+  DYNAMIC: 3,
+  DIFFERENCING: 4,
+}
+
+exports.PARENT_LOCATOR_ENTRIES = 8
+
+exports.PLATFORMS = {
+  __proto__: null,
+
+  NONE: 0,
+  WI2R: 0x57693272,
+  WI2K: 0x5769326b,
+  W2RU: 0x57327275,
+  W2KU: 0x57326b75,
+  MAC: 0x4d616320,
+  MACX: 0x4d616358,
+}
+
+exports.FILE_FORMAT_VERSION = 1 << 16
+exports.HEADER_VERSION = 1 << 16
+
+exports.ALIAS_MAX_PATH_LENGTH = 1024

packages/vhd-lib/_createFooterHeader.integ.spec.js

@@ -0,0 +1,10 @@
+/* eslint-env jest */
+const { createFooter } = require('./_createFooterHeader')
+
+test('createFooter() does not crash', () => {
+  createFooter(104448, Math.floor(Date.now() / 1000), {
+    cylinders: 3,
+    heads: 4,
+    sectorsPerTrack: 17,
+  })
+})

packages/vhd-lib/_createFooterHeader.js

@@ -1,20 +1,20 @@
-import { v4 as generateUuid } from 'uuid'
+const { v4: generateUuid } = require('uuid')
 
-import { checksumStruct, fuFooter, fuHeader } from './_structs'
-import {
+const { checksumStruct, fuFooter, fuHeader } = require('./_structs')
+const {
   CREATOR_APPLICATION,
-  DEFAULT_BLOCK_SIZE as VHD_BLOCK_SIZE_BYTES,
-  DISK_TYPE_FIXED,
+  DEFAULT_BLOCK_SIZE: VHD_BLOCK_SIZE_BYTES,
+  DISK_TYPES,
   FILE_FORMAT_VERSION,
   FOOTER_COOKIE,
   FOOTER_SIZE,
   HEADER_COOKIE,
   HEADER_SIZE,
   HEADER_VERSION,
-  PLATFORM_WI2K,
-} from './_constants'
+  PLATFORMS,
+} = require('./_constants')
 
-export function createFooter(size, timestamp, geometry, dataOffset, diskType = DISK_TYPE_FIXED) {
+exports.createFooter = function createFooter(size, timestamp, geometry, dataOffset, diskType = DISK_TYPES.FIXED) {
   const footer = fuFooter.pack({
     cookie: FOOTER_COOKIE,
     features: 2,
@@ -22,7 +22,7 @@ export function createFooter(size, timestamp, geometry, dataOffset, diskType = D
     dataOffset,
     timestamp,
     creatorApplication: CREATOR_APPLICATION,
-    creatorHostOs: PLATFORM_WI2K, // it looks like everybody is using Wi2k
+    creatorHostOs: PLATFORMS.WI2K, // it looks like everybody is using Wi2k
     originalSize: size,
     currentSize: size,
     diskGeometry: geometry,
@@ -33,7 +33,7 @@ export function createFooter(size, timestamp, geometry, dataOffset, diskType = D
   return footer
 }
 
-export function createHeader(
+exports.createHeader = function createHeader(
   maxTableEntries,
   tableOffset = HEADER_SIZE + FOOTER_SIZE,
   blockSize = VHD_BLOCK_SIZE_BYTES

packages/vhd-lib/_getFirstAndLastBlocks.js

@@ -1,10 +1,10 @@
-import assert from 'assert'
+const assert = require('assert')
 
-import { BLOCK_UNUSED } from './_constants'
+const { BLOCK_UNUSED } = require('./_constants')
 
 // get the identifiers and first sectors of the first and last block
 // in the file
-export default bat => {
+module.exports = bat => {
   const n = bat.length
   if (n === 0) {
     return

packages/vhd-lib/_noop.js

@@ -0,0 +1 @@
+module.exports = Function.prototype

packages/vhd-lib/_resolveRelativeFromFile.js

@@ -1,5 +1,5 @@
-import { dirname, resolve } from 'path'
+const { dirname, resolve } = require('path')
 
 const resolveRelativeFromFile = (file, path) => resolve('/', dirname(file), path).slice(1)
 
-export { resolveRelativeFromFile as default }
+module.exports = resolveRelativeFromFile

packages/vhd-lib/_structs.js

@@ -1,7 +1,7 @@
-import assert from 'assert'
-import fu from 'struct-fu'
+const assert = require('assert')
+const fu = require('struct-fu')
 
-import { FOOTER_SIZE, HEADER_SIZE, PARENT_LOCATOR_ENTRIES } from './_constants'
+const { FOOTER_SIZE, HEADER_SIZE, PARENT_LOCATOR_ENTRIES } = require('./_constants')
 
 const SIZE_OF_32_BITS = Math.pow(2, 32)
 
@@ -17,7 +17,7 @@ const uint64Undefinable = fu.derive(
   _ => (_[0] === 0xffffffff && _[1] === 0xffffffff ? undefined : _[0] * SIZE_OF_32_BITS + _[1])
 )
 
-export const fuFooter = fu.struct([
+const fuFooter = fu.struct([
   fu.char('cookie', 8), // 0
   fu.uint32('features'), // 8
   fu.uint32('fileFormatVersion'), // 12
@@ -33,16 +33,17 @@ export const fuFooter = fu.struct([
     fu.uint8('heads'), // 58
     fu.uint8('sectorsPerTrackCylinder'), // 59
   ]),
-  fu.uint32('diskType'), // 60 Disk type, must be equal to HARD_DISK_TYPE_DYNAMIC/HARD_DISK_TYPE_DIFFERENCING.
+  fu.uint32('diskType'), // 60 Disk type, must be equal to DYNAMIC/DIFFERENCING.
   fu.uint32('checksum'), // 64
   fu.byte('uuid', 16), // 68
   fu.char('saved'), // 84
   fu.char('hidden'), // 85 TODO: should probably be merged in reserved
   fu.char('reserved', 426), // 86
 ])
+exports.fuFooter = fuFooter
 assert.strictEqual(fuFooter.size, FOOTER_SIZE)
 
-export const fuHeader = fu.struct([
+const fuHeader = fu.struct([
   fu.char('cookie', 8),
   uint64Undefinable('dataOffset'),
   uint64('tableOffset'),
@@ -67,15 +68,18 @@ export const fuHeader = fu.struct([
   ),
   fu.char('reserved2', 256),
 ])
+exports.fuHeader = fuHeader
+
 assert.strictEqual(fuHeader.size, HEADER_SIZE)
 
-export const packField = (field, value, buf) => {
+const packField = (field, value, buf) => {
   const { offset } = field
 
   field.pack(value, buf, typeof offset !== 'object' ? { bytes: offset, bits: 0 } : offset)
 }
+exports.packField = packField
 
-export const unpackField = (field, buf) => {
+exports.unpackField = (field, buf) => {
   const { offset } = field
 
   return field.unpack(buf, typeof offset !== 'object' ? { bytes: offset, bits: 0 } : offset)
@@ -83,7 +87,7 @@ export const unpackField = (field, buf) => {
 
 // Returns the checksum of a raw struct.
 // The raw struct (footer or header) is altered with the new sum.
-export function checksumStruct(buf, struct) {
+exports.checksumStruct = function checksumStruct(buf, struct) {
   const checksumField = struct.fields.checksum
   let sum = 0
 

packages/vhd-lib/aliases.integ.spec.js

@@ -0,0 +1,64 @@
+/* eslint-env jest */
+
+const rimraf = require('rimraf')
+const tmp = require('tmp')
+const { getSyncedHandler } = require('@xen-orchestra/fs')
+const { Disposable, pFromCallback } = require('promise-toolbox')
+
+const { isVhdAlias, resolveVhdAlias } = require('./aliases')
+const { ALIAS_MAX_PATH_LENGTH } = require('./_constants')
+
+let tempDir
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+test('is vhd alias recognize only *.alias.vhd files', () => {
+  expect(isVhdAlias('filename.alias.vhd')).toEqual(true)
+  expect(isVhdAlias('alias.vhd')).toEqual(false)
+  expect(isVhdAlias('filename.vhd')).toEqual(false)
+  expect(isVhdAlias('filename.alias.vhd.other')).toEqual(false)
+})
+
+test('resolve return the path in argument for a non alias file ', async () => {
+  expect(await resolveVhdAlias(null, 'filename.vhd')).toEqual('filename.vhd')
+})
+test('resolve get the path of the target file for an alias', async () => {
+  await Disposable.use(async function* () {
+    // same directory
+    const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
+    const alias = `alias.alias.vhd`
+    await handler.writeFile(alias, 'target.vhd')
+    await expect(await resolveVhdAlias(handler, alias)).toEqual(`target.vhd`)
+
+    // different directory
+    await handler.mkdir(`sub`)
+    await handler.writeFile(alias, 'sub/target.vhd', { flags: 'w' })
+    await expect(await resolveVhdAlias(handler, alias)).toEqual(`sub/target.vhd`)
+  })
+})
+
+test('resolve throws an error an alias to an alias', async () => {
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
+    const alias = `alias.alias.vhd`
+    const target = `target.alias.vhd`
+    await handler.writeFile(alias, target)
+    await expect(async () => await resolveVhdAlias(handler, alias)).rejects.toThrow(Error)
+  })
+})
+
+test('resolve throws an error on a file too big ', async () => {
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
+    await handler.writeFile('toobig.alias.vhd', Buffer.alloc(ALIAS_MAX_PATH_LENGTH + 1, 0))
+    await expect(async () => await resolveVhdAlias(handler, 'toobig.alias.vhd')).rejects.toThrow(Error)
+  })
+})

packages/vhd-lib/aliases.js

@@ -0,0 +1,25 @@
+const { ALIAS_MAX_PATH_LENGTH } = require('./_constants')
+const resolveRelativeFromFile = require('./_resolveRelativeFromFile')
+
+function isVhdAlias(filename) {
+  return filename.endsWith('.alias.vhd')
+}
+exports.isVhdAlias = isVhdAlias
+
+exports.resolveVhdAlias = async function resolveVhdAlias(handler, filename) {
+  if (!isVhdAlias(filename)) {
+    return filename
+  }
+  const size = await handler.getSize(filename)
+  if (size > ALIAS_MAX_PATH_LENGTH) {
+    // seems reasonnable for a relative path
+    throw new Error(`The alias file ${filename} is too big (${size} bytes)`)
+  }
+  const aliasContent = (await handler.readFile(filename)).toString().trim()
+  // also handle circular references and unreasonnably long chains
+  if (isVhdAlias(aliasContent)) {
+    throw new Error(`Chaining alias is forbidden ${filename} to ${aliasContent}`)
+  }
+  // the target is relative to the alias location
+  return resolveRelativeFromFile(filename, aliasContent)
+}

packages/vhd-lib/chain.js

@@ -0,0 +1,30 @@
+const { dirname, relative } = require('path')
+
+const { openVhd } = require('./openVhd')
+const { DISK_TYPES } = require('./_constants')
+const { Disposable } = require('promise-toolbox')
+
+module.exports = async function chain(parentHandler, parentPath, childHandler, childPath, force = false) {
+  await Disposable.use(
+    [openVhd(parentHandler, parentPath), openVhd(childHandler, childPath)],
+    async ([parentVhd, childVhd]) => {
+      await childVhd.readHeaderAndFooter()
+      const { header, footer } = childVhd
+
+      if (footer.diskType !== DISK_TYPES.DIFFERENCING) {
+        if (!force) {
+          throw new Error('cannot chain disk of type ' + footer.diskType)
+        }
+        footer.diskType = DISK_TYPES.DIFFERENCING
+      }
+      await childVhd.readBlockAllocationTable()
+
+      const parentName = relative(dirname(childPath), parentPath)
+      header.parentUuid = parentVhd.footer.uuid
+      header.parentUnicodeName = parentName
+      await childVhd.setUniqueParentLocator(parentName)
+      await childVhd.writeHeader()
+      await childVhd.writeFooter()
+    }
+  )
+}

packages/vhd-lib/checkChain.js

@@ -0,0 +1,14 @@
+const { openVhd } = require('./openVhd')
+const resolveRelativeFromFile = require('./_resolveRelativeFromFile')
+const { DISK_TYPES } = require('./_constants')
+const { Disposable } = require('promise-toolbox')
+
+module.exports = async function checkChain(handler, path) {
+  await Disposable.use(function* () {
+    let vhd
+    do {
+      vhd = yield openVhd(handler, path)
+      path = resolveRelativeFromFile(path, vhd.header.parentUnicodeName)
+    } while (vhd.footer.diskType !== DISK_TYPES.DYNAMIC)
+  })
+}

packages/vhd-lib/checkFooter.js

@@ -0,0 +1,11 @@
+const assert = require('assert')
+
+const { DISK_TYPES, FILE_FORMAT_VERSION, FOOTER_COOKIE, FOOTER_SIZE } = require('./_constants')
+
+module.exports = footer => {
+  assert.strictEqual(footer.cookie, FOOTER_COOKIE)
+  assert.strictEqual(footer.dataOffset, FOOTER_SIZE)
+  assert.strictEqual(footer.fileFormatVersion, FILE_FORMAT_VERSION)
+  assert(footer.originalSize <= footer.currentSize)
+  assert(footer.diskType === DISK_TYPES.DIFFERENCING || footer.diskType === DISK_TYPES.DYNAMIC)
+}

packages/vhd-lib/createReadableSparseStream.js

@@ -1,19 +1,19 @@
-import assert from 'assert'
-import asyncIteratorToStream from 'async-iterator-to-stream'
-import { forEachRight } from 'lodash'
+const assert = require('assert')
+const asyncIteratorToStream = require('async-iterator-to-stream')
+const { forEachRight } = require('lodash')
 
-import computeGeometryForSize from './_computeGeometryForSize'
-import { createFooter, createHeader } from './_createFooterHeader'
-import {
+const computeGeometryForSize = require('./_computeGeometryForSize')
+const { createFooter, createHeader } = require('./_createFooterHeader')
+const {
   BLOCK_UNUSED,
-  DEFAULT_BLOCK_SIZE as VHD_BLOCK_SIZE_BYTES,
-  DISK_TYPE_DYNAMIC,
+  DEFAULT_BLOCK_SIZE: VHD_BLOCK_SIZE_BYTES,
+  DISK_TYPES,
   FOOTER_SIZE,
   HEADER_SIZE,
   SECTOR_SIZE,
-} from './_constants'
+} = require('./_constants')
 
-import { set as setBitmap } from './_bitmap'
+const { set: setBitmap } = require('./_bitmap')
 
 const VHD_BLOCK_SIZE_SECTORS = VHD_BLOCK_SIZE_BYTES / SECTOR_SIZE
 
@@ -55,7 +55,12 @@ function createBAT({ firstBlockPosition, fragmentLogicAddressList, fragmentSize,
  * @returns {Promise<Function>}
  */
 
-export default async function createReadableStream(diskSize, fragmentSize, fragmentLogicAddressList, fragmentIterator) {
+module.exports = async function createReadableStream(
+  diskSize,
+  fragmentSize,
+  fragmentLogicAddressList,
+  fragmentIterator
+) {
   const ratio = VHD_BLOCK_SIZE_BYTES / fragmentSize
   if (ratio % 1 !== 0) {
     throw new Error(
@@ -73,7 +78,7 @@ export default async function createReadableStream(diskSize, fragmentSize, fragm
   const firstBlockPosition = batPosition + tablePhysicalSizeBytes
   const geometry = computeGeometryForSize(diskSize)
   const actualSize = geometry.actualSize
-  const footer = createFooter(actualSize, Math.floor(Date.now() / 1000), geometry, FOOTER_SIZE, DISK_TYPE_DYNAMIC)
+  const footer = createFooter(actualSize, Math.floor(Date.now() / 1000), geometry, FOOTER_SIZE, DISK_TYPES.DYNAMIC)
   const header = createHeader(maxTableEntries, batPosition, VHD_BLOCK_SIZE_BYTES)
   const bitmapSize = Math.ceil(VHD_BLOCK_SIZE_SECTORS / 8 / SECTOR_SIZE) * SECTOR_SIZE
   const bat = Buffer.alloc(tablePhysicalSizeBytes, 0xff)

packages/vhd-lib/createVhdDirectoryFromStream.js

@@ -0,0 +1,54 @@
+const { parseVhdStream } = require('./parseVhdStream.js')
+const { VhdDirectory } = require('./Vhd/VhdDirectory.js')
+const { Disposable } = require('promise-toolbox')
+const { asyncEach } = require('@vates/async-each')
+
+const buildVhd = Disposable.wrap(async function* (handler, path, inputStream, { concurrency, compression }) {
+  const vhd = yield VhdDirectory.create(handler, path, { compression })
+  await asyncEach(
+    parseVhdStream(inputStream),
+    async function (item) {
+      switch (item.type) {
+        case 'footer':
+          vhd.footer = item.footer
+          break
+        case 'header':
+          vhd.header = item.header
+          break
+        case 'parentLocator':
+          await vhd.writeParentLocator({ ...item, data: item.buffer })
+          break
+        case 'block':
+          await vhd.writeEntireBlock(item)
+          break
+        case 'bat':
+          // it exists but  I don't care
+          break
+        default:
+          throw new Error(`unhandled type of block generated by parser : ${item.type} while generating ${path}`)
+      }
+    },
+    {
+      concurrency,
+    }
+  )
+  await Promise.all([vhd.writeFooter(), vhd.writeHeader(), vhd.writeBlockAllocationTable()])
+})
+
+exports.createVhdDirectoryFromStream = async function createVhdDirectoryFromStream(
+  handler,
+  path,
+  inputStream,
+  { validator, concurrency = 16, compression } = {}
+) {
+  try {
+    await buildVhd(handler, path, inputStream, { concurrency, compression })
+    if (validator !== undefined) {
+      await validator.call(this, path)
+    }
+  } catch (error) {
+    // cleanup on error
+    await handler.rmtree(path)
+    throw error
+  }
+}

packages/vhd-lib/createVhdStreamWithLength.integ.spec.js

@@ -1,17 +1,17 @@
 /* eslint-env jest */
 
-import execa from 'execa'
-import fs from 'fs-extra'
-import rimraf from 'rimraf'
-import getStream from 'get-stream'
-import tmp from 'tmp'
-import { createReadStream, createWriteStream } from 'fs'
-import { pFromCallback } from 'promise-toolbox'
-import { pipeline } from 'readable-stream'
+const execa = require('execa')
+const fs = require('fs-extra')
+const rimraf = require('rimraf')
+const getStream = require('get-stream')
+const tmp = require('tmp')
+const { createReadStream, createWriteStream } = require('fs')
+const { pFromCallback } = require('promise-toolbox')
+const { pipeline } = require('readable-stream')
 
-import { createVhdStreamWithLength } from '.'
-import { FOOTER_SIZE } from './_constants'
-import { createRandomFile, convertFromRawToVhd, convertFromVhdToRaw } from './tests/utils'
+const { createVhdStreamWithLength } = require('./createVhdStreamWithLength.js')
+const { FOOTER_SIZE } = require('./_constants')
+const { createRandomFile, convertFromRawToVhd, convertFromVhdToRaw } = require('./tests/utils')
 
 let tempDir = null
 

packages/vhd-lib/createVhdStreamWithLength.js

@@ -1,13 +1,13 @@
-import assert from 'assert'
-import { pipeline, Transform } from 'readable-stream'
-import { readChunk } from '@vates/read-chunk'
+const assert = require('assert')
+const { pipeline, Transform } = require('readable-stream')
+const { readChunk } = require('@vates/read-chunk')
 
-import checkFooter from './checkFooter'
-import checkHeader from './_checkHeader'
-import noop from './_noop'
-import getFirstAndLastBlocks from './_getFirstAndLastBlocks'
-import { FOOTER_SIZE, HEADER_SIZE, SECTOR_SIZE } from './_constants'
-import { fuFooter, fuHeader } from './_structs'
+const checkFooter = require('./checkFooter')
+const checkHeader = require('./_checkHeader')
+const noop = require('./_noop')
+const getFirstAndLastBlocks = require('./_getFirstAndLastBlocks')
+const { FOOTER_SIZE, HEADER_SIZE, SECTOR_SIZE } = require('./_constants')
+const { fuFooter, fuHeader } = require('./_structs')
 
 class EndCutterStream extends Transform {
   constructor(footerOffset, footerBuffer) {
@@ -35,7 +35,7 @@ class EndCutterStream extends Transform {
   }
 }
 
-export default async function createVhdStreamWithLength(stream) {
+module.exports = async function createVhdStreamWithLength(stream) {
   const readBuffers = []
   let streamPosition = 0
 

packages/vhd-lib/index.js

@@ -0,0 +1,14 @@
+exports.chainVhd = require('./chain')
+exports.checkFooter = require('./checkFooter')
+exports.checkVhdChain = require('./checkChain')
+exports.createReadableSparseStream = require('./createReadableSparseStream')
+exports.createVhdStreamWithLength = require('./createVhdStreamWithLength')
+exports.createVhdDirectoryFromStream = require('./createVhdDirectoryFromStream').createVhdDirectoryFromStream
+exports.mergeVhd = require('./merge')
+exports.peekFooterFromVhdStream = require('./peekFooterFromVhdStream')
+exports.openVhd = require('./openVhd').openVhd
+exports.VhdAbstract = require('./Vhd/VhdAbstract').VhdAbstract
+exports.VhdDirectory = require('./Vhd/VhdDirectory').VhdDirectory
+exports.VhdFile = require('./Vhd/VhdFile').VhdFile
+exports.VhdSynthetic = require('./Vhd/VhdSynthetic').VhdSynthetic
+exports.Constants = require('./_constants')

packages/vhd-lib/merge.integ.spec.js

@@ -0,0 +1,157 @@
+/* eslint-env jest */
+
+const fs = require('fs-extra')
+const rimraf = require('rimraf')
+const tmp = require('tmp')
+const { getHandler } = require('@xen-orchestra/fs')
+const { pFromCallback } = require('promise-toolbox')
+
+const { VhdFile, chainVhd, mergeVhd: vhdMerge } = require('./index')
+
+const { checkFile, createRandomFile, convertFromRawToVhd } = require('./tests/utils')
+
+let tempDir = null
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+test('merge works in normal cases', async () => {
+  const mbOfFather = 8
+  const mbOfChildren = 4
+  const parentRandomFileName = `${tempDir}/randomfile`
+  const childRandomFileName = `${tempDir}/small_randomfile`
+  const parentFileName = `${tempDir}/parent.vhd`
+  const child1FileName = `${tempDir}/child1.vhd`
+  const handler = getHandler({ url: 'file://' })
+
+  await createRandomFile(parentRandomFileName, mbOfFather)
+  await convertFromRawToVhd(parentRandomFileName, parentFileName)
+
+  await createRandomFile(childRandomFileName, mbOfChildren)
+  await convertFromRawToVhd(childRandomFileName, child1FileName)
+  await chainVhd(handler, parentFileName, handler, child1FileName, true)
+
+  // merge
+  await vhdMerge(handler, parentFileName, handler, child1FileName)
+
+  // check that vhd is still valid
+  await checkFile(parentFileName)
+
+  const parentVhd = new VhdFile(handler, parentFileName)
+  await parentVhd.readHeaderAndFooter()
+  await parentVhd.readBlockAllocationTable()
+
+  let offset = 0
+  // check that the data are the same as source
+  for await (const block of parentVhd.blocks()) {
+    const blockContent = block.data
+    const file = offset < mbOfChildren * 1024 * 1024 ? childRandomFileName : parentRandomFileName
+    const buffer = Buffer.alloc(blockContent.length)
+    const fd = await fs.open(file, 'r')
+    await fs.read(fd, buffer, 0, buffer.length, offset)
+
+    expect(buffer.equals(blockContent)).toEqual(true)
+    offset += parentVhd.header.blockSize
+  }
+})
+
+test('it can resume a merge ', async () => {
+  const mbOfFather = 8
+  const mbOfChildren = 4
+  const parentRandomFileName = `${tempDir}/randomfile`
+  const childRandomFileName = `${tempDir}/small_randomfile`
+  const handler = getHandler({ url: `file://${tempDir}` })
+
+  await createRandomFile(`${tempDir}/randomfile`, mbOfFather)
+  await convertFromRawToVhd(`${tempDir}/randomfile`, `${tempDir}/parent.vhd`)
+  const parentVhd = new VhdFile(handler, 'parent.vhd')
+  await parentVhd.readHeaderAndFooter()
+
+  await createRandomFile(`${tempDir}/small_randomfile`, mbOfChildren)
+  await convertFromRawToVhd(`${tempDir}/small_randomfile`, `${tempDir}/child1.vhd`)
+  await chainVhd(handler, 'parent.vhd', handler, 'child1.vhd', true)
+  const childVhd = new VhdFile(handler, 'child1.vhd')
+  await childVhd.readHeaderAndFooter()
+
+  await handler.writeFile(
+    '.parent.vhd.merge.json',
+    JSON.stringify({
+      parent: {
+        header: parentVhd.header.checksum,
+      },
+      child: {
+        header: 'NOT CHILD HEADER ',
+      },
+    })
+  )
+  // expect merge to fail since child header is not ok
+  await expect(async () => await vhdMerge(handler, 'parent.vhd', handler, 'child1.vhd')).rejects.toThrow()
+
+  await handler.unlink('.parent.vhd.merge.json')
+  await handler.writeFile(
+    '.parent.vhd.merge.json',
+    JSON.stringify({
+      parent: {
+        header: 'NOT PARENT HEADER',
+      },
+      child: {
+        header: childVhd.header.checksum,
+      },
+    })
+  )
+  // expect merge to fail since parent header is not ok
+  await expect(async () => await vhdMerge(handler, 'parent.vhd', handler, 'child1.vhd')).rejects.toThrow()
+
+  // break the end footer of parent
+  const size = await handler.getSize('parent.vhd')
+  const fd = await handler.openFile('parent.vhd', 'r+')
+  const buffer = Buffer.alloc(512, 0)
+  // add a fake footer at the end
+  handler.write(fd, buffer, size)
+  await handler.closeFile(fd)
+  // check vhd should fail
+  await expect(async () => await parentVhd.readHeaderAndFooter()).rejects.toThrow()
+
+  await handler.unlink('.parent.vhd.merge.json')
+  await handler.writeFile(
+    '.parent.vhd.merge.json',
+    JSON.stringify({
+      parent: {
+        header: parentVhd.header.checksum,
+      },
+      child: {
+        header: childVhd.header.checksum,
+      },
+      currentBlock: 1,
+    })
+  )
+
+  // really merge
+  await vhdMerge(handler, 'parent.vhd', handler, 'child1.vhd')
+
+  // reload header footer and block allocation table , they should succed
+  await parentVhd.readHeaderAndFooter()
+  await parentVhd.readBlockAllocationTable()
+  let offset = 0
+  // check that the data are the same as source
+  for await (const block of parentVhd.blocks()) {
+    const blockContent = block.data
+    // first block is marked as already merged, should not be modified
+    // second block should come from children
+    // then two block only in parent
+    const file = block.id === 1 ? childRandomFileName : parentRandomFileName
+    const buffer = Buffer.alloc(blockContent.length)
+    const fd = await fs.open(file, 'r')
+    await fs.read(fd, buffer, 0, buffer.length, offset)
+
+    expect(buffer.equals(blockContent)).toEqual(true)
+    offset += parentVhd.header.blockSize
+  }
+})

packages/vhd-lib/merge.js

@@ -0,0 +1,148 @@
+// TODO: remove once completely merged in vhd.js
+
+const assert = require('assert')
+const noop = require('./_noop')
+const { createLogger } = require('@xen-orchestra/log')
+const { limitConcurrency } = require('limit-concurrency-decorator')
+
+const { openVhd } = require('./openVhd')
+const { basename, dirname } = require('path')
+const { DISK_TYPES } = require('./_constants')
+const { Disposable } = require('promise-toolbox')
+const { asyncEach } = require('@vates/async-each')
+const { VhdDirectory } = require('./Vhd/VhdDirectory')
+
+const { warn } = createLogger('vhd-lib:merge')
+
+function makeThrottledWriter(handler, path, delay) {
+  let lastWrite = Date.now()
+  return async json => {
+    const now = Date.now()
+    if (now - lastWrite > delay) {
+      lastWrite = now
+      await handler.writeFile(path, JSON.stringify(json), { flags: 'w' }).catch(warn)
+    }
+  }
+}
+
+// Merge vhd child into vhd parent.
+//
+// TODO: rename the VHD file during the merge
+module.exports = limitConcurrency(2)(async function merge(
+  parentHandler,
+  parentPath,
+  childHandler,
+  childPath,
+  { onProgress = noop } = {}
+) {
+  const mergeStatePath = dirname(parentPath) + '/' + '.' + basename(parentPath) + '.merge.json'
+
+  return await Disposable.use(async function* () {
+    let mergeState
+    try {
+      const mergeStateContent = await parentHandler.readFile(mergeStatePath)
+      mergeState = JSON.parse(mergeStateContent)
+    } catch (error) {
+      if (error.code !== 'ENOENT') {
+        warn('problem while checking the merge state', { error })
+      }
+    }
+
+    // during merging, the end footer of the parent can be overwritten by new blocks
+    // we should use it as a way to check vhd health
+    const parentVhd = yield openVhd(parentHandler, parentPath, {
+      flags: 'r+',
+      checkSecondFooter: mergeState === undefined,
+    })
+    const childVhd = yield openVhd(childHandler, childPath)
+
+    const concurrency = childVhd instanceof VhdDirectory ? 16 : 1
+    if (mergeState === undefined) {
+      assert.strictEqual(childVhd.header.blockSize, parentVhd.header.blockSize)
+
+      const parentDiskType = parentVhd.footer.diskType
+      assert(parentDiskType === DISK_TYPES.DIFFERENCING || parentDiskType === DISK_TYPES.DYNAMIC)
+      assert.strictEqual(childVhd.footer.diskType, DISK_TYPES.DIFFERENCING)
+    } else {
+      assert.strictEqual(parentVhd.header.checksum, mergeState.parent.header)
+      assert.strictEqual(childVhd.header.checksum, mergeState.child.header)
+    }
+
+    // Read allocation table of child/parent.
+    await Promise.all([parentVhd.readBlockAllocationTable(), childVhd.readBlockAllocationTable()])
+
+    const { maxTableEntries } = childVhd.header
+
+    if (mergeState === undefined) {
+      await parentVhd.ensureBatSize(childVhd.header.maxTableEntries)
+
+      mergeState = {
+        child: { header: childVhd.header.checksum },
+        parent: { header: parentVhd.header.checksum },
+        currentBlock: 0,
+        mergedDataSize: 0,
+      }
+
+      // finds first allocated block for the 2 following loops
+      while (mergeState.currentBlock < maxTableEntries && !childVhd.containsBlock(mergeState.currentBlock)) {
+        ++mergeState.currentBlock
+      }
+    }
+
+    // counts number of allocated blocks
+    const toMerge = []
+    for (let block = mergeState.currentBlock; block < maxTableEntries; block++) {
+      if (childVhd.containsBlock(block)) {
+        toMerge.push(block)
+      }
+    }
+    const nBlocks = toMerge.length
+    onProgress({ total: nBlocks, done: 0 })
+
+    const merging = new Set()
+    let counter = 0
+
+    const mergeStateWriter = makeThrottledWriter(parentHandler, mergeStatePath, 10e3)
+
+    await asyncEach(
+      toMerge,
+      async blockId => {
+        merging.add(blockId)
+        mergeState.mergedDataSize += await parentVhd.coalesceBlock(childVhd, blockId)
+        merging.delete(blockId)
+
+        onProgress({
+          total: nBlocks,
+          done: counter + 1,
+        })
+        counter++
+        mergeState.currentBlock = Math.min(...merging)
+        mergeStateWriter(mergeState)
+      },
+      {
+        concurrency,
+      }
+    )
+    onProgress({ total: nBlocks, done: nBlocks })
+    // some blocks could have been created or moved in parent : write bat
+    await parentVhd.writeBlockAllocationTable()
+
+    const cFooter = childVhd.footer
+    const pFooter = parentVhd.footer
+
+    pFooter.currentSize = cFooter.currentSize
+    pFooter.diskGeometry = { ...cFooter.diskGeometry }
+    pFooter.originalSize = cFooter.originalSize
+    pFooter.timestamp = cFooter.timestamp
+    pFooter.uuid = cFooter.uuid
+
+    // necessary to update values and to recreate the footer after block
+    // creation
+    await parentVhd.writeFooter()
+
+    // should be a disposable
+    parentHandler.unlink(mergeStatePath).catch(warn)
+
+    return mergeState.mergedDataSize
+  })
+})

packages/vhd-lib/openVhd.integ.spec.js

@@ -0,0 +1,62 @@
+/* eslint-env jest */
+
+const rimraf = require('rimraf')
+const tmp = require('tmp')
+const { getSyncedHandler } = require('@xen-orchestra/fs')
+const { Disposable, pFromCallback } = require('promise-toolbox')
+
+const { openVhd } = require('./index')
+const { createRandomFile, convertFromRawToVhd, createRandomVhdDirectory } = require('./tests/utils')
+
+const { VhdAbstract } = require('./Vhd/VhdAbstract')
+
+let tempDir
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+test('It opens a vhd file ( alias or not)', async () => {
+  const initalSize = 4
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSize)
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file://' })
+    const vhd = yield openVhd(handler, vhdFileName)
+    expect(vhd.header.cookie).toEqual('cxsparse')
+    expect(vhd.footer.cookie).toEqual('conectix')
+
+    const aliasFileName = `${tempDir}/out.alias.vhd`
+    await VhdAbstract.createAlias(handler, aliasFileName, vhdFileName)
+    const alias = yield openVhd(handler, aliasFileName)
+    expect(alias.header.cookie).toEqual('cxsparse')
+    expect(alias.footer.cookie).toEqual('conectix')
+  })
+})
+
+test('It opens a vhd directory', async () => {
+  const initalSize = 4
+  const vhdDirectory = `${tempDir}/randomfile.dir`
+  await createRandomVhdDirectory(vhdDirectory, initalSize)
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file://' })
+    const vhd = yield openVhd(handler, vhdDirectory)
+    expect(vhd.header.cookie).toEqual('cxsparse')
+    expect(vhd.footer.cookie).toEqual('conectix')
+
+    const aliasFileName = `${tempDir}/out.alias.vhd`
+    await VhdAbstract.createAlias(handler, aliasFileName, vhdDirectory)
+    const alias = yield openVhd(handler, aliasFileName)
+    expect(alias.header.cookie).toEqual('cxsparse')
+    expect(alias.footer.cookie).toEqual('conectix')
+  })
+})

packages/vhd-lib/openVhd.js

@@ -0,0 +1,15 @@
+const { resolveVhdAlias } = require('./aliases')
+const { VhdDirectory } = require('./Vhd/VhdDirectory.js')
+const { VhdFile } = require('./Vhd/VhdFile.js')
+
+exports.openVhd = async function openVhd(handler, path, opts) {
+  const resolved = await resolveVhdAlias(handler, path)
+  try {
+    return await VhdFile.open(handler, resolved, opts)
+  } catch (e) {
+    if (e.code !== 'EISDIR') {
+      throw e
+    }
+    return await VhdDirectory.open(handler, resolved, opts)
+  }
+}

packages/vhd-lib/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "vhd-lib",
-  "version": "1.3.0",
+  "version": "3.0.0",
   "license": "AGPL-3.0-or-later",
   "description": "Primitives for VHD file handling",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-lib",
@@ -11,11 +11,11 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "main": "dist/",
   "engines": {
-    "node": ">=10"
+    "node": ">=12"
   },
   "dependencies": {
+    "@vates/async-each": "^0.1.0",
     "@vates/read-chunk": "^0.1.2",
     "@xen-orchestra/log": "^0.3.0",
     "async-iterator-to-stream": "^1.0.2",
@@ -27,25 +27,12 @@
     "uuid": "^8.3.1"
   },
   "devDependencies": {
-    "@babel/cli": "^7.0.0",
-    "@babel/core": "^7.0.0",
-    "@babel/preset-env": "^7.0.0",
-    "@xen-orchestra/fs": "^0.18.0",
-    "babel-plugin-lodash": "^3.3.2",
-    "cross-env": "^7.0.2",
-    "execa": "^5.0.0",
+    "@xen-orchestra/fs": "^0.19.3",
     "get-stream": "^6.0.0",
     "readable-stream": "^3.0.6",
-    "rimraf": "^3.0.0",
     "tmp": "^0.2.1"
   },
   "scripts": {
-    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
-    "clean": "rimraf dist/",
-    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
-    "prebuild": "yarn run clean",
-    "predev": "yarn run clean",
-    "prepublishOnly": "yarn run build",
     "postversion": "npm publish"
   },
   "author": {