WiP: feat(xen-api/getCachedRecord): getRecord + cache + events

Fixes #5088

.eslintrc.js

@@ -1,3 +1,5 @@
+'use strict'
+
 module.exports = {
   extends: ['plugin:eslint-comments/recommended', 'standard', 'standard-jsx', 'prettier'],
   globals: {
@@ -18,8 +20,19 @@ module.exports = {
         'no-console': 'off',
       },
     },
+    {
+      files: ['*.mjs'],
+      parserOptions: {
+        sourceType: 'module',
+      },
+    },
   ],
 
+  parserOptions: {
+    ecmaVersion: 13,
+    sourceType: 'script',
+  },
+
   rules: {
     // disabled because XAPI objects are using camel case
     camelcase: ['off'],
@@ -34,5 +47,7 @@ module.exports = {
     'lines-between-class-members': 'off',
 
     'no-console': ['error', { allow: ['warn', 'error'] }],
+
+    strict: 'error',
   },
 }

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,34 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: 'status: triaging :triangular_flag_on_post:, type: bug :bug:'
+assignees: ''
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+
+- Node: [e.g. 16.12.1]
+- xo-server: [e.g. 5.82.3]
+- xo-web: [e.g. 5.87.0]
+- hypervisor: [e.g. XCP-ng 8.2.0]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.gitignore

@@ -35,3 +35,6 @@ pnpm-debug.log.*
 yarn-error.log
 yarn-error.log.*
 .env
+
+# nyc test coverage
+.nyc_output

@vates/async-each/.USAGE.md

@@ -0,0 +1,35 @@
+### `asyncEach(iterable, iteratee, [opts])`
+
+Executes `iteratee` in order for each value yielded by `iterable`.
+
+Returns a promise wich rejects as soon as a call to `iteratee` throws or a promise returned by it rejects, and which resolves when all promises returned by `iteratee` have resolved.
+
+`iterable` must be an iterable or async iterable.
+
+`iteratee` is called with the same `this` value as `asyncEach`, and with the following arguments:
+
+- `value`: the value yielded by `iterable`
+- `index`: the 0-based index for this value
+- `iterable`: the iterable itself
+
+`opts` is an object that can contains the following options:
+
+- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
+- `signal`: an abort signal to stop the iteration
+- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
+
+```js
+import { asyncEach } from '@vates/async-each'
+
+const contents = []
+await asyncEach(
+  ['foo.txt', 'bar.txt', 'baz.txt'],
+  async function (filename, i) {
+    contents[i] = await readFile(filename)
+  },
+  {
+    // reads two files at a time
+    concurrency: 2,
+  }
+)
+```

@vates/async-each/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/async-each/README.md

@@ -0,0 +1,68 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/async-each
+
+[![Package Version](https://badgen.net/npm/v/@vates/async-each)](https://npmjs.org/package/@vates/async-each) ![License](https://badgen.net/npm/license/@vates/async-each) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/async-each)](https://bundlephobia.com/result?p=@vates/async-each) [![Node compatibility](https://badgen.net/npm/node/@vates/async-each)](https://npmjs.org/package/@vates/async-each)
+
+> Run async fn for each item in (async) iterable
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/async-each):
+
+```
+> npm install --save @vates/async-each
+```
+
+## Usage
+
+### `asyncEach(iterable, iteratee, [opts])`
+
+Executes `iteratee` in order for each value yielded by `iterable`.
+
+Returns a promise wich rejects as soon as a call to `iteratee` throws or a promise returned by it rejects, and which resolves when all promises returned by `iteratee` have resolved.
+
+`iterable` must be an iterable or async iterable.
+
+`iteratee` is called with the same `this` value as `asyncEach`, and with the following arguments:
+
+- `value`: the value yielded by `iterable`
+- `index`: the 0-based index for this value
+- `iterable`: the iterable itself
+
+`opts` is an object that can contains the following options:
+
+- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
+- `signal`: an abort signal to stop the iteration
+- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
+
+```js
+import { asyncEach } from '@vates/async-each'
+
+const contents = []
+await asyncEach(
+  ['foo.txt', 'bar.txt', 'baz.txt'],
+  async function (filename, i) {
+    contents[i] = await readFile(filename)
+  },
+  {
+    // reads two files at a time
+    concurrency: 2,
+  }
+)
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/async-each/index.js

@@ -0,0 +1,99 @@
+'use strict'
+
+const noop = Function.prototype
+
+class AggregateError extends Error {
+  constructor(errors, message) {
+    super(message)
+    this.errors = errors
+  }
+}
+
+exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 1, signal, stopOnError = true } = {}) {
+  return new Promise((resolve, reject) => {
+    const it = (iterable[Symbol.iterator] || iterable[Symbol.asyncIterator]).call(iterable)
+    const errors = []
+    let running = 0
+    let index = 0
+
+    let onAbort
+    if (signal !== undefined) {
+      onAbort = () => {
+        onRejectedWrapper(new Error('asyncEach aborted'))
+      }
+      signal.addEventListener('abort', onAbort)
+    }
+
+    const clean = () => {
+      onFulfilled = onRejected = noop
+      if (onAbort !== undefined) {
+        signal.removeEventListener('abort', onAbort)
+      }
+    }
+
+    resolve = (resolve =>
+      function resolveAndClean(value) {
+        resolve(value)
+        clean()
+      })(resolve)
+    reject = (reject =>
+      function rejectAndClean(reason) {
+        reject(reason)
+        clean()
+      })(reject)
+
+    let onFulfilled = value => {
+      --running
+      next()
+    }
+    const onFulfilledWrapper = value => onFulfilled(value)
+
+    let onRejected = stopOnError
+      ? reject
+      : error => {
+          --running
+          errors.push(error)
+          next()
+        }
+    const onRejectedWrapper = reason => onRejected(reason)
+
+    let nextIsRunning = false
+    let next = async () => {
+      if (nextIsRunning) {
+        return
+      }
+      nextIsRunning = true
+      if (running < concurrency) {
+        const cursor = await it.next()
+        if (cursor.done) {
+          next = () => {
+            if (running === 0) {
+              if (errors.length === 0) {
+                resolve()
+              } else {
+                reject(new AggregateError(errors))
+              }
+            }
+          }
+        } else {
+          ++running
+          try {
+            const result = iteratee.call(this, cursor.value, index++, iterable)
+            let then
+            if (result != null && typeof result === 'object' && typeof (then = result.then) === 'function') {
+              then.call(result, onFulfilledWrapper, onRejectedWrapper)
+            } else {
+              onFulfilled(result)
+            }
+          } catch (error) {
+            onRejected(error)
+          }
+        }
+        nextIsRunning = false
+        return next()
+      }
+      nextIsRunning = false
+    }
+    next()
+  })
+}

@vates/async-each/index.spec.js

@@ -0,0 +1,99 @@
+'use strict'
+
+/* eslint-env jest */
+
+const { asyncEach } = require('./')
+
+const randomDelay = (max = 10) =>
+  new Promise(resolve => {
+    setTimeout(resolve, Math.floor(Math.random() * max + 1))
+  })
+
+const rejectionOf = p =>
+  new Promise((resolve, reject) => {
+    p.then(reject, resolve)
+  })
+
+describe('asyncEach', () => {
+  const thisArg = 'qux'
+  const values = ['foo', 'bar', 'baz']
+
+  Object.entries({
+    'sync iterable': () => values,
+    'async iterable': async function* () {
+      for (const value of values) {
+        await randomDelay()
+        yield value
+      }
+    },
+  }).forEach(([what, getIterable]) =>
+    describe('with ' + what, () => {
+      let iterable
+      beforeEach(() => {
+        iterable = getIterable()
+      })
+
+      it('works', async () => {
+        const iteratee = jest.fn(async () => {})
+
+        await asyncEach.call(thisArg, iterable, iteratee)
+
+        expect(iteratee.mock.instances).toEqual(Array.from(values, () => thisArg))
+        expect(iteratee.mock.calls).toEqual(Array.from(values, (value, index) => [value, index, iterable]))
+      })
+      ;[1, 2, 4].forEach(concurrency => {
+        it('respects a concurrency of ' + concurrency, async () => {
+          let running = 0
+
+          await asyncEach(
+            values,
+            async () => {
+              ++running
+              expect(running).toBeLessThanOrEqual(concurrency)
+              await randomDelay()
+              --running
+            },
+            { concurrency }
+          )
+        })
+      })
+
+      it('stops on first error when stopOnError is true', async () => {
+        const error = new Error()
+        const iteratee = jest.fn((_, i) => {
+          if (i === 1) {
+            throw error
+          }
+        })
+
+        expect(await rejectionOf(asyncEach(iterable, iteratee, { stopOnError: true }))).toBe(error)
+        expect(iteratee).toHaveBeenCalledTimes(2)
+      })
+
+      it('rejects AggregateError when stopOnError is false', async () => {
+        const errors = []
+        const iteratee = jest.fn(() => {
+          const error = new Error()
+          errors.push(error)
+          throw error
+        })
+
+        const error = await rejectionOf(asyncEach(iterable, iteratee, { stopOnError: false }))
+        expect(error.errors).toEqual(errors)
+        expect(iteratee.mock.calls).toEqual(Array.from(values, (value, index) => [value, index, iterable]))
+      })
+
+      it('can be interrupted with an AbortSignal', async () => {
+        const ac = new AbortController()
+        const iteratee = jest.fn((_, i) => {
+          if (i === 1) {
+            ac.abort()
+          }
+        })
+
+        await expect(asyncEach(iterable, iteratee, { signal: ac.signal })).rejects.toThrow('asyncEach aborted')
+        expect(iteratee).toHaveBeenCalledTimes(2)
+      })
+    })
+  )
+})

@vates/async-each/package.json

@@ -0,0 +1,34 @@
+{
+  "private": false,
+  "name": "@vates/async-each",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/async-each",
+  "description": "Run async fn for each item in (async) iterable",
+  "keywords": [
+    "array",
+    "async",
+    "collection",
+    "each",
+    "for",
+    "foreach",
+    "iterable",
+    "iterator"
+  ],
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/async-each",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "0.1.0",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@vates/coalesce-calls/index.js

@@ -1,3 +1,5 @@
+'use strict'
+
 exports.coalesceCalls = function (fn) {
   let promise
   const clean = () => {

@vates/coalesce-calls/index.spec.js

@@ -1,3 +1,5 @@
+'use strict'
+
 /* eslint-env jest */
 
 const { coalesceCalls } = require('./')

@vates/compose/.USAGE.md

@@ -46,3 +46,20 @@ const f = compose(
   [add2, mul3]
 )
 ```
+
+Functions can receive extra parameters:
+
+```js
+const isIn = (value, min, max) => min <= value && value <= max
+
+// Only compatible when `fns` is passed as an array!
+const f = compose([
+  [add, 2],
+  [isIn, 3, 10],
+])
+
+console.log(f(1))
+// → true
+```
+
+> Note: if the first function is defined with extra parameters, it will only receive the first value passed to the composed function, instead of all the parameters.

@vates/compose/README.md

@@ -65,6 +65,23 @@ const f = compose(
 )
 ```
 
+Functions can receive extra parameters:
+
+```js
+const isIn = (value, min, max) => min <= value && value <= max
+
+// Only compatible when `fns` is passed as an array!
+const f = compose([
+  [add, 2],
+  [isIn, 3, 10],
+])
+
+console.log(f(1))
+// → true
+```
+
+> Note: if the first function is defined with extra parameters, it will only receive the first value passed to the composed function, instead of all the parameters.
+
 ## Contributions
 
 Contributions are _very_ welcomed, either on the documentation or on

@vates/compose/index.js

@@ -4,11 +4,13 @@ const defaultOpts = { async: false, right: false }
 
 exports.compose = function compose(opts, fns) {
   if (Array.isArray(opts)) {
-    fns = opts
+    fns = opts.slice() // don't mutate passed array
     opts = defaultOpts
   } else if (typeof opts === 'object') {
     opts = Object.assign({}, defaultOpts, opts)
-    if (!Array.isArray(fns)) {
+    if (Array.isArray(fns)) {
+      fns = fns.slice() // don't mutate passed array
+    } else {
       fns = Array.prototype.slice.call(arguments, 1)
     }
   } else {
@@ -20,6 +22,24 @@ exports.compose = function compose(opts, fns) {
   if (n === 0) {
     throw new TypeError('at least one function must be passed')
   }
+
+  for (let i = 0; i < n; ++i) {
+    const entry = fns[i]
+    if (Array.isArray(entry)) {
+      const fn = entry[0]
+      const args = entry.slice()
+      args[0] = undefined
+      fns[i] = function composeWithArgs(value) {
+        args[0] = value
+        try {
+          return fn.apply(this, args)
+        } finally {
+          args[0] = undefined
+        }
+      }
+    }
+  }
+
   if (n === 1) {
     return fns[0]
   }

@vates/compose/index.spec.js

@@ -1,3 +1,5 @@
+'use strict'
+
 /* eslint-env jest */
 
 const { compose } = require('./')

@vates/compose/package.json

@@ -14,7 +14,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "engines": {
     "node": ">=7.6"
   },

@vates/decorate-with/.USAGE.md

@@ -40,3 +40,33 @@ decorateMethodsWith(Foo, {
 ```
 
 The decorated class is returned, so you can export it directly.
+
+To apply multiple transforms to a method, you can either call `decorateMethodsWith` multiple times or use [`@vates/compose`](https://www.npmjs.com/package/@vates/compose):
+
+```js
+decorateMethodsWith(Foo, {
+  bar: compose([
+    [lodash.debounce, 150]
+    lodash.curry,
+  ])
+})
+```
+
+### `perInstance(fn, ...args)`
+
+Helper to decorate the method by instance instead of for the whole class.
+
+This is often necessary for caching or deduplicating calls.
+
+```js
+import { perInstance } from '@vates/decorateWith'
+
+class Foo {
+  @decorateWith(perInstance, lodash.memoize)
+  bar() {
+    // body
+  }
+}
+```
+
+Because it's a normal function, it can also be used with `decorateMethodsWith`, with `compose` or even by itself.

@vates/decorate-with/README.md

@@ -59,6 +59,36 @@ decorateMethodsWith(Foo, {
 
 The decorated class is returned, so you can export it directly.
 
+To apply multiple transforms to a method, you can either call `decorateMethodsWith` multiple times or use [`@vates/compose`](https://www.npmjs.com/package/@vates/compose):
+
+```js
+decorateMethodsWith(Foo, {
+  bar: compose([
+    [lodash.debounce, 150]
+    lodash.curry,
+  ])
+})
+```
+
+### `perInstance(fn, ...args)`
+
+Helper to decorate the method by instance instead of for the whole class.
+
+This is often necessary for caching or deduplicating calls.
+
+```js
+import { perInstance } from '@vates/decorateWith'
+
+class Foo {
+  @decorateWith(perInstance, lodash.memoize)
+  bar() {
+    // body
+  }
+}
+```
+
+Because it's a normal function, it can also be used with `decorateMethodsWith`, with `compose` or even by itself.
+
 ## Contributions
 
 Contributions are _very_ welcomed, either on the documentation or on

@vates/decorate-with/index.js

@@ -1,3 +1,5 @@
+'use strict'
+
 exports.decorateWith = function decorateWith(fn, ...args) {
   return (target, name, descriptor) => ({
     ...descriptor,
@@ -19,3 +21,15 @@ exports.decorateMethodsWith = function decorateMethodsWith(klass, map) {
   }
   return klass
 }
+
+exports.perInstance = function perInstance(fn, decorator, ...args) {
+  const map = new WeakMap()
+  return function () {
+    let decorated = map.get(this)
+    if (decorated === undefined) {
+      decorated = decorator(fn, ...args)
+      map.set(this, decorated)
+    }
+    return decorated.apply(this, arguments)
+  }
+}

@vates/decorate-with/index.spec.js

@@ -0,0 +1,100 @@
+'use strict'
+
+const assert = require('assert')
+const { describe, it } = require('tap').mocha
+
+const { decorateWith, decorateMethodsWith, perInstance } = require('./')
+
+describe('decorateWith', () => {
+  it('works', () => {
+    const expectedArgs = [Math.random(), Math.random()]
+    const expectedFn = Function.prototype
+    const newFn = () => {}
+
+    const decorator = decorateWith(function wrapper(fn, ...args) {
+      assert.deepStrictEqual(fn, expectedFn)
+      assert.deepStrictEqual(args, expectedArgs)
+
+      return newFn
+    }, ...expectedArgs)
+
+    const descriptor = {
+      configurable: true,
+      enumerable: false,
+      value: expectedFn,
+      writable: true,
+    }
+    assert.deepStrictEqual(decorator({}, 'foo', descriptor), {
+      ...descriptor,
+      value: newFn,
+    })
+  })
+})
+
+describe('decorateMethodsWith', () => {
+  it('works', () => {
+    class C {
+      foo() {}
+      bar() {}
+    }
+
+    const expectedArgs = [Math.random(), Math.random()]
+    const P = C.prototype
+
+    const descriptors = Object.getOwnPropertyDescriptors(P)
+
+    const newFoo = () => {}
+    const newBar = () => {}
+
+    decorateMethodsWith(C, {
+      foo(method) {
+        assert.strictEqual(arguments.length, 1)
+        assert.strictEqual(method, P.foo)
+        return newFoo
+      },
+      bar: [
+        function (method, ...args) {
+          assert.strictEqual(method, P.bar)
+          assert.deepStrictEqual(args, expectedArgs)
+          return newBar
+        },
+        ...expectedArgs,
+      ],
+    })
+
+    const newDescriptors = Object.getOwnPropertyDescriptors(P)
+    assert.deepStrictEqual(newDescriptors.foo, { ...descriptors.foo, value: newFoo })
+    assert.deepStrictEqual(newDescriptors.bar, { ...descriptors.bar, value: newBar })
+  })
+})
+
+describe('perInstance', () => {
+  it('works', () => {
+    let calls = 0
+
+    const expectedArgs = [Math.random(), Math.random()]
+    const expectedFn = Function.prototype
+    function wrapper(fn, ...args) {
+      assert.strictEqual(fn, expectedFn)
+      assert.deepStrictEqual(args, expectedArgs)
+      const i = ++calls
+      return () => i
+    }
+
+    const wrapped = perInstance(expectedFn, wrapper, ...expectedArgs)
+
+    // decorator is not called before decorated called
+    assert.strictEqual(calls, 0)
+
+    const o1 = {}
+    const o2 = {}
+
+    assert.strictEqual(wrapped.call(o1), 1)
+
+    // the same decorated function is returned for the same instance
+    assert.strictEqual(wrapped.call(o1), 1)
+
+    // a new decorated function is returned for another instance
+    assert.strictEqual(wrapped.call(o2), 2)
+  })
+})

@vates/decorate-with/package.json

@@ -20,11 +20,15 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "0.1.0",
+  "version": "1.0.0",
   "engines": {
     "node": ">=8.10"
   },
   "scripts": {
-    "postversion": "npm publish --access public"
+    "postversion": "npm publish --access public",
+    "test": "tap"
+  },
+  "devDependencies": {
+    "tap": "^15.1.6"
   }
 }

@vates/disposable/debounceResource.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { createLogger } = require('@xen-orchestra/log')
 

@vates/disposable/debounceResource.spec.js

@@ -1,3 +1,5 @@
+'use strict'
+
 /* eslint-env jest */
 
 const { createDebounceResource } = require('./debounceResource')

@vates/disposable/deduped.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const ensureArray = require('ensure-array')
 const { MultiKeyMap } = require('@vates/multi-key-map')
 

@vates/disposable/deduped.spec.js

@@ -1,3 +1,5 @@
+'use strict'
+
 /* eslint-env jest */
 
 const { deduped } = require('./deduped')

@vates/multi-key-map/index.js

@@ -1,3 +1,5 @@
+'use strict'
+
 class Node {
   constructor(value) {
     this.children = new Map()

@vates/multi-key-map/index.spec.js

@@ -1,3 +1,5 @@
+'use strict'
+
 /* eslint-env jest */
 
 const { MultiKeyMap } = require('./')

@vates/parse-duration/index.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const ms = require('ms')
 
 exports.parseDuration = value => {

@vates/predicates/.USAGE.md

@@ -0,0 +1,57 @@
+`undefined` predicates are ignored and `undefined` is returned if all predicates are `undefined`, this permits the most efficient composition:
+
+```js
+const compositePredicate = every(undefined, some(predicate2, undefined))
+
+// ends up as
+
+const compositePredicate = predicate2
+```
+
+Predicates can also be passed wrapped in an array:
+
+```js
+const compositePredicate = every([predicate1, some([predicate2, predicate3])])
+```
+
+`this` and all arguments are passed to the nested predicates.
+
+### `every(predicates)`
+
+> Returns a predicate that returns `true` iff every predicate returns `true`.
+
+```js
+const isBetween3And7 = every(
+  n => n >= 3,
+  n => n <= 7
+)
+
+isBetween3And10(0)
+// → false
+
+isBetween3And10(5)
+// → true
+
+isBetween3And10(10)
+// → false
+```
+
+### `some(predicates)`
+
+> Returns a predicate that returns `true` iff some predicate returns `true`.
+
+```js
+const isAliceOrBob = some(
+  name => name === 'Alice',
+  name => name === 'Bob'
+)
+
+isAliceOrBob('Alice')
+// → true
+
+isAliceOrBob('Bob')
+// → true
+
+isAliceOrBob('Oscar')
+// → false
+```

@vates/predicates/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/predicates/README.md

@@ -0,0 +1,90 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/predicates
+
+[![Package Version](https://badgen.net/npm/v/@vates/predicates)](https://npmjs.org/package/@vates/predicates) ![License](https://badgen.net/npm/license/@vates/predicates) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/predicates)](https://bundlephobia.com/result?p=@vates/predicates) [![Node compatibility](https://badgen.net/npm/node/@vates/predicates)](https://npmjs.org/package/@vates/predicates)
+
+> Utilities to compose predicates
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/predicates):
+
+```
+> npm install --save @vates/predicates
+```
+
+## Usage
+
+`undefined` predicates are ignored and `undefined` is returned if all predicates are `undefined`, this permits the most efficient composition:
+
+```js
+const compositePredicate = every(undefined, some(predicate2, undefined))
+
+// ends up as
+
+const compositePredicate = predicate2
+```
+
+Predicates can also be passed wrapped in an array:
+
+```js
+const compositePredicate = every([predicate1, some([predicate2, predicate3])])
+```
+
+`this` and all arguments are passed to the nested predicates.
+
+### `every(predicates)`
+
+> Returns a predicate that returns `true` iff every predicate returns `true`.
+
+```js
+const isBetween3And7 = every(
+  n => n >= 3,
+  n => n <= 7
+)
+
+isBetween3And10(0)
+// → false
+
+isBetween3And10(5)
+// → true
+
+isBetween3And10(10)
+// → false
+```
+
+### `some(predicates)`
+
+> Returns a predicate that returns `true` iff some predicate returns `true`.
+
+```js
+const isAliceOrBob = some(
+  name => name === 'Alice',
+  name => name === 'Bob'
+)
+
+isAliceOrBob('Alice')
+// → true
+
+isAliceOrBob('Bob')
+// → true
+
+isAliceOrBob('Oscar')
+// → false
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/predicates/index.js

@@ -0,0 +1,71 @@
+'use strict'
+
+const {
+  isArray,
+  prototype: { filter },
+} = Array
+
+class InvalidPredicate extends TypeError {
+  constructor(value) {
+    super('not a valid predicate')
+    this.value = value
+  }
+}
+
+function isDefinedPredicate(value) {
+  if (value === undefined) {
+    return false
+  }
+
+  if (typeof value !== 'function') {
+    throw new InvalidPredicate(value)
+  }
+
+  return true
+}
+
+function handleArgs() {
+  let predicates
+  if (!(arguments.length === 1 && isArray((predicates = arguments[0])))) {
+    predicates = arguments
+  }
+  return filter.call(predicates, isDefinedPredicate)
+}
+
+exports.every = function every() {
+  const predicates = handleArgs.apply(this, arguments)
+  const n = predicates.length
+  if (n === 0) {
+    return
+  }
+  if (n === 1) {
+    return predicates[0]
+  }
+  return function everyPredicate() {
+    for (let i = 0; i < n; ++i) {
+      if (!predicates[i].apply(this, arguments)) {
+        return false
+      }
+    }
+    return true
+  }
+}
+
+exports.some = function some() {
+  const predicates = handleArgs.apply(this, arguments)
+  const n = predicates.length
+  if (n === 0) {
+    return
+  }
+  if (n === 1) {
+    return predicates[0]
+  }
+  return function somePredicate() {
+    for (let i = 0; i < n; ++i) {
+      if (predicates[i].apply(this, arguments)) {
+        return true
+      }
+    }
+    return false
+  }
+}

@vates/predicates/index.spec.js

@@ -0,0 +1,65 @@
+'use strict'
+
+const assert = require('assert/strict')
+const { describe, it } = require('tap').mocha
+
+const { every, some } = require('./')
+
+const T = () => true
+const F = () => false
+
+const testArgsHandling = fn => {
+  it('returns undefined if all predicates are undefined', () => {
+    assert.equal(fn(undefined), undefined)
+    assert.equal(fn([undefined]), undefined)
+  })
+
+  it('returns the predicate if only a single one is passed', () => {
+    assert.equal(fn(undefined, T), T)
+    assert.equal(fn([undefined, T]), T)
+  })
+
+  it('throws if it receives a non-predicate', () => {
+    const error = new TypeError('not a valid predicate')
+    error.value = 3
+    assert.throws(() => fn(3), error)
+  })
+
+  it('forwards this and arguments to predicates', () => {
+    const thisArg = 'qux'
+    const args = ['foo', 'bar', 'baz']
+    const predicate = function () {
+      assert.equal(this, thisArg)
+      assert.deepEqual(Array.from(arguments), args)
+    }
+    fn(predicate, predicate).apply(thisArg, args)
+  })
+}
+
+const runTests = (fn, truthTable) =>
+  it('works', () => {
+    truthTable.forEach(([result, ...predicates]) => {
+      assert.equal(fn(...predicates)(), result)
+      assert.equal(fn(predicates)(), result)
+    })
+  })
+
+describe('every', () => {
+  testArgsHandling(every)
+  runTests(every, [
+    [true, T, T],
+    [false, T, F],
+    [false, F, T],
+    [false, F, F],
+  ])
+})
+
+describe('some', () => {
+  testArgsHandling(some)
+  runTests(some, [
+    [true, T, T],
+    [true, T, F],
+    [true, F, T],
+    [false, F, F],
+  ])
+})

@vates/predicates/package.json

@@ -0,0 +1,40 @@
+{
+  "private": false,
+  "name": "@vates/predicates",
+  "description": "Utilities to compose predicates",
+  "keywords": [
+    "and",
+    "combine",
+    "compose",
+    "every",
+    "function",
+    "functions",
+    "or",
+    "predicate",
+    "predicates",
+    "some"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/predicates",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/predicates",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "1.0.0",
+  "engines": {
+    "node": ">=6"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public",
+    "test": "tap"
+  },
+  "devDependencies": {
+    "tap": "^15.1.6"
+  }
+}

@vates/read-chunk/index.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const readChunk = (stream, size) =>
   size === 0
     ? Promise.resolve(Buffer.alloc(0))

@vates/read-chunk/index.spec.js

@@ -1,3 +1,5 @@
+'use strict'
+
 /* eslint-env jest */
 
 const { Readable } = require('stream')

@vates/toggle-scripts/index.js

@@ -1,5 +1,7 @@
 #!/usr/bin/env node
 
+'use strict'
+
 const fs = require('fs')
 
 const mapKeys = (object, iteratee) => {

@xen-orchestra/async-map/index.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const wrapCall = (fn, arg, thisArg) => {
   try {
     return Promise.resolve(fn.call(thisArg, arg))

@xen-orchestra/async-map/index.spec.js

@@ -1,3 +1,5 @@
+'use strict'
+
 /* eslint-env jest */
 
 const { asyncMapSettled } = require('./')

@xen-orchestra/async-map/legacy.js

@@ -1,3 +1,5 @@
+'use strict'
+
 // type MaybePromise<T> = Promise<T> | T
 //
 // declare export function asyncMap<T1, T2>(

@xen-orchestra/audit-core/.babelrc.js

@@ -1 +1,3 @@
+'use strict'
+
 module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))

@xen-orchestra/audit-core/package.json

@@ -30,7 +30,7 @@
     "rimraf": "^3.0.0"
   },
   "dependencies": {
-    "@vates/decorate-with": "^0.1.0",
+    "@vates/decorate-with": "^1.0.0",
     "@xen-orchestra/log": "^0.3.0",
     "golike-defer": "^0.5.1",
     "object-hash": "^2.0.1"

@xen-orchestra/babel-config/index.js

@@ -46,7 +46,7 @@ module.exports = function (pkg, configs = {}) {
 
   return {
     comments: !__PROD__,
-    ignore: __PROD__ ? [/\.spec\.js$/] : undefined,
+    ignore: __PROD__ ? [/\btests?\//, /\.spec\.js$/] : undefined,
     plugins: Object.keys(plugins)
       .map(plugin => [plugin, plugins[plugin]])
       .sort(([a], [b]) => {

@xen-orchestra/backups-cli/_composeCommands.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const getopts = require('getopts')
 
 const { version } = require('./package.json')

@xen-orchestra/backups-cli/_fs.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { dirname } = require('path')
 
 const fs = require('promise-toolbox/promisifyAll')(require('fs'))

@xen-orchestra/backups-cli/commands/clean-vms.js

@@ -1,5 +1,7 @@
 #!/usr/bin/env node
 
+'use strict'
+
 // -----------------------------------------------------------------------------
 
 const asyncMap = require('lodash/curryRight')(require('@xen-orchestra/async-map').asyncMap)

@xen-orchestra/backups-cli/commands/create-symlink-index.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const filenamify = require('filenamify')
 const get = require('lodash/get')
 const { asyncMap } = require('@xen-orchestra/async-map')

@xen-orchestra/backups-cli/commands/info.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const groupBy = require('lodash/groupBy')
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { createHash } = require('crypto')

@xen-orchestra/backups-cli/index.js

@@ -1,5 +1,7 @@
 #!/usr/bin/env node
 
+'use strict'
+
 require('./_composeCommands')({
   'clean-vms': {
     get main() {

@xen-orchestra/backups-cli/package.json

@@ -7,12 +7,12 @@
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.13.0",
-    "@xen-orchestra/fs": "^0.18.0",
+    "@xen-orchestra/backups": "^0.20.0",
+    "@xen-orchestra/fs": "^0.20.0",
     "filenamify": "^4.1.0",
     "getopts": "^2.2.5",
     "lodash": "^4.17.15",
-    "promise-toolbox": "^0.19.2"
+    "promise-toolbox": "^0.21.0"
   },
   "engines": {
     "node": ">=7.10.1"
@@ -27,7 +27,7 @@
   "scripts": {
     "postversion": "npm publish --access public"
   },
-  "version": "0.6.0",
+  "version": "0.7.0",
   "license": "AGPL-3.0-or-later",
   "author": {
     "name": "Vates SAS",

@xen-orchestra/backups/Backup.js

@@ -1,6 +1,8 @@
+'use strict'
+
 const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
-const Disposable = require('promise-toolbox/Disposable.js')
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const Disposable = require('promise-toolbox/Disposable')
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { compileTemplate } = require('@xen-orchestra/template')
 const { limitConcurrency } = require('limit-concurrency-decorator')
 

@xen-orchestra/backups/DurablePartition.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { asyncMap } = require('@xen-orchestra/async-map')
 
 exports.DurablePartition = class DurablePartition {

@xen-orchestra/backups/ImportVmBackup.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const assert = require('assert')
 
 const { formatFilenameDate } = require('./_filenameDate.js')
@@ -6,9 +8,9 @@ const { Task } = require('./Task.js')
 const { watchStreamSize } = require('./_watchStreamSize.js')
 
 exports.ImportVmBackup = class ImportVmBackup {
-  constructor({ adapter, metadata, srUuid, xapi, settings: { newMacAddresses } = {} }) {
+  constructor({ adapter, metadata, srUuid, xapi, settings: { newMacAddresses, mapVdisSrs } = {} }) {
     this._adapter = adapter
-    this._importDeltaVmSettings = { newMacAddresses }
+    this._importDeltaVmSettings = { newMacAddresses, mapVdisSrs }
     this._metadata = metadata
     this._srUuid = srUuid
     this._xapi = xapi

@xen-orchestra/backups/RemoteAdapter.js

@@ -1,21 +1,26 @@
+'use strict'
+
 const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
-const Disposable = require('promise-toolbox/Disposable.js')
-const fromCallback = require('promise-toolbox/fromCallback.js')
-const fromEvent = require('promise-toolbox/fromEvent.js')
-const pDefer = require('promise-toolbox/defer.js')
-const pump = require('pump')
-const { basename, dirname, join, normalize, resolve } = require('path')
+const Disposable = require('promise-toolbox/Disposable')
+const fromCallback = require('promise-toolbox/fromCallback')
+const fromEvent = require('promise-toolbox/fromEvent')
+const pDefer = require('promise-toolbox/defer')
+const groupBy = require('lodash/groupBy.js')
+const { dirname, join, normalize, resolve } = require('path')
 const { createLogger } = require('@xen-orchestra/log')
-const { createSyntheticStream, mergeVhd, default: Vhd } = require('vhd-lib')
+const { Constants, createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } = require('vhd-lib')
 const { deduped } = require('@vates/disposable/deduped.js')
+const { decorateMethodsWith } = require('@vates/decorate-with')
+const { compose } = require('@vates/compose')
 const { execFile } = require('child_process')
 const { readdir, stat } = require('fs-extra')
+const { v4: uuidv4 } = require('uuid')
 const { ZipFile } = require('yazl')
 
 const { BACKUP_DIR } = require('./_getVmBackupDir.js')
 const { cleanVm } = require('./_cleanVm.js')
 const { getTmpDir } = require('./_getTmpDir.js')
-const { isMetadataFile, isVhdFile } = require('./_backupType.js')
+const { isMetadataFile } = require('./_backupType.js')
 const { isValidXva } = require('./_isValidXva.js')
 const { listPartitions, LVM_PARTITION_TYPE } = require('./_listPartitions.js')
 const { lvs, pvs } = require('./_lvm.js')
@@ -67,58 +72,17 @@ const debounceResourceFactory = factory =>
   }
 
 class RemoteAdapter {
-  constructor(handler, { debounceResource = res => res, dirMode } = {}) {
+  constructor(handler, { debounceResource = res => res, dirMode, vhdDirectoryCompression } = {}) {
     this._debounceResource = debounceResource
     this._dirMode = dirMode
     this._handler = handler
+    this._vhdDirectoryCompression = vhdDirectoryCompression
   }
 
   get handler() {
     return this._handler
   }
 
-  async _deleteVhd(path) {
-    const handler = this._handler
-    const vhds = await asyncMapSettled(
-      await handler.list(dirname(path), {
-        filter: isVhdFile,
-        prependDir: true,
-      }),
-      async path => {
-        try {
-          const vhd = new Vhd(handler, path)
-          await vhd.readHeaderAndFooter()
-          return {
-            footer: vhd.footer,
-            header: vhd.header,
-            path,
-          }
-        } catch (error) {
-          // Do not fail on corrupted VHDs (usually uncleaned temporary files),
-          // they are probably inconsequent to the backup process and should not
-          // fail it.
-          warn(`BackupNg#_deleteVhd ${path}`, { error })
-        }
-      }
-    )
-    const base = basename(path)
-    const child = vhds.find(_ => _ !== undefined && _.header.parentUnicodeName === base)
-    if (child === undefined) {
-      await handler.unlink(path)
-      return 0
-    }
-
-    try {
-      const childPath = child.path
-      const mergedDataSize = await mergeVhd(handler, path, handler, childPath)
-      await handler.rename(path, childPath)
-      return mergedDataSize
-    } catch (error) {
-      handler.unlink(path).catch(warn)
-      throw error
-    }
-  }
-
   async _findPartition(devicePath, partitionId) {
     const partitions = await listPartitions(devicePath)
     const partition = partitions.find(_ => _.id === partitionId)
@@ -128,9 +92,6 @@ class RemoteAdapter {
     return partition
   }
 
-  _getLvmLogicalVolumes = Disposable.factory(this._getLvmLogicalVolumes)
-  _getLvmLogicalVolumes = deduped(this._getLvmLogicalVolumes, (devicePath, pvId, vgName) => [devicePath, pvId, vgName])
-  _getLvmLogicalVolumes = debounceResourceFactory(this._getLvmLogicalVolumes)
   async *_getLvmLogicalVolumes(devicePath, pvId, vgName) {
     yield this._getLvmPhysicalVolume(devicePath, pvId && (await this._findPartition(devicePath, pvId)))
 
@@ -142,9 +103,6 @@ class RemoteAdapter {
     }
   }
 
-  _getLvmPhysicalVolume = Disposable.factory(this._getLvmPhysicalVolume)
-  _getLvmPhysicalVolume = deduped(this._getLvmPhysicalVolume, (devicePath, partition) => [devicePath, partition?.id])
-  _getLvmPhysicalVolume = debounceResourceFactory(this._getLvmPhysicalVolume)
   async *_getLvmPhysicalVolume(devicePath, partition) {
     const args = []
     if (partition !== undefined) {
@@ -165,9 +123,6 @@ class RemoteAdapter {
     }
   }
 
-  _getPartition = Disposable.factory(this._getPartition)
-  _getPartition = deduped(this._getPartition, (devicePath, partition) => [devicePath, partition?.id])
-  _getPartition = debounceResourceFactory(this._getPartition)
   async *_getPartition(devicePath, partition) {
     const options = ['loop', 'ro']
 
@@ -220,7 +175,6 @@ class RemoteAdapter {
     })
   }
 
-  _usePartitionFiles = Disposable.factory(this._usePartitionFiles)
   async *_usePartitionFiles(diskId, partitionId, paths) {
     const path = yield this.getPartition(diskId, partitionId)
 
@@ -232,6 +186,22 @@ class RemoteAdapter {
     return files
   }
 
+  // check if we will be allowed to merge a a vhd created in this adapter
+  // with the vhd at path `path`
+  async isMergeableParent(packedParentUid, path) {
+    return await Disposable.use(openVhd(this.handler, path), vhd => {
+      // this baseUuid is not linked with this vhd
+      if (!vhd.footer.uuid.equals(packedParentUid)) {
+        return false
+      }
+
+      const isVhdDirectory = vhd instanceof VhdDirectory
+      return isVhdDirectory
+        ? this.#useVhdDirectory() && this.#getCompressionType() === vhd.compressionType
+        : !this.#useVhdDirectory()
+    })
+  }
+
   fetchPartitionFiles(diskId, partitionId, paths) {
     const { promise, reject, resolve } = pDefer()
     Disposable.use(
@@ -253,16 +223,9 @@ class RemoteAdapter {
 
   async deleteDeltaVmBackups(backups) {
     const handler = this._handler
-    let mergedDataSize = 0
-    await asyncMapSettled(backups, ({ _filename, vhds }) =>
-      Promise.all([
-        handler.unlink(_filename),
-        asyncMap(Object.values(vhds), async _ => {
-          mergedDataSize += await this._deleteVhd(resolveRelativeFromFile(_filename, _))
-        }),
-      ])
-    )
-    return mergedDataSize
+
+    // this will delete the json, unused VHDs will be detected by `cleanVm`
+    await asyncMapSettled(backups, ({ _filename }) => handler.unlink(_filename))
   }
 
   async deleteMetadataBackup(backupId) {
@@ -292,22 +255,42 @@ class RemoteAdapter {
     )
   }
 
-  async deleteVmBackup(filename) {
-    const metadata = JSON.parse(String(await this._handler.readFile(filename)))
-    metadata._filename = filename
+  deleteVmBackup(file) {
+    return this.deleteVmBackups([file])
+  }
 
-    if (metadata.mode === 'delta') {
-      await this.deleteDeltaVmBackups([metadata])
-    } else if (metadata.mode === 'full') {
-      await this.deleteFullVmBackups([metadata])
-    } else {
-      throw new Error(`no deleter for backup mode ${metadata.mode}`)
+  async deleteVmBackups(files) {
+    const { delta, full, ...others } = groupBy(await asyncMap(files, file => this.readVmBackupMetadata(file)), 'mode')
+
+    const unsupportedModes = Object.keys(others)
+    if (unsupportedModes.length !== 0) {
+      throw new Error('no deleter for backup modes: ' + unsupportedModes.join(', '))
+    }
+
+    await Promise.all([
+      delta !== undefined && this.deleteDeltaVmBackups(delta),
+      full !== undefined && this.deleteFullVmBackups(full),
+    ])
+
+    const dirs = new Set(files.map(file => dirname(file)))
+    for (const dir of dirs) {
+      // don't merge in main process, unused VHDs will be merged in the next backup run
+      await this.cleanVm(dir, { remove: true, onLog: warn })
     }
   }
 
-  getDisk = Disposable.factory(this.getDisk)
-  getDisk = deduped(this.getDisk, diskId => [diskId])
-  getDisk = debounceResourceFactory(this.getDisk)
+  #getCompressionType() {
+    return this._vhdDirectoryCompression
+  }
+
+  #useVhdDirectory() {
+    return this.handler.type === 's3'
+  }
+
+  #useAlias() {
+    return this.#useVhdDirectory()
+  }
+
   async *getDisk(diskId) {
     const handler = this._handler
 
@@ -344,7 +327,6 @@ class RemoteAdapter {
   // - `<partitionId>`: partitioned disk
   // - `<pvId>/<vgName>/<lvName>`: LVM on a partitioned disk
   // - `/<vgName>/lvName>`: LVM on a raw disk
-  getPartition = Disposable.factory(this.getPartition)
   async *getPartition(diskId, partitionId) {
     const devicePath = yield this.getDisk(diskId)
     if (partitionId === undefined) {
@@ -361,13 +343,26 @@ class RemoteAdapter {
     return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId))
   }
 
+  // if we use alias on this remote, we have to name the file alias.vhd
+  getVhdFileName(baseName) {
+    if (this.#useAlias()) {
+      return `${baseName}.alias.vhd`
+    }
+    return `${baseName}.vhd`
+  }
+
   async listAllVmBackups() {
     const handler = this._handler
 
     const backups = { __proto__: null }
-    await asyncMap(await handler.list(BACKUP_DIR), async vmUuid => {
-      const vmBackups = await this.listVmBackups(vmUuid)
-      backups[vmUuid] = vmBackups
+    await asyncMap(await handler.list(BACKUP_DIR), async entry => {
+      // ignore hidden and lock files
+      if (entry[0] !== '.' && !entry.endsWith('.lock')) {
+        const vmBackups = await this.listVmBackups(entry)
+        if (vmBackups.length !== 0) {
+          backups[entry] = vmBackups
+        }
+      }
     })
 
     return backups
@@ -505,6 +500,25 @@ class RemoteAdapter {
     return backups.sort(compareTimestamp)
   }
 
+  async writeVhd(path, input, { checksum = true, validator = noop } = {}) {
+    const handler = this._handler
+
+    if (this.#useVhdDirectory()) {
+      const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
+      await createVhdDirectoryFromStream(handler, dataPath, input, {
+        concurrency: 16,
+        compression: this.#getCompressionType(),
+        async validator() {
+          await input.task
+          return validator.apply(this, arguments)
+        },
+      })
+      await VhdAbstract.createAlias(handler, path, dataPath)
+    } else {
+      await this.outputStream(path, input, { checksum, validator })
+    }
+  }
+
   async outputStream(path, input, { checksum = true, validator = noop } = {}) {
     await this._handler.outputStream(path, input, {
       checksum,
@@ -516,6 +530,52 @@ class RemoteAdapter {
     })
   }
 
+  async _createSyntheticStream(handler, paths) {
+    let disposableVhds = []
+
+    // if it's a path : open all hierarchy of parent
+    if (typeof paths === 'string') {
+      let vhd
+      let vhdPath = paths
+      do {
+        const disposable = await openVhd(handler, vhdPath)
+        vhd = disposable.value
+        disposableVhds.push(disposable)
+        vhdPath = resolveRelativeFromFile(vhdPath, vhd.header.parentUnicodeName)
+      } while (vhd.footer.diskType !== Constants.DISK_TYPES.DYNAMIC)
+    } else {
+      // only open the list of path given
+      disposableVhds = paths.map(path => openVhd(handler, path))
+    }
+
+    // I don't want the vhds to be disposed on return
+    // but only when the stream is done ( or failed )
+    const disposables = await Disposable.all(disposableVhds)
+    const vhds = disposables.value
+
+    let disposed = false
+    const disposeOnce = async () => {
+      if (!disposed) {
+        disposed = true
+
+        try {
+          await disposables.dispose()
+        } catch (error) {
+          warn('_createSyntheticStream: failed to dispose VHDs', { error })
+        }
+      }
+    }
+
+    const synthetic = new VhdSynthetic(vhds)
+    await synthetic.readHeaderAndFooter()
+    await synthetic.readBlockAllocationTable()
+    const stream = await synthetic.stream()
+    stream.on('end', disposeOnce)
+    stream.on('close', disposeOnce)
+    stream.on('error', disposeOnce)
+    return stream
+  }
+
   async readDeltaVmBackup(metadata) {
     const handler = this._handler
     const { vbds, vdis, vhds, vifs, vm } = metadata
@@ -523,7 +583,7 @@ class RemoteAdapter {
 
     const streams = {}
     await asyncMapSettled(Object.keys(vdis), async id => {
-      streams[`${id}.vhd`] = await createSyntheticStream(handler, join(dir, vhds[id]))
+      streams[`${id}.vhd`] = await this._createSyntheticStream(handler, join(dir, vhds[id]))
     })
 
     return {
@@ -556,4 +616,30 @@ Object.assign(RemoteAdapter.prototype, {
   isValidXva,
 })
 
+decorateMethodsWith(RemoteAdapter, {
+  _getLvmLogicalVolumes: compose([
+    Disposable.factory,
+    [deduped, (devicePath, pvId, vgName) => [devicePath, pvId, vgName]],
+    debounceResourceFactory,
+  ]),
+
+  _getLvmPhysicalVolume: compose([
+    Disposable.factory,
+    [deduped, (devicePath, partition) => [devicePath, partition?.id]],
+    debounceResourceFactory,
+  ]),
+
+  _getPartition: compose([
+    Disposable.factory,
+    [deduped, (devicePath, partition) => [devicePath, partition?.id]],
+    debounceResourceFactory,
+  ]),
+
+  _usePartitionFiles: Disposable.factory,
+
+  getDisk: compose([Disposable.factory, [deduped, diskId => [diskId]], debounceResourceFactory]),
+
+  getPartition: Disposable.factory,
+})
+
 exports.RemoteAdapter = RemoteAdapter

@xen-orchestra/backups/RestoreMetadataBackup.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter.js')
 const { PATH_DB_DUMP } = require('./_PoolMetadataBackup.js')
 

@xen-orchestra/backups/Task.js

@@ -1,4 +1,6 @@
-const CancelToken = require('promise-toolbox/CancelToken.js')
+'use strict'
+
+const CancelToken = require('promise-toolbox/CancelToken')
 const Zone = require('node-zone')
 
 const logAfterEnd = () => {
@@ -7,6 +9,8 @@ const logAfterEnd = () => {
 
 const noop = Function.prototype
 
+const serializeErrors = errors => (Array.isArray(errors) ? errors.map(serializeError) : errors)
+
 // Create a serializable object from an error.
 //
 // Otherwise some fields might be non-enumerable and missing from logs.
@@ -15,6 +19,7 @@ const serializeError = error =>
     ? {
         ...error, // Copy enumerable properties.
         code: error.code,
+        errors: serializeErrors(error.errors), // supports AggregateError
         message: error.message,
         name: error.name,
         stack: error.stack,

@xen-orchestra/backups/_PoolMetadataBackup.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { asyncMap } = require('@xen-orchestra/async-map')
 
 const { DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter.js')

@xen-orchestra/backups/_VmBackup.js

@@ -1,10 +1,14 @@
+'use strict'
+
 const assert = require('assert')
 const findLast = require('lodash/findLast.js')
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const groupBy = require('lodash/groupBy.js')
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const keyBy = require('lodash/keyBy.js')
 const mapValues = require('lodash/mapValues.js')
-const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
+const { asyncMap } = require('@xen-orchestra/async-map')
 const { createLogger } = require('@xen-orchestra/log')
+const { decorateMethodsWith } = require('@vates/decorate-with')
 const { defer } = require('golike-defer')
 const { formatDateTime } = require('@xen-orchestra/xapi')
 
@@ -20,6 +24,13 @@ const { watchStreamSize } = require('./_watchStreamSize.js')
 
 const { debug, warn } = createLogger('xo:backups:VmBackup')
 
+class AggregateError extends Error {
+  constructor(errors, message) {
+    super(message)
+    this.errors = errors
+  }
+}
+
 const asyncEach = async (iterable, fn, thisArg = iterable) => {
   for (const item of iterable) {
     await fn.call(thisArg, item)
@@ -33,8 +44,14 @@ const forkDeltaExport = deltaExport =>
     },
   })
 
-exports.VmBackup = class VmBackup {
+class VmBackup {
   constructor({ config, getSnapshotNameLabel, job, remoteAdapters, remotes, schedule, settings, srs, vm }) {
+    if (vm.other_config['xo:backup:job'] === job.id && 'start' in vm.blocked_operations) {
+      // don't match replicated VMs created by this very job otherwise they
+      // will be replicated again and again
+      throw new Error('cannot backup a VM created by this very job')
+    }
+
     this.config = config
     this.job = job
     this.remoteAdapters = remoteAdapters
@@ -118,16 +135,18 @@ exports.VmBackup = class VmBackup {
       return
     }
 
+    const errors = []
     await (parallel ? asyncMap : asyncEach)(writers, async function (writer) {
       try {
         await fn(writer)
       } catch (error) {
+        errors.push(error)
         this.delete(writer)
         warn(warnMessage, { error, writer: writer.constructor.name })
       }
     })
     if (writers.size === 0) {
-      throw new Error('all targets have failed, step: ' + warnMessage)
+      throw new AggregateError(errors, 'all targets have failed, step: ' + warnMessage)
     }
   }
 
@@ -284,17 +303,28 @@ exports.VmBackup = class VmBackup {
   }
 
   async _removeUnusedSnapshots() {
-    // TODO: handle all schedules (no longer existing schedules default to 0 retention)
-
-    const { scheduleId } = this
-    const scheduleSnapshots = this._jobSnapshots.filter(_ => _.other_config['xo:backup:schedule'] === scheduleId)
-
+    const jobSettings = this.job.settings
     const baseVmRef = this._baseVm?.$ref
+    const { config } = this
+    const baseSettings = {
+      ...config.defaultSettings,
+      ...config.metadata.defaultSettings,
+      ...jobSettings[''],
+    }
+
+    const snapshotsPerSchedule = groupBy(this._jobSnapshots, _ => _.other_config['xo:backup:schedule'])
     const xapi = this._xapi
-    await asyncMap(getOldEntries(this._settings.snapshotRetention, scheduleSnapshots), ({ $ref }) => {
-      if ($ref !== baseVmRef) {
-        return xapi.VM_destroy($ref)
+    await asyncMap(Object.entries(snapshotsPerSchedule), ([scheduleId, snapshots]) => {
+      const settings = {
+        ...baseSettings,
+        ...jobSettings[scheduleId],
+        ...jobSettings[this.vm.uuid],
       }
+      return asyncMap(getOldEntries(settings.snapshotRetention, snapshots), ({ $ref }) => {
+        if ($ref !== baseVmRef) {
+          return xapi.VM_destroy($ref)
+        }
+      })
     })
   }
 
@@ -303,12 +333,14 @@ exports.VmBackup = class VmBackup {
 
     let baseVm = findLast(this._jobSnapshots, _ => 'xo:backup:exported' in _.other_config)
     if (baseVm === undefined) {
+      debug('no base VM found')
       return
     }
 
     const fullInterval = this._settings.fullInterval
     const deltaChainLength = +(baseVm.other_config['xo:backup:deltaChainLength'] ?? 0) + 1
     if (!(fullInterval === 0 || fullInterval > deltaChainLength)) {
+      debug('not using base VM becaust fullInterval reached')
       return
     }
 
@@ -319,10 +351,17 @@ exports.VmBackup = class VmBackup {
 
     const baseUuidToSrcVdi = new Map()
     await asyncMap(await baseVm.$getDisks(), async baseRef => {
-      const snapshotOf = await xapi.getField('VDI', baseRef, 'snapshot_of')
+      const [baseUuid, snapshotOf] = await Promise.all([
+        xapi.getField('VDI', baseRef, 'uuid'),
+        xapi.getField('VDI', baseRef, 'snapshot_of'),
+      ])
       const srcVdi = srcVdis[snapshotOf]
       if (srcVdi !== undefined) {
-        baseUuidToSrcVdi.set(await xapi.getField('VDI', baseRef, 'uuid'), srcVdi)
+        baseUuidToSrcVdi.set(baseUuid, srcVdi)
+      } else {
+        debug('ignore snapshot VDI because no longer present on VM', {
+          vdi: baseUuid,
+        })
       }
     })
 
@@ -333,9 +372,23 @@ exports.VmBackup = class VmBackup {
       false
     )
 
+    if (presentBaseVdis.size === 0) {
+      debug('no base VM found')
+      return
+    }
+
     const fullVdisRequired = new Set()
     baseUuidToSrcVdi.forEach((srcVdi, baseUuid) => {
-      if (!presentBaseVdis.has(baseUuid)) {
+      if (presentBaseVdis.has(baseUuid)) {
+        debug('found base VDI', {
+          base: baseUuid,
+          vdi: srcVdi.uuid,
+        })
+      } else {
+        debug('missing base VDI', {
+          base: baseUuid,
+          vdi: srcVdi.uuid,
+        })
         fullVdisRequired.add(srcVdi.uuid)
       }
     })
@@ -344,7 +397,6 @@ exports.VmBackup = class VmBackup {
     this._fullVdisRequired = fullVdisRequired
   }
 
-  run = defer(this.run)
   async run($defer) {
     const settings = this._settings
     assert(
@@ -392,3 +444,8 @@ exports.VmBackup = class VmBackup {
     }
   }
 }
+exports.VmBackup = VmBackup
+
+decorateMethodsWith(VmBackup, {
+  run: defer,
+})

@xen-orchestra/backups/_XoMetadataBackup.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { asyncMap } = require('@xen-orchestra/async-map')
 
 const { DIR_XO_CONFIG_BACKUPS } = require('./RemoteAdapter.js')

@xen-orchestra/backups/_backupType.js

@@ -1,3 +1,5 @@
+'use strict'
+
 exports.isMetadataFile = filename => filename.endsWith('.json')
 exports.isVhdFile = filename => filename.endsWith('.vhd')
 exports.isXvaFile = filename => filename.endsWith('.xva')

@xen-orchestra/backups/_backupWorker.js

@@ -1,11 +1,14 @@
+'use strict'
+
 require('@xen-orchestra/log/configure.js').catchGlobalErrors(
   require('@xen-orchestra/log').createLogger('xo:backups:worker')
 )
 
-const Disposable = require('promise-toolbox/Disposable.js')
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const Disposable = require('promise-toolbox/Disposable')
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { compose } = require('@vates/compose')
 const { createDebounceResource } = require('@vates/disposable/debounceResource.js')
+const { decorateMethodsWith } = require('@vates/decorate-with')
 const { deduped } = require('@vates/disposable/deduped.js')
 const { getHandler } = require('@xen-orchestra/fs')
 const { parseDuration } = require('@vates/parse-duration')
@@ -58,11 +61,6 @@ class BackupWorker {
     }).run()
   }
 
-  getAdapter = Disposable.factory(this.getAdapter)
-  getAdapter = deduped(this.getAdapter, remote => [remote.url])
-  getAdapter = compose(this.getAdapter, function (resource) {
-    return this.debounceResource(resource)
-  })
   async *getAdapter(remote) {
     const handler = getHandler(remote, this.#remoteOptions)
     await handler.sync()
@@ -70,17 +68,13 @@ class BackupWorker {
       yield new RemoteAdapter(handler, {
         debounceResource: this.debounceResource,
         dirMode: this.#config.dirMode,
+        vhdDirectoryCompression: this.#config.vhdDirectoryCompression,
       })
     } finally {
       await handler.forget()
     }
   }
 
-  getXapi = Disposable.factory(this.getXapi)
-  getXapi = deduped(this.getXapi, ({ url }) => [url])
-  getXapi = compose(this.getXapi, function (resource) {
-    return this.debounceResource(resource)
-  })
   async *getXapi({ credentials: { username: user, password }, ...opts }) {
     const xapi = new Xapi({
       ...this.#xapiOptions,
@@ -102,6 +96,30 @@ class BackupWorker {
   }
 }
 
+decorateMethodsWith(BackupWorker, {
+  getAdapter: compose([
+    Disposable.factory,
+    [deduped, remote => [remote.url]],
+    [
+      compose,
+      function (resource) {
+        return this.debounceResource(resource)
+      },
+    ],
+  ]),
+
+  getXapi: compose([
+    Disposable.factory,
+    [deduped, xapi => [xapi.url]],
+    [
+      compose,
+      function (resource) {
+        return this.debounceResource(resource)
+      },
+    ],
+  ]),
+})
+
 // Received message:
 //
 // Message {

@xen-orchestra/backups/_cancelableMap.js

@@ -1,5 +1,7 @@
-const cancelable = require('promise-toolbox/cancelable.js')
-const CancelToken = require('promise-toolbox/CancelToken.js')
+'use strict'
+
+const cancelable = require('promise-toolbox/cancelable')
+const CancelToken = require('promise-toolbox/CancelToken')
 
 // Similar to `Promise.all` + `map` but pass a cancel token to the callback
 //

@xen-orchestra/backups/_cleanVm.integ.spec.js

@@ -0,0 +1,439 @@
+'use strict'
+
+/* eslint-env jest */
+
+const rimraf = require('rimraf')
+const tmp = require('tmp')
+const fs = require('fs-extra')
+const { getHandler } = require('@xen-orchestra/fs')
+const { pFromCallback } = require('promise-toolbox')
+const crypto = require('crypto')
+const { RemoteAdapter } = require('./RemoteAdapter')
+const { VHDFOOTER, VHDHEADER } = require('./tests.fixtures.js')
+const { VhdFile, Constants, VhdDirectory, VhdAbstract } = require('vhd-lib')
+const { checkAliases } = require('./_cleanVm')
+const { dirname, basename } = require('path')
+
+let tempDir, adapter, handler, jobId, vdiId, basePath
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+  handler = getHandler({ url: `file://${tempDir}` })
+  await handler.sync()
+  adapter = new RemoteAdapter(handler)
+  jobId = uniqueId()
+  vdiId = uniqueId()
+  basePath = `vdis/${jobId}/${vdiId}`
+  await fs.mkdirp(`${tempDir}/${basePath}`)
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+  await handler.forget()
+})
+
+const uniqueId = () => crypto.randomBytes(16).toString('hex')
+
+async function generateVhd(path, opts = {}) {
+  let vhd
+
+  let dataPath = path
+  if (opts.useAlias) {
+    await handler.mkdir(dirname(path) + '/data/')
+    dataPath = dirname(path) + '/data/' + basename(path)
+  }
+  if (opts.mode === 'directory') {
+    await handler.mkdir(dataPath)
+    vhd = new VhdDirectory(handler, dataPath)
+  } else {
+    const fd = await handler.openFile(dataPath, 'wx')
+    vhd = new VhdFile(handler, fd)
+  }
+
+  vhd.header = { ...VHDHEADER, ...opts.header }
+  vhd.footer = { ...VHDFOOTER, ...opts.footer }
+  vhd.footer.uuid = Buffer.from(crypto.randomBytes(16))
+
+  if (vhd.header.parentUnicodeName) {
+    vhd.footer.diskType = Constants.DISK_TYPES.DIFFERENCING
+  } else {
+    vhd.footer.diskType = Constants.DISK_TYPES.DYNAMIC
+  }
+
+  if (opts.useAlias === true) {
+    await VhdAbstract.createAlias(handler, path + '.alias.vhd', dataPath)
+  }
+
+  await vhd.writeBlockAllocationTable()
+  await vhd.writeHeader()
+  await vhd.writeFooter()
+  return vhd
+}
+
+test('It remove broken vhd', async () => {
+  // todo also tests a directory and an alias
+
+  await handler.writeFile(`${basePath}/notReallyAVhd.vhd`, 'I AM NOT A VHD')
+  expect((await handler.list(basePath)).length).toEqual(1)
+  let loggued = ''
+  const onLog = message => {
+    loggued += message
+  }
+  await adapter.cleanVm('/', { remove: false, onLog })
+  expect(loggued).toEqual(`error while checking the VHD with path /${basePath}/notReallyAVhd.vhd`)
+  // not removed
+  expect((await handler.list(basePath)).length).toEqual(1)
+  // really remove it
+  await adapter.cleanVm('/', { remove: true, onLog })
+  expect((await handler.list(basePath)).length).toEqual(0)
+})
+
+test('it remove vhd with missing or multiple ancestors', async () => {
+  // one with a broken parent
+  await generateVhd(`${basePath}/abandonned.vhd`, {
+    header: {
+      parentUnicodeName: 'gone.vhd',
+      parentUid: Buffer.from(crypto.randomBytes(16)),
+    },
+  })
+
+  // one orphan, which is a full vhd, no parent
+  const orphan = await generateVhd(`${basePath}/orphan.vhd`)
+  // a child to the orphan
+  await generateVhd(`${basePath}/child.vhd`, {
+    header: {
+      parentUnicodeName: 'orphan.vhd',
+      parentUid: orphan.footer.uuid,
+    },
+  })
+
+  // clean
+  let loggued = ''
+  const onLog = message => {
+    loggued += message + '\n'
+  }
+  await adapter.cleanVm('/', { remove: true, onLog })
+
+  const deletedOrphanVhd = loggued.match(/deleting orphan VHD/g) || []
+  expect(deletedOrphanVhd.length).toEqual(1) // only one vhd should have been deleted
+  const deletedAbandonnedVhd = loggued.match(/abandonned.vhd is missing/g) || []
+  expect(deletedAbandonnedVhd.length).toEqual(1) // and it must be abandonned.vhd
+
+  // we don't test the filew on disk, since they will all be marker as unused and deleted without a metadata.json file
+})
+
+test('it remove backup meta data referencing a missing vhd in delta backup', async () => {
+  // create a metadata file marking child and orphan as ok
+  await handler.writeFile(
+    `metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      vhds: [
+        `${basePath}/orphan.vhd`,
+        `${basePath}/child.vhd`,
+        // abandonned.json is not here
+      ],
+    })
+  )
+
+  await generateVhd(`${basePath}/abandonned.vhd`)
+
+  // one orphan, which is a full vhd, no parent
+  const orphan = await generateVhd(`${basePath}/orphan.vhd`)
+
+  // a child to the orphan
+  await generateVhd(`${basePath}/child.vhd`, {
+    header: {
+      parentUnicodeName: 'orphan.vhd',
+      parentUid: orphan.footer.uuid,
+    },
+  })
+
+  let loggued = ''
+  const onLog = message => {
+    loggued += message + '\n'
+  }
+  await adapter.cleanVm('/', { remove: true, onLog })
+  let matched = loggued.match(/deleting unused VHD /g) || []
+  expect(matched.length).toEqual(1) // only one vhd should have been deleted
+  matched = loggued.match(/abandonned.vhd is unused/g) || []
+  expect(matched.length).toEqual(1) // and it must be abandonned.vhd
+
+  // a missing vhd cause clean to remove all vhds
+  await handler.writeFile(
+    `metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      vhds: [
+        `${basePath}/deleted.vhd`, // in metadata but not in vhds
+        `${basePath}/orphan.vhd`,
+        `${basePath}/child.vhd`,
+        // abandonned.vhd is not here anymore
+      ],
+    }),
+    { flags: 'w' }
+  )
+  loggued = ''
+  await adapter.cleanVm('/', { remove: true, onLog })
+  matched = loggued.match(/deleting unused VHD /g) || []
+  expect(matched.length).toEqual(2) // all vhds (orphan and  child  ) should have been deleted
+})
+
+test('it merges delta of non destroyed chain', async () => {
+  await handler.writeFile(
+    `metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      size: 12000, // a size too small
+      vhds: [
+        `${basePath}/grandchild.vhd`, // grand child should not be merged
+        `${basePath}/child.vhd`,
+        // orphan is not here, he should be merged in child
+      ],
+    })
+  )
+
+  // one orphan, which is a full vhd, no parent
+  const orphan = await generateVhd(`${basePath}/orphan.vhd`)
+  // a child to the orphan
+  const child = await generateVhd(`${basePath}/child.vhd`, {
+    header: {
+      parentUnicodeName: 'orphan.vhd',
+      parentUid: orphan.footer.uuid,
+    },
+  })
+  // a grand child
+  await generateVhd(`${basePath}/grandchild.vhd`, {
+    header: {
+      parentUnicodeName: 'child.vhd',
+      parentUid: child.footer.uuid,
+    },
+  })
+
+  let loggued = []
+  const onLog = message => {
+    loggued.push(message)
+  }
+  await adapter.cleanVm('/', { remove: true, onLog })
+  expect(loggued[0]).toEqual(`the parent /${basePath}/orphan.vhd of the child /${basePath}/child.vhd is unused`)
+  expect(loggued[1]).toEqual(`incorrect size in metadata: 12000 instead of 209920`)
+
+  loggued = []
+  await adapter.cleanVm('/', { remove: true, merge: true, onLog })
+  const [unused, merging] = loggued
+  expect(unused).toEqual(`the parent /${basePath}/orphan.vhd of the child /${basePath}/child.vhd is unused`)
+  expect(merging).toEqual(`merging /${basePath}/child.vhd into /${basePath}/orphan.vhd`)
+
+  const metadata = JSON.parse(await handler.readFile(`metadata.json`))
+  // size should be the size of children + grand children after the merge
+  expect(metadata.size).toEqual(209920)
+
+  // merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
+  // only check deletion
+  const remainingVhds = await handler.list(basePath)
+  expect(remainingVhds.length).toEqual(2)
+  expect(remainingVhds.includes('child.vhd')).toEqual(true)
+  expect(remainingVhds.includes('grandchild.vhd')).toEqual(true)
+})
+
+test('it finish unterminated merge ', async () => {
+  await handler.writeFile(
+    `metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      size: 209920,
+      vhds: [`${basePath}/orphan.vhd`, `${basePath}/child.vhd`],
+    })
+  )
+
+  // one orphan, which is a full vhd, no parent
+  const orphan = await generateVhd(`${basePath}/orphan.vhd`)
+  // a child to the orphan
+  const child = await generateVhd(`${basePath}/child.vhd`, {
+    header: {
+      parentUnicodeName: 'orphan.vhd',
+      parentUid: orphan.footer.uuid,
+    },
+  })
+  // a merge in progress file
+  await handler.writeFile(
+    `${basePath}/.orphan.vhd.merge.json`,
+    JSON.stringify({
+      parent: {
+        header: orphan.header.checksum,
+      },
+      child: {
+        header: child.header.checksum,
+      },
+    })
+  )
+
+  await adapter.cleanVm('/', { remove: true, merge: true })
+  // merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
+
+  // only check deletion
+  const remainingVhds = await handler.list(basePath)
+  expect(remainingVhds.length).toEqual(1)
+  expect(remainingVhds.includes('child.vhd')).toEqual(true)
+})
+
+// each of the vhd can be a file, a directory, an alias to a file or an alias to a directory
+// the message an resulting files should be identical to the output with vhd files which is tested independantly
+
+describe('tests multiple combination ', () => {
+  for (const useAlias of [true, false]) {
+    for (const vhdMode of ['file', 'directory']) {
+      test(`alias : ${useAlias}, mode: ${vhdMode}`, async () => {
+        // a broken VHD
+        if (useAlias) {
+          await handler.mkdir(basePath + '/data')
+        }
+
+        const brokenVhdDataPath = basePath + (useAlias ? '/data/broken.vhd' : '/broken.vhd')
+
+        if (vhdMode === 'directory') {
+          await handler.mkdir(brokenVhdDataPath)
+        } else {
+          await handler.writeFile(brokenVhdDataPath, 'notreallyavhd')
+        }
+        if (useAlias) {
+          await VhdAbstract.createAlias(handler, 'broken.alias.vhd', brokenVhdDataPath)
+        }
+
+        // a vhd non referenced in metada
+        await generateVhd(`${basePath}/nonreference.vhd`, { useAlias, mode: vhdMode })
+        // an abandonded delta vhd without its parent
+        await generateVhd(`${basePath}/abandonned.vhd`, {
+          useAlias,
+          mode: vhdMode,
+          header: {
+            parentUnicodeName: 'gone.vhd',
+            parentUid: crypto.randomBytes(16),
+          },
+        })
+
+        // an ancestor of a vhd present in metadata
+        const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, {
+          useAlias,
+          mode: vhdMode,
+        })
+        const child = await generateVhd(`${basePath}/child.vhd`, {
+          useAlias,
+          mode: vhdMode,
+          header: {
+            parentUnicodeName: 'ancestor.vhd' + (useAlias ? '.alias.vhd' : ''),
+            parentUid: ancestor.footer.uuid,
+          },
+        })
+        // a grand child  vhd in metadata
+        await generateVhd(`${basePath}/grandchild.vhd`, {
+          useAlias,
+          mode: vhdMode,
+          header: {
+            parentUnicodeName: 'child.vhd' + (useAlias ? '.alias.vhd' : ''),
+            parentUid: child.footer.uuid,
+          },
+        })
+
+        // an older parent that was merging in clean
+        const cleanAncestor = await generateVhd(`${basePath}/cleanAncestor.vhd`, {
+          useAlias,
+          mode: vhdMode,
+        })
+        // a clean  vhd in metadata
+        const clean = await generateVhd(`${basePath}/clean.vhd`, {
+          useAlias,
+          mode: vhdMode,
+          header: {
+            parentUnicodeName: 'cleanAncestor.vhd' + (useAlias ? '.alias.vhd' : ''),
+            parentUid: cleanAncestor.footer.uuid,
+          },
+        })
+
+        await handler.writeFile(
+          `${basePath}/.cleanAncestor.vhd${useAlias ? '.alias.vhd' : ''}.merge.json`,
+          JSON.stringify({
+            parent: {
+              header: cleanAncestor.header.checksum,
+            },
+            child: {
+              header: clean.header.checksum,
+            },
+          })
+        )
+
+        // the metadata file
+        await handler.writeFile(
+          `metadata.json`,
+          JSON.stringify({
+            mode: 'delta',
+            vhds: [
+              `${basePath}/grandchild.vhd` + (useAlias ? '.alias.vhd' : ''), // grand child should not be merged
+              `${basePath}/child.vhd` + (useAlias ? '.alias.vhd' : ''),
+              `${basePath}/clean.vhd` + (useAlias ? '.alias.vhd' : ''),
+            ],
+          })
+        )
+
+        await adapter.cleanVm('/', { remove: true, merge: true })
+
+        const metadata = JSON.parse(await handler.readFile(`metadata.json`))
+        // size should be the size of children + grand children + clean after the merge
+        expect(metadata.size).toEqual(vhdMode === 'file' ? 314880 : undefined)
+
+        // broken vhd, non referenced, abandonned should be deleted ( alias and data)
+        // ancestor and child should be merged
+        // grand child and clean vhd should not have changed
+        const survivors = await handler.list(basePath)
+        // console.log(survivors)
+        if (useAlias) {
+          const dataSurvivors = await handler.list(basePath + '/data')
+          // the goal of the alias : do not move a full folder
+          expect(dataSurvivors).toContain('ancestor.vhd')
+          expect(dataSurvivors).toContain('grandchild.vhd')
+          expect(dataSurvivors).toContain('cleanAncestor.vhd')
+          expect(survivors).toContain('clean.vhd.alias.vhd')
+          expect(survivors).toContain('child.vhd.alias.vhd')
+          expect(survivors).toContain('grandchild.vhd.alias.vhd')
+          expect(survivors.length).toEqual(4) // the 3 ok + data
+          expect(dataSurvivors.length).toEqual(3) // the 3 ok + data
+        } else {
+          expect(survivors).toContain('clean.vhd')
+          expect(survivors).toContain('child.vhd')
+          expect(survivors).toContain('grandchild.vhd')
+          expect(survivors.length).toEqual(3)
+        }
+      })
+    }
+  }
+})
+
+test('it cleans orphan merge states ', async () => {
+  await handler.writeFile(`${basePath}/.orphan.vhd.merge.json`, '')
+
+  await adapter.cleanVm('/', { remove: true })
+
+  expect(await handler.list(basePath)).toEqual([])
+})
+
+test('check Aliases should work alone', async () => {
+  await handler.mkdir('vhds')
+  await handler.mkdir('vhds/data')
+  await generateVhd(`vhds/data/ok.vhd`)
+  await VhdAbstract.createAlias(handler, 'vhds/ok.alias.vhd', 'vhds/data/ok.vhd')
+
+  await VhdAbstract.createAlias(handler, 'vhds/missingData.alias.vhd', 'vhds/data/nonexistent.vhd')
+
+  await generateVhd(`vhds/data/missingalias.vhd`)
+
+  await checkAliases(['vhds/missingData.alias.vhd', 'vhds/ok.alias.vhd'], 'vhds/data', { remove: true, handler })
+
+  // only ok have suvived
+  const alias = (await handler.list('vhds')).filter(f => f.endsWith('.vhd'))
+  expect(alias.length).toEqual(1)
+
+  const data = await handler.list('vhds/data')
+  expect(data.length).toEqual(1)
+})

@xen-orchestra/backups/_cleanVm.js

@@ -1,17 +1,41 @@
+'use strict'
+
 const assert = require('assert')
 const sum = require('lodash/sum')
 const { asyncMap } = require('@xen-orchestra/async-map')
-const { default: Vhd, mergeVhd } = require('vhd-lib')
+const { Constants, mergeVhd, openVhd, VhdAbstract, VhdFile } = require('vhd-lib')
+const { isVhdAlias, resolveVhdAlias } = require('vhd-lib/aliases')
 const { dirname, resolve } = require('path')
-const { DISK_TYPE_DIFFERENCING } = require('vhd-lib/dist/_constants.js')
+const { DISK_TYPES } = Constants
 const { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } = require('./_backupType.js')
 const { limitConcurrency } = require('limit-concurrency-decorator')
 
+const { Task } = require('./Task.js')
+const { Disposable } = require('promise-toolbox')
+
+// checking the size of a vhd directory is costly
+// 1 Http Query per 1000 blocks
+// we only check size of all the vhd are VhdFiles
+function shouldComputeVhdsSize(vhds) {
+  return vhds.every(vhd => vhd instanceof VhdFile)
+}
+
+const computeVhdsSize = (handler, vhdPaths) =>
+  Disposable.use(
+    vhdPaths.map(vhdPath => openVhd(handler, vhdPath)),
+    async vhds => {
+      if (shouldComputeVhdsSize(vhds)) {
+        const sizes = await asyncMap(vhds, vhd => vhd.getSize())
+        return sum(sizes)
+      }
+    }
+  )
+
 // chain is an array of VHDs from child to parent
 //
 // the whole chain will be merged into parent, parent will be renamed to child
 // and all the others will deleted
-const mergeVhdChain = limitConcurrency(1)(async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
+async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
   assert(chain.length >= 2)
 
   let child = chain[0]
@@ -44,7 +68,7 @@ const mergeVhdChain = limitConcurrency(1)(async function mergeVhdChain(chain, {
       }
     }, 10e3)
 
-    await mergeVhd(
+    const mergedSize = await mergeVhd(
       handler,
       parent,
       handler,
@@ -61,26 +85,28 @@ const mergeVhdChain = limitConcurrency(1)(async function mergeVhdChain(chain, {
     )
 
     clearInterval(handle)
-
     await Promise.all([
-      handler.rename(parent, child),
+      VhdAbstract.rename(handler, parent, child),
       asyncMap(children.slice(0, -1), child => {
         onLog(`the VHD ${child} is unused`)
         if (remove) {
           onLog(`deleting unused VHD ${child}`)
-          return handler.unlink(child)
+          return VhdAbstract.unlink(handler, child)
         }
       }),
     ])
+
+    return mergedSize
   }
-})
+}
 
 const noop = Function.prototype
 
-const INTERRUPTED_VHDS_REG = /^(?:(.+)\/)?\.(.+)\.merge.json$/
+const INTERRUPTED_VHDS_REG = /^\.(.+)\.merge.json$/
 const listVhds = async (handler, vmDir) => {
-  const vhds = []
-  const interruptedVhds = new Set()
+  const vhds = new Set()
+  const aliases = {}
+  const interruptedVhds = new Map()
 
   await asyncMap(
     await handler.list(`${vmDir}/vdis`, {
@@ -95,84 +121,166 @@ const listVhds = async (handler, vmDir) => {
         async vdiDir => {
           const list = await handler.list(vdiDir, {
             filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file),
-            prependDir: true,
           })
-
+          aliases[vdiDir] = list.filter(vhd => isVhdAlias(vhd)).map(file => `${vdiDir}/${file}`)
           list.forEach(file => {
             const res = INTERRUPTED_VHDS_REG.exec(file)
             if (res === null) {
-              vhds.push(file)
+              vhds.add(`${vdiDir}/${file}`)
             } else {
-              const [, dir, file] = res
-              interruptedVhds.add(`${dir}/${file}`)
+              interruptedVhds.set(`${vdiDir}/${res[1]}`, `${vdiDir}/${file}`)
             }
           })
         }
       )
   )
 
-  return { vhds, interruptedVhds }
+  return { vhds, interruptedVhds, aliases }
 }
 
-exports.cleanVm = async function cleanVm(vmDir, { fixMetadata, remove, merge, onLog = noop }) {
+async function checkAliases(aliasPaths, targetDataRepository, { handler, onLog = noop, remove = false }) {
+  const aliasFound = []
+  for (const path of aliasPaths) {
+    const target = await resolveVhdAlias(handler, path)
+
+    if (!isVhdFile(target)) {
+      onLog(`Alias ${path} references a non vhd target:  ${target}`)
+      if (remove) {
+        await handler.unlink(target)
+        await handler.unlink(path)
+      }
+      continue
+    }
+
+    try {
+      const { dispose } = await openVhd(handler, target)
+      try {
+        await dispose()
+      } catch (e) {
+        // error during dispose should not trigger a deletion
+      }
+    } catch (error) {
+      onLog(`target ${target} of alias ${path} is missing or broken`, { error })
+      if (remove) {
+        try {
+          await VhdAbstract.unlink(handler, path)
+        } catch (e) {
+          if (e.code !== 'ENOENT') {
+            onLog(`Error while deleting target ${target} of alias ${path}`, { error: e })
+          }
+        }
+      }
+      continue
+    }
+
+    aliasFound.push(resolve('/', target))
+  }
+
+  const entries = await handler.list(targetDataRepository, {
+    ignoreMissing: true,
+    prependDir: true,
+  })
+
+  entries.forEach(async entry => {
+    if (!aliasFound.includes(entry)) {
+      onLog(`the Vhd  ${entry} is not referenced by a an alias`)
+      if (remove) {
+        await VhdAbstract.unlink(handler, entry)
+      }
+    }
+  })
+}
+exports.checkAliases = checkAliases
+
+const defaultMergeLimiter = limitConcurrency(1)
+
+exports.cleanVm = async function cleanVm(
+  vmDir,
+  { fixMetadata, remove, merge, mergeLimiter = defaultMergeLimiter, onLog = noop }
+) {
+  const limitedMergeVhdChain = mergeLimiter(mergeVhdChain)
+
   const handler = this._handler
 
-  const vhds = new Set()
+  const vhdsToJSons = new Set()
   const vhdParents = { __proto__: null }
   const vhdChildren = { __proto__: null }
 
-  const vhdsList = await listVhds(handler, vmDir)
+  const { vhds, interruptedVhds, aliases } = await listVhds(handler, vmDir)
 
   // remove broken VHDs
-  await asyncMap(vhdsList.vhds, async path => {
+  await asyncMap(vhds, async path => {
     try {
-      const vhd = new Vhd(handler, path)
-      await vhd.readHeaderAndFooter(!vhdsList.interruptedVhds.has(path))
-      vhds.add(path)
-      if (vhd.footer.diskType === DISK_TYPE_DIFFERENCING) {
-        const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
-        vhdParents[path] = parent
-        if (parent in vhdChildren) {
-          const error = new Error('this script does not support multiple VHD children')
-          error.parent = parent
-          error.child1 = vhdChildren[parent]
-          error.child2 = path
-          throw error // should we throw?
+      await Disposable.use(openVhd(handler, path, { checkSecondFooter: !interruptedVhds.has(path) }), vhd => {
+        if (vhd.footer.diskType === DISK_TYPES.DIFFERENCING) {
+          const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
+          vhdParents[path] = parent
+          if (parent in vhdChildren) {
+            const error = new Error('this script does not support multiple VHD children')
+            error.parent = parent
+            error.child1 = vhdChildren[parent]
+            error.child2 = path
+            throw error // should we throw?
+          }
+          vhdChildren[parent] = path
         }
-        vhdChildren[parent] = path
-      }
+      })
     } catch (error) {
+      vhds.delete(path)
       onLog(`error while checking the VHD with path ${path}`, { error })
       if (error?.code === 'ERR_ASSERTION' && remove) {
         onLog(`deleting broken ${path}`)
-        await handler.unlink(path)
+        return VhdAbstract.unlink(handler, path)
       }
     }
   })
 
+  // remove interrupted merge states for missing VHDs
+  for (const interruptedVhd of interruptedVhds.keys()) {
+    if (!vhds.has(interruptedVhd)) {
+      const statePath = interruptedVhds.get(interruptedVhd)
+      interruptedVhds.delete(interruptedVhd)
+
+      onLog('orphan merge state', {
+        mergeStatePath: statePath,
+        missingVhdPath: interruptedVhd,
+      })
+      if (remove) {
+        onLog(`deleting orphan merge state ${statePath}`)
+        await handler.unlink(statePath)
+      }
+    }
+  }
+
+  // check if alias are correct
+  // check if all vhd in data subfolder have a corresponding alias
+  await asyncMap(Object.keys(aliases), async dir => {
+    await checkAliases(aliases[dir], `${dir}/data`, { handler, onLog, remove })
+  })
+
   // remove VHDs with missing ancestors
   {
     const deletions = []
 
     // return true if the VHD has been deleted or is missing
-    const deleteIfOrphan = vhd => {
-      const parent = vhdParents[vhd]
+    const deleteIfOrphan = vhdPath => {
+      const parent = vhdParents[vhdPath]
       if (parent === undefined) {
         return
       }
 
       // no longer needs to be checked
-      delete vhdParents[vhd]
+      delete vhdParents[vhdPath]
 
       deleteIfOrphan(parent)
 
       if (!vhds.has(parent)) {
-        vhds.delete(vhd)
+        vhds.delete(vhdPath)
 
-        onLog(`the parent ${parent} of the VHD ${vhd} is missing`)
+        onLog(`the parent ${parent} of the VHD ${vhdPath} is missing`)
         if (remove) {
-          onLog(`deleting orphan VHD ${vhd}`)
-          deletions.push(handler.unlink(vhd))
+          onLog(`deleting orphan VHD ${vhdPath}`)
+          deletions.push(VhdAbstract.unlink(handler, vhdPath))
         }
       }
     }
@@ -188,7 +296,7 @@ exports.cleanVm = async function cleanVm(vmDir, { fixMetadata, remove, merge, on
     await Promise.all(deletions)
   }
 
-  const jsons = []
+  const jsons = new Set()
   const xvas = new Set()
   const xvaSums = []
   const entries = await handler.list(vmDir, {
@@ -196,7 +304,7 @@ exports.cleanVm = async function cleanVm(vmDir, { fixMetadata, remove, merge, on
   })
   entries.forEach(path => {
     if (isMetadataFile(path)) {
-      jsons.push(path)
+      jsons.add(path)
     } else if (isXvaFile(path)) {
       xvas.add(path)
     } else if (isXvaSumFile(path)) {
@@ -218,22 +326,25 @@ exports.cleanVm = async function cleanVm(vmDir, { fixMetadata, remove, merge, on
   // compile the list of unused XVAs and VHDs, and remove backup metadata which
   // reference a missing XVA/VHD
   await asyncMap(jsons, async json => {
-    const metadata = JSON.parse(await handler.readFile(json))
+    let metadata
+    try {
+      metadata = JSON.parse(await handler.readFile(json))
+    } catch (error) {
+      onLog(`failed to read metadata file ${json}`, { error })
+      jsons.delete(json)
+      return
+    }
+
     const { mode } = metadata
-    let size
     if (mode === 'full') {
       const linkedXva = resolve('/', vmDir, metadata.xva)
-
       if (xvas.has(linkedXva)) {
         unusedXvas.delete(linkedXva)
-
-        size = await handler.getSize(linkedXva).catch(error => {
-          onLog(`failed to get size of ${json}`, { error })
-        })
       } else {
         onLog(`the XVA linked to the metadata ${json} is missing`)
         if (remove) {
           onLog(`deleting incomplete backup ${json}`)
+          jsons.delete(json)
           await handler.unlink(json)
         }
       }
@@ -243,42 +354,29 @@ exports.cleanVm = async function cleanVm(vmDir, { fixMetadata, remove, merge, on
         return Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
       })()
 
+      const missingVhds = linkedVhds.filter(_ => !vhds.has(_))
+
       // FIXME: find better approach by keeping as much of the backup as
       // possible (existing disks) even if one disk is missing
-      if (linkedVhds.every(_ => vhds.has(_))) {
+      if (missingVhds.length === 0) {
         linkedVhds.forEach(_ => unusedVhds.delete(_))
-
-        size = await asyncMap(linkedVhds, vhd => handler.getSize(vhd)).then(sum, error => {
-          onLog(`failed to get size of ${json}`, { error })
+        linkedVhds.forEach(path => {
+          vhdsToJSons[path] = json
         })
       } else {
-        onLog(`Some VHDs linked to the metadata ${json} are missing`)
+        onLog(`Some VHDs linked to the metadata ${json} are missing`, { missingVhds })
         if (remove) {
           onLog(`deleting incomplete backup ${json}`)
+          jsons.delete(json)
           await handler.unlink(json)
         }
       }
     }
-
-    const metadataSize = metadata.size
-    if (size !== undefined && metadataSize !== size) {
-      onLog(`incorrect size in metadata: ${metadataSize ?? 'none'} instead of ${size}`)
-
-      // don't update if the the stored size is greater than found files,
-      // it can indicates a problem
-      if (fixMetadata && (metadataSize === undefined || metadataSize < size)) {
-        try {
-          metadata.size = size
-          await handler.writeFile(json, JSON.stringify(metadata), { flags: 'w' })
-        } catch (error) {
-          onLog(`failed to update size in backup metadata ${json}`, { error })
-        }
-      }
-    }
   })
 
   // TODO: parallelize by vm/job/vdi
   const unusedVhdsDeletion = []
+  const toMerge = []
   {
     // VHD chains (as list from child to ancestor) to merge indexed by last
     // ancestor
@@ -312,7 +410,7 @@ exports.cleanVm = async function cleanVm(vmDir, { fixMetadata, remove, merge, on
       onLog(`the VHD ${vhd} is unused`)
       if (remove) {
         onLog(`deleting unused VHD ${vhd}`)
-        unusedVhdsDeletion.push(handler.unlink(vhd))
+        unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))
       }
     }
 
@@ -321,22 +419,31 @@ exports.cleanVm = async function cleanVm(vmDir, { fixMetadata, remove, merge, on
     })
 
     // merge interrupted VHDs
-    if (merge) {
-      vhdsList.interruptedVhds.forEach(parent => {
-        vhdChainsToMerge[parent] = [vhdChildren[parent], parent]
-      })
+    for (const parent of interruptedVhds.keys()) {
+      vhdChainsToMerge[parent] = [vhdChildren[parent], parent]
     }
 
-    Object.keys(vhdChainsToMerge).forEach(key => {
-      const chain = vhdChainsToMerge[key]
+    Object.values(vhdChainsToMerge).forEach(chain => {
       if (chain !== undefined) {
-        unusedVhdsDeletion.push(mergeVhdChain(chain, { handler, onLog, remove, merge }))
+        toMerge.push(chain)
+      }
+    })
+  }
+
+  const metadataWithMergedVhd = {}
+  const doMerge = async () => {
+    await asyncMap(toMerge, async chain => {
+      const merged = await limitedMergeVhdChain(chain, { handler, onLog, remove, merge })
+      if (merged !== undefined) {
+        const metadataPath = vhdsToJSons[chain[0]] // all the chain should have the same metada file
+        metadataWithMergedVhd[metadataPath] = true
       }
     })
   }
 
   await Promise.all([
     ...unusedVhdsDeletion,
+    toMerge.length !== 0 && (merge ? Task.run({ name: 'merge' }, doMerge) : doMerge()),
     asyncMap(unusedXvas, path => {
       onLog(`the XVA ${path} is unused`)
       if (remove) {
@@ -355,4 +462,55 @@ exports.cleanVm = async function cleanVm(vmDir, { fixMetadata, remove, merge, on
       }
     }),
   ])
+
+  // update size for delta metadata with merged VHD
+  // check for the other that the size is the same as the real file size
+
+  await asyncMap(jsons, async metadataPath => {
+    const metadata = JSON.parse(await handler.readFile(metadataPath))
+
+    let fileSystemSize
+    const merged = metadataWithMergedVhd[metadataPath] !== undefined
+
+    const { mode, size, vhds, xva } = metadata
+
+    try {
+      if (mode === 'full') {
+        // a full backup : check size
+        const linkedXva = resolve('/', vmDir, xva)
+        fileSystemSize = await handler.getSize(linkedXva)
+      } else if (mode === 'delta') {
+        const linkedVhds = Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
+        fileSystemSize = await computeVhdsSize(handler, linkedVhds)
+
+        // the size is not computed in some cases (e.g. VhdDirectory)
+        if (fileSystemSize === undefined) {
+          return
+        }
+
+        // don't warn if the size has changed after a merge
+        if (!merged && fileSystemSize !== size) {
+          onLog(`incorrect size in metadata: ${size ?? 'none'} instead of ${fileSystemSize}`)
+        }
+      }
+    } catch (error) {
+      onLog(`failed to get size of ${metadataPath}`, { error })
+      return
+    }
+
+    // systematically update size after a merge
+    if ((merged || fixMetadata) && size !== fileSystemSize) {
+      metadata.size = fileSystemSize
+      try {
+        await handler.writeFile(metadataPath, JSON.stringify(metadata), { flags: 'w' })
+      } catch (error) {
+        onLog(`failed to update size in backup metadata ${metadataPath} after merge`, { error })
+      }
+    }
+  })
+
+  return {
+    // boolean whether some VHDs were merged (or should be merged)
+    merge: toMerge.length !== 0,
+  }
 }

@xen-orchestra/backups/_deltaVm.js

@@ -1,7 +1,9 @@
+'use strict'
+
 const compareVersions = require('compare-versions')
 const find = require('lodash/find.js')
 const groupBy = require('lodash/groupBy.js')
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const omit = require('lodash/omit.js')
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { CancelToken } = require('promise-toolbox')
@@ -17,6 +19,14 @@ const TAG_COPY_SRC = 'xo:copy_of'
 exports.TAG_COPY_SRC = TAG_COPY_SRC
 
 const ensureArray = value => (value === undefined ? [] : Array.isArray(value) ? value : [value])
+const resolveUuid = async (xapi, cache, uuid, type) => {
+  let ref = cache.get(uuid)
+  if (ref === undefined) {
+    ref = await xapi.call(`${type}.get_by_uuid`, uuid)
+    cache.set(uuid, ref)
+  }
+  return ref
+}
 
 exports.exportDeltaVm = async function exportDeltaVm(
   vm,
@@ -165,6 +175,12 @@ exports.importDeltaVm = defer(async function importDeltaVm(
     }
   }
 
+  const cache = new Map()
+  const mapVdisSrRefs = {}
+  for (const [vdiUuid, srUuid] of Object.entries(mapVdisSrs)) {
+    mapVdisSrRefs[vdiUuid] = await resolveUuid(xapi, cache, srUuid, 'SR')
+  }
+
   const baseVdis = {}
   baseVm &&
     baseVm.$VBDs.forEach(vbd => {
@@ -188,7 +204,7 @@ exports.importDeltaVm = defer(async function importDeltaVm(
           [TAG_BASE_DELTA]: undefined,
           [TAG_COPY_SRC]: vdi.uuid,
         },
-        sr: mapVdisSrs[vdi.uuid] ?? sr.$ref,
+        sr: mapVdisSrRefs[vdi.uuid] ?? sr.$ref,
       })
     )
     $defer.onFailure(() => suspendVdi.$destroy())
@@ -255,7 +271,7 @@ exports.importDeltaVm = defer(async function importDeltaVm(
             [TAG_BASE_DELTA]: undefined,
             [TAG_COPY_SRC]: vdi.uuid,
           },
-          SR: mapVdisSrs[vdi.uuid] ?? sr.$ref,
+          SR: mapVdisSrRefs[vdi.uuid] ?? sr.$ref,
         })
       )
       $defer.onFailure(() => newVdi.$destroy())

@xen-orchestra/backups/_extractIdsFromSimplePattern.js

@@ -1,3 +1,5 @@
+'use strict'
+
 exports.extractIdsFromSimplePattern = function extractIdsFromSimplePattern(pattern) {
   if (pattern === undefined) {
     return []

@xen-orchestra/backups/_filenameDate.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { utcFormat, utcParse } = require('d3-time-format')
 
 // Format a date in ISO 8601 in a safe way to be used in filenames

@xen-orchestra/backups/_forkStreamUnpipe.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const eos = require('end-of-stream')
 const { PassThrough } = require('stream')
 

@xen-orchestra/backups/_getOldEntries.js

@@ -1,3 +1,5 @@
+'use strict'
+
 // returns all entries but the last retention-th
 exports.getOldEntries = function getOldEntries(retention, entries) {
   return entries === undefined ? [] : retention > 0 ? entries.slice(0, -retention) : entries

@xen-orchestra/backups/_getTmpDir.js

@@ -1,4 +1,6 @@
-const Disposable = require('promise-toolbox/Disposable.js')
+'use strict'
+
+const Disposable = require('promise-toolbox/Disposable')
 const { join } = require('path')
 const { mkdir, rmdir } = require('fs-extra')
 const { tmpdir } = require('os')

@xen-orchestra/backups/_getVmBackupDir.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const BACKUP_DIR = 'xo-vm-backups'
 exports.BACKUP_DIR = BACKUP_DIR
 

@xen-orchestra/backups/_isValidXva.js

@@ -1,11 +1,26 @@
+'use strict'
+
 const assert = require('assert')
 
-const isGzipFile = async (handler, fd) => {
+const COMPRESSED_MAGIC_NUMBERS = [
   // https://tools.ietf.org/html/rfc1952.html#page-5
-  const magicNumber = Buffer.allocUnsafe(2)
+  Buffer.from('1F8B', 'hex'),
 
-  assert.strictEqual((await handler.read(fd, magicNumber, 0)).bytesRead, magicNumber.length)
-  return magicNumber[0] === 31 && magicNumber[1] === 139
+  // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#zstandard-frames
+  Buffer.from('28B52FFD', 'hex'),
+]
+const MAGIC_NUMBER_MAX_LENGTH = Math.max(...COMPRESSED_MAGIC_NUMBERS.map(_ => _.length))
+
+const isCompressedFile = async (handler, fd) => {
+  const header = Buffer.allocUnsafe(MAGIC_NUMBER_MAX_LENGTH)
+  assert.strictEqual((await handler.read(fd, header, 0)).bytesRead, header.length)
+
+  for (const magicNumber of COMPRESSED_MAGIC_NUMBERS) {
+    if (magicNumber.compare(header, 0, magicNumber.length) === 0) {
+      return true
+    }
+  }
+  return false
 }
 
 // TODO: better check?
@@ -43,8 +58,8 @@ async function isValidXva(path) {
         return false
       }
 
-      return (await isGzipFile(handler, fd))
-        ? true // gzip files cannot be validated at this time
+      return (await isCompressedFile(handler, fd))
+        ? true // compressed files cannot be validated at this time
         : await isValidTar(handler, size, fd)
     } finally {
       handler.closeFile(fd).catch(noop)

@xen-orchestra/backups/_listPartitions.js

@@ -1,4 +1,6 @@
-const fromCallback = require('promise-toolbox/fromCallback.js')
+'use strict'
+
+const fromCallback = require('promise-toolbox/fromCallback')
 const { createLogger } = require('@xen-orchestra/log')
 const { createParser } = require('parse-pairs')
 const { execFile } = require('child_process')

@xen-orchestra/backups/_lvm.js

@@ -1,4 +1,6 @@
-const fromCallback = require('promise-toolbox/fromCallback.js')
+'use strict'
+
+const fromCallback = require('promise-toolbox/fromCallback')
 const { createParser } = require('parse-pairs')
 const { execFile } = require('child_process')
 

@xen-orchestra/backups/_watchStreamSize.js

@@ -1,3 +1,5 @@
+'use strict'
+
 exports.watchStreamSize = function watchStreamSize(stream, container = { size: 0 }) {
   stream.on('data', data => {
     container.size += data.length

@xen-orchestra/backups/formatVmBackups.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const mapValues = require('lodash/mapValues.js')
 const { dirname } = require('path')
 

@xen-orchestra/backups/merge-worker/cli.js

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+
+'use strict'
+
+const { catchGlobalErrors } = require('@xen-orchestra/log/configure.js')
+const { createLogger } = require('@xen-orchestra/log')
+const { getSyncedHandler } = require('@xen-orchestra/fs')
+const { join } = require('path')
+const Disposable = require('promise-toolbox/Disposable')
+const min = require('lodash/min')
+
+const { getVmBackupDir } = require('../_getVmBackupDir.js')
+const { RemoteAdapter } = require('../RemoteAdapter.js')
+
+const { CLEAN_VM_QUEUE } = require('./index.js')
+
+// -------------------------------------------------------------------
+
+catchGlobalErrors(createLogger('xo:backups:mergeWorker'))
+
+const { fatal, info, warn } = createLogger('xo:backups:mergeWorker')
+
+// -------------------------------------------------------------------
+
+const main = Disposable.wrap(async function* main(args) {
+  const handler = yield getSyncedHandler({ url: 'file://' + process.cwd() })
+
+  yield handler.lock(CLEAN_VM_QUEUE)
+
+  const adapter = new RemoteAdapter(handler)
+
+  const listRetry = async () => {
+    const timeoutResolver = resolve => setTimeout(resolve, 10e3)
+    for (let i = 0; i < 10; ++i) {
+      const entries = await handler.list(CLEAN_VM_QUEUE)
+      if (entries.length !== 0) {
+        return entries
+      }
+      await new Promise(timeoutResolver)
+    }
+  }
+
+  let taskFiles
+  while ((taskFiles = await listRetry()) !== undefined) {
+    const taskFileBasename = min(taskFiles)
+    const previousTaskFile = join(CLEAN_VM_QUEUE, taskFileBasename)
+    const taskFile = join(CLEAN_VM_QUEUE, '_' + taskFileBasename)
+
+    // move this task to the end
+    try {
+      await handler.rename(previousTaskFile, taskFile)
+    } catch (error) {
+      // this error occurs if the task failed too many times (i.e. too many `_` prefixes)
+      // there is nothing more that can be done
+      if (error.code === 'ENAMETOOLONG') {
+        await handler.unlink(previousTaskFile)
+      }
+
+      throw error
+    }
+
+    try {
+      const vmDir = getVmBackupDir(String(await handler.readFile(taskFile)))
+      try {
+        await adapter.cleanVm(vmDir, { merge: true, onLog: info, remove: true })
+      } catch (error) {
+        // consider the clean successful if the VM dir is missing
+        if (error.code !== 'ENOENT') {
+          throw error
+        }
+      }
+
+      handler.unlink(taskFile).catch(error => warn('deleting task failure', { error }))
+    } catch (error) {
+      warn('failure handling task', { error })
+    }
+  }
+})
+
+info('starting')
+main(process.argv.slice(2)).then(
+  () => {
+    info('bye :-)')
+  },
+  error => {
+    fatal(error)
+
+    process.exit(1)
+  }
+)

@xen-orchestra/backups/merge-worker/index.js

@@ -0,0 +1,27 @@
+'use strict'
+
+const { join, resolve } = require('path')
+const { spawn } = require('child_process')
+const { check } = require('proper-lockfile')
+
+const CLEAN_VM_QUEUE = (exports.CLEAN_VM_QUEUE = '/xo-vm-backups/.queue/clean-vm/')
+
+const CLI_PATH = resolve(__dirname, 'cli.js')
+exports.run = async function runMergeWorker(remotePath) {
+  try {
+    // TODO: find a way to pass the acquire the lock and then pass it down the worker
+    if (await check(join(remotePath, CLEAN_VM_QUEUE))) {
+      // already locked, don't start another worker
+      return
+    }
+
+    spawn(CLI_PATH, {
+      cwd: remotePath,
+      detached: true,
+      stdio: 'inherit',
+    }).unref()
+  } catch (error) {
+    // we usually don't want to throw if the merge worker failed to start
+    return error
+  }
+}

@xen-orchestra/backups/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.13.0",
+  "version": "0.20.0",
   "engines": {
     "node": ">=14.6"
   },
@@ -16,14 +16,15 @@
     "postversion": "npm publish --access public"
   },
   "dependencies": {
-    "@vates/compose": "^2.0.0",
+    "@vates/compose": "^2.1.0",
+    "@vates/decorate-with": "^1.0.0",
     "@vates/disposable": "^0.1.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^0.18.0",
+    "@xen-orchestra/fs": "^0.20.0",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/template": "^0.1.0",
-    "compare-versions": "^3.6.0",
+    "compare-versions": "^4.0.1",
     "d3-time-format": "^3.0.0",
     "end-of-stream": "^1.4.4",
     "fs-extra": "^10.0.0",
@@ -32,13 +33,14 @@
     "lodash": "^4.17.20",
     "node-zone": "^0.4.0",
     "parse-pairs": "^1.1.0",
-    "promise-toolbox": "^0.19.2",
-    "pump": "^3.0.0",
-    "vhd-lib": "^1.2.0",
+    "promise-toolbox": "^0.21.0",
+    "proper-lockfile": "^4.1.2",
+    "uuid": "^8.3.2",
+    "vhd-lib": "^3.1.0",
     "yazl": "^2.5.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^0.7.0"
+    "@xen-orchestra/xapi": "^0.9.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

@xen-orchestra/backups/parseMetadataBackupId.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { DIR_XO_CONFIG_BACKUPS, DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter.js')
 
 exports.parseMetadataBackupId = function parseMetadataBackupId(backupId) {

@xen-orchestra/backups/runBackupWorker.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const path = require('path')
 const { createLogger } = require('@xen-orchestra/log')
 const { fork } = require('child_process')

@xen-orchestra/backups/tests.fixtures.js

@@ -0,0 +1,94 @@
+'use strict'
+
+// a valid footer of a 2
+exports.VHDFOOTER = {
+  cookie: 'conectix',
+  features: 2,
+  fileFormatVersion: 65536,
+  dataOffset: 512,
+  timestamp: 0,
+  creatorApplication: 'caml',
+  creatorVersion: 1,
+  creatorHostOs: 0,
+  originalSize: 53687091200,
+  currentSize: 53687091200,
+  diskGeometry: { cylinders: 25700, heads: 16, sectorsPerTrackCylinder: 255 },
+  diskType: 3,
+  checksum: 4294962945,
+  uuid: Buffer.from('d8dbcad85265421e8b298d99c2eec551', 'utf-8'),
+  saved: '',
+  hidden: '',
+  reserved: '',
+}
+exports.VHDHEADER = {
+  cookie: 'cxsparse',
+  dataOffset: undefined,
+  tableOffset: 2048,
+  headerVersion: 65536,
+  maxTableEntries: 25600,
+  blockSize: 2097152,
+  checksum: 4294964241,
+  parentUuid: null,
+  parentTimestamp: 0,
+  reserved1: 0,
+  parentUnicodeName: '',
+  parentLocatorEntry: [
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+  ],
+  reserved2: '',
+}

@xen-orchestra/backups/writers/DeltaBackupWriter.js

@@ -1,9 +1,11 @@
+'use strict'
+
 const assert = require('assert')
 const map = require('lodash/map.js')
 const mapValues = require('lodash/mapValues.js')
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { asyncMap } = require('@xen-orchestra/async-map')
-const { chainVhd, checkVhdChain, default: Vhd } = require('vhd-lib')
+const { chainVhd, checkVhdChain, openVhd, VhdAbstract } = require('vhd-lib')
 const { createLogger } = require('@xen-orchestra/log')
 const { dirname } = require('path')
 
@@ -16,6 +18,7 @@ const { MixinBackupWriter } = require('./_MixinBackupWriter.js')
 const { AbstractDeltaWriter } = require('./_AbstractDeltaWriter.js')
 const { checkVhd } = require('./_checkVhd.js')
 const { packUuid } = require('./_packUuid.js')
+const { Disposable } = require('promise-toolbox')
 
 const { warn } = createLogger('xo:backups:DeltaBackupWriter')
 
@@ -23,6 +26,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
   async checkBaseVdis(baseUuidToSrcVdi) {
     const { handler } = this._adapter
     const backup = this._backup
+    const adapter = this._adapter
 
     const backupDir = getVmBackupDir(backup.vm.uuid)
     const vdisDir = `${backupDir}/vdis/${backup.job.id}`
@@ -34,16 +38,21 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           filter: _ => _[0] !== '.' && _.endsWith('.vhd'),
           prependDir: true,
         })
+        const packedBaseUuid = packUuid(baseUuid)
         await asyncMap(vhds, async path => {
           try {
             await checkVhdChain(handler, path)
+            // Warning, this should not be written as found = found || await adapter.isMergeableParent(packedBaseUuid, path)
+            //
+            // since all the checks of a path are done in parallel, found would be containing
+            // only the last answer of isMergeableParent which is probably not the right one
+            // this led to the support tickets  https://help.vates.fr/#ticket/zoom/4751 , 4729, 4665 and 4300
 
-            const vhd = new Vhd(handler, path)
-            await vhd.readHeaderAndFooter()
-            found = found || vhd.footer.uuid.equals(packUuid(baseUuid))
+            const isMergeable = await adapter.isMergeableParent(packedBaseUuid, path)
+            found = found || isMergeable
           } catch (error) {
             warn('checkBaseVdis', { error })
-            await ignoreErrors.call(handler.unlink(path))
+            await ignoreErrors.call(VhdAbstract.unlink(handler, path))
           }
         })
       } catch (error) {
@@ -113,19 +122,13 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
   }
 
   async _deleteOldEntries() {
-    return Task.run({ name: 'merge' }, async () => {
-      const adapter = this._adapter
-      const oldEntries = this._oldEntries
+    const adapter = this._adapter
+    const oldEntries = this._oldEntries
 
-      let size = 0
-      // delete sequentially from newest to oldest to avoid unnecessary merges
-      for (let i = oldEntries.length; i-- > 0; ) {
-        size += await adapter.deleteDeltaVmBackups([oldEntries[i]])
-      }
-      return {
-        size,
-      }
-    })
+    // delete sequentially from newest to oldest to avoid unnecessary merges
+    for (let i = oldEntries.length; i-- > 0; ) {
+      await adapter.deleteDeltaVmBackups([oldEntries[i]])
+    }
   }
 
   async _transfer({ timestamp, deltaExport, sizeContainers }) {
@@ -150,7 +153,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
               // don't do delta for it
               vdi.uuid
             : vdi.$snapshot_of$uuid
-        }/${basename}.vhd`
+        }/${adapter.getVhdFileName(basename)}`
     )
 
     const metadataFilename = `${backupDir}/${basename}.json`
@@ -194,7 +197,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
             await checkVhd(handler, parentPath)
           }
 
-          await adapter.outputStream(path, deltaExport.streams[`${id}.vhd`], {
+          await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
             // no checksum for VHDs, because they will be invalidated by
             // merges and chainings
             checksum: false,
@@ -206,11 +209,11 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           }
 
           // set the correct UUID in the VHD
-          const vhd = new Vhd(handler, path)
-          await vhd.readHeaderAndFooter()
-          vhd.footer.uuid = packUuid(vdi.uuid)
-          await vhd.readBlockAllocationTable() // required by writeFooter()
-          await vhd.writeFooter()
+          await Disposable.use(openVhd(handler, path), async vhd => {
+            vhd.footer.uuid = packUuid(vdi.uuid)
+            await vhd.readBlockAllocationTable() // required by writeFooter()
+            await vhd.writeFooter()
+          })
         })
       )
       return {

@xen-orchestra/backups/writers/DeltaReplicationWriter.js

@@ -1,5 +1,7 @@
+'use strict'
+
 const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { formatDateTime } = require('@xen-orchestra/xapi')
 
 const { formatFilenameDate } = require('../_filenameDate.js')

@xen-orchestra/backups/writers/FullBackupWriter.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { formatFilenameDate } = require('../_filenameDate.js')
 const { getOldEntries } = require('../_getOldEntries.js')
 const { getVmBackupDir } = require('../_getVmBackupDir.js')

@xen-orchestra/backups/writers/FullReplicationWriter.js

@@ -1,4 +1,6 @@
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+'use strict'
+
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
 const { formatDateTime } = require('@xen-orchestra/xapi')
 

@xen-orchestra/backups/writers/_AbstractDeltaWriter.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { AbstractWriter } = require('./_AbstractWriter.js')
 
 exports.AbstractDeltaWriter = class AbstractDeltaWriter extends AbstractWriter {

@xen-orchestra/backups/writers/_AbstractFullWriter.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { AbstractWriter } = require('./_AbstractWriter.js')
 
 exports.AbstractFullWriter = class AbstractFullWriter extends AbstractWriter {